1 /* 2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "code/compiledIC.hpp" 29 #include "compiler/compiler_globals.hpp" 30 #include "compiler/disassembler.hpp" 31 #include "crc32c.h" 32 #include "gc/shared/barrierSet.hpp" 33 #include "gc/shared/barrierSetAssembler.hpp" 34 #include "gc/shared/collectedHeap.inline.hpp" 35 #include "gc/shared/tlab_globals.hpp" 36 #include "interpreter/bytecodeHistogram.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "jvm.h" 39 #include "memory/resourceArea.hpp" 40 #include "memory/universe.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedKlass.inline.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/klass.inline.hpp" 45 #include "prims/methodHandles.hpp" 46 #include "runtime/continuation.hpp" 47 #include "runtime/interfaceSupport.inline.hpp" 48 #include "runtime/javaThread.hpp" 49 #include "runtime/jniHandles.hpp" 50 #include "runtime/objectMonitor.hpp" 51 #include "runtime/os.hpp" 52 #include "runtime/safepoint.hpp" 53 #include "runtime/safepointMechanism.hpp" 54 #include "runtime/sharedRuntime.hpp" 55 #include "runtime/stubRoutines.hpp" 56 #include "utilities/checkedCast.hpp" 57 #include "utilities/macros.hpp" 58 59 #ifdef PRODUCT 60 #define BLOCK_COMMENT(str) /* nothing */ 61 #define STOP(error) stop(error) 62 #else 63 #define BLOCK_COMMENT(str) block_comment(str) 64 #define STOP(error) block_comment(error); stop(error) 65 #endif 66 67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 68 69 #ifdef ASSERT 70 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 71 #endif 72 73 static const Assembler::Condition reverse[] = { 74 Assembler::noOverflow /* overflow = 0x0 */ , 75 Assembler::overflow /* noOverflow = 0x1 */ , 76 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 77 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 78 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 79 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 80 Assembler::above /* belowEqual = 0x6 */ , 81 Assembler::belowEqual /* above = 0x7 */ , 82 Assembler::positive /* negative = 0x8 */ , 83 Assembler::negative /* positive = 0x9 */ , 84 Assembler::noParity /* parity = 0xa */ , 85 Assembler::parity /* noParity = 0xb */ , 86 Assembler::greaterEqual /* less = 0xc */ , 87 Assembler::less /* greaterEqual = 0xd */ , 88 Assembler::greater /* lessEqual = 0xe */ , 89 Assembler::lessEqual /* greater = 0xf, */ 90 91 }; 92 93 94 // Implementation of MacroAssembler 95 96 // First all the versions that have distinct versions depending on 32/64 bit 97 // Unless the difference is trivial (1 line or so). 98 99 #ifndef _LP64 100 101 // 32bit versions 102 103 Address MacroAssembler::as_Address(AddressLiteral adr) { 104 return Address(adr.target(), adr.rspec()); 105 } 106 107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 108 assert(rscratch == noreg, ""); 109 return Address::make_array(adr); 110 } 111 112 void MacroAssembler::call_VM_leaf_base(address entry_point, 113 int number_of_arguments) { 114 call(RuntimeAddress(entry_point)); 115 increment(rsp, number_of_arguments * wordSize); 116 } 117 118 void MacroAssembler::cmpklass(Address src1, Metadata* obj) { 119 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 120 } 121 122 123 void MacroAssembler::cmpklass(Register src1, Metadata* obj) { 124 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 125 } 126 127 void MacroAssembler::cmpoop(Address src1, jobject obj) { 128 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 129 } 130 131 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) { 132 assert(rscratch == noreg, "redundant"); 133 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 134 } 135 136 void MacroAssembler::extend_sign(Register hi, Register lo) { 137 // According to Intel Doc. AP-526, "Integer Divide", p.18. 138 if (VM_Version::is_P6() && hi == rdx && lo == rax) { 139 cdql(); 140 } else { 141 movl(hi, lo); 142 sarl(hi, 31); 143 } 144 } 145 146 void MacroAssembler::jC2(Register tmp, Label& L) { 147 // set parity bit if FPU flag C2 is set (via rax) 148 save_rax(tmp); 149 fwait(); fnstsw_ax(); 150 sahf(); 151 restore_rax(tmp); 152 // branch 153 jcc(Assembler::parity, L); 154 } 155 156 void MacroAssembler::jnC2(Register tmp, Label& L) { 157 // set parity bit if FPU flag C2 is set (via rax) 158 save_rax(tmp); 159 fwait(); fnstsw_ax(); 160 sahf(); 161 restore_rax(tmp); 162 // branch 163 jcc(Assembler::noParity, L); 164 } 165 166 // 32bit can do a case table jump in one instruction but we no longer allow the base 167 // to be installed in the Address class 168 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 169 assert(rscratch == noreg, "not needed"); 170 jmp(as_Address(entry, noreg)); 171 } 172 173 // Note: y_lo will be destroyed 174 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 175 // Long compare for Java (semantics as described in JVM spec.) 176 Label high, low, done; 177 178 cmpl(x_hi, y_hi); 179 jcc(Assembler::less, low); 180 jcc(Assembler::greater, high); 181 // x_hi is the return register 182 xorl(x_hi, x_hi); 183 cmpl(x_lo, y_lo); 184 jcc(Assembler::below, low); 185 jcc(Assembler::equal, done); 186 187 bind(high); 188 xorl(x_hi, x_hi); 189 increment(x_hi); 190 jmp(done); 191 192 bind(low); 193 xorl(x_hi, x_hi); 194 decrementl(x_hi); 195 196 bind(done); 197 } 198 199 void MacroAssembler::lea(Register dst, AddressLiteral src) { 200 mov_literal32(dst, (int32_t)src.target(), src.rspec()); 201 } 202 203 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 204 assert(rscratch == noreg, "not needed"); 205 206 // leal(dst, as_Address(adr)); 207 // see note in movl as to why we must use a move 208 mov_literal32(dst, (int32_t)adr.target(), adr.rspec()); 209 } 210 211 void MacroAssembler::leave() { 212 mov(rsp, rbp); 213 pop(rbp); 214 } 215 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { 217 // Multiplication of two Java long values stored on the stack 218 // as illustrated below. Result is in rdx:rax. 219 // 220 // rsp ---> [ ?? ] \ \ 221 // .... | y_rsp_offset | 222 // [ y_lo ] / (in bytes) | x_rsp_offset 223 // [ y_hi ] | (in bytes) 224 // .... | 225 // [ x_lo ] / 226 // [ x_hi ] 227 // .... 228 // 229 // Basic idea: lo(result) = lo(x_lo * y_lo) 230 // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) 231 Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); 232 Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); 233 Label quick; 234 // load x_hi, y_hi and check if quick 235 // multiplication is possible 236 movl(rbx, x_hi); 237 movl(rcx, y_hi); 238 movl(rax, rbx); 239 orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 240 jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply 241 // do full multiplication 242 // 1st step 243 mull(y_lo); // x_hi * y_lo 244 movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, 245 // 2nd step 246 movl(rax, x_lo); 247 mull(rcx); // x_lo * y_hi 248 addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, 249 // 3rd step 250 bind(quick); // note: rbx, = 0 if quick multiply! 251 movl(rax, x_lo); 252 mull(y_lo); // x_lo * y_lo 253 addl(rdx, rbx); // correct hi(x_lo * y_lo) 254 } 255 256 void MacroAssembler::lneg(Register hi, Register lo) { 257 negl(lo); 258 adcl(hi, 0); 259 negl(hi); 260 } 261 262 void MacroAssembler::lshl(Register hi, Register lo) { 263 // Java shift left long support (semantics as described in JVM spec., p.305) 264 // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) 265 // shift value is in rcx ! 266 assert(hi != rcx, "must not use rcx"); 267 assert(lo != rcx, "must not use rcx"); 268 const Register s = rcx; // shift count 269 const int n = BitsPerWord; 270 Label L; 271 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 272 cmpl(s, n); // if (s < n) 273 jcc(Assembler::less, L); // else (s >= n) 274 movl(hi, lo); // x := x << n 275 xorl(lo, lo); 276 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 277 bind(L); // s (mod n) < n 278 shldl(hi, lo); // x := x << s 279 shll(lo); 280 } 281 282 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { 284 // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) 285 // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) 286 assert(hi != rcx, "must not use rcx"); 287 assert(lo != rcx, "must not use rcx"); 288 const Register s = rcx; // shift count 289 const int n = BitsPerWord; 290 Label L; 291 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 292 cmpl(s, n); // if (s < n) 293 jcc(Assembler::less, L); // else (s >= n) 294 movl(lo, hi); // x := x >> n 295 if (sign_extension) sarl(hi, 31); 296 else xorl(hi, hi); 297 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 298 bind(L); // s (mod n) < n 299 shrdl(lo, hi); // x := x >> s 300 if (sign_extension) sarl(hi); 301 else shrl(hi); 302 } 303 304 void MacroAssembler::movoop(Register dst, jobject obj) { 305 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 306 } 307 308 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 309 assert(rscratch == noreg, "redundant"); 310 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 311 } 312 313 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 314 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 315 } 316 317 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 318 assert(rscratch == noreg, "redundant"); 319 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 320 } 321 322 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 323 if (src.is_lval()) { 324 mov_literal32(dst, (intptr_t)src.target(), src.rspec()); 325 } else { 326 movl(dst, as_Address(src)); 327 } 328 } 329 330 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 331 assert(rscratch == noreg, "redundant"); 332 movl(as_Address(dst, noreg), src); 333 } 334 335 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 336 movl(dst, as_Address(src, noreg)); 337 } 338 339 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 340 assert(rscratch == noreg, "redundant"); 341 movl(dst, src); 342 } 343 344 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 345 assert(rscratch == noreg, "redundant"); 346 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); 347 } 348 349 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 350 assert(rscratch == noreg, "redundant"); 351 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); 352 } 353 354 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 355 assert(rscratch == noreg, "redundant"); 356 if (src.is_lval()) { 357 push_literal32((int32_t)src.target(), src.rspec()); 358 } else { 359 pushl(as_Address(src)); 360 } 361 } 362 363 static void pass_arg0(MacroAssembler* masm, Register arg) { 364 masm->push(arg); 365 } 366 367 static void pass_arg1(MacroAssembler* masm, Register arg) { 368 masm->push(arg); 369 } 370 371 static void pass_arg2(MacroAssembler* masm, Register arg) { 372 masm->push(arg); 373 } 374 375 static void pass_arg3(MacroAssembler* masm, Register arg) { 376 masm->push(arg); 377 } 378 379 #ifndef PRODUCT 380 extern "C" void findpc(intptr_t x); 381 #endif 382 383 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { 384 // In order to get locks to work, we need to fake a in_VM state 385 JavaThread* thread = JavaThread::current(); 386 JavaThreadState saved_state = thread->thread_state(); 387 thread->set_thread_state(_thread_in_vm); 388 if (ShowMessageBoxOnError) { 389 JavaThread* thread = JavaThread::current(); 390 JavaThreadState saved_state = thread->thread_state(); 391 thread->set_thread_state(_thread_in_vm); 392 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 393 ttyLocker ttyl; 394 BytecodeCounter::print(); 395 } 396 // To see where a verify_oop failed, get $ebx+40/X for this frame. 397 // This is the value of eip which points to where verify_oop will return. 398 if (os::message_box(msg, "Execution stopped, print registers?")) { 399 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); 400 BREAKPOINT; 401 } 402 } 403 fatal("DEBUG MESSAGE: %s", msg); 404 } 405 406 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { 407 ttyLocker ttyl; 408 DebuggingContext debugging{}; 409 tty->print_cr("eip = 0x%08x", eip); 410 #ifndef PRODUCT 411 if ((WizardMode || Verbose) && PrintMiscellaneous) { 412 tty->cr(); 413 findpc(eip); 414 tty->cr(); 415 } 416 #endif 417 #define PRINT_REG(rax) \ 418 { tty->print("%s = ", #rax); os::print_location(tty, rax); } 419 PRINT_REG(rax); 420 PRINT_REG(rbx); 421 PRINT_REG(rcx); 422 PRINT_REG(rdx); 423 PRINT_REG(rdi); 424 PRINT_REG(rsi); 425 PRINT_REG(rbp); 426 PRINT_REG(rsp); 427 #undef PRINT_REG 428 // Print some words near top of staack. 429 int* dump_sp = (int*) rsp; 430 for (int col1 = 0; col1 < 8; col1++) { 431 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 432 os::print_location(tty, *dump_sp++); 433 } 434 for (int row = 0; row < 16; row++) { 435 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 436 for (int col = 0; col < 8; col++) { 437 tty->print(" 0x%08x", *dump_sp++); 438 } 439 tty->cr(); 440 } 441 // Print some instructions around pc: 442 Disassembler::decode((address)eip-64, (address)eip); 443 tty->print_cr("--------"); 444 Disassembler::decode((address)eip, (address)eip+32); 445 } 446 447 void MacroAssembler::stop(const char* msg) { 448 // push address of message 449 ExternalAddress message((address)msg); 450 pushptr(message.addr(), noreg); 451 { Label L; call(L, relocInfo::none); bind(L); } // push eip 452 pusha(); // push registers 453 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); 454 hlt(); 455 } 456 457 void MacroAssembler::warn(const char* msg) { 458 push_CPU_state(); 459 460 // push address of message 461 ExternalAddress message((address)msg); 462 pushptr(message.addr(), noreg); 463 464 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 465 addl(rsp, wordSize); // discard argument 466 pop_CPU_state(); 467 } 468 469 void MacroAssembler::print_state() { 470 { Label L; call(L, relocInfo::none); bind(L); } // push eip 471 pusha(); // push registers 472 473 push_CPU_state(); 474 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); 475 pop_CPU_state(); 476 477 popa(); 478 addl(rsp, wordSize); 479 } 480 481 #else // _LP64 482 483 // 64 bit versions 484 485 Address MacroAssembler::as_Address(AddressLiteral adr) { 486 // amd64 always does this as a pc-rel 487 // we can be absolute or disp based on the instruction type 488 // jmp/call are displacements others are absolute 489 assert(!adr.is_lval(), "must be rval"); 490 assert(reachable(adr), "must be"); 491 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 492 493 } 494 495 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 496 AddressLiteral base = adr.base(); 497 lea(rscratch, base); 498 Address index = adr.index(); 499 assert(index._disp == 0, "must not have disp"); // maybe it can? 500 Address array(rscratch, index._index, index._scale, index._disp); 501 return array; 502 } 503 504 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 505 Label L, E; 506 507 #ifdef _WIN64 508 // Windows always allocates space for it's register args 509 assert(num_args <= 4, "only register arguments supported"); 510 subq(rsp, frame::arg_reg_save_area_bytes); 511 #endif 512 513 // Align stack if necessary 514 testl(rsp, 15); 515 jcc(Assembler::zero, L); 516 517 subq(rsp, 8); 518 call(RuntimeAddress(entry_point)); 519 addq(rsp, 8); 520 jmp(E); 521 522 bind(L); 523 call(RuntimeAddress(entry_point)); 524 525 bind(E); 526 527 #ifdef _WIN64 528 // restore stack pointer 529 addq(rsp, frame::arg_reg_save_area_bytes); 530 #endif 531 532 } 533 534 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 535 assert(!src2.is_lval(), "should use cmpptr"); 536 assert(rscratch != noreg || always_reachable(src2), "missing"); 537 538 if (reachable(src2)) { 539 cmpq(src1, as_Address(src2)); 540 } else { 541 lea(rscratch, src2); 542 Assembler::cmpq(src1, Address(rscratch, 0)); 543 } 544 } 545 546 int MacroAssembler::corrected_idivq(Register reg) { 547 // Full implementation of Java ldiv and lrem; checks for special 548 // case as described in JVM spec., p.243 & p.271. The function 549 // returns the (pc) offset of the idivl instruction - may be needed 550 // for implicit exceptions. 551 // 552 // normal case special case 553 // 554 // input : rax: dividend min_long 555 // reg: divisor (may not be eax/edx) -1 556 // 557 // output: rax: quotient (= rax idiv reg) min_long 558 // rdx: remainder (= rax irem reg) 0 559 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 560 static const int64_t min_long = 0x8000000000000000; 561 Label normal_case, special_case; 562 563 // check for special case 564 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 565 jcc(Assembler::notEqual, normal_case); 566 xorl(rdx, rdx); // prepare rdx for possible special case (where 567 // remainder = 0) 568 cmpq(reg, -1); 569 jcc(Assembler::equal, special_case); 570 571 // handle normal case 572 bind(normal_case); 573 cdqq(); 574 int idivq_offset = offset(); 575 idivq(reg); 576 577 // normal and special case exit 578 bind(special_case); 579 580 return idivq_offset; 581 } 582 583 void MacroAssembler::decrementq(Register reg, int value) { 584 if (value == min_jint) { subq(reg, value); return; } 585 if (value < 0) { incrementq(reg, -value); return; } 586 if (value == 0) { ; return; } 587 if (value == 1 && UseIncDec) { decq(reg) ; return; } 588 /* else */ { subq(reg, value) ; return; } 589 } 590 591 void MacroAssembler::decrementq(Address dst, int value) { 592 if (value == min_jint) { subq(dst, value); return; } 593 if (value < 0) { incrementq(dst, -value); return; } 594 if (value == 0) { ; return; } 595 if (value == 1 && UseIncDec) { decq(dst) ; return; } 596 /* else */ { subq(dst, value) ; return; } 597 } 598 599 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 600 assert(rscratch != noreg || always_reachable(dst), "missing"); 601 602 if (reachable(dst)) { 603 incrementq(as_Address(dst)); 604 } else { 605 lea(rscratch, dst); 606 incrementq(Address(rscratch, 0)); 607 } 608 } 609 610 void MacroAssembler::incrementq(Register reg, int value) { 611 if (value == min_jint) { addq(reg, value); return; } 612 if (value < 0) { decrementq(reg, -value); return; } 613 if (value == 0) { ; return; } 614 if (value == 1 && UseIncDec) { incq(reg) ; return; } 615 /* else */ { addq(reg, value) ; return; } 616 } 617 618 void MacroAssembler::incrementq(Address dst, int value) { 619 if (value == min_jint) { addq(dst, value); return; } 620 if (value < 0) { decrementq(dst, -value); return; } 621 if (value == 0) { ; return; } 622 if (value == 1 && UseIncDec) { incq(dst) ; return; } 623 /* else */ { addq(dst, value) ; return; } 624 } 625 626 // 32bit can do a case table jump in one instruction but we no longer allow the base 627 // to be installed in the Address class 628 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 629 lea(rscratch, entry.base()); 630 Address dispatch = entry.index(); 631 assert(dispatch._base == noreg, "must be"); 632 dispatch._base = rscratch; 633 jmp(dispatch); 634 } 635 636 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 637 ShouldNotReachHere(); // 64bit doesn't use two regs 638 cmpq(x_lo, y_lo); 639 } 640 641 void MacroAssembler::lea(Register dst, AddressLiteral src) { 642 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 643 } 644 645 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 646 lea(rscratch, adr); 647 movptr(dst, rscratch); 648 } 649 650 void MacroAssembler::leave() { 651 // %%% is this really better? Why not on 32bit too? 652 emit_int8((unsigned char)0xC9); // LEAVE 653 } 654 655 void MacroAssembler::lneg(Register hi, Register lo) { 656 ShouldNotReachHere(); // 64bit doesn't use two regs 657 negq(lo); 658 } 659 660 void MacroAssembler::movoop(Register dst, jobject obj) { 661 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 662 } 663 664 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 665 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 666 movq(dst, rscratch); 667 } 668 669 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 670 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 671 } 672 673 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 674 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 675 movq(dst, rscratch); 676 } 677 678 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 679 if (src.is_lval()) { 680 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 681 } else { 682 if (reachable(src)) { 683 movq(dst, as_Address(src)); 684 } else { 685 lea(dst, src); 686 movq(dst, Address(dst, 0)); 687 } 688 } 689 } 690 691 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 692 movq(as_Address(dst, rscratch), src); 693 } 694 695 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 696 movq(dst, as_Address(src, dst /*rscratch*/)); 697 } 698 699 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 700 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 701 if (is_simm32(src)) { 702 movptr(dst, checked_cast<int32_t>(src)); 703 } else { 704 mov64(rscratch, src); 705 movq(dst, rscratch); 706 } 707 } 708 709 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 710 movoop(rscratch, obj); 711 push(rscratch); 712 } 713 714 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 715 mov_metadata(rscratch, obj); 716 push(rscratch); 717 } 718 719 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 720 lea(rscratch, src); 721 if (src.is_lval()) { 722 push(rscratch); 723 } else { 724 pushq(Address(rscratch, 0)); 725 } 726 } 727 728 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 729 reset_last_Java_frame(r15_thread, clear_fp); 730 } 731 732 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 733 Register last_java_fp, 734 address last_java_pc, 735 Register rscratch) { 736 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch); 737 } 738 739 static void pass_arg0(MacroAssembler* masm, Register arg) { 740 if (c_rarg0 != arg ) { 741 masm->mov(c_rarg0, arg); 742 } 743 } 744 745 static void pass_arg1(MacroAssembler* masm, Register arg) { 746 if (c_rarg1 != arg ) { 747 masm->mov(c_rarg1, arg); 748 } 749 } 750 751 static void pass_arg2(MacroAssembler* masm, Register arg) { 752 if (c_rarg2 != arg ) { 753 masm->mov(c_rarg2, arg); 754 } 755 } 756 757 static void pass_arg3(MacroAssembler* masm, Register arg) { 758 if (c_rarg3 != arg ) { 759 masm->mov(c_rarg3, arg); 760 } 761 } 762 763 void MacroAssembler::stop(const char* msg) { 764 if (ShowMessageBoxOnError) { 765 address rip = pc(); 766 pusha(); // get regs on stack 767 lea(c_rarg1, InternalAddress(rip)); 768 movq(c_rarg2, rsp); // pass pointer to regs array 769 } 770 lea(c_rarg0, ExternalAddress((address) msg)); 771 andq(rsp, -16); // align stack as required by ABI 772 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 773 hlt(); 774 } 775 776 void MacroAssembler::warn(const char* msg) { 777 push(rbp); 778 movq(rbp, rsp); 779 andq(rsp, -16); // align stack as required by push_CPU_state and call 780 push_CPU_state(); // keeps alignment at 16 bytes 781 782 lea(c_rarg0, ExternalAddress((address) msg)); 783 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 784 785 pop_CPU_state(); 786 mov(rsp, rbp); 787 pop(rbp); 788 } 789 790 void MacroAssembler::print_state() { 791 address rip = pc(); 792 pusha(); // get regs on stack 793 push(rbp); 794 movq(rbp, rsp); 795 andq(rsp, -16); // align stack as required by push_CPU_state and call 796 push_CPU_state(); // keeps alignment at 16 bytes 797 798 lea(c_rarg0, InternalAddress(rip)); 799 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 800 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 801 802 pop_CPU_state(); 803 mov(rsp, rbp); 804 pop(rbp); 805 popa(); 806 } 807 808 #ifndef PRODUCT 809 extern "C" void findpc(intptr_t x); 810 #endif 811 812 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 813 // In order to get locks to work, we need to fake a in_VM state 814 if (ShowMessageBoxOnError) { 815 JavaThread* thread = JavaThread::current(); 816 JavaThreadState saved_state = thread->thread_state(); 817 thread->set_thread_state(_thread_in_vm); 818 #ifndef PRODUCT 819 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 820 ttyLocker ttyl; 821 BytecodeCounter::print(); 822 } 823 #endif 824 // To see where a verify_oop failed, get $ebx+40/X for this frame. 825 // XXX correct this offset for amd64 826 // This is the value of eip which points to where verify_oop will return. 827 if (os::message_box(msg, "Execution stopped, print registers?")) { 828 print_state64(pc, regs); 829 BREAKPOINT; 830 } 831 } 832 fatal("DEBUG MESSAGE: %s", msg); 833 } 834 835 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 836 ttyLocker ttyl; 837 DebuggingContext debugging{}; 838 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 839 #ifndef PRODUCT 840 tty->cr(); 841 findpc(pc); 842 tty->cr(); 843 #endif 844 #define PRINT_REG(rax, value) \ 845 { tty->print("%s = ", #rax); os::print_location(tty, value); } 846 PRINT_REG(rax, regs[15]); 847 PRINT_REG(rbx, regs[12]); 848 PRINT_REG(rcx, regs[14]); 849 PRINT_REG(rdx, regs[13]); 850 PRINT_REG(rdi, regs[8]); 851 PRINT_REG(rsi, regs[9]); 852 PRINT_REG(rbp, regs[10]); 853 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 854 PRINT_REG(rsp, (intptr_t)(®s[16])); 855 PRINT_REG(r8 , regs[7]); 856 PRINT_REG(r9 , regs[6]); 857 PRINT_REG(r10, regs[5]); 858 PRINT_REG(r11, regs[4]); 859 PRINT_REG(r12, regs[3]); 860 PRINT_REG(r13, regs[2]); 861 PRINT_REG(r14, regs[1]); 862 PRINT_REG(r15, regs[0]); 863 #undef PRINT_REG 864 // Print some words near the top of the stack. 865 int64_t* rsp = ®s[16]; 866 int64_t* dump_sp = rsp; 867 for (int col1 = 0; col1 < 8; col1++) { 868 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 869 os::print_location(tty, *dump_sp++); 870 } 871 for (int row = 0; row < 25; row++) { 872 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 873 for (int col = 0; col < 4; col++) { 874 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 875 } 876 tty->cr(); 877 } 878 // Print some instructions around pc: 879 Disassembler::decode((address)pc-64, (address)pc); 880 tty->print_cr("--------"); 881 Disassembler::decode((address)pc, (address)pc+32); 882 } 883 884 // The java_calling_convention describes stack locations as ideal slots on 885 // a frame with no abi restrictions. Since we must observe abi restrictions 886 // (like the placement of the register window) the slots must be biased by 887 // the following value. 888 static int reg2offset_in(VMReg r) { 889 // Account for saved rbp and return address 890 // This should really be in_preserve_stack_slots 891 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 892 } 893 894 static int reg2offset_out(VMReg r) { 895 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 896 } 897 898 // A long move 899 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 900 901 // The calling conventions assures us that each VMregpair is either 902 // all really one physical register or adjacent stack slots. 903 904 if (src.is_single_phys_reg() ) { 905 if (dst.is_single_phys_reg()) { 906 if (dst.first() != src.first()) { 907 mov(dst.first()->as_Register(), src.first()->as_Register()); 908 } 909 } else { 910 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 911 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 912 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 913 } 914 } else if (dst.is_single_phys_reg()) { 915 assert(src.is_single_reg(), "not a stack pair"); 916 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 917 } else { 918 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 919 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 920 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 921 } 922 } 923 924 // A double move 925 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 926 927 // The calling conventions assures us that each VMregpair is either 928 // all really one physical register or adjacent stack slots. 929 930 if (src.is_single_phys_reg() ) { 931 if (dst.is_single_phys_reg()) { 932 // In theory these overlap but the ordering is such that this is likely a nop 933 if ( src.first() != dst.first()) { 934 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 935 } 936 } else { 937 assert(dst.is_single_reg(), "not a stack pair"); 938 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 939 } 940 } else if (dst.is_single_phys_reg()) { 941 assert(src.is_single_reg(), "not a stack pair"); 942 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 943 } else { 944 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 945 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 946 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 947 } 948 } 949 950 951 // A float arg may have to do float reg int reg conversion 952 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 953 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 954 955 // The calling conventions assures us that each VMregpair is either 956 // all really one physical register or adjacent stack slots. 957 958 if (src.first()->is_stack()) { 959 if (dst.first()->is_stack()) { 960 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 961 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 962 } else { 963 // stack to reg 964 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 965 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 966 } 967 } else if (dst.first()->is_stack()) { 968 // reg to stack 969 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 970 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 971 } else { 972 // reg to reg 973 // In theory these overlap but the ordering is such that this is likely a nop 974 if ( src.first() != dst.first()) { 975 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 976 } 977 } 978 } 979 980 // On 64 bit we will store integer like items to the stack as 981 // 64 bits items (x86_32/64 abi) even though java would only store 982 // 32bits for a parameter. On 32bit it will simply be 32 bits 983 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 984 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 985 if (src.first()->is_stack()) { 986 if (dst.first()->is_stack()) { 987 // stack to stack 988 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 989 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 990 } else { 991 // stack to reg 992 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 993 } 994 } else if (dst.first()->is_stack()) { 995 // reg to stack 996 // Do we really have to sign extend??? 997 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 998 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 999 } else { 1000 // Do we really have to sign extend??? 1001 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 1002 if (dst.first() != src.first()) { 1003 movq(dst.first()->as_Register(), src.first()->as_Register()); 1004 } 1005 } 1006 } 1007 1008 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 1009 if (src.first()->is_stack()) { 1010 if (dst.first()->is_stack()) { 1011 // stack to stack 1012 movq(rax, Address(rbp, reg2offset_in(src.first()))); 1013 movq(Address(rsp, reg2offset_out(dst.first())), rax); 1014 } else { 1015 // stack to reg 1016 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 1017 } 1018 } else if (dst.first()->is_stack()) { 1019 // reg to stack 1020 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 1021 } else { 1022 if (dst.first() != src.first()) { 1023 movq(dst.first()->as_Register(), src.first()->as_Register()); 1024 } 1025 } 1026 } 1027 1028 // An oop arg. Must pass a handle not the oop itself 1029 void MacroAssembler::object_move(OopMap* map, 1030 int oop_handle_offset, 1031 int framesize_in_slots, 1032 VMRegPair src, 1033 VMRegPair dst, 1034 bool is_receiver, 1035 int* receiver_offset) { 1036 1037 // must pass a handle. First figure out the location we use as a handle 1038 1039 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 1040 1041 // See if oop is null if it is we need no handle 1042 1043 if (src.first()->is_stack()) { 1044 1045 // Oop is already on the stack as an argument 1046 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 1047 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 1048 if (is_receiver) { 1049 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 1050 } 1051 1052 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 1053 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 1054 // conditionally move a null 1055 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 1056 } else { 1057 1058 // Oop is in a register we must store it to the space we reserve 1059 // on the stack for oop_handles and pass a handle if oop is non-null 1060 1061 const Register rOop = src.first()->as_Register(); 1062 int oop_slot; 1063 if (rOop == j_rarg0) 1064 oop_slot = 0; 1065 else if (rOop == j_rarg1) 1066 oop_slot = 1; 1067 else if (rOop == j_rarg2) 1068 oop_slot = 2; 1069 else if (rOop == j_rarg3) 1070 oop_slot = 3; 1071 else if (rOop == j_rarg4) 1072 oop_slot = 4; 1073 else { 1074 assert(rOop == j_rarg5, "wrong register"); 1075 oop_slot = 5; 1076 } 1077 1078 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 1079 int offset = oop_slot*VMRegImpl::stack_slot_size; 1080 1081 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 1082 // Store oop in handle area, may be null 1083 movptr(Address(rsp, offset), rOop); 1084 if (is_receiver) { 1085 *receiver_offset = offset; 1086 } 1087 1088 cmpptr(rOop, NULL_WORD); 1089 lea(rHandle, Address(rsp, offset)); 1090 // conditionally move a null from the handle area where it was just stored 1091 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 1092 } 1093 1094 // If arg is on the stack then place it otherwise it is already in correct reg. 1095 if (dst.first()->is_stack()) { 1096 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 1097 } 1098 } 1099 1100 #endif // _LP64 1101 1102 // Now versions that are common to 32/64 bit 1103 1104 void MacroAssembler::addptr(Register dst, int32_t imm32) { 1105 LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); 1106 } 1107 1108 void MacroAssembler::addptr(Register dst, Register src) { 1109 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1110 } 1111 1112 void MacroAssembler::addptr(Address dst, Register src) { 1113 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1114 } 1115 1116 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1117 assert(rscratch != noreg || always_reachable(src), "missing"); 1118 1119 if (reachable(src)) { 1120 Assembler::addsd(dst, as_Address(src)); 1121 } else { 1122 lea(rscratch, src); 1123 Assembler::addsd(dst, Address(rscratch, 0)); 1124 } 1125 } 1126 1127 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1128 assert(rscratch != noreg || always_reachable(src), "missing"); 1129 1130 if (reachable(src)) { 1131 addss(dst, as_Address(src)); 1132 } else { 1133 lea(rscratch, src); 1134 addss(dst, Address(rscratch, 0)); 1135 } 1136 } 1137 1138 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1139 assert(rscratch != noreg || always_reachable(src), "missing"); 1140 1141 if (reachable(src)) { 1142 Assembler::addpd(dst, as_Address(src)); 1143 } else { 1144 lea(rscratch, src); 1145 Assembler::addpd(dst, Address(rscratch, 0)); 1146 } 1147 } 1148 1149 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 1150 // Stub code is generated once and never copied. 1151 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 1152 void MacroAssembler::align64() { 1153 align(64, (unsigned long long) pc()); 1154 } 1155 1156 void MacroAssembler::align32() { 1157 align(32, (unsigned long long) pc()); 1158 } 1159 1160 void MacroAssembler::align(int modulus) { 1161 // 8273459: Ensure alignment is possible with current segment alignment 1162 assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 1163 align(modulus, offset()); 1164 } 1165 1166 void MacroAssembler::align(int modulus, int target) { 1167 if (target % modulus != 0) { 1168 nop(modulus - (target % modulus)); 1169 } 1170 } 1171 1172 void MacroAssembler::push_f(XMMRegister r) { 1173 subptr(rsp, wordSize); 1174 movflt(Address(rsp, 0), r); 1175 } 1176 1177 void MacroAssembler::pop_f(XMMRegister r) { 1178 movflt(r, Address(rsp, 0)); 1179 addptr(rsp, wordSize); 1180 } 1181 1182 void MacroAssembler::push_d(XMMRegister r) { 1183 subptr(rsp, 2 * wordSize); 1184 movdbl(Address(rsp, 0), r); 1185 } 1186 1187 void MacroAssembler::pop_d(XMMRegister r) { 1188 movdbl(r, Address(rsp, 0)); 1189 addptr(rsp, 2 * Interpreter::stackElementSize); 1190 } 1191 1192 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1193 // Used in sign-masking with aligned address. 1194 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1195 assert(rscratch != noreg || always_reachable(src), "missing"); 1196 1197 if (reachable(src)) { 1198 Assembler::andpd(dst, as_Address(src)); 1199 } else { 1200 lea(rscratch, src); 1201 Assembler::andpd(dst, Address(rscratch, 0)); 1202 } 1203 } 1204 1205 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 1206 // Used in sign-masking with aligned address. 1207 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1208 assert(rscratch != noreg || always_reachable(src), "missing"); 1209 1210 if (reachable(src)) { 1211 Assembler::andps(dst, as_Address(src)); 1212 } else { 1213 lea(rscratch, src); 1214 Assembler::andps(dst, Address(rscratch, 0)); 1215 } 1216 } 1217 1218 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1219 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1220 } 1221 1222 #ifdef _LP64 1223 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 1224 assert(rscratch != noreg || always_reachable(src), "missing"); 1225 1226 if (reachable(src)) { 1227 andq(dst, as_Address(src)); 1228 } else { 1229 lea(rscratch, src); 1230 andq(dst, Address(rscratch, 0)); 1231 } 1232 } 1233 #endif 1234 1235 void MacroAssembler::atomic_incl(Address counter_addr) { 1236 lock(); 1237 incrementl(counter_addr); 1238 } 1239 1240 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 1241 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1242 1243 if (reachable(counter_addr)) { 1244 atomic_incl(as_Address(counter_addr)); 1245 } else { 1246 lea(rscratch, counter_addr); 1247 atomic_incl(Address(rscratch, 0)); 1248 } 1249 } 1250 1251 #ifdef _LP64 1252 void MacroAssembler::atomic_incq(Address counter_addr) { 1253 lock(); 1254 incrementq(counter_addr); 1255 } 1256 1257 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 1258 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1259 1260 if (reachable(counter_addr)) { 1261 atomic_incq(as_Address(counter_addr)); 1262 } else { 1263 lea(rscratch, counter_addr); 1264 atomic_incq(Address(rscratch, 0)); 1265 } 1266 } 1267 #endif 1268 1269 // Writes to stack successive pages until offset reached to check for 1270 // stack overflow + shadow pages. This clobbers tmp. 1271 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 1272 movptr(tmp, rsp); 1273 // Bang stack for total size given plus shadow page size. 1274 // Bang one page at a time because large size can bang beyond yellow and 1275 // red zones. 1276 Label loop; 1277 bind(loop); 1278 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 1279 subptr(tmp, (int)os::vm_page_size()); 1280 subl(size, (int)os::vm_page_size()); 1281 jcc(Assembler::greater, loop); 1282 1283 // Bang down shadow pages too. 1284 // At this point, (tmp-0) is the last address touched, so don't 1285 // touch it again. (It was touched as (tmp-pagesize) but then tmp 1286 // was post-decremented.) Skip this address by starting at i=1, and 1287 // touch a few more pages below. N.B. It is important to touch all 1288 // the way down including all pages in the shadow zone. 1289 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 1290 // this could be any sized move but this is can be a debugging crumb 1291 // so the bigger the better. 1292 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 1293 } 1294 } 1295 1296 void MacroAssembler::reserved_stack_check() { 1297 // testing if reserved zone needs to be enabled 1298 Label no_reserved_zone_enabling; 1299 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 1300 NOT_LP64(get_thread(rsi);) 1301 1302 cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); 1303 jcc(Assembler::below, no_reserved_zone_enabling); 1304 1305 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); 1306 jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); 1307 should_not_reach_here(); 1308 1309 bind(no_reserved_zone_enabling); 1310 } 1311 1312 void MacroAssembler::c2bool(Register x) { 1313 // implements x == 0 ? 0 : 1 1314 // note: must only look at least-significant byte of x 1315 // since C-style booleans are stored in one byte 1316 // only! (was bug) 1317 andl(x, 0xFF); 1318 setb(Assembler::notZero, x); 1319 } 1320 1321 // Wouldn't need if AddressLiteral version had new name 1322 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 1323 Assembler::call(L, rtype); 1324 } 1325 1326 void MacroAssembler::call(Register entry) { 1327 Assembler::call(entry); 1328 } 1329 1330 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 1331 assert(rscratch != noreg || always_reachable(entry), "missing"); 1332 1333 if (reachable(entry)) { 1334 Assembler::call_literal(entry.target(), entry.rspec()); 1335 } else { 1336 lea(rscratch, entry); 1337 Assembler::call(rscratch); 1338 } 1339 } 1340 1341 void MacroAssembler::ic_call(address entry, jint method_index) { 1342 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1343 #ifdef _LP64 1344 // Needs full 64-bit immediate for later patching. 1345 mov64(rax, (int64_t)Universe::non_oop_word()); 1346 #else 1347 movptr(rax, (intptr_t)Universe::non_oop_word()); 1348 #endif 1349 call(AddressLiteral(entry, rh)); 1350 } 1351 1352 int MacroAssembler::ic_check_size() { 1353 return LP64_ONLY(14) NOT_LP64(12); 1354 } 1355 1356 int MacroAssembler::ic_check(int end_alignment) { 1357 Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx); 1358 Register data = rax; 1359 Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx); 1360 1361 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 1362 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 1363 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 1364 // before the inline cache check here, and not after 1365 align(end_alignment, offset() + ic_check_size()); 1366 1367 int uep_offset = offset(); 1368 1369 if (UseCompressedClassPointers) { 1370 movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1371 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 1372 } else { 1373 movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1374 cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset())); 1375 } 1376 1377 // if inline cache check fails, then jump to runtime routine 1378 jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1379 assert((offset() % end_alignment) == 0, "Misaligned verified entry point"); 1380 1381 return uep_offset; 1382 } 1383 1384 void MacroAssembler::emit_static_call_stub() { 1385 // Static stub relocation also tags the Method* in the code-stream. 1386 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 1387 // This is recognized as unresolved by relocs/nativeinst/ic code. 1388 jump(RuntimeAddress(pc())); 1389 } 1390 1391 // Implementation of call_VM versions 1392 1393 void MacroAssembler::call_VM(Register oop_result, 1394 address entry_point, 1395 bool check_exceptions) { 1396 Label C, E; 1397 call(C, relocInfo::none); 1398 jmp(E); 1399 1400 bind(C); 1401 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1402 ret(0); 1403 1404 bind(E); 1405 } 1406 1407 void MacroAssembler::call_VM(Register oop_result, 1408 address entry_point, 1409 Register arg_1, 1410 bool check_exceptions) { 1411 Label C, E; 1412 call(C, relocInfo::none); 1413 jmp(E); 1414 1415 bind(C); 1416 pass_arg1(this, arg_1); 1417 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1418 ret(0); 1419 1420 bind(E); 1421 } 1422 1423 void MacroAssembler::call_VM(Register oop_result, 1424 address entry_point, 1425 Register arg_1, 1426 Register arg_2, 1427 bool check_exceptions) { 1428 Label C, E; 1429 call(C, relocInfo::none); 1430 jmp(E); 1431 1432 bind(C); 1433 1434 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1435 1436 pass_arg2(this, arg_2); 1437 pass_arg1(this, arg_1); 1438 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1439 ret(0); 1440 1441 bind(E); 1442 } 1443 1444 void MacroAssembler::call_VM(Register oop_result, 1445 address entry_point, 1446 Register arg_1, 1447 Register arg_2, 1448 Register arg_3, 1449 bool check_exceptions) { 1450 Label C, E; 1451 call(C, relocInfo::none); 1452 jmp(E); 1453 1454 bind(C); 1455 1456 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1457 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1458 pass_arg3(this, arg_3); 1459 pass_arg2(this, arg_2); 1460 pass_arg1(this, arg_1); 1461 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1462 ret(0); 1463 1464 bind(E); 1465 } 1466 1467 void MacroAssembler::call_VM(Register oop_result, 1468 Register last_java_sp, 1469 address entry_point, 1470 int number_of_arguments, 1471 bool check_exceptions) { 1472 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1473 call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1474 } 1475 1476 void MacroAssembler::call_VM(Register oop_result, 1477 Register last_java_sp, 1478 address entry_point, 1479 Register arg_1, 1480 bool check_exceptions) { 1481 pass_arg1(this, arg_1); 1482 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1483 } 1484 1485 void MacroAssembler::call_VM(Register oop_result, 1486 Register last_java_sp, 1487 address entry_point, 1488 Register arg_1, 1489 Register arg_2, 1490 bool check_exceptions) { 1491 1492 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1493 pass_arg2(this, arg_2); 1494 pass_arg1(this, arg_1); 1495 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1496 } 1497 1498 void MacroAssembler::call_VM(Register oop_result, 1499 Register last_java_sp, 1500 address entry_point, 1501 Register arg_1, 1502 Register arg_2, 1503 Register arg_3, 1504 bool check_exceptions) { 1505 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1506 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1507 pass_arg3(this, arg_3); 1508 pass_arg2(this, arg_2); 1509 pass_arg1(this, arg_1); 1510 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1511 } 1512 1513 void MacroAssembler::super_call_VM(Register oop_result, 1514 Register last_java_sp, 1515 address entry_point, 1516 int number_of_arguments, 1517 bool check_exceptions) { 1518 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1519 MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1520 } 1521 1522 void MacroAssembler::super_call_VM(Register oop_result, 1523 Register last_java_sp, 1524 address entry_point, 1525 Register arg_1, 1526 bool check_exceptions) { 1527 pass_arg1(this, arg_1); 1528 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1529 } 1530 1531 void MacroAssembler::super_call_VM(Register oop_result, 1532 Register last_java_sp, 1533 address entry_point, 1534 Register arg_1, 1535 Register arg_2, 1536 bool check_exceptions) { 1537 1538 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1539 pass_arg2(this, arg_2); 1540 pass_arg1(this, arg_1); 1541 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1542 } 1543 1544 void MacroAssembler::super_call_VM(Register oop_result, 1545 Register last_java_sp, 1546 address entry_point, 1547 Register arg_1, 1548 Register arg_2, 1549 Register arg_3, 1550 bool check_exceptions) { 1551 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1552 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1553 pass_arg3(this, arg_3); 1554 pass_arg2(this, arg_2); 1555 pass_arg1(this, arg_1); 1556 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1557 } 1558 1559 void MacroAssembler::call_VM_base(Register oop_result, 1560 Register java_thread, 1561 Register last_java_sp, 1562 address entry_point, 1563 int number_of_arguments, 1564 bool check_exceptions) { 1565 // determine java_thread register 1566 if (!java_thread->is_valid()) { 1567 #ifdef _LP64 1568 java_thread = r15_thread; 1569 #else 1570 java_thread = rdi; 1571 get_thread(java_thread); 1572 #endif // LP64 1573 } 1574 // determine last_java_sp register 1575 if (!last_java_sp->is_valid()) { 1576 last_java_sp = rsp; 1577 } 1578 // debugging support 1579 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1580 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); 1581 #ifdef ASSERT 1582 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1583 // r12 is the heapbase. 1584 LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) 1585 #endif // ASSERT 1586 1587 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1588 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1589 1590 // push java thread (becomes first argument of C function) 1591 1592 NOT_LP64(push(java_thread); number_of_arguments++); 1593 LP64_ONLY(mov(c_rarg0, r15_thread)); 1594 1595 // set last Java frame before call 1596 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1597 1598 // Only interpreter should have to set fp 1599 set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1); 1600 1601 // do the call, remove parameters 1602 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1603 1604 // restore the thread (cannot use the pushed argument since arguments 1605 // may be overwritten by C code generated by an optimizing compiler); 1606 // however can use the register value directly if it is callee saved. 1607 if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { 1608 // rdi & rsi (also r15) are callee saved -> nothing to do 1609 #ifdef ASSERT 1610 guarantee(java_thread != rax, "change this code"); 1611 push(rax); 1612 { Label L; 1613 get_thread(rax); 1614 cmpptr(java_thread, rax); 1615 jcc(Assembler::equal, L); 1616 STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); 1617 bind(L); 1618 } 1619 pop(rax); 1620 #endif 1621 } else { 1622 get_thread(java_thread); 1623 } 1624 // reset last Java frame 1625 // Only interpreter should have to clear fp 1626 reset_last_Java_frame(java_thread, true); 1627 1628 // C++ interp handles this in the interpreter 1629 check_and_handle_popframe(java_thread); 1630 check_and_handle_earlyret(java_thread); 1631 1632 if (check_exceptions) { 1633 // check for pending exceptions (java_thread is set upon return) 1634 cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD); 1635 #ifndef _LP64 1636 jump_cc(Assembler::notEqual, 1637 RuntimeAddress(StubRoutines::forward_exception_entry())); 1638 #else 1639 // This used to conditionally jump to forward_exception however it is 1640 // possible if we relocate that the branch will not reach. So we must jump 1641 // around so we can always reach 1642 1643 Label ok; 1644 jcc(Assembler::equal, ok); 1645 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1646 bind(ok); 1647 #endif // LP64 1648 } 1649 1650 // get oop result if there is one and reset the value in the thread 1651 if (oop_result->is_valid()) { 1652 get_vm_result(oop_result, java_thread); 1653 } 1654 } 1655 1656 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1657 1658 // Calculate the value for last_Java_sp 1659 // somewhat subtle. call_VM does an intermediate call 1660 // which places a return address on the stack just under the 1661 // stack pointer as the user finished with it. This allows 1662 // use to retrieve last_Java_pc from last_Java_sp[-1]. 1663 // On 32bit we then have to push additional args on the stack to accomplish 1664 // the actual requested call. On 64bit call_VM only can use register args 1665 // so the only extra space is the return address that call_VM created. 1666 // This hopefully explains the calculations here. 1667 1668 #ifdef _LP64 1669 // We've pushed one address, correct last_Java_sp 1670 lea(rax, Address(rsp, wordSize)); 1671 #else 1672 lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); 1673 #endif // LP64 1674 1675 call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); 1676 1677 } 1678 1679 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1680 void MacroAssembler::call_VM_leaf0(address entry_point) { 1681 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1682 } 1683 1684 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1685 call_VM_leaf_base(entry_point, number_of_arguments); 1686 } 1687 1688 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1689 pass_arg0(this, arg_0); 1690 call_VM_leaf(entry_point, 1); 1691 } 1692 1693 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1694 1695 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1696 pass_arg1(this, arg_1); 1697 pass_arg0(this, arg_0); 1698 call_VM_leaf(entry_point, 2); 1699 } 1700 1701 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1702 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1703 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1704 pass_arg2(this, arg_2); 1705 pass_arg1(this, arg_1); 1706 pass_arg0(this, arg_0); 1707 call_VM_leaf(entry_point, 3); 1708 } 1709 1710 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1711 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1712 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1713 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1714 pass_arg3(this, arg_3); 1715 pass_arg2(this, arg_2); 1716 pass_arg1(this, arg_1); 1717 pass_arg0(this, arg_0); 1718 call_VM_leaf(entry_point, 3); 1719 } 1720 1721 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1722 pass_arg0(this, arg_0); 1723 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1724 } 1725 1726 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1727 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1728 pass_arg1(this, arg_1); 1729 pass_arg0(this, arg_0); 1730 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1731 } 1732 1733 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1734 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1735 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1736 pass_arg2(this, arg_2); 1737 pass_arg1(this, arg_1); 1738 pass_arg0(this, arg_0); 1739 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1740 } 1741 1742 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1743 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1744 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1745 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1746 pass_arg3(this, arg_3); 1747 pass_arg2(this, arg_2); 1748 pass_arg1(this, arg_1); 1749 pass_arg0(this, arg_0); 1750 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1751 } 1752 1753 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1754 movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1755 movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); 1756 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1757 } 1758 1759 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1760 movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1761 movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); 1762 } 1763 1764 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { 1765 } 1766 1767 void MacroAssembler::check_and_handle_popframe(Register java_thread) { 1768 } 1769 1770 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1771 assert(rscratch != noreg || always_reachable(src1), "missing"); 1772 1773 if (reachable(src1)) { 1774 cmpl(as_Address(src1), imm); 1775 } else { 1776 lea(rscratch, src1); 1777 cmpl(Address(rscratch, 0), imm); 1778 } 1779 } 1780 1781 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1782 assert(!src2.is_lval(), "use cmpptr"); 1783 assert(rscratch != noreg || always_reachable(src2), "missing"); 1784 1785 if (reachable(src2)) { 1786 cmpl(src1, as_Address(src2)); 1787 } else { 1788 lea(rscratch, src2); 1789 cmpl(src1, Address(rscratch, 0)); 1790 } 1791 } 1792 1793 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1794 Assembler::cmpl(src1, imm); 1795 } 1796 1797 void MacroAssembler::cmp32(Register src1, Address src2) { 1798 Assembler::cmpl(src1, src2); 1799 } 1800 1801 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1802 ucomisd(opr1, opr2); 1803 1804 Label L; 1805 if (unordered_is_less) { 1806 movl(dst, -1); 1807 jcc(Assembler::parity, L); 1808 jcc(Assembler::below , L); 1809 movl(dst, 0); 1810 jcc(Assembler::equal , L); 1811 increment(dst); 1812 } else { // unordered is greater 1813 movl(dst, 1); 1814 jcc(Assembler::parity, L); 1815 jcc(Assembler::above , L); 1816 movl(dst, 0); 1817 jcc(Assembler::equal , L); 1818 decrementl(dst); 1819 } 1820 bind(L); 1821 } 1822 1823 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1824 ucomiss(opr1, opr2); 1825 1826 Label L; 1827 if (unordered_is_less) { 1828 movl(dst, -1); 1829 jcc(Assembler::parity, L); 1830 jcc(Assembler::below , L); 1831 movl(dst, 0); 1832 jcc(Assembler::equal , L); 1833 increment(dst); 1834 } else { // unordered is greater 1835 movl(dst, 1); 1836 jcc(Assembler::parity, L); 1837 jcc(Assembler::above , L); 1838 movl(dst, 0); 1839 jcc(Assembler::equal , L); 1840 decrementl(dst); 1841 } 1842 bind(L); 1843 } 1844 1845 1846 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1847 assert(rscratch != noreg || always_reachable(src1), "missing"); 1848 1849 if (reachable(src1)) { 1850 cmpb(as_Address(src1), imm); 1851 } else { 1852 lea(rscratch, src1); 1853 cmpb(Address(rscratch, 0), imm); 1854 } 1855 } 1856 1857 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1858 #ifdef _LP64 1859 assert(rscratch != noreg || always_reachable(src2), "missing"); 1860 1861 if (src2.is_lval()) { 1862 movptr(rscratch, src2); 1863 Assembler::cmpq(src1, rscratch); 1864 } else if (reachable(src2)) { 1865 cmpq(src1, as_Address(src2)); 1866 } else { 1867 lea(rscratch, src2); 1868 Assembler::cmpq(src1, Address(rscratch, 0)); 1869 } 1870 #else 1871 assert(rscratch == noreg, "not needed"); 1872 if (src2.is_lval()) { 1873 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1874 } else { 1875 cmpl(src1, as_Address(src2)); 1876 } 1877 #endif // _LP64 1878 } 1879 1880 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1881 assert(src2.is_lval(), "not a mem-mem compare"); 1882 #ifdef _LP64 1883 // moves src2's literal address 1884 movptr(rscratch, src2); 1885 Assembler::cmpq(src1, rscratch); 1886 #else 1887 assert(rscratch == noreg, "not needed"); 1888 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1889 #endif // _LP64 1890 } 1891 1892 void MacroAssembler::cmpoop(Register src1, Register src2) { 1893 cmpptr(src1, src2); 1894 } 1895 1896 void MacroAssembler::cmpoop(Register src1, Address src2) { 1897 cmpptr(src1, src2); 1898 } 1899 1900 #ifdef _LP64 1901 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1902 movoop(rscratch, src2); 1903 cmpptr(src1, rscratch); 1904 } 1905 #endif 1906 1907 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1908 assert(rscratch != noreg || always_reachable(adr), "missing"); 1909 1910 if (reachable(adr)) { 1911 lock(); 1912 cmpxchgptr(reg, as_Address(adr)); 1913 } else { 1914 lea(rscratch, adr); 1915 lock(); 1916 cmpxchgptr(reg, Address(rscratch, 0)); 1917 } 1918 } 1919 1920 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1921 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); 1922 } 1923 1924 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1925 assert(rscratch != noreg || always_reachable(src), "missing"); 1926 1927 if (reachable(src)) { 1928 Assembler::comisd(dst, as_Address(src)); 1929 } else { 1930 lea(rscratch, src); 1931 Assembler::comisd(dst, Address(rscratch, 0)); 1932 } 1933 } 1934 1935 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1936 assert(rscratch != noreg || always_reachable(src), "missing"); 1937 1938 if (reachable(src)) { 1939 Assembler::comiss(dst, as_Address(src)); 1940 } else { 1941 lea(rscratch, src); 1942 Assembler::comiss(dst, Address(rscratch, 0)); 1943 } 1944 } 1945 1946 1947 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 1948 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1949 1950 Condition negated_cond = negate_condition(cond); 1951 Label L; 1952 jcc(negated_cond, L); 1953 pushf(); // Preserve flags 1954 atomic_incl(counter_addr, rscratch); 1955 popf(); 1956 bind(L); 1957 } 1958 1959 int MacroAssembler::corrected_idivl(Register reg) { 1960 // Full implementation of Java idiv and irem; checks for 1961 // special case as described in JVM spec., p.243 & p.271. 1962 // The function returns the (pc) offset of the idivl 1963 // instruction - may be needed for implicit exceptions. 1964 // 1965 // normal case special case 1966 // 1967 // input : rax,: dividend min_int 1968 // reg: divisor (may not be rax,/rdx) -1 1969 // 1970 // output: rax,: quotient (= rax, idiv reg) min_int 1971 // rdx: remainder (= rax, irem reg) 0 1972 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 1973 const int min_int = 0x80000000; 1974 Label normal_case, special_case; 1975 1976 // check for special case 1977 cmpl(rax, min_int); 1978 jcc(Assembler::notEqual, normal_case); 1979 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 1980 cmpl(reg, -1); 1981 jcc(Assembler::equal, special_case); 1982 1983 // handle normal case 1984 bind(normal_case); 1985 cdql(); 1986 int idivl_offset = offset(); 1987 idivl(reg); 1988 1989 // normal and special case exit 1990 bind(special_case); 1991 1992 return idivl_offset; 1993 } 1994 1995 1996 1997 void MacroAssembler::decrementl(Register reg, int value) { 1998 if (value == min_jint) {subl(reg, value) ; return; } 1999 if (value < 0) { incrementl(reg, -value); return; } 2000 if (value == 0) { ; return; } 2001 if (value == 1 && UseIncDec) { decl(reg) ; return; } 2002 /* else */ { subl(reg, value) ; return; } 2003 } 2004 2005 void MacroAssembler::decrementl(Address dst, int value) { 2006 if (value == min_jint) {subl(dst, value) ; return; } 2007 if (value < 0) { incrementl(dst, -value); return; } 2008 if (value == 0) { ; return; } 2009 if (value == 1 && UseIncDec) { decl(dst) ; return; } 2010 /* else */ { subl(dst, value) ; return; } 2011 } 2012 2013 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 2014 assert(shift_value > 0, "illegal shift value"); 2015 Label _is_positive; 2016 testl (reg, reg); 2017 jcc (Assembler::positive, _is_positive); 2018 int offset = (1 << shift_value) - 1 ; 2019 2020 if (offset == 1) { 2021 incrementl(reg); 2022 } else { 2023 addl(reg, offset); 2024 } 2025 2026 bind (_is_positive); 2027 sarl(reg, shift_value); 2028 } 2029 2030 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2031 assert(rscratch != noreg || always_reachable(src), "missing"); 2032 2033 if (reachable(src)) { 2034 Assembler::divsd(dst, as_Address(src)); 2035 } else { 2036 lea(rscratch, src); 2037 Assembler::divsd(dst, Address(rscratch, 0)); 2038 } 2039 } 2040 2041 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2042 assert(rscratch != noreg || always_reachable(src), "missing"); 2043 2044 if (reachable(src)) { 2045 Assembler::divss(dst, as_Address(src)); 2046 } else { 2047 lea(rscratch, src); 2048 Assembler::divss(dst, Address(rscratch, 0)); 2049 } 2050 } 2051 2052 void MacroAssembler::enter() { 2053 push(rbp); 2054 mov(rbp, rsp); 2055 } 2056 2057 void MacroAssembler::post_call_nop() { 2058 if (!Continuations::enabled()) { 2059 return; 2060 } 2061 InstructionMark im(this); 2062 relocate(post_call_nop_Relocation::spec()); 2063 InlineSkippedInstructionsCounter skipCounter(this); 2064 emit_int8((uint8_t)0x0f); 2065 emit_int8((uint8_t)0x1f); 2066 emit_int8((uint8_t)0x84); 2067 emit_int8((uint8_t)0x00); 2068 emit_int32(0x00); 2069 } 2070 2071 // A 5 byte nop that is safe for patching (see patch_verified_entry) 2072 void MacroAssembler::fat_nop() { 2073 if (UseAddressNop) { 2074 addr_nop_5(); 2075 } else { 2076 emit_int8((uint8_t)0x26); // es: 2077 emit_int8((uint8_t)0x2e); // cs: 2078 emit_int8((uint8_t)0x64); // fs: 2079 emit_int8((uint8_t)0x65); // gs: 2080 emit_int8((uint8_t)0x90); 2081 } 2082 } 2083 2084 #ifndef _LP64 2085 void MacroAssembler::fcmp(Register tmp) { 2086 fcmp(tmp, 1, true, true); 2087 } 2088 2089 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { 2090 assert(!pop_right || pop_left, "usage error"); 2091 if (VM_Version::supports_cmov()) { 2092 assert(tmp == noreg, "unneeded temp"); 2093 if (pop_left) { 2094 fucomip(index); 2095 } else { 2096 fucomi(index); 2097 } 2098 if (pop_right) { 2099 fpop(); 2100 } 2101 } else { 2102 assert(tmp != noreg, "need temp"); 2103 if (pop_left) { 2104 if (pop_right) { 2105 fcompp(); 2106 } else { 2107 fcomp(index); 2108 } 2109 } else { 2110 fcom(index); 2111 } 2112 // convert FPU condition into eflags condition via rax, 2113 save_rax(tmp); 2114 fwait(); fnstsw_ax(); 2115 sahf(); 2116 restore_rax(tmp); 2117 } 2118 // condition codes set as follows: 2119 // 2120 // CF (corresponds to C0) if x < y 2121 // PF (corresponds to C2) if unordered 2122 // ZF (corresponds to C3) if x = y 2123 } 2124 2125 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { 2126 fcmp2int(dst, unordered_is_less, 1, true, true); 2127 } 2128 2129 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { 2130 fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); 2131 Label L; 2132 if (unordered_is_less) { 2133 movl(dst, -1); 2134 jcc(Assembler::parity, L); 2135 jcc(Assembler::below , L); 2136 movl(dst, 0); 2137 jcc(Assembler::equal , L); 2138 increment(dst); 2139 } else { // unordered is greater 2140 movl(dst, 1); 2141 jcc(Assembler::parity, L); 2142 jcc(Assembler::above , L); 2143 movl(dst, 0); 2144 jcc(Assembler::equal , L); 2145 decrementl(dst); 2146 } 2147 bind(L); 2148 } 2149 2150 void MacroAssembler::fld_d(AddressLiteral src) { 2151 fld_d(as_Address(src)); 2152 } 2153 2154 void MacroAssembler::fld_s(AddressLiteral src) { 2155 fld_s(as_Address(src)); 2156 } 2157 2158 void MacroAssembler::fldcw(AddressLiteral src) { 2159 fldcw(as_Address(src)); 2160 } 2161 2162 void MacroAssembler::fpop() { 2163 ffree(); 2164 fincstp(); 2165 } 2166 2167 void MacroAssembler::fremr(Register tmp) { 2168 save_rax(tmp); 2169 { Label L; 2170 bind(L); 2171 fprem(); 2172 fwait(); fnstsw_ax(); 2173 sahf(); 2174 jcc(Assembler::parity, L); 2175 } 2176 restore_rax(tmp); 2177 // Result is in ST0. 2178 // Note: fxch & fpop to get rid of ST1 2179 // (otherwise FPU stack could overflow eventually) 2180 fxch(1); 2181 fpop(); 2182 } 2183 2184 void MacroAssembler::empty_FPU_stack() { 2185 if (VM_Version::supports_mmx()) { 2186 emms(); 2187 } else { 2188 for (int i = 8; i-- > 0; ) ffree(i); 2189 } 2190 } 2191 #endif // !LP64 2192 2193 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2194 assert(rscratch != noreg || always_reachable(src), "missing"); 2195 if (reachable(src)) { 2196 Assembler::mulpd(dst, as_Address(src)); 2197 } else { 2198 lea(rscratch, src); 2199 Assembler::mulpd(dst, Address(rscratch, 0)); 2200 } 2201 } 2202 2203 void MacroAssembler::load_float(Address src) { 2204 #ifdef _LP64 2205 movflt(xmm0, src); 2206 #else 2207 if (UseSSE >= 1) { 2208 movflt(xmm0, src); 2209 } else { 2210 fld_s(src); 2211 } 2212 #endif // LP64 2213 } 2214 2215 void MacroAssembler::store_float(Address dst) { 2216 #ifdef _LP64 2217 movflt(dst, xmm0); 2218 #else 2219 if (UseSSE >= 1) { 2220 movflt(dst, xmm0); 2221 } else { 2222 fstp_s(dst); 2223 } 2224 #endif // LP64 2225 } 2226 2227 void MacroAssembler::load_double(Address src) { 2228 #ifdef _LP64 2229 movdbl(xmm0, src); 2230 #else 2231 if (UseSSE >= 2) { 2232 movdbl(xmm0, src); 2233 } else { 2234 fld_d(src); 2235 } 2236 #endif // LP64 2237 } 2238 2239 void MacroAssembler::store_double(Address dst) { 2240 #ifdef _LP64 2241 movdbl(dst, xmm0); 2242 #else 2243 if (UseSSE >= 2) { 2244 movdbl(dst, xmm0); 2245 } else { 2246 fstp_d(dst); 2247 } 2248 #endif // LP64 2249 } 2250 2251 // dst = c = a * b + c 2252 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2253 Assembler::vfmadd231sd(c, a, b); 2254 if (dst != c) { 2255 movdbl(dst, c); 2256 } 2257 } 2258 2259 // dst = c = a * b + c 2260 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2261 Assembler::vfmadd231ss(c, a, b); 2262 if (dst != c) { 2263 movflt(dst, c); 2264 } 2265 } 2266 2267 // dst = c = a * b + c 2268 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2269 Assembler::vfmadd231pd(c, a, b, vector_len); 2270 if (dst != c) { 2271 vmovdqu(dst, c); 2272 } 2273 } 2274 2275 // dst = c = a * b + c 2276 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2277 Assembler::vfmadd231ps(c, a, b, vector_len); 2278 if (dst != c) { 2279 vmovdqu(dst, c); 2280 } 2281 } 2282 2283 // dst = c = a * b + c 2284 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2285 Assembler::vfmadd231pd(c, a, b, vector_len); 2286 if (dst != c) { 2287 vmovdqu(dst, c); 2288 } 2289 } 2290 2291 // dst = c = a * b + c 2292 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2293 Assembler::vfmadd231ps(c, a, b, vector_len); 2294 if (dst != c) { 2295 vmovdqu(dst, c); 2296 } 2297 } 2298 2299 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 2300 assert(rscratch != noreg || always_reachable(dst), "missing"); 2301 2302 if (reachable(dst)) { 2303 incrementl(as_Address(dst)); 2304 } else { 2305 lea(rscratch, dst); 2306 incrementl(Address(rscratch, 0)); 2307 } 2308 } 2309 2310 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 2311 incrementl(as_Address(dst, rscratch)); 2312 } 2313 2314 void MacroAssembler::incrementl(Register reg, int value) { 2315 if (value == min_jint) {addl(reg, value) ; return; } 2316 if (value < 0) { decrementl(reg, -value); return; } 2317 if (value == 0) { ; return; } 2318 if (value == 1 && UseIncDec) { incl(reg) ; return; } 2319 /* else */ { addl(reg, value) ; return; } 2320 } 2321 2322 void MacroAssembler::incrementl(Address dst, int value) { 2323 if (value == min_jint) {addl(dst, value) ; return; } 2324 if (value < 0) { decrementl(dst, -value); return; } 2325 if (value == 0) { ; return; } 2326 if (value == 1 && UseIncDec) { incl(dst) ; return; } 2327 /* else */ { addl(dst, value) ; return; } 2328 } 2329 2330 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 2331 assert(rscratch != noreg || always_reachable(dst), "missing"); 2332 2333 if (reachable(dst)) { 2334 jmp_literal(dst.target(), dst.rspec()); 2335 } else { 2336 lea(rscratch, dst); 2337 jmp(rscratch); 2338 } 2339 } 2340 2341 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 2342 assert(rscratch != noreg || always_reachable(dst), "missing"); 2343 2344 if (reachable(dst)) { 2345 InstructionMark im(this); 2346 relocate(dst.reloc()); 2347 const int short_size = 2; 2348 const int long_size = 6; 2349 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 2350 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 2351 // 0111 tttn #8-bit disp 2352 emit_int8(0x70 | cc); 2353 emit_int8((offs - short_size) & 0xFF); 2354 } else { 2355 // 0000 1111 1000 tttn #32-bit disp 2356 emit_int8(0x0F); 2357 emit_int8((unsigned char)(0x80 | cc)); 2358 emit_int32(offs - long_size); 2359 } 2360 } else { 2361 #ifdef ASSERT 2362 warning("reversing conditional branch"); 2363 #endif /* ASSERT */ 2364 Label skip; 2365 jccb(reverse[cc], skip); 2366 lea(rscratch, dst); 2367 Assembler::jmp(rscratch); 2368 bind(skip); 2369 } 2370 } 2371 2372 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 2373 assert(rscratch != noreg || always_reachable(src), "missing"); 2374 2375 if (reachable(src)) { 2376 Assembler::ldmxcsr(as_Address(src)); 2377 } else { 2378 lea(rscratch, src); 2379 Assembler::ldmxcsr(Address(rscratch, 0)); 2380 } 2381 } 2382 2383 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2384 int off; 2385 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2386 off = offset(); 2387 movsbl(dst, src); // movsxb 2388 } else { 2389 off = load_unsigned_byte(dst, src); 2390 shll(dst, 24); 2391 sarl(dst, 24); 2392 } 2393 return off; 2394 } 2395 2396 // Note: load_signed_short used to be called load_signed_word. 2397 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 2398 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 2399 // The term "word" in HotSpot means a 32- or 64-bit machine word. 2400 int MacroAssembler::load_signed_short(Register dst, Address src) { 2401 int off; 2402 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2403 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 2404 // version but this is what 64bit has always done. This seems to imply 2405 // that users are only using 32bits worth. 2406 off = offset(); 2407 movswl(dst, src); // movsxw 2408 } else { 2409 off = load_unsigned_short(dst, src); 2410 shll(dst, 16); 2411 sarl(dst, 16); 2412 } 2413 return off; 2414 } 2415 2416 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2417 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2418 // and "3.9 Partial Register Penalties", p. 22). 2419 int off; 2420 if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { 2421 off = offset(); 2422 movzbl(dst, src); // movzxb 2423 } else { 2424 xorl(dst, dst); 2425 off = offset(); 2426 movb(dst, src); 2427 } 2428 return off; 2429 } 2430 2431 // Note: load_unsigned_short used to be called load_unsigned_word. 2432 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2433 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2434 // and "3.9 Partial Register Penalties", p. 22). 2435 int off; 2436 if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { 2437 off = offset(); 2438 movzwl(dst, src); // movzxw 2439 } else { 2440 xorl(dst, dst); 2441 off = offset(); 2442 movw(dst, src); 2443 } 2444 return off; 2445 } 2446 2447 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 2448 switch (size_in_bytes) { 2449 #ifndef _LP64 2450 case 8: 2451 assert(dst2 != noreg, "second dest register required"); 2452 movl(dst, src); 2453 movl(dst2, src.plus_disp(BytesPerInt)); 2454 break; 2455 #else 2456 case 8: movq(dst, src); break; 2457 #endif 2458 case 4: movl(dst, src); break; 2459 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2460 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2461 default: ShouldNotReachHere(); 2462 } 2463 } 2464 2465 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 2466 switch (size_in_bytes) { 2467 #ifndef _LP64 2468 case 8: 2469 assert(src2 != noreg, "second source register required"); 2470 movl(dst, src); 2471 movl(dst.plus_disp(BytesPerInt), src2); 2472 break; 2473 #else 2474 case 8: movq(dst, src); break; 2475 #endif 2476 case 4: movl(dst, src); break; 2477 case 2: movw(dst, src); break; 2478 case 1: movb(dst, src); break; 2479 default: ShouldNotReachHere(); 2480 } 2481 } 2482 2483 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 2484 assert(rscratch != noreg || always_reachable(dst), "missing"); 2485 2486 if (reachable(dst)) { 2487 movl(as_Address(dst), src); 2488 } else { 2489 lea(rscratch, dst); 2490 movl(Address(rscratch, 0), src); 2491 } 2492 } 2493 2494 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 2495 if (reachable(src)) { 2496 movl(dst, as_Address(src)); 2497 } else { 2498 lea(dst, src); 2499 movl(dst, Address(dst, 0)); 2500 } 2501 } 2502 2503 // C++ bool manipulation 2504 2505 void MacroAssembler::movbool(Register dst, Address src) { 2506 if(sizeof(bool) == 1) 2507 movb(dst, src); 2508 else if(sizeof(bool) == 2) 2509 movw(dst, src); 2510 else if(sizeof(bool) == 4) 2511 movl(dst, src); 2512 else 2513 // unsupported 2514 ShouldNotReachHere(); 2515 } 2516 2517 void MacroAssembler::movbool(Address dst, bool boolconst) { 2518 if(sizeof(bool) == 1) 2519 movb(dst, (int) boolconst); 2520 else if(sizeof(bool) == 2) 2521 movw(dst, (int) boolconst); 2522 else if(sizeof(bool) == 4) 2523 movl(dst, (int) boolconst); 2524 else 2525 // unsupported 2526 ShouldNotReachHere(); 2527 } 2528 2529 void MacroAssembler::movbool(Address dst, Register src) { 2530 if(sizeof(bool) == 1) 2531 movb(dst, src); 2532 else if(sizeof(bool) == 2) 2533 movw(dst, src); 2534 else if(sizeof(bool) == 4) 2535 movl(dst, src); 2536 else 2537 // unsupported 2538 ShouldNotReachHere(); 2539 } 2540 2541 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2542 assert(rscratch != noreg || always_reachable(src), "missing"); 2543 2544 if (reachable(src)) { 2545 movdl(dst, as_Address(src)); 2546 } else { 2547 lea(rscratch, src); 2548 movdl(dst, Address(rscratch, 0)); 2549 } 2550 } 2551 2552 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 2553 assert(rscratch != noreg || always_reachable(src), "missing"); 2554 2555 if (reachable(src)) { 2556 movq(dst, as_Address(src)); 2557 } else { 2558 lea(rscratch, src); 2559 movq(dst, Address(rscratch, 0)); 2560 } 2561 } 2562 2563 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2564 assert(rscratch != noreg || always_reachable(src), "missing"); 2565 2566 if (reachable(src)) { 2567 if (UseXmmLoadAndClearUpper) { 2568 movsd (dst, as_Address(src)); 2569 } else { 2570 movlpd(dst, as_Address(src)); 2571 } 2572 } else { 2573 lea(rscratch, src); 2574 if (UseXmmLoadAndClearUpper) { 2575 movsd (dst, Address(rscratch, 0)); 2576 } else { 2577 movlpd(dst, Address(rscratch, 0)); 2578 } 2579 } 2580 } 2581 2582 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 2583 assert(rscratch != noreg || always_reachable(src), "missing"); 2584 2585 if (reachable(src)) { 2586 movss(dst, as_Address(src)); 2587 } else { 2588 lea(rscratch, src); 2589 movss(dst, Address(rscratch, 0)); 2590 } 2591 } 2592 2593 void MacroAssembler::movptr(Register dst, Register src) { 2594 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2595 } 2596 2597 void MacroAssembler::movptr(Register dst, Address src) { 2598 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2599 } 2600 2601 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 2602 void MacroAssembler::movptr(Register dst, intptr_t src) { 2603 #ifdef _LP64 2604 if (is_uimm32(src)) { 2605 movl(dst, checked_cast<uint32_t>(src)); 2606 } else if (is_simm32(src)) { 2607 movq(dst, checked_cast<int32_t>(src)); 2608 } else { 2609 mov64(dst, src); 2610 } 2611 #else 2612 movl(dst, src); 2613 #endif 2614 } 2615 2616 void MacroAssembler::movptr(Address dst, Register src) { 2617 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2618 } 2619 2620 void MacroAssembler::movptr(Address dst, int32_t src) { 2621 LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); 2622 } 2623 2624 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 2625 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2626 Assembler::movdqu(dst, src); 2627 } 2628 2629 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 2630 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2631 Assembler::movdqu(dst, src); 2632 } 2633 2634 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 2635 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2636 Assembler::movdqu(dst, src); 2637 } 2638 2639 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2640 assert(rscratch != noreg || always_reachable(src), "missing"); 2641 2642 if (reachable(src)) { 2643 movdqu(dst, as_Address(src)); 2644 } else { 2645 lea(rscratch, src); 2646 movdqu(dst, Address(rscratch, 0)); 2647 } 2648 } 2649 2650 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2651 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2652 Assembler::vmovdqu(dst, src); 2653 } 2654 2655 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2656 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2657 Assembler::vmovdqu(dst, src); 2658 } 2659 2660 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2661 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2662 Assembler::vmovdqu(dst, src); 2663 } 2664 2665 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2666 assert(rscratch != noreg || always_reachable(src), "missing"); 2667 2668 if (reachable(src)) { 2669 vmovdqu(dst, as_Address(src)); 2670 } 2671 else { 2672 lea(rscratch, src); 2673 vmovdqu(dst, Address(rscratch, 0)); 2674 } 2675 } 2676 2677 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2678 assert(rscratch != noreg || always_reachable(src), "missing"); 2679 2680 if (vector_len == AVX_512bit) { 2681 evmovdquq(dst, src, AVX_512bit, rscratch); 2682 } else if (vector_len == AVX_256bit) { 2683 vmovdqu(dst, src, rscratch); 2684 } else { 2685 movdqu(dst, src, rscratch); 2686 } 2687 } 2688 2689 void MacroAssembler::kmov(KRegister dst, Address src) { 2690 if (VM_Version::supports_avx512bw()) { 2691 kmovql(dst, src); 2692 } else { 2693 assert(VM_Version::supports_evex(), ""); 2694 kmovwl(dst, src); 2695 } 2696 } 2697 2698 void MacroAssembler::kmov(Address dst, KRegister src) { 2699 if (VM_Version::supports_avx512bw()) { 2700 kmovql(dst, src); 2701 } else { 2702 assert(VM_Version::supports_evex(), ""); 2703 kmovwl(dst, src); 2704 } 2705 } 2706 2707 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2708 if (VM_Version::supports_avx512bw()) { 2709 kmovql(dst, src); 2710 } else { 2711 assert(VM_Version::supports_evex(), ""); 2712 kmovwl(dst, src); 2713 } 2714 } 2715 2716 void MacroAssembler::kmov(Register dst, KRegister src) { 2717 if (VM_Version::supports_avx512bw()) { 2718 kmovql(dst, src); 2719 } else { 2720 assert(VM_Version::supports_evex(), ""); 2721 kmovwl(dst, src); 2722 } 2723 } 2724 2725 void MacroAssembler::kmov(KRegister dst, Register src) { 2726 if (VM_Version::supports_avx512bw()) { 2727 kmovql(dst, src); 2728 } else { 2729 assert(VM_Version::supports_evex(), ""); 2730 kmovwl(dst, src); 2731 } 2732 } 2733 2734 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2735 assert(rscratch != noreg || always_reachable(src), "missing"); 2736 2737 if (reachable(src)) { 2738 kmovql(dst, as_Address(src)); 2739 } else { 2740 lea(rscratch, src); 2741 kmovql(dst, Address(rscratch, 0)); 2742 } 2743 } 2744 2745 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2746 assert(rscratch != noreg || always_reachable(src), "missing"); 2747 2748 if (reachable(src)) { 2749 kmovwl(dst, as_Address(src)); 2750 } else { 2751 lea(rscratch, src); 2752 kmovwl(dst, Address(rscratch, 0)); 2753 } 2754 } 2755 2756 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2757 int vector_len, Register rscratch) { 2758 assert(rscratch != noreg || always_reachable(src), "missing"); 2759 2760 if (reachable(src)) { 2761 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2762 } else { 2763 lea(rscratch, src); 2764 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2765 } 2766 } 2767 2768 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2769 int vector_len, Register rscratch) { 2770 assert(rscratch != noreg || always_reachable(src), "missing"); 2771 2772 if (reachable(src)) { 2773 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2774 } else { 2775 lea(rscratch, src); 2776 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2777 } 2778 } 2779 2780 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2781 assert(rscratch != noreg || always_reachable(src), "missing"); 2782 2783 if (reachable(src)) { 2784 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2785 } else { 2786 lea(rscratch, src); 2787 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2788 } 2789 } 2790 2791 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2792 assert(rscratch != noreg || always_reachable(src), "missing"); 2793 2794 if (reachable(src)) { 2795 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2796 } else { 2797 lea(rscratch, src); 2798 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2799 } 2800 } 2801 2802 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2803 assert(rscratch != noreg || always_reachable(src), "missing"); 2804 2805 if (reachable(src)) { 2806 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2807 } else { 2808 lea(rscratch, src); 2809 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2810 } 2811 } 2812 2813 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2814 assert(rscratch != noreg || always_reachable(src), "missing"); 2815 2816 if (reachable(src)) { 2817 Assembler::movdqa(dst, as_Address(src)); 2818 } else { 2819 lea(rscratch, src); 2820 Assembler::movdqa(dst, Address(rscratch, 0)); 2821 } 2822 } 2823 2824 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2825 assert(rscratch != noreg || always_reachable(src), "missing"); 2826 2827 if (reachable(src)) { 2828 Assembler::movsd(dst, as_Address(src)); 2829 } else { 2830 lea(rscratch, src); 2831 Assembler::movsd(dst, Address(rscratch, 0)); 2832 } 2833 } 2834 2835 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2836 assert(rscratch != noreg || always_reachable(src), "missing"); 2837 2838 if (reachable(src)) { 2839 Assembler::movss(dst, as_Address(src)); 2840 } else { 2841 lea(rscratch, src); 2842 Assembler::movss(dst, Address(rscratch, 0)); 2843 } 2844 } 2845 2846 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2847 assert(rscratch != noreg || always_reachable(src), "missing"); 2848 2849 if (reachable(src)) { 2850 Assembler::movddup(dst, as_Address(src)); 2851 } else { 2852 lea(rscratch, src); 2853 Assembler::movddup(dst, Address(rscratch, 0)); 2854 } 2855 } 2856 2857 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2858 assert(rscratch != noreg || always_reachable(src), "missing"); 2859 2860 if (reachable(src)) { 2861 Assembler::vmovddup(dst, as_Address(src), vector_len); 2862 } else { 2863 lea(rscratch, src); 2864 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2865 } 2866 } 2867 2868 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2869 assert(rscratch != noreg || always_reachable(src), "missing"); 2870 2871 if (reachable(src)) { 2872 Assembler::mulsd(dst, as_Address(src)); 2873 } else { 2874 lea(rscratch, src); 2875 Assembler::mulsd(dst, Address(rscratch, 0)); 2876 } 2877 } 2878 2879 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2880 assert(rscratch != noreg || always_reachable(src), "missing"); 2881 2882 if (reachable(src)) { 2883 Assembler::mulss(dst, as_Address(src)); 2884 } else { 2885 lea(rscratch, src); 2886 Assembler::mulss(dst, Address(rscratch, 0)); 2887 } 2888 } 2889 2890 void MacroAssembler::null_check(Register reg, int offset) { 2891 if (needs_explicit_null_check(offset)) { 2892 // provoke OS null exception if reg is null by 2893 // accessing M[reg] w/o changing any (non-CC) registers 2894 // NOTE: cmpl is plenty here to provoke a segv 2895 cmpptr(rax, Address(reg, 0)); 2896 // Note: should probably use testl(rax, Address(reg, 0)); 2897 // may be shorter code (however, this version of 2898 // testl needs to be implemented first) 2899 } else { 2900 // nothing to do, (later) access of M[reg + offset] 2901 // will provoke OS null exception if reg is null 2902 } 2903 } 2904 2905 void MacroAssembler::os_breakpoint() { 2906 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2907 // (e.g., MSVC can't call ps() otherwise) 2908 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2909 } 2910 2911 void MacroAssembler::unimplemented(const char* what) { 2912 const char* buf = nullptr; 2913 { 2914 ResourceMark rm; 2915 stringStream ss; 2916 ss.print("unimplemented: %s", what); 2917 buf = code_string(ss.as_string()); 2918 } 2919 stop(buf); 2920 } 2921 2922 #ifdef _LP64 2923 #define XSTATE_BV 0x200 2924 #endif 2925 2926 void MacroAssembler::pop_CPU_state() { 2927 pop_FPU_state(); 2928 pop_IU_state(); 2929 } 2930 2931 void MacroAssembler::pop_FPU_state() { 2932 #ifndef _LP64 2933 frstor(Address(rsp, 0)); 2934 #else 2935 fxrstor(Address(rsp, 0)); 2936 #endif 2937 addptr(rsp, FPUStateSizeInWords * wordSize); 2938 } 2939 2940 void MacroAssembler::pop_IU_state() { 2941 popa(); 2942 LP64_ONLY(addq(rsp, 8)); 2943 popf(); 2944 } 2945 2946 // Save Integer and Float state 2947 // Warning: Stack must be 16 byte aligned (64bit) 2948 void MacroAssembler::push_CPU_state() { 2949 push_IU_state(); 2950 push_FPU_state(); 2951 } 2952 2953 void MacroAssembler::push_FPU_state() { 2954 subptr(rsp, FPUStateSizeInWords * wordSize); 2955 #ifndef _LP64 2956 fnsave(Address(rsp, 0)); 2957 fwait(); 2958 #else 2959 fxsave(Address(rsp, 0)); 2960 #endif // LP64 2961 } 2962 2963 void MacroAssembler::push_IU_state() { 2964 // Push flags first because pusha kills them 2965 pushf(); 2966 // Make sure rsp stays 16-byte aligned 2967 LP64_ONLY(subq(rsp, 8)); 2968 pusha(); 2969 } 2970 2971 void MacroAssembler::push_cont_fastpath() { 2972 if (!Continuations::enabled()) return; 2973 2974 #ifndef _LP64 2975 Register rthread = rax; 2976 Register rrealsp = rbx; 2977 push(rthread); 2978 push(rrealsp); 2979 2980 get_thread(rthread); 2981 2982 // The code below wants the original RSP. 2983 // Move it back after the pushes above. 2984 movptr(rrealsp, rsp); 2985 addptr(rrealsp, 2*wordSize); 2986 #else 2987 Register rthread = r15_thread; 2988 Register rrealsp = rsp; 2989 #endif 2990 2991 Label done; 2992 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 2993 jccb(Assembler::belowEqual, done); 2994 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp); 2995 bind(done); 2996 2997 #ifndef _LP64 2998 pop(rrealsp); 2999 pop(rthread); 3000 #endif 3001 } 3002 3003 void MacroAssembler::pop_cont_fastpath() { 3004 if (!Continuations::enabled()) return; 3005 3006 #ifndef _LP64 3007 Register rthread = rax; 3008 Register rrealsp = rbx; 3009 push(rthread); 3010 push(rrealsp); 3011 3012 get_thread(rthread); 3013 3014 // The code below wants the original RSP. 3015 // Move it back after the pushes above. 3016 movptr(rrealsp, rsp); 3017 addptr(rrealsp, 2*wordSize); 3018 #else 3019 Register rthread = r15_thread; 3020 Register rrealsp = rsp; 3021 #endif 3022 3023 Label done; 3024 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3025 jccb(Assembler::below, done); 3026 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0); 3027 bind(done); 3028 3029 #ifndef _LP64 3030 pop(rrealsp); 3031 pop(rthread); 3032 #endif 3033 } 3034 3035 void MacroAssembler::inc_held_monitor_count() { 3036 #ifndef _LP64 3037 Register thread = rax; 3038 push(thread); 3039 get_thread(thread); 3040 incrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3041 pop(thread); 3042 #else // LP64 3043 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3044 #endif 3045 } 3046 3047 void MacroAssembler::dec_held_monitor_count() { 3048 #ifndef _LP64 3049 Register thread = rax; 3050 push(thread); 3051 get_thread(thread); 3052 decrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3053 pop(thread); 3054 #else // LP64 3055 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3056 #endif 3057 } 3058 3059 #ifdef ASSERT 3060 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 3061 #ifdef _LP64 3062 Label no_cont; 3063 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 3064 testl(cont, cont); 3065 jcc(Assembler::zero, no_cont); 3066 stop(name); 3067 bind(no_cont); 3068 #else 3069 Unimplemented(); 3070 #endif 3071 } 3072 #endif 3073 3074 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register 3075 if (!java_thread->is_valid()) { 3076 java_thread = rdi; 3077 get_thread(java_thread); 3078 } 3079 // we must set sp to zero to clear frame 3080 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 3081 // must clear fp, so that compiled frames are not confused; it is 3082 // possible that we need it only for debugging 3083 if (clear_fp) { 3084 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 3085 } 3086 // Always clear the pc because it could have been set by make_walkable() 3087 movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 3088 vzeroupper(); 3089 } 3090 3091 void MacroAssembler::restore_rax(Register tmp) { 3092 if (tmp == noreg) pop(rax); 3093 else if (tmp != rax) mov(rax, tmp); 3094 } 3095 3096 void MacroAssembler::round_to(Register reg, int modulus) { 3097 addptr(reg, modulus - 1); 3098 andptr(reg, -modulus); 3099 } 3100 3101 void MacroAssembler::save_rax(Register tmp) { 3102 if (tmp == noreg) push(rax); 3103 else if (tmp != rax) mov(tmp, rax); 3104 } 3105 3106 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) { 3107 if (at_return) { 3108 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 3109 // we may safely use rsp instead to perform the stack watermark check. 3110 cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset())); 3111 jcc(Assembler::above, slow_path); 3112 return; 3113 } 3114 testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 3115 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 3116 } 3117 3118 // Calls to C land 3119 // 3120 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 3121 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 3122 // has to be reset to 0. This is required to allow proper stack traversal. 3123 void MacroAssembler::set_last_Java_frame(Register java_thread, 3124 Register last_java_sp, 3125 Register last_java_fp, 3126 address last_java_pc, 3127 Register rscratch) { 3128 vzeroupper(); 3129 // determine java_thread register 3130 if (!java_thread->is_valid()) { 3131 java_thread = rdi; 3132 get_thread(java_thread); 3133 } 3134 // determine last_java_sp register 3135 if (!last_java_sp->is_valid()) { 3136 last_java_sp = rsp; 3137 } 3138 // last_java_fp is optional 3139 if (last_java_fp->is_valid()) { 3140 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 3141 } 3142 // last_java_pc is optional 3143 if (last_java_pc != nullptr) { 3144 Address java_pc(java_thread, 3145 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 3146 lea(java_pc, InternalAddress(last_java_pc), rscratch); 3147 } 3148 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 3149 } 3150 3151 void MacroAssembler::shlptr(Register dst, int imm8) { 3152 LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); 3153 } 3154 3155 void MacroAssembler::shrptr(Register dst, int imm8) { 3156 LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); 3157 } 3158 3159 void MacroAssembler::sign_extend_byte(Register reg) { 3160 if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { 3161 movsbl(reg, reg); // movsxb 3162 } else { 3163 shll(reg, 24); 3164 sarl(reg, 24); 3165 } 3166 } 3167 3168 void MacroAssembler::sign_extend_short(Register reg) { 3169 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 3170 movswl(reg, reg); // movsxw 3171 } else { 3172 shll(reg, 16); 3173 sarl(reg, 16); 3174 } 3175 } 3176 3177 void MacroAssembler::testl(Address dst, int32_t imm32) { 3178 if (imm32 >= 0 && is8bit(imm32)) { 3179 testb(dst, imm32); 3180 } else { 3181 Assembler::testl(dst, imm32); 3182 } 3183 } 3184 3185 void MacroAssembler::testl(Register dst, int32_t imm32) { 3186 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 3187 testb(dst, imm32); 3188 } else { 3189 Assembler::testl(dst, imm32); 3190 } 3191 } 3192 3193 void MacroAssembler::testl(Register dst, AddressLiteral src) { 3194 assert(always_reachable(src), "Address should be reachable"); 3195 testl(dst, as_Address(src)); 3196 } 3197 3198 #ifdef _LP64 3199 3200 void MacroAssembler::testq(Address dst, int32_t imm32) { 3201 if (imm32 >= 0) { 3202 testl(dst, imm32); 3203 } else { 3204 Assembler::testq(dst, imm32); 3205 } 3206 } 3207 3208 void MacroAssembler::testq(Register dst, int32_t imm32) { 3209 if (imm32 >= 0) { 3210 testl(dst, imm32); 3211 } else { 3212 Assembler::testq(dst, imm32); 3213 } 3214 } 3215 3216 #endif 3217 3218 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 3219 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3220 Assembler::pcmpeqb(dst, src); 3221 } 3222 3223 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 3224 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3225 Assembler::pcmpeqw(dst, src); 3226 } 3227 3228 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 3229 assert((dst->encoding() < 16),"XMM register should be 0-15"); 3230 Assembler::pcmpestri(dst, src, imm8); 3231 } 3232 3233 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 3234 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3235 Assembler::pcmpestri(dst, src, imm8); 3236 } 3237 3238 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 3239 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3240 Assembler::pmovzxbw(dst, src); 3241 } 3242 3243 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 3244 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3245 Assembler::pmovzxbw(dst, src); 3246 } 3247 3248 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 3249 assert((src->encoding() < 16),"XMM register should be 0-15"); 3250 Assembler::pmovmskb(dst, src); 3251 } 3252 3253 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 3254 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3255 Assembler::ptest(dst, src); 3256 } 3257 3258 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3259 assert(rscratch != noreg || always_reachable(src), "missing"); 3260 3261 if (reachable(src)) { 3262 Assembler::sqrtss(dst, as_Address(src)); 3263 } else { 3264 lea(rscratch, src); 3265 Assembler::sqrtss(dst, Address(rscratch, 0)); 3266 } 3267 } 3268 3269 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3270 assert(rscratch != noreg || always_reachable(src), "missing"); 3271 3272 if (reachable(src)) { 3273 Assembler::subsd(dst, as_Address(src)); 3274 } else { 3275 lea(rscratch, src); 3276 Assembler::subsd(dst, Address(rscratch, 0)); 3277 } 3278 } 3279 3280 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 3281 assert(rscratch != noreg || always_reachable(src), "missing"); 3282 3283 if (reachable(src)) { 3284 Assembler::roundsd(dst, as_Address(src), rmode); 3285 } else { 3286 lea(rscratch, src); 3287 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 3288 } 3289 } 3290 3291 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3292 assert(rscratch != noreg || always_reachable(src), "missing"); 3293 3294 if (reachable(src)) { 3295 Assembler::subss(dst, as_Address(src)); 3296 } else { 3297 lea(rscratch, src); 3298 Assembler::subss(dst, Address(rscratch, 0)); 3299 } 3300 } 3301 3302 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3303 assert(rscratch != noreg || always_reachable(src), "missing"); 3304 3305 if (reachable(src)) { 3306 Assembler::ucomisd(dst, as_Address(src)); 3307 } else { 3308 lea(rscratch, src); 3309 Assembler::ucomisd(dst, Address(rscratch, 0)); 3310 } 3311 } 3312 3313 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3314 assert(rscratch != noreg || always_reachable(src), "missing"); 3315 3316 if (reachable(src)) { 3317 Assembler::ucomiss(dst, as_Address(src)); 3318 } else { 3319 lea(rscratch, src); 3320 Assembler::ucomiss(dst, Address(rscratch, 0)); 3321 } 3322 } 3323 3324 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3325 assert(rscratch != noreg || always_reachable(src), "missing"); 3326 3327 // Used in sign-bit flipping with aligned address. 3328 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3329 if (reachable(src)) { 3330 Assembler::xorpd(dst, as_Address(src)); 3331 } else { 3332 lea(rscratch, src); 3333 Assembler::xorpd(dst, Address(rscratch, 0)); 3334 } 3335 } 3336 3337 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3338 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3339 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3340 } 3341 else { 3342 Assembler::xorpd(dst, src); 3343 } 3344 } 3345 3346 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3347 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3348 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3349 } else { 3350 Assembler::xorps(dst, src); 3351 } 3352 } 3353 3354 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 3355 assert(rscratch != noreg || always_reachable(src), "missing"); 3356 3357 // Used in sign-bit flipping with aligned address. 3358 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3359 if (reachable(src)) { 3360 Assembler::xorps(dst, as_Address(src)); 3361 } else { 3362 lea(rscratch, src); 3363 Assembler::xorps(dst, Address(rscratch, 0)); 3364 } 3365 } 3366 3367 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 3368 assert(rscratch != noreg || always_reachable(src), "missing"); 3369 3370 // Used in sign-bit flipping with aligned address. 3371 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3372 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3373 if (reachable(src)) { 3374 Assembler::pshufb(dst, as_Address(src)); 3375 } else { 3376 lea(rscratch, src); 3377 Assembler::pshufb(dst, Address(rscratch, 0)); 3378 } 3379 } 3380 3381 // AVX 3-operands instructions 3382 3383 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3384 assert(rscratch != noreg || always_reachable(src), "missing"); 3385 3386 if (reachable(src)) { 3387 vaddsd(dst, nds, as_Address(src)); 3388 } else { 3389 lea(rscratch, src); 3390 vaddsd(dst, nds, Address(rscratch, 0)); 3391 } 3392 } 3393 3394 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3395 assert(rscratch != noreg || always_reachable(src), "missing"); 3396 3397 if (reachable(src)) { 3398 vaddss(dst, nds, as_Address(src)); 3399 } else { 3400 lea(rscratch, src); 3401 vaddss(dst, nds, Address(rscratch, 0)); 3402 } 3403 } 3404 3405 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3406 assert(UseAVX > 0, "requires some form of AVX"); 3407 assert(rscratch != noreg || always_reachable(src), "missing"); 3408 3409 if (reachable(src)) { 3410 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 3411 } else { 3412 lea(rscratch, src); 3413 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 3414 } 3415 } 3416 3417 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3418 assert(UseAVX > 0, "requires some form of AVX"); 3419 assert(rscratch != noreg || always_reachable(src), "missing"); 3420 3421 if (reachable(src)) { 3422 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 3423 } else { 3424 lea(rscratch, src); 3425 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 3426 } 3427 } 3428 3429 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3430 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3431 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3432 3433 vandps(dst, nds, negate_field, vector_len, rscratch); 3434 } 3435 3436 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3437 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3438 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3439 3440 vandpd(dst, nds, negate_field, vector_len, rscratch); 3441 } 3442 3443 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3444 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3445 Assembler::vpaddb(dst, nds, src, vector_len); 3446 } 3447 3448 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3449 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3450 Assembler::vpaddb(dst, nds, src, vector_len); 3451 } 3452 3453 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3454 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3455 Assembler::vpaddw(dst, nds, src, vector_len); 3456 } 3457 3458 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3459 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3460 Assembler::vpaddw(dst, nds, src, vector_len); 3461 } 3462 3463 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3464 assert(rscratch != noreg || always_reachable(src), "missing"); 3465 3466 if (reachable(src)) { 3467 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3468 } else { 3469 lea(rscratch, src); 3470 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 3471 } 3472 } 3473 3474 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3475 assert(rscratch != noreg || always_reachable(src), "missing"); 3476 3477 if (reachable(src)) { 3478 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 3479 } else { 3480 lea(rscratch, src); 3481 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 3482 } 3483 } 3484 3485 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3486 assert(rscratch != noreg || always_reachable(src), "missing"); 3487 3488 if (reachable(src)) { 3489 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 3490 } else { 3491 lea(rscratch, src); 3492 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 3493 } 3494 } 3495 3496 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3497 assert(rscratch != noreg || always_reachable(src), "missing"); 3498 3499 if (reachable(src)) { 3500 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 3501 } else { 3502 lea(rscratch, src); 3503 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 3504 } 3505 } 3506 3507 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3508 assert(rscratch != noreg || always_reachable(src), "missing"); 3509 3510 if (reachable(src)) { 3511 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 3512 } else { 3513 lea(rscratch, src); 3514 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 3515 } 3516 } 3517 3518 // Vector float blend 3519 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3520 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3521 // WARN: Allow dst == (src1|src2), mask == scratch 3522 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3523 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst; 3524 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3525 if (blend_emulation && scratch_available && dst_available) { 3526 if (compute_mask) { 3527 vpsrad(scratch, mask, 32, vector_len); 3528 mask = scratch; 3529 } 3530 if (dst == src1) { 3531 vpandn(dst, mask, src1, vector_len); // if mask == 0, src1 3532 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3533 } else { 3534 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3535 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1 3536 } 3537 vpor(dst, dst, scratch, vector_len); 3538 } else { 3539 Assembler::vblendvps(dst, src1, src2, mask, vector_len); 3540 } 3541 } 3542 3543 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3544 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3545 // WARN: Allow dst == (src1|src2), mask == scratch 3546 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3547 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask); 3548 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3549 if (blend_emulation && scratch_available && dst_available) { 3550 if (compute_mask) { 3551 vpxor(scratch, scratch, scratch, vector_len); 3552 vpcmpgtq(scratch, scratch, mask, vector_len); 3553 mask = scratch; 3554 } 3555 if (dst == src1) { 3556 vpandn(dst, mask, src1, vector_len); // if mask == 0, src 3557 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3558 } else { 3559 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3560 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src 3561 } 3562 vpor(dst, dst, scratch, vector_len); 3563 } else { 3564 Assembler::vblendvpd(dst, src1, src2, mask, vector_len); 3565 } 3566 } 3567 3568 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3569 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3570 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3571 } 3572 3573 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3574 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3575 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3576 } 3577 3578 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3579 assert(rscratch != noreg || always_reachable(src), "missing"); 3580 3581 if (reachable(src)) { 3582 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 3583 } else { 3584 lea(rscratch, src); 3585 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 3586 } 3587 } 3588 3589 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3590 int comparison, bool is_signed, int vector_len, Register rscratch) { 3591 assert(rscratch != noreg || always_reachable(src), "missing"); 3592 3593 if (reachable(src)) { 3594 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3595 } else { 3596 lea(rscratch, src); 3597 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3598 } 3599 } 3600 3601 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3602 int comparison, bool is_signed, int vector_len, Register rscratch) { 3603 assert(rscratch != noreg || always_reachable(src), "missing"); 3604 3605 if (reachable(src)) { 3606 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3607 } else { 3608 lea(rscratch, src); 3609 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3610 } 3611 } 3612 3613 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3614 int comparison, bool is_signed, int vector_len, Register rscratch) { 3615 assert(rscratch != noreg || always_reachable(src), "missing"); 3616 3617 if (reachable(src)) { 3618 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3619 } else { 3620 lea(rscratch, src); 3621 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3622 } 3623 } 3624 3625 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3626 int comparison, bool is_signed, int vector_len, Register rscratch) { 3627 assert(rscratch != noreg || always_reachable(src), "missing"); 3628 3629 if (reachable(src)) { 3630 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3631 } else { 3632 lea(rscratch, src); 3633 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3634 } 3635 } 3636 3637 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3638 if (width == Assembler::Q) { 3639 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3640 } else { 3641 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3642 } 3643 } 3644 3645 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3646 int eq_cond_enc = 0x29; 3647 int gt_cond_enc = 0x37; 3648 if (width != Assembler::Q) { 3649 eq_cond_enc = 0x74 + width; 3650 gt_cond_enc = 0x64 + width; 3651 } 3652 switch (cond) { 3653 case eq: 3654 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3655 break; 3656 case neq: 3657 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3658 vallones(xtmp, vector_len); 3659 vpxor(dst, xtmp, dst, vector_len); 3660 break; 3661 case le: 3662 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3663 vallones(xtmp, vector_len); 3664 vpxor(dst, xtmp, dst, vector_len); 3665 break; 3666 case nlt: 3667 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3668 vallones(xtmp, vector_len); 3669 vpxor(dst, xtmp, dst, vector_len); 3670 break; 3671 case lt: 3672 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3673 break; 3674 case nle: 3675 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3676 break; 3677 default: 3678 assert(false, "Should not reach here"); 3679 } 3680 } 3681 3682 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3683 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3684 Assembler::vpmovzxbw(dst, src, vector_len); 3685 } 3686 3687 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3688 assert((src->encoding() < 16),"XMM register should be 0-15"); 3689 Assembler::vpmovmskb(dst, src, vector_len); 3690 } 3691 3692 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3693 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3694 Assembler::vpmullw(dst, nds, src, vector_len); 3695 } 3696 3697 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3698 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3699 Assembler::vpmullw(dst, nds, src, vector_len); 3700 } 3701 3702 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3703 assert((UseAVX > 0), "AVX support is needed"); 3704 assert(rscratch != noreg || always_reachable(src), "missing"); 3705 3706 if (reachable(src)) { 3707 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3708 } else { 3709 lea(rscratch, src); 3710 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3711 } 3712 } 3713 3714 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3715 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3716 Assembler::vpsubb(dst, nds, src, vector_len); 3717 } 3718 3719 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3720 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3721 Assembler::vpsubb(dst, nds, src, vector_len); 3722 } 3723 3724 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3725 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3726 Assembler::vpsubw(dst, nds, src, vector_len); 3727 } 3728 3729 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3730 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3731 Assembler::vpsubw(dst, nds, src, vector_len); 3732 } 3733 3734 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3735 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3736 Assembler::vpsraw(dst, nds, shift, vector_len); 3737 } 3738 3739 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3740 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3741 Assembler::vpsraw(dst, nds, shift, vector_len); 3742 } 3743 3744 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3745 assert(UseAVX > 2,""); 3746 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3747 vector_len = 2; 3748 } 3749 Assembler::evpsraq(dst, nds, shift, vector_len); 3750 } 3751 3752 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3753 assert(UseAVX > 2,""); 3754 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3755 vector_len = 2; 3756 } 3757 Assembler::evpsraq(dst, nds, shift, vector_len); 3758 } 3759 3760 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3761 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3762 Assembler::vpsrlw(dst, nds, shift, vector_len); 3763 } 3764 3765 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3766 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3767 Assembler::vpsrlw(dst, nds, shift, vector_len); 3768 } 3769 3770 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3771 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3772 Assembler::vpsllw(dst, nds, shift, vector_len); 3773 } 3774 3775 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3776 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3777 Assembler::vpsllw(dst, nds, shift, vector_len); 3778 } 3779 3780 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3781 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3782 Assembler::vptest(dst, src); 3783 } 3784 3785 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3786 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3787 Assembler::punpcklbw(dst, src); 3788 } 3789 3790 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3791 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3792 Assembler::pshufd(dst, src, mode); 3793 } 3794 3795 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3796 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3797 Assembler::pshuflw(dst, src, mode); 3798 } 3799 3800 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3801 assert(rscratch != noreg || always_reachable(src), "missing"); 3802 3803 if (reachable(src)) { 3804 vandpd(dst, nds, as_Address(src), vector_len); 3805 } else { 3806 lea(rscratch, src); 3807 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3808 } 3809 } 3810 3811 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3812 assert(rscratch != noreg || always_reachable(src), "missing"); 3813 3814 if (reachable(src)) { 3815 vandps(dst, nds, as_Address(src), vector_len); 3816 } else { 3817 lea(rscratch, src); 3818 vandps(dst, nds, Address(rscratch, 0), vector_len); 3819 } 3820 } 3821 3822 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3823 bool merge, int vector_len, Register rscratch) { 3824 assert(rscratch != noreg || always_reachable(src), "missing"); 3825 3826 if (reachable(src)) { 3827 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3828 } else { 3829 lea(rscratch, src); 3830 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3831 } 3832 } 3833 3834 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3835 assert(rscratch != noreg || always_reachable(src), "missing"); 3836 3837 if (reachable(src)) { 3838 vdivsd(dst, nds, as_Address(src)); 3839 } else { 3840 lea(rscratch, src); 3841 vdivsd(dst, nds, Address(rscratch, 0)); 3842 } 3843 } 3844 3845 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3846 assert(rscratch != noreg || always_reachable(src), "missing"); 3847 3848 if (reachable(src)) { 3849 vdivss(dst, nds, as_Address(src)); 3850 } else { 3851 lea(rscratch, src); 3852 vdivss(dst, nds, Address(rscratch, 0)); 3853 } 3854 } 3855 3856 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3857 assert(rscratch != noreg || always_reachable(src), "missing"); 3858 3859 if (reachable(src)) { 3860 vmulsd(dst, nds, as_Address(src)); 3861 } else { 3862 lea(rscratch, src); 3863 vmulsd(dst, nds, Address(rscratch, 0)); 3864 } 3865 } 3866 3867 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3868 assert(rscratch != noreg || always_reachable(src), "missing"); 3869 3870 if (reachable(src)) { 3871 vmulss(dst, nds, as_Address(src)); 3872 } else { 3873 lea(rscratch, src); 3874 vmulss(dst, nds, Address(rscratch, 0)); 3875 } 3876 } 3877 3878 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3879 assert(rscratch != noreg || always_reachable(src), "missing"); 3880 3881 if (reachable(src)) { 3882 vsubsd(dst, nds, as_Address(src)); 3883 } else { 3884 lea(rscratch, src); 3885 vsubsd(dst, nds, Address(rscratch, 0)); 3886 } 3887 } 3888 3889 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3890 assert(rscratch != noreg || always_reachable(src), "missing"); 3891 3892 if (reachable(src)) { 3893 vsubss(dst, nds, as_Address(src)); 3894 } else { 3895 lea(rscratch, src); 3896 vsubss(dst, nds, Address(rscratch, 0)); 3897 } 3898 } 3899 3900 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3901 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3902 assert(rscratch != noreg || always_reachable(src), "missing"); 3903 3904 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3905 } 3906 3907 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3908 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3909 assert(rscratch != noreg || always_reachable(src), "missing"); 3910 3911 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3912 } 3913 3914 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3915 assert(rscratch != noreg || always_reachable(src), "missing"); 3916 3917 if (reachable(src)) { 3918 vxorpd(dst, nds, as_Address(src), vector_len); 3919 } else { 3920 lea(rscratch, src); 3921 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3922 } 3923 } 3924 3925 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3926 assert(rscratch != noreg || always_reachable(src), "missing"); 3927 3928 if (reachable(src)) { 3929 vxorps(dst, nds, as_Address(src), vector_len); 3930 } else { 3931 lea(rscratch, src); 3932 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3933 } 3934 } 3935 3936 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3937 assert(rscratch != noreg || always_reachable(src), "missing"); 3938 3939 if (UseAVX > 1 || (vector_len < 1)) { 3940 if (reachable(src)) { 3941 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3942 } else { 3943 lea(rscratch, src); 3944 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3945 } 3946 } else { 3947 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3948 } 3949 } 3950 3951 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3952 assert(rscratch != noreg || always_reachable(src), "missing"); 3953 3954 if (reachable(src)) { 3955 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 3956 } else { 3957 lea(rscratch, src); 3958 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 3959 } 3960 } 3961 3962 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 3963 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 3964 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 3965 // The inverted mask is sign-extended 3966 andptr(possibly_non_local, inverted_mask); 3967 } 3968 3969 void MacroAssembler::resolve_jobject(Register value, 3970 Register thread, 3971 Register tmp) { 3972 assert_different_registers(value, thread, tmp); 3973 Label done, tagged, weak_tagged; 3974 testptr(value, value); 3975 jcc(Assembler::zero, done); // Use null as-is. 3976 testptr(value, JNIHandles::tag_mask); // Test for tag. 3977 jcc(Assembler::notZero, tagged); 3978 3979 // Resolve local handle 3980 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread); 3981 verify_oop(value); 3982 jmp(done); 3983 3984 bind(tagged); 3985 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 3986 jcc(Assembler::notZero, weak_tagged); 3987 3988 // Resolve global handle 3989 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 3990 verify_oop(value); 3991 jmp(done); 3992 3993 bind(weak_tagged); 3994 // Resolve jweak. 3995 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 3996 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread); 3997 verify_oop(value); 3998 3999 bind(done); 4000 } 4001 4002 void MacroAssembler::resolve_global_jobject(Register value, 4003 Register thread, 4004 Register tmp) { 4005 assert_different_registers(value, thread, tmp); 4006 Label done; 4007 4008 testptr(value, value); 4009 jcc(Assembler::zero, done); // Use null as-is. 4010 4011 #ifdef ASSERT 4012 { 4013 Label valid_global_tag; 4014 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 4015 jcc(Assembler::notZero, valid_global_tag); 4016 stop("non global jobject using resolve_global_jobject"); 4017 bind(valid_global_tag); 4018 } 4019 #endif 4020 4021 // Resolve global handle 4022 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4023 verify_oop(value); 4024 4025 bind(done); 4026 } 4027 4028 void MacroAssembler::subptr(Register dst, int32_t imm32) { 4029 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 4030 } 4031 4032 // Force generation of a 4 byte immediate value even if it fits into 8bit 4033 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 4034 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 4035 } 4036 4037 void MacroAssembler::subptr(Register dst, Register src) { 4038 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 4039 } 4040 4041 // C++ bool manipulation 4042 void MacroAssembler::testbool(Register dst) { 4043 if(sizeof(bool) == 1) 4044 testb(dst, 0xff); 4045 else if(sizeof(bool) == 2) { 4046 // testw implementation needed for two byte bools 4047 ShouldNotReachHere(); 4048 } else if(sizeof(bool) == 4) 4049 testl(dst, dst); 4050 else 4051 // unsupported 4052 ShouldNotReachHere(); 4053 } 4054 4055 void MacroAssembler::testptr(Register dst, Register src) { 4056 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); 4057 } 4058 4059 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 4060 void MacroAssembler::tlab_allocate(Register thread, Register obj, 4061 Register var_size_in_bytes, 4062 int con_size_in_bytes, 4063 Register t1, 4064 Register t2, 4065 Label& slow_case) { 4066 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 4067 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 4068 } 4069 4070 RegSet MacroAssembler::call_clobbered_gp_registers() { 4071 RegSet regs; 4072 #ifdef _LP64 4073 regs += RegSet::of(rax, rcx, rdx); 4074 #ifndef WINDOWS 4075 regs += RegSet::of(rsi, rdi); 4076 #endif 4077 regs += RegSet::range(r8, r11); 4078 #else 4079 regs += RegSet::of(rax, rcx, rdx); 4080 #endif 4081 return regs; 4082 } 4083 4084 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 4085 int num_xmm_registers = XMMRegister::available_xmm_registers(); 4086 #if defined(WINDOWS) && defined(_LP64) 4087 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 4088 if (num_xmm_registers > 16) { 4089 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 4090 } 4091 return result; 4092 #else 4093 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 4094 #endif 4095 } 4096 4097 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor 4098 4099 #ifndef _LP64 4100 static bool use_x87_registers() { return UseSSE < 2; } 4101 #endif 4102 static bool use_xmm_registers() { return UseSSE >= 1; } 4103 4104 // C1 only ever uses the first double/float of the XMM register. 4105 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); } 4106 4107 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4108 if (UseSSE == 1) { 4109 masm->movflt(Address(rsp, offset), reg); 4110 } else { 4111 masm->movdbl(Address(rsp, offset), reg); 4112 } 4113 } 4114 4115 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4116 if (UseSSE == 1) { 4117 masm->movflt(reg, Address(rsp, offset)); 4118 } else { 4119 masm->movdbl(reg, Address(rsp, offset)); 4120 } 4121 } 4122 4123 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, 4124 bool save_fpu, int& gp_area_size, 4125 int& fp_area_size, int& xmm_area_size) { 4126 4127 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 4128 StackAlignmentInBytes); 4129 #ifdef _LP64 4130 fp_area_size = 0; 4131 #else 4132 fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0; 4133 #endif 4134 xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0; 4135 4136 return gp_area_size + fp_area_size + xmm_area_size; 4137 } 4138 4139 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 4140 block_comment("push_call_clobbered_registers start"); 4141 // Regular registers 4142 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 4143 4144 int gp_area_size; 4145 int fp_area_size; 4146 int xmm_area_size; 4147 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 4148 gp_area_size, fp_area_size, xmm_area_size); 4149 subptr(rsp, total_save_size); 4150 4151 push_set(gp_registers_to_push, 0); 4152 4153 #ifndef _LP64 4154 if (save_fpu && use_x87_registers()) { 4155 fnsave(Address(rsp, gp_area_size)); 4156 fwait(); 4157 } 4158 #endif 4159 if (save_fpu && use_xmm_registers()) { 4160 push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4161 } 4162 4163 block_comment("push_call_clobbered_registers end"); 4164 } 4165 4166 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 4167 block_comment("pop_call_clobbered_registers start"); 4168 4169 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 4170 4171 int gp_area_size; 4172 int fp_area_size; 4173 int xmm_area_size; 4174 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 4175 gp_area_size, fp_area_size, xmm_area_size); 4176 4177 if (restore_fpu && use_xmm_registers()) { 4178 pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4179 } 4180 #ifndef _LP64 4181 if (restore_fpu && use_x87_registers()) { 4182 frstor(Address(rsp, gp_area_size)); 4183 } 4184 #endif 4185 4186 pop_set(gp_registers_to_pop, 0); 4187 4188 addptr(rsp, total_save_size); 4189 4190 vzeroupper(); 4191 4192 block_comment("pop_call_clobbered_registers end"); 4193 } 4194 4195 void MacroAssembler::push_set(XMMRegSet set, int offset) { 4196 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 4197 int spill_offset = offset; 4198 4199 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 4200 save_xmm_register(this, spill_offset, *it); 4201 spill_offset += xmm_save_size(); 4202 } 4203 } 4204 4205 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 4206 int restore_size = set.size() * xmm_save_size(); 4207 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 4208 4209 int restore_offset = offset + restore_size - xmm_save_size(); 4210 4211 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 4212 restore_xmm_register(this, restore_offset, *it); 4213 restore_offset -= xmm_save_size(); 4214 } 4215 } 4216 4217 void MacroAssembler::push_set(RegSet set, int offset) { 4218 int spill_offset; 4219 if (offset == -1) { 4220 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4221 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4222 subptr(rsp, aligned_size); 4223 spill_offset = 0; 4224 } else { 4225 spill_offset = offset; 4226 } 4227 4228 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 4229 movptr(Address(rsp, spill_offset), *it); 4230 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4231 } 4232 } 4233 4234 void MacroAssembler::pop_set(RegSet set, int offset) { 4235 4236 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4237 int restore_size = set.size() * gp_reg_size; 4238 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 4239 4240 int restore_offset; 4241 if (offset == -1) { 4242 restore_offset = restore_size - gp_reg_size; 4243 } else { 4244 restore_offset = offset + restore_size - gp_reg_size; 4245 } 4246 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 4247 movptr(*it, Address(rsp, restore_offset)); 4248 restore_offset -= gp_reg_size; 4249 } 4250 4251 if (offset == -1) { 4252 addptr(rsp, aligned_size); 4253 } 4254 } 4255 4256 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 4257 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 4258 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 4259 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 4260 Label done; 4261 4262 testptr(length_in_bytes, length_in_bytes); 4263 jcc(Assembler::zero, done); 4264 4265 // initialize topmost word, divide index by 2, check if odd and test if zero 4266 // note: for the remaining code to work, index must be a multiple of BytesPerWord 4267 #ifdef ASSERT 4268 { 4269 Label L; 4270 testptr(length_in_bytes, BytesPerWord - 1); 4271 jcc(Assembler::zero, L); 4272 stop("length must be a multiple of BytesPerWord"); 4273 bind(L); 4274 } 4275 #endif 4276 Register index = length_in_bytes; 4277 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 4278 if (UseIncDec) { 4279 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 4280 } else { 4281 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 4282 shrptr(index, 1); 4283 } 4284 #ifndef _LP64 4285 // index could have not been a multiple of 8 (i.e., bit 2 was set) 4286 { 4287 Label even; 4288 // note: if index was a multiple of 8, then it cannot 4289 // be 0 now otherwise it must have been 0 before 4290 // => if it is even, we don't need to check for 0 again 4291 jcc(Assembler::carryClear, even); 4292 // clear topmost word (no jump would be needed if conditional assignment worked here) 4293 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); 4294 // index could be 0 now, must check again 4295 jcc(Assembler::zero, done); 4296 bind(even); 4297 } 4298 #endif // !_LP64 4299 // initialize remaining object fields: index is a multiple of 2 now 4300 { 4301 Label loop; 4302 bind(loop); 4303 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 4304 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) 4305 decrement(index); 4306 jcc(Assembler::notZero, loop); 4307 } 4308 4309 bind(done); 4310 } 4311 4312 // Look up the method for a megamorphic invokeinterface call. 4313 // The target method is determined by <intf_klass, itable_index>. 4314 // The receiver klass is in recv_klass. 4315 // On success, the result will be in method_result, and execution falls through. 4316 // On failure, execution transfers to the given label. 4317 void MacroAssembler::lookup_interface_method(Register recv_klass, 4318 Register intf_klass, 4319 RegisterOrConstant itable_index, 4320 Register method_result, 4321 Register scan_temp, 4322 Label& L_no_such_interface, 4323 bool return_method) { 4324 assert_different_registers(recv_klass, intf_klass, scan_temp); 4325 assert_different_registers(method_result, intf_klass, scan_temp); 4326 assert(recv_klass != method_result || !return_method, 4327 "recv_klass can be destroyed when method isn't needed"); 4328 4329 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 4330 "caller must use same register for non-constant itable index as for method"); 4331 4332 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 4333 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4334 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4335 int scan_step = itableOffsetEntry::size() * wordSize; 4336 int vte_size = vtableEntry::size_in_bytes(); 4337 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4338 assert(vte_size == wordSize, "else adjust times_vte_scale"); 4339 4340 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4341 4342 // %%% Could store the aligned, prescaled offset in the klassoop. 4343 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 4344 4345 if (return_method) { 4346 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 4347 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4348 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 4349 } 4350 4351 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 4352 // if (scan->interface() == intf) { 4353 // result = (klass + scan->offset() + itable_index); 4354 // } 4355 // } 4356 Label search, found_method; 4357 4358 for (int peel = 1; peel >= 0; peel--) { 4359 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 4360 cmpptr(intf_klass, method_result); 4361 4362 if (peel) { 4363 jccb(Assembler::equal, found_method); 4364 } else { 4365 jccb(Assembler::notEqual, search); 4366 // (invert the test to fall through to found_method...) 4367 } 4368 4369 if (!peel) break; 4370 4371 bind(search); 4372 4373 // Check that the previous entry is non-null. A null entry means that 4374 // the receiver class doesn't implement the interface, and wasn't the 4375 // same as when the caller was compiled. 4376 testptr(method_result, method_result); 4377 jcc(Assembler::zero, L_no_such_interface); 4378 addptr(scan_temp, scan_step); 4379 } 4380 4381 bind(found_method); 4382 4383 if (return_method) { 4384 // Got a hit. 4385 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 4386 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 4387 } 4388 } 4389 4390 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 4391 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 4392 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 4393 // The target method is determined by <holder_klass, itable_index>. 4394 // The receiver klass is in recv_klass. 4395 // On success, the result will be in method_result, and execution falls through. 4396 // On failure, execution transfers to the given label. 4397 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 4398 Register holder_klass, 4399 Register resolved_klass, 4400 Register method_result, 4401 Register scan_temp, 4402 Register temp_reg2, 4403 Register receiver, 4404 int itable_index, 4405 Label& L_no_such_interface) { 4406 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 4407 Register temp_itbl_klass = method_result; 4408 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 4409 4410 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4411 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4412 int scan_step = itableOffsetEntry::size() * wordSize; 4413 int vte_size = vtableEntry::size_in_bytes(); 4414 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 4415 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 4416 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4417 assert(vte_size == wordSize, "adjust times_vte_scale"); 4418 4419 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 4420 4421 // temp_itbl_klass = recv_klass.itable[0] 4422 // scan_temp = &recv_klass.itable[0] + step 4423 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4424 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 4425 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 4426 xorptr(temp_reg, temp_reg); 4427 4428 // Initial checks: 4429 // - if (holder_klass != resolved_klass), go to "scan for resolved" 4430 // - if (itable[0] == 0), no such interface 4431 // - if (itable[0] == holder_klass), shortcut to "holder found" 4432 cmpptr(holder_klass, resolved_klass); 4433 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 4434 testptr(temp_itbl_klass, temp_itbl_klass); 4435 jccb(Assembler::zero, L_no_such_interface); 4436 cmpptr(holder_klass, temp_itbl_klass); 4437 jccb(Assembler::equal, L_holder_found); 4438 4439 // Loop: Look for holder_klass record in itable 4440 // do { 4441 // tmp = itable[index]; 4442 // index += step; 4443 // if (tmp == holder_klass) { 4444 // goto L_holder_found; // Found! 4445 // } 4446 // } while (tmp != 0); 4447 // goto L_no_such_interface // Not found. 4448 Label L_scan_holder; 4449 bind(L_scan_holder); 4450 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4451 addptr(scan_temp, scan_step); 4452 cmpptr(holder_klass, temp_itbl_klass); 4453 jccb(Assembler::equal, L_holder_found); 4454 testptr(temp_itbl_klass, temp_itbl_klass); 4455 jccb(Assembler::notZero, L_scan_holder); 4456 4457 jmpb(L_no_such_interface); 4458 4459 // Loop: Look for resolved_class record in itable 4460 // do { 4461 // tmp = itable[index]; 4462 // index += step; 4463 // if (tmp == holder_klass) { 4464 // // Also check if we have met a holder klass 4465 // holder_tmp = itable[index-step-ioffset]; 4466 // } 4467 // if (tmp == resolved_klass) { 4468 // goto L_resolved_found; // Found! 4469 // } 4470 // } while (tmp != 0); 4471 // goto L_no_such_interface // Not found. 4472 // 4473 Label L_loop_scan_resolved; 4474 bind(L_loop_scan_resolved); 4475 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4476 addptr(scan_temp, scan_step); 4477 bind(L_loop_scan_resolved_entry); 4478 cmpptr(holder_klass, temp_itbl_klass); 4479 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4480 cmpptr(resolved_klass, temp_itbl_klass); 4481 jccb(Assembler::equal, L_resolved_found); 4482 testptr(temp_itbl_klass, temp_itbl_klass); 4483 jccb(Assembler::notZero, L_loop_scan_resolved); 4484 4485 jmpb(L_no_such_interface); 4486 4487 Label L_ready; 4488 4489 // See if we already have a holder klass. If not, go and scan for it. 4490 bind(L_resolved_found); 4491 testptr(temp_reg, temp_reg); 4492 jccb(Assembler::zero, L_scan_holder); 4493 jmpb(L_ready); 4494 4495 bind(L_holder_found); 4496 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4497 4498 // Finally, temp_reg contains holder_klass vtable offset 4499 bind(L_ready); 4500 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4501 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 4502 load_klass(scan_temp, receiver, noreg); 4503 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4504 } else { 4505 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4506 } 4507 } 4508 4509 4510 // virtual method calling 4511 void MacroAssembler::lookup_virtual_method(Register recv_klass, 4512 RegisterOrConstant vtable_index, 4513 Register method_result) { 4514 const ByteSize base = Klass::vtable_start_offset(); 4515 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 4516 Address vtable_entry_addr(recv_klass, 4517 vtable_index, Address::times_ptr, 4518 base + vtableEntry::method_offset()); 4519 movptr(method_result, vtable_entry_addr); 4520 } 4521 4522 4523 void MacroAssembler::check_klass_subtype(Register sub_klass, 4524 Register super_klass, 4525 Register temp_reg, 4526 Label& L_success) { 4527 Label L_failure; 4528 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 4529 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 4530 bind(L_failure); 4531 } 4532 4533 4534 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 4535 Register super_klass, 4536 Register temp_reg, 4537 Label* L_success, 4538 Label* L_failure, 4539 Label* L_slow_path, 4540 RegisterOrConstant super_check_offset) { 4541 assert_different_registers(sub_klass, super_klass, temp_reg); 4542 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 4543 if (super_check_offset.is_register()) { 4544 assert_different_registers(sub_klass, super_klass, 4545 super_check_offset.as_register()); 4546 } else if (must_load_sco) { 4547 assert(temp_reg != noreg, "supply either a temp or a register offset"); 4548 } 4549 4550 Label L_fallthrough; 4551 int label_nulls = 0; 4552 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4553 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4554 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 4555 assert(label_nulls <= 1, "at most one null in the batch"); 4556 4557 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4558 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 4559 Address super_check_offset_addr(super_klass, sco_offset); 4560 4561 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 4562 // range of a jccb. If this routine grows larger, reconsider at 4563 // least some of these. 4564 #define local_jcc(assembler_cond, label) \ 4565 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 4566 else jcc( assembler_cond, label) /*omit semi*/ 4567 4568 // Hacked jmp, which may only be used just before L_fallthrough. 4569 #define final_jmp(label) \ 4570 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 4571 else jmp(label) /*omit semi*/ 4572 4573 // If the pointers are equal, we are done (e.g., String[] elements). 4574 // This self-check enables sharing of secondary supertype arrays among 4575 // non-primary types such as array-of-interface. Otherwise, each such 4576 // type would need its own customized SSA. 4577 // We move this check to the front of the fast path because many 4578 // type checks are in fact trivially successful in this manner, 4579 // so we get a nicely predicted branch right at the start of the check. 4580 cmpptr(sub_klass, super_klass); 4581 local_jcc(Assembler::equal, *L_success); 4582 4583 // Check the supertype display: 4584 if (must_load_sco) { 4585 // Positive movl does right thing on LP64. 4586 movl(temp_reg, super_check_offset_addr); 4587 super_check_offset = RegisterOrConstant(temp_reg); 4588 } 4589 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 4590 cmpptr(super_klass, super_check_addr); // load displayed supertype 4591 4592 // This check has worked decisively for primary supers. 4593 // Secondary supers are sought in the super_cache ('super_cache_addr'). 4594 // (Secondary supers are interfaces and very deeply nested subtypes.) 4595 // This works in the same check above because of a tricky aliasing 4596 // between the super_cache and the primary super display elements. 4597 // (The 'super_check_addr' can address either, as the case requires.) 4598 // Note that the cache is updated below if it does not help us find 4599 // what we need immediately. 4600 // So if it was a primary super, we can just fail immediately. 4601 // Otherwise, it's the slow path for us (no success at this point). 4602 4603 if (super_check_offset.is_register()) { 4604 local_jcc(Assembler::equal, *L_success); 4605 cmpl(super_check_offset.as_register(), sc_offset); 4606 if (L_failure == &L_fallthrough) { 4607 local_jcc(Assembler::equal, *L_slow_path); 4608 } else { 4609 local_jcc(Assembler::notEqual, *L_failure); 4610 final_jmp(*L_slow_path); 4611 } 4612 } else if (super_check_offset.as_constant() == sc_offset) { 4613 // Need a slow path; fast failure is impossible. 4614 if (L_slow_path == &L_fallthrough) { 4615 local_jcc(Assembler::equal, *L_success); 4616 } else { 4617 local_jcc(Assembler::notEqual, *L_slow_path); 4618 final_jmp(*L_success); 4619 } 4620 } else { 4621 // No slow path; it's a fast decision. 4622 if (L_failure == &L_fallthrough) { 4623 local_jcc(Assembler::equal, *L_success); 4624 } else { 4625 local_jcc(Assembler::notEqual, *L_failure); 4626 final_jmp(*L_success); 4627 } 4628 } 4629 4630 bind(L_fallthrough); 4631 4632 #undef local_jcc 4633 #undef final_jmp 4634 } 4635 4636 4637 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4638 Register super_klass, 4639 Register temp_reg, 4640 Register temp2_reg, 4641 Label* L_success, 4642 Label* L_failure, 4643 bool set_cond_codes) { 4644 assert_different_registers(sub_klass, super_klass, temp_reg); 4645 if (temp2_reg != noreg) 4646 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 4647 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 4648 4649 Label L_fallthrough; 4650 int label_nulls = 0; 4651 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4652 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4653 assert(label_nulls <= 1, "at most one null in the batch"); 4654 4655 // a couple of useful fields in sub_klass: 4656 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 4657 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4658 Address secondary_supers_addr(sub_klass, ss_offset); 4659 Address super_cache_addr( sub_klass, sc_offset); 4660 4661 // Do a linear scan of the secondary super-klass chain. 4662 // This code is rarely used, so simplicity is a virtue here. 4663 // The repne_scan instruction uses fixed registers, which we must spill. 4664 // Don't worry too much about pre-existing connections with the input regs. 4665 4666 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 4667 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 4668 4669 // Get super_klass value into rax (even if it was in rdi or rcx). 4670 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 4671 if (super_klass != rax) { 4672 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 4673 mov(rax, super_klass); 4674 } 4675 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 4676 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 4677 4678 #ifndef PRODUCT 4679 uint* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4680 ExternalAddress pst_counter_addr((address) pst_counter); 4681 NOT_LP64( incrementl(pst_counter_addr) ); 4682 LP64_ONLY( lea(rcx, pst_counter_addr) ); 4683 LP64_ONLY( incrementl(Address(rcx, 0)) ); 4684 #endif //PRODUCT 4685 4686 // We will consult the secondary-super array. 4687 movptr(rdi, secondary_supers_addr); 4688 // Load the array length. (Positive movl does right thing on LP64.) 4689 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4690 // Skip to start of data. 4691 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4692 4693 // Scan RCX words at [RDI] for an occurrence of RAX. 4694 // Set NZ/Z based on last compare. 4695 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4696 // not change flags (only scas instruction which is repeated sets flags). 4697 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4698 4699 testptr(rax,rax); // Set Z = 0 4700 repne_scan(); 4701 4702 // Unspill the temp. registers: 4703 if (pushed_rdi) pop(rdi); 4704 if (pushed_rcx) pop(rcx); 4705 if (pushed_rax) pop(rax); 4706 4707 if (set_cond_codes) { 4708 // Special hack for the AD files: rdi is guaranteed non-zero. 4709 assert(!pushed_rdi, "rdi must be left non-null"); 4710 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4711 } 4712 4713 if (L_failure == &L_fallthrough) 4714 jccb(Assembler::notEqual, *L_failure); 4715 else jcc(Assembler::notEqual, *L_failure); 4716 4717 // Success. Cache the super we found and proceed in triumph. 4718 movptr(super_cache_addr, super_klass); 4719 4720 if (L_success != &L_fallthrough) { 4721 jmp(*L_success); 4722 } 4723 4724 #undef IS_A_TEMP 4725 4726 bind(L_fallthrough); 4727 } 4728 4729 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { 4730 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 4731 4732 Label L_fallthrough; 4733 if (L_fast_path == nullptr) { 4734 L_fast_path = &L_fallthrough; 4735 } else if (L_slow_path == nullptr) { 4736 L_slow_path = &L_fallthrough; 4737 } 4738 4739 // Fast path check: class is fully initialized 4740 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 4741 jcc(Assembler::equal, *L_fast_path); 4742 4743 // Fast path check: current thread is initializer thread 4744 cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); 4745 if (L_slow_path == &L_fallthrough) { 4746 jcc(Assembler::equal, *L_fast_path); 4747 bind(*L_slow_path); 4748 } else if (L_fast_path == &L_fallthrough) { 4749 jcc(Assembler::notEqual, *L_slow_path); 4750 bind(*L_fast_path); 4751 } else { 4752 Unimplemented(); 4753 } 4754 } 4755 4756 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 4757 if (VM_Version::supports_cmov()) { 4758 cmovl(cc, dst, src); 4759 } else { 4760 Label L; 4761 jccb(negate_condition(cc), L); 4762 movl(dst, src); 4763 bind(L); 4764 } 4765 } 4766 4767 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 4768 if (VM_Version::supports_cmov()) { 4769 cmovl(cc, dst, src); 4770 } else { 4771 Label L; 4772 jccb(negate_condition(cc), L); 4773 movl(dst, src); 4774 bind(L); 4775 } 4776 } 4777 4778 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 4779 if (!VerifyOops) return; 4780 4781 BLOCK_COMMENT("verify_oop {"); 4782 #ifdef _LP64 4783 push(rscratch1); 4784 #endif 4785 push(rax); // save rax 4786 push(reg); // pass register argument 4787 4788 // Pass register number to verify_oop_subroutine 4789 const char* b = nullptr; 4790 { 4791 ResourceMark rm; 4792 stringStream ss; 4793 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 4794 b = code_string(ss.as_string()); 4795 } 4796 ExternalAddress buffer((address) b); 4797 pushptr(buffer.addr(), rscratch1); 4798 4799 // call indirectly to solve generation ordering problem 4800 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4801 call(rax); 4802 // Caller pops the arguments (oop, message) and restores rax, r10 4803 BLOCK_COMMENT("} verify_oop"); 4804 } 4805 4806 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 4807 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 4808 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 4809 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 4810 vpternlogd(dst, 0xFF, dst, dst, vector_len); 4811 } else if (VM_Version::supports_avx()) { 4812 vpcmpeqd(dst, dst, dst, vector_len); 4813 } else { 4814 assert(VM_Version::supports_sse2(), ""); 4815 pcmpeqd(dst, dst); 4816 } 4817 } 4818 4819 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 4820 int extra_slot_offset) { 4821 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 4822 int stackElementSize = Interpreter::stackElementSize; 4823 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 4824 #ifdef ASSERT 4825 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 4826 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 4827 #endif 4828 Register scale_reg = noreg; 4829 Address::ScaleFactor scale_factor = Address::no_scale; 4830 if (arg_slot.is_constant()) { 4831 offset += arg_slot.as_constant() * stackElementSize; 4832 } else { 4833 scale_reg = arg_slot.as_register(); 4834 scale_factor = Address::times(stackElementSize); 4835 } 4836 offset += wordSize; // return PC is on stack 4837 return Address(rsp, scale_reg, scale_factor, offset); 4838 } 4839 4840 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 4841 if (!VerifyOops) return; 4842 4843 #ifdef _LP64 4844 push(rscratch1); 4845 #endif 4846 push(rax); // save rax, 4847 // addr may contain rsp so we will have to adjust it based on the push 4848 // we just did (and on 64 bit we do two pushes) 4849 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 4850 // stores rax into addr which is backwards of what was intended. 4851 if (addr.uses(rsp)) { 4852 lea(rax, addr); 4853 pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); 4854 } else { 4855 pushptr(addr); 4856 } 4857 4858 // Pass register number to verify_oop_subroutine 4859 const char* b = nullptr; 4860 { 4861 ResourceMark rm; 4862 stringStream ss; 4863 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 4864 b = code_string(ss.as_string()); 4865 } 4866 ExternalAddress buffer((address) b); 4867 pushptr(buffer.addr(), rscratch1); 4868 4869 // call indirectly to solve generation ordering problem 4870 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4871 call(rax); 4872 // Caller pops the arguments (addr, message) and restores rax, r10. 4873 } 4874 4875 void MacroAssembler::verify_tlab() { 4876 #ifdef ASSERT 4877 if (UseTLAB && VerifyOops) { 4878 Label next, ok; 4879 Register t1 = rsi; 4880 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); 4881 4882 push(t1); 4883 NOT_LP64(push(thread_reg)); 4884 NOT_LP64(get_thread(thread_reg)); 4885 4886 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4887 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); 4888 jcc(Assembler::aboveEqual, next); 4889 STOP("assert(top >= start)"); 4890 should_not_reach_here(); 4891 4892 bind(next); 4893 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); 4894 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4895 jcc(Assembler::aboveEqual, ok); 4896 STOP("assert(top <= end)"); 4897 should_not_reach_here(); 4898 4899 bind(ok); 4900 NOT_LP64(pop(thread_reg)); 4901 pop(t1); 4902 } 4903 #endif 4904 } 4905 4906 class ControlWord { 4907 public: 4908 int32_t _value; 4909 4910 int rounding_control() const { return (_value >> 10) & 3 ; } 4911 int precision_control() const { return (_value >> 8) & 3 ; } 4912 bool precision() const { return ((_value >> 5) & 1) != 0; } 4913 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4914 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4915 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4916 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4917 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4918 4919 void print() const { 4920 // rounding control 4921 const char* rc; 4922 switch (rounding_control()) { 4923 case 0: rc = "round near"; break; 4924 case 1: rc = "round down"; break; 4925 case 2: rc = "round up "; break; 4926 case 3: rc = "chop "; break; 4927 default: 4928 rc = nullptr; // silence compiler warnings 4929 fatal("Unknown rounding control: %d", rounding_control()); 4930 }; 4931 // precision control 4932 const char* pc; 4933 switch (precision_control()) { 4934 case 0: pc = "24 bits "; break; 4935 case 1: pc = "reserved"; break; 4936 case 2: pc = "53 bits "; break; 4937 case 3: pc = "64 bits "; break; 4938 default: 4939 pc = nullptr; // silence compiler warnings 4940 fatal("Unknown precision control: %d", precision_control()); 4941 }; 4942 // flags 4943 char f[9]; 4944 f[0] = ' '; 4945 f[1] = ' '; 4946 f[2] = (precision ()) ? 'P' : 'p'; 4947 f[3] = (underflow ()) ? 'U' : 'u'; 4948 f[4] = (overflow ()) ? 'O' : 'o'; 4949 f[5] = (zero_divide ()) ? 'Z' : 'z'; 4950 f[6] = (denormalized()) ? 'D' : 'd'; 4951 f[7] = (invalid ()) ? 'I' : 'i'; 4952 f[8] = '\x0'; 4953 // output 4954 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 4955 } 4956 4957 }; 4958 4959 class StatusWord { 4960 public: 4961 int32_t _value; 4962 4963 bool busy() const { return ((_value >> 15) & 1) != 0; } 4964 bool C3() const { return ((_value >> 14) & 1) != 0; } 4965 bool C2() const { return ((_value >> 10) & 1) != 0; } 4966 bool C1() const { return ((_value >> 9) & 1) != 0; } 4967 bool C0() const { return ((_value >> 8) & 1) != 0; } 4968 int top() const { return (_value >> 11) & 7 ; } 4969 bool error_status() const { return ((_value >> 7) & 1) != 0; } 4970 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 4971 bool precision() const { return ((_value >> 5) & 1) != 0; } 4972 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4973 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4974 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4975 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4976 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4977 4978 void print() const { 4979 // condition codes 4980 char c[5]; 4981 c[0] = (C3()) ? '3' : '-'; 4982 c[1] = (C2()) ? '2' : '-'; 4983 c[2] = (C1()) ? '1' : '-'; 4984 c[3] = (C0()) ? '0' : '-'; 4985 c[4] = '\x0'; 4986 // flags 4987 char f[9]; 4988 f[0] = (error_status()) ? 'E' : '-'; 4989 f[1] = (stack_fault ()) ? 'S' : '-'; 4990 f[2] = (precision ()) ? 'P' : '-'; 4991 f[3] = (underflow ()) ? 'U' : '-'; 4992 f[4] = (overflow ()) ? 'O' : '-'; 4993 f[5] = (zero_divide ()) ? 'Z' : '-'; 4994 f[6] = (denormalized()) ? 'D' : '-'; 4995 f[7] = (invalid ()) ? 'I' : '-'; 4996 f[8] = '\x0'; 4997 // output 4998 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 4999 } 5000 5001 }; 5002 5003 class TagWord { 5004 public: 5005 int32_t _value; 5006 5007 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 5008 5009 void print() const { 5010 printf("%04x", _value & 0xFFFF); 5011 } 5012 5013 }; 5014 5015 class FPU_Register { 5016 public: 5017 int32_t _m0; 5018 int32_t _m1; 5019 int16_t _ex; 5020 5021 bool is_indefinite() const { 5022 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 5023 } 5024 5025 void print() const { 5026 char sign = (_ex < 0) ? '-' : '+'; 5027 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 5028 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 5029 }; 5030 5031 }; 5032 5033 class FPU_State { 5034 public: 5035 enum { 5036 register_size = 10, 5037 number_of_registers = 8, 5038 register_mask = 7 5039 }; 5040 5041 ControlWord _control_word; 5042 StatusWord _status_word; 5043 TagWord _tag_word; 5044 int32_t _error_offset; 5045 int32_t _error_selector; 5046 int32_t _data_offset; 5047 int32_t _data_selector; 5048 int8_t _register[register_size * number_of_registers]; 5049 5050 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 5051 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 5052 5053 const char* tag_as_string(int tag) const { 5054 switch (tag) { 5055 case 0: return "valid"; 5056 case 1: return "zero"; 5057 case 2: return "special"; 5058 case 3: return "empty"; 5059 } 5060 ShouldNotReachHere(); 5061 return nullptr; 5062 } 5063 5064 void print() const { 5065 // print computation registers 5066 { int t = _status_word.top(); 5067 for (int i = 0; i < number_of_registers; i++) { 5068 int j = (i - t) & register_mask; 5069 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 5070 st(j)->print(); 5071 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 5072 } 5073 } 5074 printf("\n"); 5075 // print control registers 5076 printf("ctrl = "); _control_word.print(); printf("\n"); 5077 printf("stat = "); _status_word .print(); printf("\n"); 5078 printf("tags = "); _tag_word .print(); printf("\n"); 5079 } 5080 5081 }; 5082 5083 class Flag_Register { 5084 public: 5085 int32_t _value; 5086 5087 bool overflow() const { return ((_value >> 11) & 1) != 0; } 5088 bool direction() const { return ((_value >> 10) & 1) != 0; } 5089 bool sign() const { return ((_value >> 7) & 1) != 0; } 5090 bool zero() const { return ((_value >> 6) & 1) != 0; } 5091 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 5092 bool parity() const { return ((_value >> 2) & 1) != 0; } 5093 bool carry() const { return ((_value >> 0) & 1) != 0; } 5094 5095 void print() const { 5096 // flags 5097 char f[8]; 5098 f[0] = (overflow ()) ? 'O' : '-'; 5099 f[1] = (direction ()) ? 'D' : '-'; 5100 f[2] = (sign ()) ? 'S' : '-'; 5101 f[3] = (zero ()) ? 'Z' : '-'; 5102 f[4] = (auxiliary_carry()) ? 'A' : '-'; 5103 f[5] = (parity ()) ? 'P' : '-'; 5104 f[6] = (carry ()) ? 'C' : '-'; 5105 f[7] = '\x0'; 5106 // output 5107 printf("%08x flags = %s", _value, f); 5108 } 5109 5110 }; 5111 5112 class IU_Register { 5113 public: 5114 int32_t _value; 5115 5116 void print() const { 5117 printf("%08x %11d", _value, _value); 5118 } 5119 5120 }; 5121 5122 class IU_State { 5123 public: 5124 Flag_Register _eflags; 5125 IU_Register _rdi; 5126 IU_Register _rsi; 5127 IU_Register _rbp; 5128 IU_Register _rsp; 5129 IU_Register _rbx; 5130 IU_Register _rdx; 5131 IU_Register _rcx; 5132 IU_Register _rax; 5133 5134 void print() const { 5135 // computation registers 5136 printf("rax, = "); _rax.print(); printf("\n"); 5137 printf("rbx, = "); _rbx.print(); printf("\n"); 5138 printf("rcx = "); _rcx.print(); printf("\n"); 5139 printf("rdx = "); _rdx.print(); printf("\n"); 5140 printf("rdi = "); _rdi.print(); printf("\n"); 5141 printf("rsi = "); _rsi.print(); printf("\n"); 5142 printf("rbp, = "); _rbp.print(); printf("\n"); 5143 printf("rsp = "); _rsp.print(); printf("\n"); 5144 printf("\n"); 5145 // control registers 5146 printf("flgs = "); _eflags.print(); printf("\n"); 5147 } 5148 }; 5149 5150 5151 class CPU_State { 5152 public: 5153 FPU_State _fpu_state; 5154 IU_State _iu_state; 5155 5156 void print() const { 5157 printf("--------------------------------------------------\n"); 5158 _iu_state .print(); 5159 printf("\n"); 5160 _fpu_state.print(); 5161 printf("--------------------------------------------------\n"); 5162 } 5163 5164 }; 5165 5166 5167 static void _print_CPU_state(CPU_State* state) { 5168 state->print(); 5169 }; 5170 5171 5172 void MacroAssembler::print_CPU_state() { 5173 push_CPU_state(); 5174 push(rsp); // pass CPU state 5175 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5176 addptr(rsp, wordSize); // discard argument 5177 pop_CPU_state(); 5178 } 5179 5180 5181 #ifndef _LP64 5182 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { 5183 static int counter = 0; 5184 FPU_State* fs = &state->_fpu_state; 5185 counter++; 5186 // For leaf calls, only verify that the top few elements remain empty. 5187 // We only need 1 empty at the top for C2 code. 5188 if( stack_depth < 0 ) { 5189 if( fs->tag_for_st(7) != 3 ) { 5190 printf("FPR7 not empty\n"); 5191 state->print(); 5192 assert(false, "error"); 5193 return false; 5194 } 5195 return true; // All other stack states do not matter 5196 } 5197 5198 assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(), 5199 "bad FPU control word"); 5200 5201 // compute stack depth 5202 int i = 0; 5203 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; 5204 int d = i; 5205 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; 5206 // verify findings 5207 if (i != FPU_State::number_of_registers) { 5208 // stack not contiguous 5209 printf("%s: stack not contiguous at ST%d\n", s, i); 5210 state->print(); 5211 assert(false, "error"); 5212 return false; 5213 } 5214 // check if computed stack depth corresponds to expected stack depth 5215 if (stack_depth < 0) { 5216 // expected stack depth is -stack_depth or less 5217 if (d > -stack_depth) { 5218 // too many elements on the stack 5219 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); 5220 state->print(); 5221 assert(false, "error"); 5222 return false; 5223 } 5224 } else { 5225 // expected stack depth is stack_depth 5226 if (d != stack_depth) { 5227 // wrong stack depth 5228 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); 5229 state->print(); 5230 assert(false, "error"); 5231 return false; 5232 } 5233 } 5234 // everything is cool 5235 return true; 5236 } 5237 5238 void MacroAssembler::verify_FPU(int stack_depth, const char* s) { 5239 if (!VerifyFPU) return; 5240 push_CPU_state(); 5241 push(rsp); // pass CPU state 5242 ExternalAddress msg((address) s); 5243 // pass message string s 5244 pushptr(msg.addr(), noreg); 5245 push(stack_depth); // pass stack depth 5246 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); 5247 addptr(rsp, 3 * wordSize); // discard arguments 5248 // check for error 5249 { Label L; 5250 testl(rax, rax); 5251 jcc(Assembler::notZero, L); 5252 int3(); // break if error condition 5253 bind(L); 5254 } 5255 pop_CPU_state(); 5256 } 5257 #endif // _LP64 5258 5259 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5260 // Either restore the MXCSR register after returning from the JNI Call 5261 // or verify that it wasn't changed (with -Xcheck:jni flag). 5262 if (VM_Version::supports_sse()) { 5263 if (RestoreMXCSROnJNICalls) { 5264 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5265 } else if (CheckJNICalls) { 5266 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5267 } 5268 } 5269 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5270 vzeroupper(); 5271 5272 #ifndef _LP64 5273 // Either restore the x87 floating pointer control word after returning 5274 // from the JNI call or verify that it wasn't changed. 5275 if (CheckJNICalls) { 5276 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); 5277 } 5278 #endif // _LP64 5279 } 5280 5281 // ((OopHandle)result).resolve(); 5282 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5283 assert_different_registers(result, tmp); 5284 5285 // Only 64 bit platforms support GCs that require a tmp register 5286 // Only IN_HEAP loads require a thread_tmp register 5287 // OopHandle::resolve is an indirection like jobject. 5288 access_load_at(T_OBJECT, IN_NATIVE, 5289 result, Address(result, 0), tmp, /*tmp_thread*/noreg); 5290 } 5291 5292 // ((WeakHandle)result).resolve(); 5293 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5294 assert_different_registers(rresult, rtmp); 5295 Label resolved; 5296 5297 // A null weak handle resolves to null. 5298 cmpptr(rresult, 0); 5299 jcc(Assembler::equal, resolved); 5300 5301 // Only 64 bit platforms support GCs that require a tmp register 5302 // Only IN_HEAP loads require a thread_tmp register 5303 // WeakHandle::resolve is an indirection like jweak. 5304 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5305 rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); 5306 bind(resolved); 5307 } 5308 5309 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5310 // get mirror 5311 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5312 load_method_holder(mirror, method); 5313 movptr(mirror, Address(mirror, mirror_offset)); 5314 resolve_oop_handle(mirror, tmp); 5315 } 5316 5317 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5318 load_method_holder(rresult, rmethod); 5319 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5320 } 5321 5322 void MacroAssembler::load_method_holder(Register holder, Register method) { 5323 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5324 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5325 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5326 } 5327 5328 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5329 assert_different_registers(src, tmp); 5330 assert_different_registers(dst, tmp); 5331 #ifdef _LP64 5332 if (UseCompressedClassPointers) { 5333 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5334 decode_klass_not_null(dst, tmp); 5335 } else 5336 #endif 5337 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5338 } 5339 5340 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 5341 assert_different_registers(src, tmp); 5342 assert_different_registers(dst, tmp); 5343 #ifdef _LP64 5344 if (UseCompressedClassPointers) { 5345 encode_klass_not_null(src, tmp); 5346 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5347 } else 5348 #endif 5349 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5350 } 5351 5352 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 5353 Register tmp1, Register thread_tmp) { 5354 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5355 decorators = AccessInternal::decorator_fixup(decorators, type); 5356 bool as_raw = (decorators & AS_RAW) != 0; 5357 if (as_raw) { 5358 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5359 } else { 5360 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5361 } 5362 } 5363 5364 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 5365 Register tmp1, Register tmp2, Register tmp3) { 5366 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5367 decorators = AccessInternal::decorator_fixup(decorators, type); 5368 bool as_raw = (decorators & AS_RAW) != 0; 5369 if (as_raw) { 5370 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5371 } else { 5372 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5373 } 5374 } 5375 5376 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5377 Register thread_tmp, DecoratorSet decorators) { 5378 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); 5379 } 5380 5381 // Doesn't do verification, generates fixed size code 5382 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5383 Register thread_tmp, DecoratorSet decorators) { 5384 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); 5385 } 5386 5387 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5388 Register tmp2, Register tmp3, DecoratorSet decorators) { 5389 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5390 } 5391 5392 // Used for storing nulls. 5393 void MacroAssembler::store_heap_oop_null(Address dst) { 5394 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5395 } 5396 5397 #ifdef _LP64 5398 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5399 if (UseCompressedClassPointers) { 5400 // Store to klass gap in destination 5401 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 5402 } 5403 } 5404 5405 #ifdef ASSERT 5406 void MacroAssembler::verify_heapbase(const char* msg) { 5407 assert (UseCompressedOops, "should be compressed"); 5408 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5409 if (CheckCompressedOops) { 5410 Label ok; 5411 ExternalAddress src2(CompressedOops::ptrs_base_addr()); 5412 const bool is_src2_reachable = reachable(src2); 5413 if (!is_src2_reachable) { 5414 push(rscratch1); // cmpptr trashes rscratch1 5415 } 5416 cmpptr(r12_heapbase, src2, rscratch1); 5417 jcc(Assembler::equal, ok); 5418 STOP(msg); 5419 bind(ok); 5420 if (!is_src2_reachable) { 5421 pop(rscratch1); 5422 } 5423 } 5424 } 5425 #endif 5426 5427 // Algorithm must match oop.inline.hpp encode_heap_oop. 5428 void MacroAssembler::encode_heap_oop(Register r) { 5429 #ifdef ASSERT 5430 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5431 #endif 5432 verify_oop_msg(r, "broken oop in encode_heap_oop"); 5433 if (CompressedOops::base() == nullptr) { 5434 if (CompressedOops::shift() != 0) { 5435 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5436 shrq(r, LogMinObjAlignmentInBytes); 5437 } 5438 return; 5439 } 5440 testq(r, r); 5441 cmovq(Assembler::equal, r, r12_heapbase); 5442 subq(r, r12_heapbase); 5443 shrq(r, LogMinObjAlignmentInBytes); 5444 } 5445 5446 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5447 #ifdef ASSERT 5448 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5449 if (CheckCompressedOops) { 5450 Label ok; 5451 testq(r, r); 5452 jcc(Assembler::notEqual, ok); 5453 STOP("null oop passed to encode_heap_oop_not_null"); 5454 bind(ok); 5455 } 5456 #endif 5457 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5458 if (CompressedOops::base() != nullptr) { 5459 subq(r, r12_heapbase); 5460 } 5461 if (CompressedOops::shift() != 0) { 5462 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5463 shrq(r, LogMinObjAlignmentInBytes); 5464 } 5465 } 5466 5467 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5468 #ifdef ASSERT 5469 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5470 if (CheckCompressedOops) { 5471 Label ok; 5472 testq(src, src); 5473 jcc(Assembler::notEqual, ok); 5474 STOP("null oop passed to encode_heap_oop_not_null2"); 5475 bind(ok); 5476 } 5477 #endif 5478 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5479 if (dst != src) { 5480 movq(dst, src); 5481 } 5482 if (CompressedOops::base() != nullptr) { 5483 subq(dst, r12_heapbase); 5484 } 5485 if (CompressedOops::shift() != 0) { 5486 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5487 shrq(dst, LogMinObjAlignmentInBytes); 5488 } 5489 } 5490 5491 void MacroAssembler::decode_heap_oop(Register r) { 5492 #ifdef ASSERT 5493 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5494 #endif 5495 if (CompressedOops::base() == nullptr) { 5496 if (CompressedOops::shift() != 0) { 5497 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5498 shlq(r, LogMinObjAlignmentInBytes); 5499 } 5500 } else { 5501 Label done; 5502 shlq(r, LogMinObjAlignmentInBytes); 5503 jccb(Assembler::equal, done); 5504 addq(r, r12_heapbase); 5505 bind(done); 5506 } 5507 verify_oop_msg(r, "broken oop in decode_heap_oop"); 5508 } 5509 5510 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5511 // Note: it will change flags 5512 assert (UseCompressedOops, "should only be used for compressed headers"); 5513 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5514 // Cannot assert, unverified entry point counts instructions (see .ad file) 5515 // vtableStubs also counts instructions in pd_code_size_limit. 5516 // Also do not verify_oop as this is called by verify_oop. 5517 if (CompressedOops::shift() != 0) { 5518 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5519 shlq(r, LogMinObjAlignmentInBytes); 5520 if (CompressedOops::base() != nullptr) { 5521 addq(r, r12_heapbase); 5522 } 5523 } else { 5524 assert (CompressedOops::base() == nullptr, "sanity"); 5525 } 5526 } 5527 5528 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5529 // Note: it will change flags 5530 assert (UseCompressedOops, "should only be used for compressed headers"); 5531 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5532 // Cannot assert, unverified entry point counts instructions (see .ad file) 5533 // vtableStubs also counts instructions in pd_code_size_limit. 5534 // Also do not verify_oop as this is called by verify_oop. 5535 if (CompressedOops::shift() != 0) { 5536 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5537 if (LogMinObjAlignmentInBytes == Address::times_8) { 5538 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 5539 } else { 5540 if (dst != src) { 5541 movq(dst, src); 5542 } 5543 shlq(dst, LogMinObjAlignmentInBytes); 5544 if (CompressedOops::base() != nullptr) { 5545 addq(dst, r12_heapbase); 5546 } 5547 } 5548 } else { 5549 assert (CompressedOops::base() == nullptr, "sanity"); 5550 if (dst != src) { 5551 movq(dst, src); 5552 } 5553 } 5554 } 5555 5556 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 5557 assert_different_registers(r, tmp); 5558 if (CompressedKlassPointers::base() != nullptr) { 5559 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5560 subq(r, tmp); 5561 } 5562 if (CompressedKlassPointers::shift() != 0) { 5563 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5564 shrq(r, LogKlassAlignmentInBytes); 5565 } 5566 } 5567 5568 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 5569 assert_different_registers(src, dst); 5570 if (CompressedKlassPointers::base() != nullptr) { 5571 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 5572 addq(dst, src); 5573 } else { 5574 movptr(dst, src); 5575 } 5576 if (CompressedKlassPointers::shift() != 0) { 5577 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5578 shrq(dst, LogKlassAlignmentInBytes); 5579 } 5580 } 5581 5582 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 5583 assert_different_registers(r, tmp); 5584 // Note: it will change flags 5585 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 5586 // Cannot assert, unverified entry point counts instructions (see .ad file) 5587 // vtableStubs also counts instructions in pd_code_size_limit. 5588 // Also do not verify_oop as this is called by verify_oop. 5589 if (CompressedKlassPointers::shift() != 0) { 5590 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5591 shlq(r, LogKlassAlignmentInBytes); 5592 } 5593 if (CompressedKlassPointers::base() != nullptr) { 5594 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5595 addq(r, tmp); 5596 } 5597 } 5598 5599 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 5600 assert_different_registers(src, dst); 5601 // Note: it will change flags 5602 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5603 // Cannot assert, unverified entry point counts instructions (see .ad file) 5604 // vtableStubs also counts instructions in pd_code_size_limit. 5605 // Also do not verify_oop as this is called by verify_oop. 5606 5607 if (CompressedKlassPointers::base() == nullptr && 5608 CompressedKlassPointers::shift() == 0) { 5609 // The best case scenario is that there is no base or shift. Then it is already 5610 // a pointer that needs nothing but a register rename. 5611 movl(dst, src); 5612 } else { 5613 if (CompressedKlassPointers::base() != nullptr) { 5614 mov64(dst, (int64_t)CompressedKlassPointers::base()); 5615 } else { 5616 xorq(dst, dst); 5617 } 5618 if (CompressedKlassPointers::shift() != 0) { 5619 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5620 assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); 5621 leaq(dst, Address(dst, src, Address::times_8, 0)); 5622 } else { 5623 addq(dst, src); 5624 } 5625 } 5626 } 5627 5628 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5629 assert (UseCompressedOops, "should only be used for compressed headers"); 5630 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5631 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5632 int oop_index = oop_recorder()->find_index(obj); 5633 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5634 mov_narrow_oop(dst, oop_index, rspec); 5635 } 5636 5637 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 5638 assert (UseCompressedOops, "should only be used for compressed headers"); 5639 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5640 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5641 int oop_index = oop_recorder()->find_index(obj); 5642 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5643 mov_narrow_oop(dst, oop_index, rspec); 5644 } 5645 5646 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5647 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5648 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5649 int klass_index = oop_recorder()->find_index(k); 5650 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5651 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5652 } 5653 5654 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 5655 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5656 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5657 int klass_index = oop_recorder()->find_index(k); 5658 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5659 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5660 } 5661 5662 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 5663 assert (UseCompressedOops, "should only be used for compressed headers"); 5664 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5665 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5666 int oop_index = oop_recorder()->find_index(obj); 5667 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5668 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5669 } 5670 5671 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 5672 assert (UseCompressedOops, "should only be used for compressed headers"); 5673 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5674 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5675 int oop_index = oop_recorder()->find_index(obj); 5676 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5677 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5678 } 5679 5680 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 5681 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5682 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5683 int klass_index = oop_recorder()->find_index(k); 5684 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5685 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5686 } 5687 5688 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 5689 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5690 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5691 int klass_index = oop_recorder()->find_index(k); 5692 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5693 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5694 } 5695 5696 void MacroAssembler::reinit_heapbase() { 5697 if (UseCompressedOops) { 5698 if (Universe::heap() != nullptr) { 5699 if (CompressedOops::base() == nullptr) { 5700 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 5701 } else { 5702 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base()); 5703 } 5704 } else { 5705 movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr())); 5706 } 5707 } 5708 } 5709 5710 #endif // _LP64 5711 5712 #if COMPILER2_OR_JVMCI 5713 5714 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 5715 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5716 // cnt - number of qwords (8-byte words). 5717 // base - start address, qword aligned. 5718 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 5719 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 5720 if (use64byteVector) { 5721 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 5722 } else if (MaxVectorSize >= 32) { 5723 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 5724 } else { 5725 pxor(xtmp, xtmp); 5726 } 5727 jmp(L_zero_64_bytes); 5728 5729 BIND(L_loop); 5730 if (MaxVectorSize >= 32) { 5731 fill64(base, 0, xtmp, use64byteVector); 5732 } else { 5733 movdqu(Address(base, 0), xtmp); 5734 movdqu(Address(base, 16), xtmp); 5735 movdqu(Address(base, 32), xtmp); 5736 movdqu(Address(base, 48), xtmp); 5737 } 5738 addptr(base, 64); 5739 5740 BIND(L_zero_64_bytes); 5741 subptr(cnt, 8); 5742 jccb(Assembler::greaterEqual, L_loop); 5743 5744 // Copy trailing 64 bytes 5745 if (use64byteVector) { 5746 addptr(cnt, 8); 5747 jccb(Assembler::equal, L_end); 5748 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 5749 jmp(L_end); 5750 } else { 5751 addptr(cnt, 4); 5752 jccb(Assembler::less, L_tail); 5753 if (MaxVectorSize >= 32) { 5754 vmovdqu(Address(base, 0), xtmp); 5755 } else { 5756 movdqu(Address(base, 0), xtmp); 5757 movdqu(Address(base, 16), xtmp); 5758 } 5759 } 5760 addptr(base, 32); 5761 subptr(cnt, 4); 5762 5763 BIND(L_tail); 5764 addptr(cnt, 4); 5765 jccb(Assembler::lessEqual, L_end); 5766 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 5767 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 5768 } else { 5769 decrement(cnt); 5770 5771 BIND(L_sloop); 5772 movq(Address(base, 0), xtmp); 5773 addptr(base, 8); 5774 decrement(cnt); 5775 jccb(Assembler::greaterEqual, L_sloop); 5776 } 5777 BIND(L_end); 5778 } 5779 5780 // Clearing constant sized memory using YMM/ZMM registers. 5781 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5782 assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), ""); 5783 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 5784 5785 int vector64_count = (cnt & (~0x7)) >> 3; 5786 cnt = cnt & 0x7; 5787 const int fill64_per_loop = 4; 5788 const int max_unrolled_fill64 = 8; 5789 5790 // 64 byte initialization loop. 5791 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 5792 int start64 = 0; 5793 if (vector64_count > max_unrolled_fill64) { 5794 Label LOOP; 5795 Register index = rtmp; 5796 5797 start64 = vector64_count - (vector64_count % fill64_per_loop); 5798 5799 movl(index, 0); 5800 BIND(LOOP); 5801 for (int i = 0; i < fill64_per_loop; i++) { 5802 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 5803 } 5804 addl(index, fill64_per_loop * 64); 5805 cmpl(index, start64 * 64); 5806 jccb(Assembler::less, LOOP); 5807 } 5808 for (int i = start64; i < vector64_count; i++) { 5809 fill64(base, i * 64, xtmp, use64byteVector); 5810 } 5811 5812 // Clear remaining 64 byte tail. 5813 int disp = vector64_count * 64; 5814 if (cnt) { 5815 switch (cnt) { 5816 case 1: 5817 movq(Address(base, disp), xtmp); 5818 break; 5819 case 2: 5820 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 5821 break; 5822 case 3: 5823 movl(rtmp, 0x7); 5824 kmovwl(mask, rtmp); 5825 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 5826 break; 5827 case 4: 5828 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5829 break; 5830 case 5: 5831 if (use64byteVector) { 5832 movl(rtmp, 0x1F); 5833 kmovwl(mask, rtmp); 5834 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5835 } else { 5836 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5837 movq(Address(base, disp + 32), xtmp); 5838 } 5839 break; 5840 case 6: 5841 if (use64byteVector) { 5842 movl(rtmp, 0x3F); 5843 kmovwl(mask, rtmp); 5844 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5845 } else { 5846 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5847 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 5848 } 5849 break; 5850 case 7: 5851 if (use64byteVector) { 5852 movl(rtmp, 0x7F); 5853 kmovwl(mask, rtmp); 5854 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5855 } else { 5856 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5857 movl(rtmp, 0x7); 5858 kmovwl(mask, rtmp); 5859 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 5860 } 5861 break; 5862 default: 5863 fatal("Unexpected length : %d\n",cnt); 5864 break; 5865 } 5866 } 5867 } 5868 5869 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 5870 bool is_large, KRegister mask) { 5871 // cnt - number of qwords (8-byte words). 5872 // base - start address, qword aligned. 5873 // is_large - if optimizers know cnt is larger than InitArrayShortSize 5874 assert(base==rdi, "base register must be edi for rep stos"); 5875 assert(tmp==rax, "tmp register must be eax for rep stos"); 5876 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 5877 assert(InitArrayShortSize % BytesPerLong == 0, 5878 "InitArrayShortSize should be the multiple of BytesPerLong"); 5879 5880 Label DONE; 5881 if (!is_large || !UseXMMForObjInit) { 5882 xorptr(tmp, tmp); 5883 } 5884 5885 if (!is_large) { 5886 Label LOOP, LONG; 5887 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 5888 jccb(Assembler::greater, LONG); 5889 5890 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5891 5892 decrement(cnt); 5893 jccb(Assembler::negative, DONE); // Zero length 5894 5895 // Use individual pointer-sized stores for small counts: 5896 BIND(LOOP); 5897 movptr(Address(base, cnt, Address::times_ptr), tmp); 5898 decrement(cnt); 5899 jccb(Assembler::greaterEqual, LOOP); 5900 jmpb(DONE); 5901 5902 BIND(LONG); 5903 } 5904 5905 // Use longer rep-prefixed ops for non-small counts: 5906 if (UseFastStosb) { 5907 shlptr(cnt, 3); // convert to number of bytes 5908 rep_stosb(); 5909 } else if (UseXMMForObjInit) { 5910 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 5911 } else { 5912 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5913 rep_stos(); 5914 } 5915 5916 BIND(DONE); 5917 } 5918 5919 #endif //COMPILER2_OR_JVMCI 5920 5921 5922 void MacroAssembler::generate_fill(BasicType t, bool aligned, 5923 Register to, Register value, Register count, 5924 Register rtmp, XMMRegister xtmp) { 5925 ShortBranchVerifier sbv(this); 5926 assert_different_registers(to, value, count, rtmp); 5927 Label L_exit; 5928 Label L_fill_2_bytes, L_fill_4_bytes; 5929 5930 #if defined(COMPILER2) && defined(_LP64) 5931 if(MaxVectorSize >=32 && 5932 VM_Version::supports_avx512vlbw() && 5933 VM_Version::supports_bmi2()) { 5934 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 5935 return; 5936 } 5937 #endif 5938 5939 int shift = -1; 5940 switch (t) { 5941 case T_BYTE: 5942 shift = 2; 5943 break; 5944 case T_SHORT: 5945 shift = 1; 5946 break; 5947 case T_INT: 5948 shift = 0; 5949 break; 5950 default: ShouldNotReachHere(); 5951 } 5952 5953 if (t == T_BYTE) { 5954 andl(value, 0xff); 5955 movl(rtmp, value); 5956 shll(rtmp, 8); 5957 orl(value, rtmp); 5958 } 5959 if (t == T_SHORT) { 5960 andl(value, 0xffff); 5961 } 5962 if (t == T_BYTE || t == T_SHORT) { 5963 movl(rtmp, value); 5964 shll(rtmp, 16); 5965 orl(value, rtmp); 5966 } 5967 5968 cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 5969 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 5970 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 5971 Label L_skip_align2; 5972 // align source address at 4 bytes address boundary 5973 if (t == T_BYTE) { 5974 Label L_skip_align1; 5975 // One byte misalignment happens only for byte arrays 5976 testptr(to, 1); 5977 jccb(Assembler::zero, L_skip_align1); 5978 movb(Address(to, 0), value); 5979 increment(to); 5980 decrement(count); 5981 BIND(L_skip_align1); 5982 } 5983 // Two bytes misalignment happens only for byte and short (char) arrays 5984 testptr(to, 2); 5985 jccb(Assembler::zero, L_skip_align2); 5986 movw(Address(to, 0), value); 5987 addptr(to, 2); 5988 subl(count, 1<<(shift-1)); 5989 BIND(L_skip_align2); 5990 } 5991 if (UseSSE < 2) { 5992 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 5993 // Fill 32-byte chunks 5994 subl(count, 8 << shift); 5995 jcc(Assembler::less, L_check_fill_8_bytes); 5996 align(16); 5997 5998 BIND(L_fill_32_bytes_loop); 5999 6000 for (int i = 0; i < 32; i += 4) { 6001 movl(Address(to, i), value); 6002 } 6003 6004 addptr(to, 32); 6005 subl(count, 8 << shift); 6006 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6007 BIND(L_check_fill_8_bytes); 6008 addl(count, 8 << shift); 6009 jccb(Assembler::zero, L_exit); 6010 jmpb(L_fill_8_bytes); 6011 6012 // 6013 // length is too short, just fill qwords 6014 // 6015 BIND(L_fill_8_bytes_loop); 6016 movl(Address(to, 0), value); 6017 movl(Address(to, 4), value); 6018 addptr(to, 8); 6019 BIND(L_fill_8_bytes); 6020 subl(count, 1 << (shift + 1)); 6021 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6022 // fall through to fill 4 bytes 6023 } else { 6024 Label L_fill_32_bytes; 6025 if (!UseUnalignedLoadStores) { 6026 // align to 8 bytes, we know we are 4 byte aligned to start 6027 testptr(to, 4); 6028 jccb(Assembler::zero, L_fill_32_bytes); 6029 movl(Address(to, 0), value); 6030 addptr(to, 4); 6031 subl(count, 1<<shift); 6032 } 6033 BIND(L_fill_32_bytes); 6034 { 6035 assert( UseSSE >= 2, "supported cpu only" ); 6036 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6037 movdl(xtmp, value); 6038 if (UseAVX >= 2 && UseUnalignedLoadStores) { 6039 Label L_check_fill_32_bytes; 6040 if (UseAVX > 2) { 6041 // Fill 64-byte chunks 6042 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 6043 6044 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 6045 cmpl(count, VM_Version::avx3_threshold()); 6046 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 6047 6048 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 6049 6050 subl(count, 16 << shift); 6051 jccb(Assembler::less, L_check_fill_32_bytes); 6052 align(16); 6053 6054 BIND(L_fill_64_bytes_loop_avx3); 6055 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 6056 addptr(to, 64); 6057 subl(count, 16 << shift); 6058 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 6059 jmpb(L_check_fill_32_bytes); 6060 6061 BIND(L_check_fill_64_bytes_avx2); 6062 } 6063 // Fill 64-byte chunks 6064 Label L_fill_64_bytes_loop; 6065 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 6066 6067 subl(count, 16 << shift); 6068 jcc(Assembler::less, L_check_fill_32_bytes); 6069 align(16); 6070 6071 BIND(L_fill_64_bytes_loop); 6072 vmovdqu(Address(to, 0), xtmp); 6073 vmovdqu(Address(to, 32), xtmp); 6074 addptr(to, 64); 6075 subl(count, 16 << shift); 6076 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 6077 6078 BIND(L_check_fill_32_bytes); 6079 addl(count, 8 << shift); 6080 jccb(Assembler::less, L_check_fill_8_bytes); 6081 vmovdqu(Address(to, 0), xtmp); 6082 addptr(to, 32); 6083 subl(count, 8 << shift); 6084 6085 BIND(L_check_fill_8_bytes); 6086 // clean upper bits of YMM registers 6087 movdl(xtmp, value); 6088 pshufd(xtmp, xtmp, 0); 6089 } else { 6090 // Fill 32-byte chunks 6091 pshufd(xtmp, xtmp, 0); 6092 6093 subl(count, 8 << shift); 6094 jcc(Assembler::less, L_check_fill_8_bytes); 6095 align(16); 6096 6097 BIND(L_fill_32_bytes_loop); 6098 6099 if (UseUnalignedLoadStores) { 6100 movdqu(Address(to, 0), xtmp); 6101 movdqu(Address(to, 16), xtmp); 6102 } else { 6103 movq(Address(to, 0), xtmp); 6104 movq(Address(to, 8), xtmp); 6105 movq(Address(to, 16), xtmp); 6106 movq(Address(to, 24), xtmp); 6107 } 6108 6109 addptr(to, 32); 6110 subl(count, 8 << shift); 6111 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6112 6113 BIND(L_check_fill_8_bytes); 6114 } 6115 addl(count, 8 << shift); 6116 jccb(Assembler::zero, L_exit); 6117 jmpb(L_fill_8_bytes); 6118 6119 // 6120 // length is too short, just fill qwords 6121 // 6122 BIND(L_fill_8_bytes_loop); 6123 movq(Address(to, 0), xtmp); 6124 addptr(to, 8); 6125 BIND(L_fill_8_bytes); 6126 subl(count, 1 << (shift + 1)); 6127 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6128 } 6129 } 6130 // fill trailing 4 bytes 6131 BIND(L_fill_4_bytes); 6132 testl(count, 1<<shift); 6133 jccb(Assembler::zero, L_fill_2_bytes); 6134 movl(Address(to, 0), value); 6135 if (t == T_BYTE || t == T_SHORT) { 6136 Label L_fill_byte; 6137 addptr(to, 4); 6138 BIND(L_fill_2_bytes); 6139 // fill trailing 2 bytes 6140 testl(count, 1<<(shift-1)); 6141 jccb(Assembler::zero, L_fill_byte); 6142 movw(Address(to, 0), value); 6143 if (t == T_BYTE) { 6144 addptr(to, 2); 6145 BIND(L_fill_byte); 6146 // fill trailing byte 6147 testl(count, 1); 6148 jccb(Assembler::zero, L_exit); 6149 movb(Address(to, 0), value); 6150 } else { 6151 BIND(L_fill_byte); 6152 } 6153 } else { 6154 BIND(L_fill_2_bytes); 6155 } 6156 BIND(L_exit); 6157 } 6158 6159 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 6160 switch(type) { 6161 case T_BYTE: 6162 case T_BOOLEAN: 6163 evpbroadcastb(dst, src, vector_len); 6164 break; 6165 case T_SHORT: 6166 case T_CHAR: 6167 evpbroadcastw(dst, src, vector_len); 6168 break; 6169 case T_INT: 6170 case T_FLOAT: 6171 evpbroadcastd(dst, src, vector_len); 6172 break; 6173 case T_LONG: 6174 case T_DOUBLE: 6175 evpbroadcastq(dst, src, vector_len); 6176 break; 6177 default: 6178 fatal("Unhandled type : %s", type2name(type)); 6179 break; 6180 } 6181 } 6182 6183 // encode char[] to byte[] in ISO_8859_1 or ASCII 6184 //@IntrinsicCandidate 6185 //private static int implEncodeISOArray(byte[] sa, int sp, 6186 //byte[] da, int dp, int len) { 6187 // int i = 0; 6188 // for (; i < len; i++) { 6189 // char c = StringUTF16.getChar(sa, sp++); 6190 // if (c > '\u00FF') 6191 // break; 6192 // da[dp++] = (byte)c; 6193 // } 6194 // return i; 6195 //} 6196 // 6197 //@IntrinsicCandidate 6198 //private static int implEncodeAsciiArray(char[] sa, int sp, 6199 // byte[] da, int dp, int len) { 6200 // int i = 0; 6201 // for (; i < len; i++) { 6202 // char c = sa[sp++]; 6203 // if (c >= '\u0080') 6204 // break; 6205 // da[dp++] = (byte)c; 6206 // } 6207 // return i; 6208 //} 6209 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6210 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6211 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6212 Register tmp5, Register result, bool ascii) { 6213 6214 // rsi: src 6215 // rdi: dst 6216 // rdx: len 6217 // rcx: tmp5 6218 // rax: result 6219 ShortBranchVerifier sbv(this); 6220 assert_different_registers(src, dst, len, tmp5, result); 6221 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6222 6223 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6224 int short_mask = ascii ? 0xff80 : 0xff00; 6225 6226 // set result 6227 xorl(result, result); 6228 // check for zero length 6229 testl(len, len); 6230 jcc(Assembler::zero, L_done); 6231 6232 movl(result, len); 6233 6234 // Setup pointers 6235 lea(src, Address(src, len, Address::times_2)); // char[] 6236 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6237 negptr(len); 6238 6239 if (UseSSE42Intrinsics || UseAVX >= 2) { 6240 Label L_copy_8_chars, L_copy_8_chars_exit; 6241 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6242 6243 if (UseAVX >= 2) { 6244 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6245 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6246 movdl(tmp1Reg, tmp5); 6247 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6248 jmp(L_chars_32_check); 6249 6250 bind(L_copy_32_chars); 6251 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6252 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6253 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6254 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6255 jccb(Assembler::notZero, L_copy_32_chars_exit); 6256 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6257 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6258 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6259 6260 bind(L_chars_32_check); 6261 addptr(len, 32); 6262 jcc(Assembler::lessEqual, L_copy_32_chars); 6263 6264 bind(L_copy_32_chars_exit); 6265 subptr(len, 16); 6266 jccb(Assembler::greater, L_copy_16_chars_exit); 6267 6268 } else if (UseSSE42Intrinsics) { 6269 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6270 movdl(tmp1Reg, tmp5); 6271 pshufd(tmp1Reg, tmp1Reg, 0); 6272 jmpb(L_chars_16_check); 6273 } 6274 6275 bind(L_copy_16_chars); 6276 if (UseAVX >= 2) { 6277 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6278 vptest(tmp2Reg, tmp1Reg); 6279 jcc(Assembler::notZero, L_copy_16_chars_exit); 6280 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6281 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6282 } else { 6283 if (UseAVX > 0) { 6284 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6285 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6286 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6287 } else { 6288 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6289 por(tmp2Reg, tmp3Reg); 6290 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6291 por(tmp2Reg, tmp4Reg); 6292 } 6293 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6294 jccb(Assembler::notZero, L_copy_16_chars_exit); 6295 packuswb(tmp3Reg, tmp4Reg); 6296 } 6297 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 6298 6299 bind(L_chars_16_check); 6300 addptr(len, 16); 6301 jcc(Assembler::lessEqual, L_copy_16_chars); 6302 6303 bind(L_copy_16_chars_exit); 6304 if (UseAVX >= 2) { 6305 // clean upper bits of YMM registers 6306 vpxor(tmp2Reg, tmp2Reg); 6307 vpxor(tmp3Reg, tmp3Reg); 6308 vpxor(tmp4Reg, tmp4Reg); 6309 movdl(tmp1Reg, tmp5); 6310 pshufd(tmp1Reg, tmp1Reg, 0); 6311 } 6312 subptr(len, 8); 6313 jccb(Assembler::greater, L_copy_8_chars_exit); 6314 6315 bind(L_copy_8_chars); 6316 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 6317 ptest(tmp3Reg, tmp1Reg); 6318 jccb(Assembler::notZero, L_copy_8_chars_exit); 6319 packuswb(tmp3Reg, tmp1Reg); 6320 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 6321 addptr(len, 8); 6322 jccb(Assembler::lessEqual, L_copy_8_chars); 6323 6324 bind(L_copy_8_chars_exit); 6325 subptr(len, 8); 6326 jccb(Assembler::zero, L_done); 6327 } 6328 6329 bind(L_copy_1_char); 6330 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 6331 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 6332 jccb(Assembler::notZero, L_copy_1_char_exit); 6333 movb(Address(dst, len, Address::times_1, 0), tmp5); 6334 addptr(len, 1); 6335 jccb(Assembler::less, L_copy_1_char); 6336 6337 bind(L_copy_1_char_exit); 6338 addptr(result, len); // len is negative count of not processed elements 6339 6340 bind(L_done); 6341 } 6342 6343 #ifdef _LP64 6344 /** 6345 * Helper for multiply_to_len(). 6346 */ 6347 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 6348 addq(dest_lo, src1); 6349 adcq(dest_hi, 0); 6350 addq(dest_lo, src2); 6351 adcq(dest_hi, 0); 6352 } 6353 6354 /** 6355 * Multiply 64 bit by 64 bit first loop. 6356 */ 6357 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 6358 Register y, Register y_idx, Register z, 6359 Register carry, Register product, 6360 Register idx, Register kdx) { 6361 // 6362 // jlong carry, x[], y[], z[]; 6363 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6364 // huge_128 product = y[idx] * x[xstart] + carry; 6365 // z[kdx] = (jlong)product; 6366 // carry = (jlong)(product >>> 64); 6367 // } 6368 // z[xstart] = carry; 6369 // 6370 6371 Label L_first_loop, L_first_loop_exit; 6372 Label L_one_x, L_one_y, L_multiply; 6373 6374 decrementl(xstart); 6375 jcc(Assembler::negative, L_one_x); 6376 6377 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6378 rorq(x_xstart, 32); // convert big-endian to little-endian 6379 6380 bind(L_first_loop); 6381 decrementl(idx); 6382 jcc(Assembler::negative, L_first_loop_exit); 6383 decrementl(idx); 6384 jcc(Assembler::negative, L_one_y); 6385 movq(y_idx, Address(y, idx, Address::times_4, 0)); 6386 rorq(y_idx, 32); // convert big-endian to little-endian 6387 bind(L_multiply); 6388 movq(product, x_xstart); 6389 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 6390 addq(product, carry); 6391 adcq(rdx, 0); 6392 subl(kdx, 2); 6393 movl(Address(z, kdx, Address::times_4, 4), product); 6394 shrq(product, 32); 6395 movl(Address(z, kdx, Address::times_4, 0), product); 6396 movq(carry, rdx); 6397 jmp(L_first_loop); 6398 6399 bind(L_one_y); 6400 movl(y_idx, Address(y, 0)); 6401 jmp(L_multiply); 6402 6403 bind(L_one_x); 6404 movl(x_xstart, Address(x, 0)); 6405 jmp(L_first_loop); 6406 6407 bind(L_first_loop_exit); 6408 } 6409 6410 /** 6411 * Multiply 64 bit by 64 bit and add 128 bit. 6412 */ 6413 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 6414 Register yz_idx, Register idx, 6415 Register carry, Register product, int offset) { 6416 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 6417 // z[kdx] = (jlong)product; 6418 6419 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 6420 rorq(yz_idx, 32); // convert big-endian to little-endian 6421 movq(product, x_xstart); 6422 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6423 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 6424 rorq(yz_idx, 32); // convert big-endian to little-endian 6425 6426 add2_with_carry(rdx, product, carry, yz_idx); 6427 6428 movl(Address(z, idx, Address::times_4, offset+4), product); 6429 shrq(product, 32); 6430 movl(Address(z, idx, Address::times_4, offset), product); 6431 6432 } 6433 6434 /** 6435 * Multiply 128 bit by 128 bit. Unrolled inner loop. 6436 */ 6437 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 6438 Register yz_idx, Register idx, Register jdx, 6439 Register carry, Register product, 6440 Register carry2) { 6441 // jlong carry, x[], y[], z[]; 6442 // int kdx = ystart+1; 6443 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6444 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 6445 // z[kdx+idx+1] = (jlong)product; 6446 // jlong carry2 = (jlong)(product >>> 64); 6447 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 6448 // z[kdx+idx] = (jlong)product; 6449 // carry = (jlong)(product >>> 64); 6450 // } 6451 // idx += 2; 6452 // if (idx > 0) { 6453 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 6454 // z[kdx+idx] = (jlong)product; 6455 // carry = (jlong)(product >>> 64); 6456 // } 6457 // 6458 6459 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6460 6461 movl(jdx, idx); 6462 andl(jdx, 0xFFFFFFFC); 6463 shrl(jdx, 2); 6464 6465 bind(L_third_loop); 6466 subl(jdx, 1); 6467 jcc(Assembler::negative, L_third_loop_exit); 6468 subl(idx, 4); 6469 6470 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 6471 movq(carry2, rdx); 6472 6473 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 6474 movq(carry, rdx); 6475 jmp(L_third_loop); 6476 6477 bind (L_third_loop_exit); 6478 6479 andl (idx, 0x3); 6480 jcc(Assembler::zero, L_post_third_loop_done); 6481 6482 Label L_check_1; 6483 subl(idx, 2); 6484 jcc(Assembler::negative, L_check_1); 6485 6486 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 6487 movq(carry, rdx); 6488 6489 bind (L_check_1); 6490 addl (idx, 0x2); 6491 andl (idx, 0x1); 6492 subl(idx, 1); 6493 jcc(Assembler::negative, L_post_third_loop_done); 6494 6495 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 6496 movq(product, x_xstart); 6497 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6498 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 6499 6500 add2_with_carry(rdx, product, yz_idx, carry); 6501 6502 movl(Address(z, idx, Address::times_4, 0), product); 6503 shrq(product, 32); 6504 6505 shlq(rdx, 32); 6506 orq(product, rdx); 6507 movq(carry, product); 6508 6509 bind(L_post_third_loop_done); 6510 } 6511 6512 /** 6513 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 6514 * 6515 */ 6516 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 6517 Register carry, Register carry2, 6518 Register idx, Register jdx, 6519 Register yz_idx1, Register yz_idx2, 6520 Register tmp, Register tmp3, Register tmp4) { 6521 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 6522 6523 // jlong carry, x[], y[], z[]; 6524 // int kdx = ystart+1; 6525 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6526 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 6527 // jlong carry2 = (jlong)(tmp3 >>> 64); 6528 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 6529 // carry = (jlong)(tmp4 >>> 64); 6530 // z[kdx+idx+1] = (jlong)tmp3; 6531 // z[kdx+idx] = (jlong)tmp4; 6532 // } 6533 // idx += 2; 6534 // if (idx > 0) { 6535 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 6536 // z[kdx+idx] = (jlong)yz_idx1; 6537 // carry = (jlong)(yz_idx1 >>> 64); 6538 // } 6539 // 6540 6541 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6542 6543 movl(jdx, idx); 6544 andl(jdx, 0xFFFFFFFC); 6545 shrl(jdx, 2); 6546 6547 bind(L_third_loop); 6548 subl(jdx, 1); 6549 jcc(Assembler::negative, L_third_loop_exit); 6550 subl(idx, 4); 6551 6552 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 6553 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 6554 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 6555 rorxq(yz_idx2, yz_idx2, 32); 6556 6557 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6558 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 6559 6560 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 6561 rorxq(yz_idx1, yz_idx1, 32); 6562 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6563 rorxq(yz_idx2, yz_idx2, 32); 6564 6565 if (VM_Version::supports_adx()) { 6566 adcxq(tmp3, carry); 6567 adoxq(tmp3, yz_idx1); 6568 6569 adcxq(tmp4, tmp); 6570 adoxq(tmp4, yz_idx2); 6571 6572 movl(carry, 0); // does not affect flags 6573 adcxq(carry2, carry); 6574 adoxq(carry2, carry); 6575 } else { 6576 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 6577 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 6578 } 6579 movq(carry, carry2); 6580 6581 movl(Address(z, idx, Address::times_4, 12), tmp3); 6582 shrq(tmp3, 32); 6583 movl(Address(z, idx, Address::times_4, 8), tmp3); 6584 6585 movl(Address(z, idx, Address::times_4, 4), tmp4); 6586 shrq(tmp4, 32); 6587 movl(Address(z, idx, Address::times_4, 0), tmp4); 6588 6589 jmp(L_third_loop); 6590 6591 bind (L_third_loop_exit); 6592 6593 andl (idx, 0x3); 6594 jcc(Assembler::zero, L_post_third_loop_done); 6595 6596 Label L_check_1; 6597 subl(idx, 2); 6598 jcc(Assembler::negative, L_check_1); 6599 6600 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 6601 rorxq(yz_idx1, yz_idx1, 32); 6602 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6603 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6604 rorxq(yz_idx2, yz_idx2, 32); 6605 6606 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 6607 6608 movl(Address(z, idx, Address::times_4, 4), tmp3); 6609 shrq(tmp3, 32); 6610 movl(Address(z, idx, Address::times_4, 0), tmp3); 6611 movq(carry, tmp4); 6612 6613 bind (L_check_1); 6614 addl (idx, 0x2); 6615 andl (idx, 0x1); 6616 subl(idx, 1); 6617 jcc(Assembler::negative, L_post_third_loop_done); 6618 movl(tmp4, Address(y, idx, Address::times_4, 0)); 6619 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 6620 movl(tmp4, Address(z, idx, Address::times_4, 0)); 6621 6622 add2_with_carry(carry2, tmp3, tmp4, carry); 6623 6624 movl(Address(z, idx, Address::times_4, 0), tmp3); 6625 shrq(tmp3, 32); 6626 6627 shlq(carry2, 32); 6628 orq(tmp3, carry2); 6629 movq(carry, tmp3); 6630 6631 bind(L_post_third_loop_done); 6632 } 6633 6634 /** 6635 * Code for BigInteger::multiplyToLen() intrinsic. 6636 * 6637 * rdi: x 6638 * rax: xlen 6639 * rsi: y 6640 * rcx: ylen 6641 * r8: z 6642 * r11: zlen 6643 * r12: tmp1 6644 * r13: tmp2 6645 * r14: tmp3 6646 * r15: tmp4 6647 * rbx: tmp5 6648 * 6649 */ 6650 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, 6651 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 6652 ShortBranchVerifier sbv(this); 6653 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 6654 6655 push(tmp1); 6656 push(tmp2); 6657 push(tmp3); 6658 push(tmp4); 6659 push(tmp5); 6660 6661 push(xlen); 6662 push(zlen); 6663 6664 const Register idx = tmp1; 6665 const Register kdx = tmp2; 6666 const Register xstart = tmp3; 6667 6668 const Register y_idx = tmp4; 6669 const Register carry = tmp5; 6670 const Register product = xlen; 6671 const Register x_xstart = zlen; // reuse register 6672 6673 // First Loop. 6674 // 6675 // final static long LONG_MASK = 0xffffffffL; 6676 // int xstart = xlen - 1; 6677 // int ystart = ylen - 1; 6678 // long carry = 0; 6679 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6680 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 6681 // z[kdx] = (int)product; 6682 // carry = product >>> 32; 6683 // } 6684 // z[xstart] = (int)carry; 6685 // 6686 6687 movl(idx, ylen); // idx = ylen; 6688 movl(kdx, zlen); // kdx = xlen+ylen; 6689 xorq(carry, carry); // carry = 0; 6690 6691 Label L_done; 6692 6693 movl(xstart, xlen); 6694 decrementl(xstart); 6695 jcc(Assembler::negative, L_done); 6696 6697 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 6698 6699 Label L_second_loop; 6700 testl(kdx, kdx); 6701 jcc(Assembler::zero, L_second_loop); 6702 6703 Label L_carry; 6704 subl(kdx, 1); 6705 jcc(Assembler::zero, L_carry); 6706 6707 movl(Address(z, kdx, Address::times_4, 0), carry); 6708 shrq(carry, 32); 6709 subl(kdx, 1); 6710 6711 bind(L_carry); 6712 movl(Address(z, kdx, Address::times_4, 0), carry); 6713 6714 // Second and third (nested) loops. 6715 // 6716 // for (int i = xstart-1; i >= 0; i--) { // Second loop 6717 // carry = 0; 6718 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 6719 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 6720 // (z[k] & LONG_MASK) + carry; 6721 // z[k] = (int)product; 6722 // carry = product >>> 32; 6723 // } 6724 // z[i] = (int)carry; 6725 // } 6726 // 6727 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 6728 6729 const Register jdx = tmp1; 6730 6731 bind(L_second_loop); 6732 xorl(carry, carry); // carry = 0; 6733 movl(jdx, ylen); // j = ystart+1 6734 6735 subl(xstart, 1); // i = xstart-1; 6736 jcc(Assembler::negative, L_done); 6737 6738 push (z); 6739 6740 Label L_last_x; 6741 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 6742 subl(xstart, 1); // i = xstart-1; 6743 jcc(Assembler::negative, L_last_x); 6744 6745 if (UseBMI2Instructions) { 6746 movq(rdx, Address(x, xstart, Address::times_4, 0)); 6747 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 6748 } else { 6749 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6750 rorq(x_xstart, 32); // convert big-endian to little-endian 6751 } 6752 6753 Label L_third_loop_prologue; 6754 bind(L_third_loop_prologue); 6755 6756 push (x); 6757 push (xstart); 6758 push (ylen); 6759 6760 6761 if (UseBMI2Instructions) { 6762 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 6763 } else { // !UseBMI2Instructions 6764 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 6765 } 6766 6767 pop(ylen); 6768 pop(xlen); 6769 pop(x); 6770 pop(z); 6771 6772 movl(tmp3, xlen); 6773 addl(tmp3, 1); 6774 movl(Address(z, tmp3, Address::times_4, 0), carry); 6775 subl(tmp3, 1); 6776 jccb(Assembler::negative, L_done); 6777 6778 shrq(carry, 32); 6779 movl(Address(z, tmp3, Address::times_4, 0), carry); 6780 jmp(L_second_loop); 6781 6782 // Next infrequent code is moved outside loops. 6783 bind(L_last_x); 6784 if (UseBMI2Instructions) { 6785 movl(rdx, Address(x, 0)); 6786 } else { 6787 movl(x_xstart, Address(x, 0)); 6788 } 6789 jmp(L_third_loop_prologue); 6790 6791 bind(L_done); 6792 6793 pop(zlen); 6794 pop(xlen); 6795 6796 pop(tmp5); 6797 pop(tmp4); 6798 pop(tmp3); 6799 pop(tmp2); 6800 pop(tmp1); 6801 } 6802 6803 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 6804 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 6805 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 6806 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 6807 Label VECTOR8_TAIL, VECTOR4_TAIL; 6808 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 6809 Label SAME_TILL_END, DONE; 6810 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 6811 6812 //scale is in rcx in both Win64 and Unix 6813 ShortBranchVerifier sbv(this); 6814 6815 shlq(length); 6816 xorq(result, result); 6817 6818 if ((AVX3Threshold == 0) && (UseAVX > 2) && 6819 VM_Version::supports_avx512vlbw()) { 6820 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 6821 6822 cmpq(length, 64); 6823 jcc(Assembler::less, VECTOR32_TAIL); 6824 6825 movq(tmp1, length); 6826 andq(tmp1, 0x3F); // tail count 6827 andq(length, ~(0x3F)); //vector count 6828 6829 bind(VECTOR64_LOOP); 6830 // AVX512 code to compare 64 byte vectors. 6831 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 6832 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 6833 kortestql(k7, k7); 6834 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 6835 addq(result, 64); 6836 subq(length, 64); 6837 jccb(Assembler::notZero, VECTOR64_LOOP); 6838 6839 //bind(VECTOR64_TAIL); 6840 testq(tmp1, tmp1); 6841 jcc(Assembler::zero, SAME_TILL_END); 6842 6843 //bind(VECTOR64_TAIL); 6844 // AVX512 code to compare up to 63 byte vectors. 6845 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 6846 shlxq(tmp2, tmp2, tmp1); 6847 notq(tmp2); 6848 kmovql(k3, tmp2); 6849 6850 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 6851 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 6852 6853 ktestql(k7, k3); 6854 jcc(Assembler::below, SAME_TILL_END); // not mismatch 6855 6856 bind(VECTOR64_NOT_EQUAL); 6857 kmovql(tmp1, k7); 6858 notq(tmp1); 6859 tzcntq(tmp1, tmp1); 6860 addq(result, tmp1); 6861 shrq(result); 6862 jmp(DONE); 6863 bind(VECTOR32_TAIL); 6864 } 6865 6866 cmpq(length, 8); 6867 jcc(Assembler::equal, VECTOR8_LOOP); 6868 jcc(Assembler::less, VECTOR4_TAIL); 6869 6870 if (UseAVX >= 2) { 6871 Label VECTOR16_TAIL, VECTOR32_LOOP; 6872 6873 cmpq(length, 16); 6874 jcc(Assembler::equal, VECTOR16_LOOP); 6875 jcc(Assembler::less, VECTOR8_LOOP); 6876 6877 cmpq(length, 32); 6878 jccb(Assembler::less, VECTOR16_TAIL); 6879 6880 subq(length, 32); 6881 bind(VECTOR32_LOOP); 6882 vmovdqu(rymm0, Address(obja, result)); 6883 vmovdqu(rymm1, Address(objb, result)); 6884 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 6885 vptest(rymm2, rymm2); 6886 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 6887 addq(result, 32); 6888 subq(length, 32); 6889 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 6890 addq(length, 32); 6891 jcc(Assembler::equal, SAME_TILL_END); 6892 //falling through if less than 32 bytes left //close the branch here. 6893 6894 bind(VECTOR16_TAIL); 6895 cmpq(length, 16); 6896 jccb(Assembler::less, VECTOR8_TAIL); 6897 bind(VECTOR16_LOOP); 6898 movdqu(rymm0, Address(obja, result)); 6899 movdqu(rymm1, Address(objb, result)); 6900 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 6901 ptest(rymm2, rymm2); 6902 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6903 addq(result, 16); 6904 subq(length, 16); 6905 jcc(Assembler::equal, SAME_TILL_END); 6906 //falling through if less than 16 bytes left 6907 } else {//regular intrinsics 6908 6909 cmpq(length, 16); 6910 jccb(Assembler::less, VECTOR8_TAIL); 6911 6912 subq(length, 16); 6913 bind(VECTOR16_LOOP); 6914 movdqu(rymm0, Address(obja, result)); 6915 movdqu(rymm1, Address(objb, result)); 6916 pxor(rymm0, rymm1); 6917 ptest(rymm0, rymm0); 6918 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6919 addq(result, 16); 6920 subq(length, 16); 6921 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 6922 addq(length, 16); 6923 jcc(Assembler::equal, SAME_TILL_END); 6924 //falling through if less than 16 bytes left 6925 } 6926 6927 bind(VECTOR8_TAIL); 6928 cmpq(length, 8); 6929 jccb(Assembler::less, VECTOR4_TAIL); 6930 bind(VECTOR8_LOOP); 6931 movq(tmp1, Address(obja, result)); 6932 movq(tmp2, Address(objb, result)); 6933 xorq(tmp1, tmp2); 6934 testq(tmp1, tmp1); 6935 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 6936 addq(result, 8); 6937 subq(length, 8); 6938 jcc(Assembler::equal, SAME_TILL_END); 6939 //falling through if less than 8 bytes left 6940 6941 bind(VECTOR4_TAIL); 6942 cmpq(length, 4); 6943 jccb(Assembler::less, BYTES_TAIL); 6944 bind(VECTOR4_LOOP); 6945 movl(tmp1, Address(obja, result)); 6946 xorl(tmp1, Address(objb, result)); 6947 testl(tmp1, tmp1); 6948 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 6949 addq(result, 4); 6950 subq(length, 4); 6951 jcc(Assembler::equal, SAME_TILL_END); 6952 //falling through if less than 4 bytes left 6953 6954 bind(BYTES_TAIL); 6955 bind(BYTES_LOOP); 6956 load_unsigned_byte(tmp1, Address(obja, result)); 6957 load_unsigned_byte(tmp2, Address(objb, result)); 6958 xorl(tmp1, tmp2); 6959 testl(tmp1, tmp1); 6960 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6961 decq(length); 6962 jcc(Assembler::zero, SAME_TILL_END); 6963 incq(result); 6964 load_unsigned_byte(tmp1, Address(obja, result)); 6965 load_unsigned_byte(tmp2, Address(objb, result)); 6966 xorl(tmp1, tmp2); 6967 testl(tmp1, tmp1); 6968 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6969 decq(length); 6970 jcc(Assembler::zero, SAME_TILL_END); 6971 incq(result); 6972 load_unsigned_byte(tmp1, Address(obja, result)); 6973 load_unsigned_byte(tmp2, Address(objb, result)); 6974 xorl(tmp1, tmp2); 6975 testl(tmp1, tmp1); 6976 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6977 jmp(SAME_TILL_END); 6978 6979 if (UseAVX >= 2) { 6980 bind(VECTOR32_NOT_EQUAL); 6981 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 6982 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 6983 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 6984 vpmovmskb(tmp1, rymm0); 6985 bsfq(tmp1, tmp1); 6986 addq(result, tmp1); 6987 shrq(result); 6988 jmp(DONE); 6989 } 6990 6991 bind(VECTOR16_NOT_EQUAL); 6992 if (UseAVX >= 2) { 6993 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 6994 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 6995 pxor(rymm0, rymm2); 6996 } else { 6997 pcmpeqb(rymm2, rymm2); 6998 pxor(rymm0, rymm1); 6999 pcmpeqb(rymm0, rymm1); 7000 pxor(rymm0, rymm2); 7001 } 7002 pmovmskb(tmp1, rymm0); 7003 bsfq(tmp1, tmp1); 7004 addq(result, tmp1); 7005 shrq(result); 7006 jmpb(DONE); 7007 7008 bind(VECTOR8_NOT_EQUAL); 7009 bind(VECTOR4_NOT_EQUAL); 7010 bsfq(tmp1, tmp1); 7011 shrq(tmp1, 3); 7012 addq(result, tmp1); 7013 bind(BYTES_NOT_EQUAL); 7014 shrq(result); 7015 jmpb(DONE); 7016 7017 bind(SAME_TILL_END); 7018 mov64(result, -1); 7019 7020 bind(DONE); 7021 } 7022 7023 //Helper functions for square_to_len() 7024 7025 /** 7026 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 7027 * Preserves x and z and modifies rest of the registers. 7028 */ 7029 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7030 // Perform square and right shift by 1 7031 // Handle odd xlen case first, then for even xlen do the following 7032 // jlong carry = 0; 7033 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 7034 // huge_128 product = x[j:j+1] * x[j:j+1]; 7035 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 7036 // z[i+2:i+3] = (jlong)(product >>> 1); 7037 // carry = (jlong)product; 7038 // } 7039 7040 xorq(tmp5, tmp5); // carry 7041 xorq(rdxReg, rdxReg); 7042 xorl(tmp1, tmp1); // index for x 7043 xorl(tmp4, tmp4); // index for z 7044 7045 Label L_first_loop, L_first_loop_exit; 7046 7047 testl(xlen, 1); 7048 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 7049 7050 // Square and right shift by 1 the odd element using 32 bit multiply 7051 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 7052 imulq(raxReg, raxReg); 7053 shrq(raxReg, 1); 7054 adcq(tmp5, 0); 7055 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 7056 incrementl(tmp1); 7057 addl(tmp4, 2); 7058 7059 // Square and right shift by 1 the rest using 64 bit multiply 7060 bind(L_first_loop); 7061 cmpptr(tmp1, xlen); 7062 jccb(Assembler::equal, L_first_loop_exit); 7063 7064 // Square 7065 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 7066 rorq(raxReg, 32); // convert big-endian to little-endian 7067 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 7068 7069 // Right shift by 1 and save carry 7070 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 7071 rcrq(rdxReg, 1); 7072 rcrq(raxReg, 1); 7073 adcq(tmp5, 0); 7074 7075 // Store result in z 7076 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 7077 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 7078 7079 // Update indices for x and z 7080 addl(tmp1, 2); 7081 addl(tmp4, 4); 7082 jmp(L_first_loop); 7083 7084 bind(L_first_loop_exit); 7085 } 7086 7087 7088 /** 7089 * Perform the following multiply add operation using BMI2 instructions 7090 * carry:sum = sum + op1*op2 + carry 7091 * op2 should be in rdx 7092 * op2 is preserved, all other registers are modified 7093 */ 7094 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 7095 // assert op2 is rdx 7096 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 7097 addq(sum, carry); 7098 adcq(tmp2, 0); 7099 addq(sum, op1); 7100 adcq(tmp2, 0); 7101 movq(carry, tmp2); 7102 } 7103 7104 /** 7105 * Perform the following multiply add operation: 7106 * carry:sum = sum + op1*op2 + carry 7107 * Preserves op1, op2 and modifies rest of registers 7108 */ 7109 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 7110 // rdx:rax = op1 * op2 7111 movq(raxReg, op2); 7112 mulq(op1); 7113 7114 // rdx:rax = sum + carry + rdx:rax 7115 addq(sum, carry); 7116 adcq(rdxReg, 0); 7117 addq(sum, raxReg); 7118 adcq(rdxReg, 0); 7119 7120 // carry:sum = rdx:sum 7121 movq(carry, rdxReg); 7122 } 7123 7124 /** 7125 * Add 64 bit long carry into z[] with carry propagation. 7126 * Preserves z and carry register values and modifies rest of registers. 7127 * 7128 */ 7129 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 7130 Label L_fourth_loop, L_fourth_loop_exit; 7131 7132 movl(tmp1, 1); 7133 subl(zlen, 2); 7134 addq(Address(z, zlen, Address::times_4, 0), carry); 7135 7136 bind(L_fourth_loop); 7137 jccb(Assembler::carryClear, L_fourth_loop_exit); 7138 subl(zlen, 2); 7139 jccb(Assembler::negative, L_fourth_loop_exit); 7140 addq(Address(z, zlen, Address::times_4, 0), tmp1); 7141 jmp(L_fourth_loop); 7142 bind(L_fourth_loop_exit); 7143 } 7144 7145 /** 7146 * Shift z[] left by 1 bit. 7147 * Preserves x, len, z and zlen registers and modifies rest of the registers. 7148 * 7149 */ 7150 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 7151 7152 Label L_fifth_loop, L_fifth_loop_exit; 7153 7154 // Fifth loop 7155 // Perform primitiveLeftShift(z, zlen, 1) 7156 7157 const Register prev_carry = tmp1; 7158 const Register new_carry = tmp4; 7159 const Register value = tmp2; 7160 const Register zidx = tmp3; 7161 7162 // int zidx, carry; 7163 // long value; 7164 // carry = 0; 7165 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 7166 // (carry:value) = (z[i] << 1) | carry ; 7167 // z[i] = value; 7168 // } 7169 7170 movl(zidx, zlen); 7171 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 7172 7173 bind(L_fifth_loop); 7174 decl(zidx); // Use decl to preserve carry flag 7175 decl(zidx); 7176 jccb(Assembler::negative, L_fifth_loop_exit); 7177 7178 if (UseBMI2Instructions) { 7179 movq(value, Address(z, zidx, Address::times_4, 0)); 7180 rclq(value, 1); 7181 rorxq(value, value, 32); 7182 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7183 } 7184 else { 7185 // clear new_carry 7186 xorl(new_carry, new_carry); 7187 7188 // Shift z[i] by 1, or in previous carry and save new carry 7189 movq(value, Address(z, zidx, Address::times_4, 0)); 7190 shlq(value, 1); 7191 adcl(new_carry, 0); 7192 7193 orq(value, prev_carry); 7194 rorq(value, 0x20); 7195 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7196 7197 // Set previous carry = new carry 7198 movl(prev_carry, new_carry); 7199 } 7200 jmp(L_fifth_loop); 7201 7202 bind(L_fifth_loop_exit); 7203 } 7204 7205 7206 /** 7207 * Code for BigInteger::squareToLen() intrinsic 7208 * 7209 * rdi: x 7210 * rsi: len 7211 * r8: z 7212 * rcx: zlen 7213 * r12: tmp1 7214 * r13: tmp2 7215 * r14: tmp3 7216 * r15: tmp4 7217 * rbx: tmp5 7218 * 7219 */ 7220 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7221 7222 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7223 push(tmp1); 7224 push(tmp2); 7225 push(tmp3); 7226 push(tmp4); 7227 push(tmp5); 7228 7229 // First loop 7230 // Store the squares, right shifted one bit (i.e., divided by 2). 7231 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7232 7233 // Add in off-diagonal sums. 7234 // 7235 // Second, third (nested) and fourth loops. 7236 // zlen +=2; 7237 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7238 // carry = 0; 7239 // long op2 = x[xidx:xidx+1]; 7240 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7241 // k -= 2; 7242 // long op1 = x[j:j+1]; 7243 // long sum = z[k:k+1]; 7244 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7245 // z[k:k+1] = sum; 7246 // } 7247 // add_one_64(z, k, carry, tmp_regs); 7248 // } 7249 7250 const Register carry = tmp5; 7251 const Register sum = tmp3; 7252 const Register op1 = tmp4; 7253 Register op2 = tmp2; 7254 7255 push(zlen); 7256 push(len); 7257 addl(zlen,2); 7258 bind(L_second_loop); 7259 xorq(carry, carry); 7260 subl(zlen, 4); 7261 subl(len, 2); 7262 push(zlen); 7263 push(len); 7264 cmpl(len, 0); 7265 jccb(Assembler::lessEqual, L_second_loop_exit); 7266 7267 // Multiply an array by one 64 bit long. 7268 if (UseBMI2Instructions) { 7269 op2 = rdxReg; 7270 movq(op2, Address(x, len, Address::times_4, 0)); 7271 rorxq(op2, op2, 32); 7272 } 7273 else { 7274 movq(op2, Address(x, len, Address::times_4, 0)); 7275 rorq(op2, 32); 7276 } 7277 7278 bind(L_third_loop); 7279 decrementl(len); 7280 jccb(Assembler::negative, L_third_loop_exit); 7281 decrementl(len); 7282 jccb(Assembler::negative, L_last_x); 7283 7284 movq(op1, Address(x, len, Address::times_4, 0)); 7285 rorq(op1, 32); 7286 7287 bind(L_multiply); 7288 subl(zlen, 2); 7289 movq(sum, Address(z, zlen, Address::times_4, 0)); 7290 7291 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7292 if (UseBMI2Instructions) { 7293 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 7294 } 7295 else { 7296 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7297 } 7298 7299 movq(Address(z, zlen, Address::times_4, 0), sum); 7300 7301 jmp(L_third_loop); 7302 bind(L_third_loop_exit); 7303 7304 // Fourth loop 7305 // Add 64 bit long carry into z with carry propagation. 7306 // Uses offsetted zlen. 7307 add_one_64(z, zlen, carry, tmp1); 7308 7309 pop(len); 7310 pop(zlen); 7311 jmp(L_second_loop); 7312 7313 // Next infrequent code is moved outside loops. 7314 bind(L_last_x); 7315 movl(op1, Address(x, 0)); 7316 jmp(L_multiply); 7317 7318 bind(L_second_loop_exit); 7319 pop(len); 7320 pop(zlen); 7321 pop(len); 7322 pop(zlen); 7323 7324 // Fifth loop 7325 // Shift z left 1 bit. 7326 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 7327 7328 // z[zlen-1] |= x[len-1] & 1; 7329 movl(tmp3, Address(x, len, Address::times_4, -4)); 7330 andl(tmp3, 1); 7331 orl(Address(z, zlen, Address::times_4, -4), tmp3); 7332 7333 pop(tmp5); 7334 pop(tmp4); 7335 pop(tmp3); 7336 pop(tmp2); 7337 pop(tmp1); 7338 } 7339 7340 /** 7341 * Helper function for mul_add() 7342 * Multiply the in[] by int k and add to out[] starting at offset offs using 7343 * 128 bit by 32 bit multiply and return the carry in tmp5. 7344 * Only quad int aligned length of in[] is operated on in this function. 7345 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 7346 * This function preserves out, in and k registers. 7347 * len and offset point to the appropriate index in "in" & "out" correspondingly 7348 * tmp5 has the carry. 7349 * other registers are temporary and are modified. 7350 * 7351 */ 7352 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 7353 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 7354 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7355 7356 Label L_first_loop, L_first_loop_exit; 7357 7358 movl(tmp1, len); 7359 shrl(tmp1, 2); 7360 7361 bind(L_first_loop); 7362 subl(tmp1, 1); 7363 jccb(Assembler::negative, L_first_loop_exit); 7364 7365 subl(len, 4); 7366 subl(offset, 4); 7367 7368 Register op2 = tmp2; 7369 const Register sum = tmp3; 7370 const Register op1 = tmp4; 7371 const Register carry = tmp5; 7372 7373 if (UseBMI2Instructions) { 7374 op2 = rdxReg; 7375 } 7376 7377 movq(op1, Address(in, len, Address::times_4, 8)); 7378 rorq(op1, 32); 7379 movq(sum, Address(out, offset, Address::times_4, 8)); 7380 rorq(sum, 32); 7381 if (UseBMI2Instructions) { 7382 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7383 } 7384 else { 7385 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7386 } 7387 // Store back in big endian from little endian 7388 rorq(sum, 0x20); 7389 movq(Address(out, offset, Address::times_4, 8), sum); 7390 7391 movq(op1, Address(in, len, Address::times_4, 0)); 7392 rorq(op1, 32); 7393 movq(sum, Address(out, offset, Address::times_4, 0)); 7394 rorq(sum, 32); 7395 if (UseBMI2Instructions) { 7396 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7397 } 7398 else { 7399 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7400 } 7401 // Store back in big endian from little endian 7402 rorq(sum, 0x20); 7403 movq(Address(out, offset, Address::times_4, 0), sum); 7404 7405 jmp(L_first_loop); 7406 bind(L_first_loop_exit); 7407 } 7408 7409 /** 7410 * Code for BigInteger::mulAdd() intrinsic 7411 * 7412 * rdi: out 7413 * rsi: in 7414 * r11: offs (out.length - offset) 7415 * rcx: len 7416 * r8: k 7417 * r12: tmp1 7418 * r13: tmp2 7419 * r14: tmp3 7420 * r15: tmp4 7421 * rbx: tmp5 7422 * Multiply the in[] by word k and add to out[], return the carry in rax 7423 */ 7424 void MacroAssembler::mul_add(Register out, Register in, Register offs, 7425 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 7426 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7427 7428 Label L_carry, L_last_in, L_done; 7429 7430 // carry = 0; 7431 // for (int j=len-1; j >= 0; j--) { 7432 // long product = (in[j] & LONG_MASK) * kLong + 7433 // (out[offs] & LONG_MASK) + carry; 7434 // out[offs--] = (int)product; 7435 // carry = product >>> 32; 7436 // } 7437 // 7438 push(tmp1); 7439 push(tmp2); 7440 push(tmp3); 7441 push(tmp4); 7442 push(tmp5); 7443 7444 Register op2 = tmp2; 7445 const Register sum = tmp3; 7446 const Register op1 = tmp4; 7447 const Register carry = tmp5; 7448 7449 if (UseBMI2Instructions) { 7450 op2 = rdxReg; 7451 movl(op2, k); 7452 } 7453 else { 7454 movl(op2, k); 7455 } 7456 7457 xorq(carry, carry); 7458 7459 //First loop 7460 7461 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 7462 //The carry is in tmp5 7463 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 7464 7465 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 7466 decrementl(len); 7467 jccb(Assembler::negative, L_carry); 7468 decrementl(len); 7469 jccb(Assembler::negative, L_last_in); 7470 7471 movq(op1, Address(in, len, Address::times_4, 0)); 7472 rorq(op1, 32); 7473 7474 subl(offs, 2); 7475 movq(sum, Address(out, offs, Address::times_4, 0)); 7476 rorq(sum, 32); 7477 7478 if (UseBMI2Instructions) { 7479 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7480 } 7481 else { 7482 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7483 } 7484 7485 // Store back in big endian from little endian 7486 rorq(sum, 0x20); 7487 movq(Address(out, offs, Address::times_4, 0), sum); 7488 7489 testl(len, len); 7490 jccb(Assembler::zero, L_carry); 7491 7492 //Multiply the last in[] entry, if any 7493 bind(L_last_in); 7494 movl(op1, Address(in, 0)); 7495 movl(sum, Address(out, offs, Address::times_4, -4)); 7496 7497 movl(raxReg, k); 7498 mull(op1); //tmp4 * eax -> edx:eax 7499 addl(sum, carry); 7500 adcl(rdxReg, 0); 7501 addl(sum, raxReg); 7502 adcl(rdxReg, 0); 7503 movl(carry, rdxReg); 7504 7505 movl(Address(out, offs, Address::times_4, -4), sum); 7506 7507 bind(L_carry); 7508 //return tmp5/carry as carry in rax 7509 movl(rax, carry); 7510 7511 bind(L_done); 7512 pop(tmp5); 7513 pop(tmp4); 7514 pop(tmp3); 7515 pop(tmp2); 7516 pop(tmp1); 7517 } 7518 #endif 7519 7520 /** 7521 * Emits code to update CRC-32 with a byte value according to constants in table 7522 * 7523 * @param [in,out]crc Register containing the crc. 7524 * @param [in]val Register containing the byte to fold into the CRC. 7525 * @param [in]table Register containing the table of crc constants. 7526 * 7527 * uint32_t crc; 7528 * val = crc_table[(val ^ crc) & 0xFF]; 7529 * crc = val ^ (crc >> 8); 7530 * 7531 */ 7532 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 7533 xorl(val, crc); 7534 andl(val, 0xFF); 7535 shrl(crc, 8); // unsigned shift 7536 xorl(crc, Address(table, val, Address::times_4, 0)); 7537 } 7538 7539 /** 7540 * Fold 128-bit data chunk 7541 */ 7542 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 7543 if (UseAVX > 0) { 7544 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 7545 vpclmulldq(xcrc, xK, xcrc); // [63:0] 7546 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 7547 pxor(xcrc, xtmp); 7548 } else { 7549 movdqa(xtmp, xcrc); 7550 pclmulhdq(xtmp, xK); // [123:64] 7551 pclmulldq(xcrc, xK); // [63:0] 7552 pxor(xcrc, xtmp); 7553 movdqu(xtmp, Address(buf, offset)); 7554 pxor(xcrc, xtmp); 7555 } 7556 } 7557 7558 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 7559 if (UseAVX > 0) { 7560 vpclmulhdq(xtmp, xK, xcrc); 7561 vpclmulldq(xcrc, xK, xcrc); 7562 pxor(xcrc, xbuf); 7563 pxor(xcrc, xtmp); 7564 } else { 7565 movdqa(xtmp, xcrc); 7566 pclmulhdq(xtmp, xK); 7567 pclmulldq(xcrc, xK); 7568 pxor(xcrc, xbuf); 7569 pxor(xcrc, xtmp); 7570 } 7571 } 7572 7573 /** 7574 * 8-bit folds to compute 32-bit CRC 7575 * 7576 * uint64_t xcrc; 7577 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 7578 */ 7579 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 7580 movdl(tmp, xcrc); 7581 andl(tmp, 0xFF); 7582 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 7583 psrldq(xcrc, 1); // unsigned shift one byte 7584 pxor(xcrc, xtmp); 7585 } 7586 7587 /** 7588 * uint32_t crc; 7589 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 7590 */ 7591 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 7592 movl(tmp, crc); 7593 andl(tmp, 0xFF); 7594 shrl(crc, 8); 7595 xorl(crc, Address(table, tmp, Address::times_4, 0)); 7596 } 7597 7598 /** 7599 * @param crc register containing existing CRC (32-bit) 7600 * @param buf register pointing to input byte buffer (byte*) 7601 * @param len register containing number of bytes 7602 * @param table register that will contain address of CRC table 7603 * @param tmp scratch register 7604 */ 7605 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 7606 assert_different_registers(crc, buf, len, table, tmp, rax); 7607 7608 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7609 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7610 7611 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7612 // context for the registers used, where all instructions below are using 128-bit mode 7613 // On EVEX without VL and BW, these instructions will all be AVX. 7614 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 7615 notl(crc); // ~crc 7616 cmpl(len, 16); 7617 jcc(Assembler::less, L_tail); 7618 7619 // Align buffer to 16 bytes 7620 movl(tmp, buf); 7621 andl(tmp, 0xF); 7622 jccb(Assembler::zero, L_aligned); 7623 subl(tmp, 16); 7624 addl(len, tmp); 7625 7626 align(4); 7627 BIND(L_align_loop); 7628 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7629 update_byte_crc32(crc, rax, table); 7630 increment(buf); 7631 incrementl(tmp); 7632 jccb(Assembler::less, L_align_loop); 7633 7634 BIND(L_aligned); 7635 movl(tmp, len); // save 7636 shrl(len, 4); 7637 jcc(Assembler::zero, L_tail_restore); 7638 7639 // Fold crc into first bytes of vector 7640 movdqa(xmm1, Address(buf, 0)); 7641 movdl(rax, xmm1); 7642 xorl(crc, rax); 7643 if (VM_Version::supports_sse4_1()) { 7644 pinsrd(xmm1, crc, 0); 7645 } else { 7646 pinsrw(xmm1, crc, 0); 7647 shrl(crc, 16); 7648 pinsrw(xmm1, crc, 1); 7649 } 7650 addptr(buf, 16); 7651 subl(len, 4); // len > 0 7652 jcc(Assembler::less, L_fold_tail); 7653 7654 movdqa(xmm2, Address(buf, 0)); 7655 movdqa(xmm3, Address(buf, 16)); 7656 movdqa(xmm4, Address(buf, 32)); 7657 addptr(buf, 48); 7658 subl(len, 3); 7659 jcc(Assembler::lessEqual, L_fold_512b); 7660 7661 // Fold total 512 bits of polynomial on each iteration, 7662 // 128 bits per each of 4 parallel streams. 7663 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 7664 7665 align32(); 7666 BIND(L_fold_512b_loop); 7667 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7668 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 7669 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 7670 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 7671 addptr(buf, 64); 7672 subl(len, 4); 7673 jcc(Assembler::greater, L_fold_512b_loop); 7674 7675 // Fold 512 bits to 128 bits. 7676 BIND(L_fold_512b); 7677 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7678 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 7679 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 7680 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 7681 7682 // Fold the rest of 128 bits data chunks 7683 BIND(L_fold_tail); 7684 addl(len, 3); 7685 jccb(Assembler::lessEqual, L_fold_128b); 7686 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7687 7688 BIND(L_fold_tail_loop); 7689 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7690 addptr(buf, 16); 7691 decrementl(len); 7692 jccb(Assembler::greater, L_fold_tail_loop); 7693 7694 // Fold 128 bits in xmm1 down into 32 bits in crc register. 7695 BIND(L_fold_128b); 7696 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 7697 if (UseAVX > 0) { 7698 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 7699 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 7700 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 7701 } else { 7702 movdqa(xmm2, xmm0); 7703 pclmulqdq(xmm2, xmm1, 0x1); 7704 movdqa(xmm3, xmm0); 7705 pand(xmm3, xmm2); 7706 pclmulqdq(xmm0, xmm3, 0x1); 7707 } 7708 psrldq(xmm1, 8); 7709 psrldq(xmm2, 4); 7710 pxor(xmm0, xmm1); 7711 pxor(xmm0, xmm2); 7712 7713 // 8 8-bit folds to compute 32-bit CRC. 7714 for (int j = 0; j < 4; j++) { 7715 fold_8bit_crc32(xmm0, table, xmm1, rax); 7716 } 7717 movdl(crc, xmm0); // mov 32 bits to general register 7718 for (int j = 0; j < 4; j++) { 7719 fold_8bit_crc32(crc, table, rax); 7720 } 7721 7722 BIND(L_tail_restore); 7723 movl(len, tmp); // restore 7724 BIND(L_tail); 7725 andl(len, 0xf); 7726 jccb(Assembler::zero, L_exit); 7727 7728 // Fold the rest of bytes 7729 align(4); 7730 BIND(L_tail_loop); 7731 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7732 update_byte_crc32(crc, rax, table); 7733 increment(buf); 7734 decrementl(len); 7735 jccb(Assembler::greater, L_tail_loop); 7736 7737 BIND(L_exit); 7738 notl(crc); // ~c 7739 } 7740 7741 #ifdef _LP64 7742 // Helper function for AVX 512 CRC32 7743 // Fold 512-bit data chunks 7744 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 7745 Register pos, int offset) { 7746 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 7747 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 7748 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 7749 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 7750 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 7751 } 7752 7753 // Helper function for AVX 512 CRC32 7754 // Compute CRC32 for < 256B buffers 7755 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 7756 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 7757 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 7758 7759 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 7760 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 7761 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 7762 7763 // check if there is enough buffer to be able to fold 16B at a time 7764 cmpl(len, 32); 7765 jcc(Assembler::less, L_less_than_32); 7766 7767 // if there is, load the constants 7768 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 7769 movdl(xmm0, crc); // get the initial crc value 7770 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7771 pxor(xmm7, xmm0); 7772 7773 // update the buffer pointer 7774 addl(pos, 16); 7775 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 7776 subl(len, 32); 7777 jmp(L_16B_reduction_loop); 7778 7779 bind(L_less_than_32); 7780 //mov initial crc to the return value. this is necessary for zero - length buffers. 7781 movl(rax, crc); 7782 testl(len, len); 7783 jcc(Assembler::equal, L_cleanup); 7784 7785 movdl(xmm0, crc); //get the initial crc value 7786 7787 cmpl(len, 16); 7788 jcc(Assembler::equal, L_exact_16_left); 7789 jcc(Assembler::less, L_less_than_16_left); 7790 7791 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7792 pxor(xmm7, xmm0); //xor the initial crc value 7793 addl(pos, 16); 7794 subl(len, 16); 7795 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 7796 jmp(L_get_last_two_xmms); 7797 7798 bind(L_less_than_16_left); 7799 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 7800 pxor(xmm1, xmm1); 7801 movptr(tmp1, rsp); 7802 movdqu(Address(tmp1, 0 * 16), xmm1); 7803 7804 cmpl(len, 4); 7805 jcc(Assembler::less, L_only_less_than_4); 7806 7807 //backup the counter value 7808 movl(tmp2, len); 7809 cmpl(len, 8); 7810 jcc(Assembler::less, L_less_than_8_left); 7811 7812 //load 8 Bytes 7813 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 7814 movq(Address(tmp1, 0 * 16), rax); 7815 addptr(tmp1, 8); 7816 subl(len, 8); 7817 addl(pos, 8); 7818 7819 bind(L_less_than_8_left); 7820 cmpl(len, 4); 7821 jcc(Assembler::less, L_less_than_4_left); 7822 7823 //load 4 Bytes 7824 movl(rax, Address(buf, pos, Address::times_1, 0)); 7825 movl(Address(tmp1, 0 * 16), rax); 7826 addptr(tmp1, 4); 7827 subl(len, 4); 7828 addl(pos, 4); 7829 7830 bind(L_less_than_4_left); 7831 cmpl(len, 2); 7832 jcc(Assembler::less, L_less_than_2_left); 7833 7834 // load 2 Bytes 7835 movw(rax, Address(buf, pos, Address::times_1, 0)); 7836 movl(Address(tmp1, 0 * 16), rax); 7837 addptr(tmp1, 2); 7838 subl(len, 2); 7839 addl(pos, 2); 7840 7841 bind(L_less_than_2_left); 7842 cmpl(len, 1); 7843 jcc(Assembler::less, L_zero_left); 7844 7845 // load 1 Byte 7846 movb(rax, Address(buf, pos, Address::times_1, 0)); 7847 movb(Address(tmp1, 0 * 16), rax); 7848 7849 bind(L_zero_left); 7850 movdqu(xmm7, Address(rsp, 0)); 7851 pxor(xmm7, xmm0); //xor the initial crc value 7852 7853 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7854 movdqu(xmm0, Address(rax, tmp2)); 7855 pshufb(xmm7, xmm0); 7856 jmp(L_128_done); 7857 7858 bind(L_exact_16_left); 7859 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 7860 pxor(xmm7, xmm0); //xor the initial crc value 7861 jmp(L_128_done); 7862 7863 bind(L_only_less_than_4); 7864 cmpl(len, 3); 7865 jcc(Assembler::less, L_only_less_than_3); 7866 7867 // load 3 Bytes 7868 movb(rax, Address(buf, pos, Address::times_1, 0)); 7869 movb(Address(tmp1, 0), rax); 7870 7871 movb(rax, Address(buf, pos, Address::times_1, 1)); 7872 movb(Address(tmp1, 1), rax); 7873 7874 movb(rax, Address(buf, pos, Address::times_1, 2)); 7875 movb(Address(tmp1, 2), rax); 7876 7877 movdqu(xmm7, Address(rsp, 0)); 7878 pxor(xmm7, xmm0); //xor the initial crc value 7879 7880 pslldq(xmm7, 0x5); 7881 jmp(L_barrett); 7882 bind(L_only_less_than_3); 7883 cmpl(len, 2); 7884 jcc(Assembler::less, L_only_less_than_2); 7885 7886 // load 2 Bytes 7887 movb(rax, Address(buf, pos, Address::times_1, 0)); 7888 movb(Address(tmp1, 0), rax); 7889 7890 movb(rax, Address(buf, pos, Address::times_1, 1)); 7891 movb(Address(tmp1, 1), rax); 7892 7893 movdqu(xmm7, Address(rsp, 0)); 7894 pxor(xmm7, xmm0); //xor the initial crc value 7895 7896 pslldq(xmm7, 0x6); 7897 jmp(L_barrett); 7898 7899 bind(L_only_less_than_2); 7900 //load 1 Byte 7901 movb(rax, Address(buf, pos, Address::times_1, 0)); 7902 movb(Address(tmp1, 0), rax); 7903 7904 movdqu(xmm7, Address(rsp, 0)); 7905 pxor(xmm7, xmm0); //xor the initial crc value 7906 7907 pslldq(xmm7, 0x7); 7908 } 7909 7910 /** 7911 * Compute CRC32 using AVX512 instructions 7912 * param crc register containing existing CRC (32-bit) 7913 * param buf register pointing to input byte buffer (byte*) 7914 * param len register containing number of bytes 7915 * param table address of crc or crc32c table 7916 * param tmp1 scratch register 7917 * param tmp2 scratch register 7918 * return rax result register 7919 * 7920 * This routine is identical for crc32c with the exception of the precomputed constant 7921 * table which will be passed as the table argument. The calculation steps are 7922 * the same for both variants. 7923 */ 7924 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 7925 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 7926 7927 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7928 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7929 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 7930 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 7931 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 7932 7933 const Register pos = r12; 7934 push(r12); 7935 subptr(rsp, 16 * 2 + 8); 7936 7937 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7938 // context for the registers used, where all instructions below are using 128-bit mode 7939 // On EVEX without VL and BW, these instructions will all be AVX. 7940 movl(pos, 0); 7941 7942 // check if smaller than 256B 7943 cmpl(len, 256); 7944 jcc(Assembler::less, L_less_than_256); 7945 7946 // load the initial crc value 7947 movdl(xmm10, crc); 7948 7949 // receive the initial 64B data, xor the initial crc value 7950 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 7951 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 7952 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 7953 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 7954 7955 subl(len, 256); 7956 cmpl(len, 256); 7957 jcc(Assembler::less, L_fold_128_B_loop); 7958 7959 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 7960 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 7961 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 7962 subl(len, 256); 7963 7964 bind(L_fold_256_B_loop); 7965 addl(pos, 256); 7966 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 7967 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 7968 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 7969 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 7970 7971 subl(len, 256); 7972 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 7973 7974 // Fold 256 into 128 7975 addl(pos, 256); 7976 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 7977 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 7978 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 7979 7980 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 7981 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 7982 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 7983 7984 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 7985 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 7986 7987 addl(len, 128); 7988 jmp(L_fold_128_B_register); 7989 7990 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 7991 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 7992 7993 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 7994 bind(L_fold_128_B_loop); 7995 addl(pos, 128); 7996 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 7997 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 7998 7999 subl(len, 128); 8000 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 8001 8002 addl(pos, 128); 8003 8004 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 8005 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 8006 bind(L_fold_128_B_register); 8007 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 8008 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 8009 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 8010 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 8011 // save last that has no multiplicand 8012 vextracti64x2(xmm7, xmm4, 3); 8013 8014 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 8015 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 8016 // Needed later in reduction loop 8017 movdqu(xmm10, Address(table, 1 * 16)); 8018 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 8019 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 8020 8021 // Swap 1,0,3,2 - 01 00 11 10 8022 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 8023 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 8024 vextracti128(xmm5, xmm8, 1); 8025 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 8026 8027 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 8028 // instead of a cmp instruction, we use the negative flag with the jl instruction 8029 addl(len, 128 - 16); 8030 jcc(Assembler::less, L_final_reduction_for_128); 8031 8032 bind(L_16B_reduction_loop); 8033 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8034 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8035 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8036 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 8037 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8038 addl(pos, 16); 8039 subl(len, 16); 8040 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 8041 8042 bind(L_final_reduction_for_128); 8043 addl(len, 16); 8044 jcc(Assembler::equal, L_128_done); 8045 8046 bind(L_get_last_two_xmms); 8047 movdqu(xmm2, xmm7); 8048 addl(pos, len); 8049 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 8050 subl(pos, len); 8051 8052 // get rid of the extra data that was loaded before 8053 // load the shift constant 8054 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8055 movdqu(xmm0, Address(rax, len)); 8056 addl(rax, len); 8057 8058 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8059 //Change mask to 512 8060 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 8061 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 8062 8063 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 8064 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8065 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8066 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8067 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 8068 8069 bind(L_128_done); 8070 // compute crc of a 128-bit value 8071 movdqu(xmm10, Address(table, 3 * 16)); 8072 movdqu(xmm0, xmm7); 8073 8074 // 64b fold 8075 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 8076 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 8077 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8078 8079 // 32b fold 8080 movdqu(xmm0, xmm7); 8081 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 8082 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8083 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8084 jmp(L_barrett); 8085 8086 bind(L_less_than_256); 8087 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 8088 8089 //barrett reduction 8090 bind(L_barrett); 8091 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 8092 movdqu(xmm1, xmm7); 8093 movdqu(xmm2, xmm7); 8094 movdqu(xmm10, Address(table, 4 * 16)); 8095 8096 pclmulqdq(xmm7, xmm10, 0x0); 8097 pxor(xmm7, xmm2); 8098 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 8099 movdqu(xmm2, xmm7); 8100 pclmulqdq(xmm7, xmm10, 0x10); 8101 pxor(xmm7, xmm2); 8102 pxor(xmm7, xmm1); 8103 pextrd(crc, xmm7, 2); 8104 8105 bind(L_cleanup); 8106 addptr(rsp, 16 * 2 + 8); 8107 pop(r12); 8108 } 8109 8110 // S. Gueron / Information Processing Letters 112 (2012) 184 8111 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 8112 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 8113 // Output: the 64-bit carry-less product of B * CONST 8114 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 8115 Register tmp1, Register tmp2, Register tmp3) { 8116 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8117 if (n > 0) { 8118 addq(tmp3, n * 256 * 8); 8119 } 8120 // Q1 = TABLEExt[n][B & 0xFF]; 8121 movl(tmp1, in); 8122 andl(tmp1, 0x000000FF); 8123 shll(tmp1, 3); 8124 addq(tmp1, tmp3); 8125 movq(tmp1, Address(tmp1, 0)); 8126 8127 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8128 movl(tmp2, in); 8129 shrl(tmp2, 8); 8130 andl(tmp2, 0x000000FF); 8131 shll(tmp2, 3); 8132 addq(tmp2, tmp3); 8133 movq(tmp2, Address(tmp2, 0)); 8134 8135 shlq(tmp2, 8); 8136 xorq(tmp1, tmp2); 8137 8138 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8139 movl(tmp2, in); 8140 shrl(tmp2, 16); 8141 andl(tmp2, 0x000000FF); 8142 shll(tmp2, 3); 8143 addq(tmp2, tmp3); 8144 movq(tmp2, Address(tmp2, 0)); 8145 8146 shlq(tmp2, 16); 8147 xorq(tmp1, tmp2); 8148 8149 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8150 shrl(in, 24); 8151 andl(in, 0x000000FF); 8152 shll(in, 3); 8153 addq(in, tmp3); 8154 movq(in, Address(in, 0)); 8155 8156 shlq(in, 24); 8157 xorq(in, tmp1); 8158 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8159 } 8160 8161 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8162 Register in_out, 8163 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8164 XMMRegister w_xtmp2, 8165 Register tmp1, 8166 Register n_tmp2, Register n_tmp3) { 8167 if (is_pclmulqdq_supported) { 8168 movdl(w_xtmp1, in_out); // modified blindly 8169 8170 movl(tmp1, const_or_pre_comp_const_index); 8171 movdl(w_xtmp2, tmp1); 8172 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8173 8174 movdq(in_out, w_xtmp1); 8175 } else { 8176 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 8177 } 8178 } 8179 8180 // Recombination Alternative 2: No bit-reflections 8181 // T1 = (CRC_A * U1) << 1 8182 // T2 = (CRC_B * U2) << 1 8183 // C1 = T1 >> 32 8184 // C2 = T2 >> 32 8185 // T1 = T1 & 0xFFFFFFFF 8186 // T2 = T2 & 0xFFFFFFFF 8187 // T1 = CRC32(0, T1) 8188 // T2 = CRC32(0, T2) 8189 // C1 = C1 ^ T1 8190 // C2 = C2 ^ T2 8191 // CRC = C1 ^ C2 ^ CRC_C 8192 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8193 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8194 Register tmp1, Register tmp2, 8195 Register n_tmp3) { 8196 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8197 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8198 shlq(in_out, 1); 8199 movl(tmp1, in_out); 8200 shrq(in_out, 32); 8201 xorl(tmp2, tmp2); 8202 crc32(tmp2, tmp1, 4); 8203 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8204 shlq(in1, 1); 8205 movl(tmp1, in1); 8206 shrq(in1, 32); 8207 xorl(tmp2, tmp2); 8208 crc32(tmp2, tmp1, 4); 8209 xorl(in1, tmp2); 8210 xorl(in_out, in1); 8211 xorl(in_out, in2); 8212 } 8213 8214 // Set N to predefined value 8215 // Subtract from a length of a buffer 8216 // execute in a loop: 8217 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8218 // for i = 1 to N do 8219 // CRC_A = CRC32(CRC_A, A[i]) 8220 // CRC_B = CRC32(CRC_B, B[i]) 8221 // CRC_C = CRC32(CRC_C, C[i]) 8222 // end for 8223 // Recombine 8224 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8225 Register in_out1, Register in_out2, Register in_out3, 8226 Register tmp1, Register tmp2, Register tmp3, 8227 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8228 Register tmp4, Register tmp5, 8229 Register n_tmp6) { 8230 Label L_processPartitions; 8231 Label L_processPartition; 8232 Label L_exit; 8233 8234 bind(L_processPartitions); 8235 cmpl(in_out1, 3 * size); 8236 jcc(Assembler::less, L_exit); 8237 xorl(tmp1, tmp1); 8238 xorl(tmp2, tmp2); 8239 movq(tmp3, in_out2); 8240 addq(tmp3, size); 8241 8242 bind(L_processPartition); 8243 crc32(in_out3, Address(in_out2, 0), 8); 8244 crc32(tmp1, Address(in_out2, size), 8); 8245 crc32(tmp2, Address(in_out2, size * 2), 8); 8246 addq(in_out2, 8); 8247 cmpq(in_out2, tmp3); 8248 jcc(Assembler::less, L_processPartition); 8249 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8250 w_xtmp1, w_xtmp2, w_xtmp3, 8251 tmp4, tmp5, 8252 n_tmp6); 8253 addq(in_out2, 2 * size); 8254 subl(in_out1, 3 * size); 8255 jmp(L_processPartitions); 8256 8257 bind(L_exit); 8258 } 8259 #else 8260 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, 8261 Register tmp1, Register tmp2, Register tmp3, 8262 XMMRegister xtmp1, XMMRegister xtmp2) { 8263 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8264 if (n > 0) { 8265 addl(tmp3, n * 256 * 8); 8266 } 8267 // Q1 = TABLEExt[n][B & 0xFF]; 8268 movl(tmp1, in_out); 8269 andl(tmp1, 0x000000FF); 8270 shll(tmp1, 3); 8271 addl(tmp1, tmp3); 8272 movq(xtmp1, Address(tmp1, 0)); 8273 8274 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8275 movl(tmp2, in_out); 8276 shrl(tmp2, 8); 8277 andl(tmp2, 0x000000FF); 8278 shll(tmp2, 3); 8279 addl(tmp2, tmp3); 8280 movq(xtmp2, Address(tmp2, 0)); 8281 8282 psllq(xtmp2, 8); 8283 pxor(xtmp1, xtmp2); 8284 8285 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8286 movl(tmp2, in_out); 8287 shrl(tmp2, 16); 8288 andl(tmp2, 0x000000FF); 8289 shll(tmp2, 3); 8290 addl(tmp2, tmp3); 8291 movq(xtmp2, Address(tmp2, 0)); 8292 8293 psllq(xtmp2, 16); 8294 pxor(xtmp1, xtmp2); 8295 8296 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8297 shrl(in_out, 24); 8298 andl(in_out, 0x000000FF); 8299 shll(in_out, 3); 8300 addl(in_out, tmp3); 8301 movq(xtmp2, Address(in_out, 0)); 8302 8303 psllq(xtmp2, 24); 8304 pxor(xtmp1, xtmp2); // Result in CXMM 8305 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8306 } 8307 8308 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8309 Register in_out, 8310 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8311 XMMRegister w_xtmp2, 8312 Register tmp1, 8313 Register n_tmp2, Register n_tmp3) { 8314 if (is_pclmulqdq_supported) { 8315 movdl(w_xtmp1, in_out); 8316 8317 movl(tmp1, const_or_pre_comp_const_index); 8318 movdl(w_xtmp2, tmp1); 8319 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8320 // Keep result in XMM since GPR is 32 bit in length 8321 } else { 8322 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); 8323 } 8324 } 8325 8326 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8327 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8328 Register tmp1, Register tmp2, 8329 Register n_tmp3) { 8330 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8331 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8332 8333 psllq(w_xtmp1, 1); 8334 movdl(tmp1, w_xtmp1); 8335 psrlq(w_xtmp1, 32); 8336 movdl(in_out, w_xtmp1); 8337 8338 xorl(tmp2, tmp2); 8339 crc32(tmp2, tmp1, 4); 8340 xorl(in_out, tmp2); 8341 8342 psllq(w_xtmp2, 1); 8343 movdl(tmp1, w_xtmp2); 8344 psrlq(w_xtmp2, 32); 8345 movdl(in1, w_xtmp2); 8346 8347 xorl(tmp2, tmp2); 8348 crc32(tmp2, tmp1, 4); 8349 xorl(in1, tmp2); 8350 xorl(in_out, in1); 8351 xorl(in_out, in2); 8352 } 8353 8354 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8355 Register in_out1, Register in_out2, Register in_out3, 8356 Register tmp1, Register tmp2, Register tmp3, 8357 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8358 Register tmp4, Register tmp5, 8359 Register n_tmp6) { 8360 Label L_processPartitions; 8361 Label L_processPartition; 8362 Label L_exit; 8363 8364 bind(L_processPartitions); 8365 cmpl(in_out1, 3 * size); 8366 jcc(Assembler::less, L_exit); 8367 xorl(tmp1, tmp1); 8368 xorl(tmp2, tmp2); 8369 movl(tmp3, in_out2); 8370 addl(tmp3, size); 8371 8372 bind(L_processPartition); 8373 crc32(in_out3, Address(in_out2, 0), 4); 8374 crc32(tmp1, Address(in_out2, size), 4); 8375 crc32(tmp2, Address(in_out2, size*2), 4); 8376 crc32(in_out3, Address(in_out2, 0+4), 4); 8377 crc32(tmp1, Address(in_out2, size+4), 4); 8378 crc32(tmp2, Address(in_out2, size*2+4), 4); 8379 addl(in_out2, 8); 8380 cmpl(in_out2, tmp3); 8381 jcc(Assembler::less, L_processPartition); 8382 8383 push(tmp3); 8384 push(in_out1); 8385 push(in_out2); 8386 tmp4 = tmp3; 8387 tmp5 = in_out1; 8388 n_tmp6 = in_out2; 8389 8390 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8391 w_xtmp1, w_xtmp2, w_xtmp3, 8392 tmp4, tmp5, 8393 n_tmp6); 8394 8395 pop(in_out2); 8396 pop(in_out1); 8397 pop(tmp3); 8398 8399 addl(in_out2, 2 * size); 8400 subl(in_out1, 3 * size); 8401 jmp(L_processPartitions); 8402 8403 bind(L_exit); 8404 } 8405 #endif //LP64 8406 8407 #ifdef _LP64 8408 // Algorithm 2: Pipelined usage of the CRC32 instruction. 8409 // Input: A buffer I of L bytes. 8410 // Output: the CRC32C value of the buffer. 8411 // Notations: 8412 // Write L = 24N + r, with N = floor (L/24). 8413 // r = L mod 24 (0 <= r < 24). 8414 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 8415 // N quadwords, and R consists of r bytes. 8416 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 8417 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 8418 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 8419 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 8420 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8421 Register tmp1, Register tmp2, Register tmp3, 8422 Register tmp4, Register tmp5, Register tmp6, 8423 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8424 bool is_pclmulqdq_supported) { 8425 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8426 Label L_wordByWord; 8427 Label L_byteByByteProlog; 8428 Label L_byteByByte; 8429 Label L_exit; 8430 8431 if (is_pclmulqdq_supported ) { 8432 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8433 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); 8434 8435 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8436 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8437 8438 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8439 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8440 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 8441 } else { 8442 const_or_pre_comp_const_index[0] = 1; 8443 const_or_pre_comp_const_index[1] = 0; 8444 8445 const_or_pre_comp_const_index[2] = 3; 8446 const_or_pre_comp_const_index[3] = 2; 8447 8448 const_or_pre_comp_const_index[4] = 5; 8449 const_or_pre_comp_const_index[5] = 4; 8450 } 8451 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8452 in2, in1, in_out, 8453 tmp1, tmp2, tmp3, 8454 w_xtmp1, w_xtmp2, w_xtmp3, 8455 tmp4, tmp5, 8456 tmp6); 8457 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8458 in2, in1, in_out, 8459 tmp1, tmp2, tmp3, 8460 w_xtmp1, w_xtmp2, w_xtmp3, 8461 tmp4, tmp5, 8462 tmp6); 8463 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8464 in2, in1, in_out, 8465 tmp1, tmp2, tmp3, 8466 w_xtmp1, w_xtmp2, w_xtmp3, 8467 tmp4, tmp5, 8468 tmp6); 8469 movl(tmp1, in2); 8470 andl(tmp1, 0x00000007); 8471 negl(tmp1); 8472 addl(tmp1, in2); 8473 addq(tmp1, in1); 8474 8475 cmpq(in1, tmp1); 8476 jccb(Assembler::greaterEqual, L_byteByByteProlog); 8477 align(16); 8478 BIND(L_wordByWord); 8479 crc32(in_out, Address(in1, 0), 8); 8480 addq(in1, 8); 8481 cmpq(in1, tmp1); 8482 jcc(Assembler::less, L_wordByWord); 8483 8484 BIND(L_byteByByteProlog); 8485 andl(in2, 0x00000007); 8486 movl(tmp2, 1); 8487 8488 cmpl(tmp2, in2); 8489 jccb(Assembler::greater, L_exit); 8490 BIND(L_byteByByte); 8491 crc32(in_out, Address(in1, 0), 1); 8492 incq(in1); 8493 incl(tmp2); 8494 cmpl(tmp2, in2); 8495 jcc(Assembler::lessEqual, L_byteByByte); 8496 8497 BIND(L_exit); 8498 } 8499 #else 8500 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8501 Register tmp1, Register tmp2, Register tmp3, 8502 Register tmp4, Register tmp5, Register tmp6, 8503 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8504 bool is_pclmulqdq_supported) { 8505 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8506 Label L_wordByWord; 8507 Label L_byteByByteProlog; 8508 Label L_byteByByte; 8509 Label L_exit; 8510 8511 if (is_pclmulqdq_supported) { 8512 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8513 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); 8514 8515 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8516 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8517 8518 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8519 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8520 } else { 8521 const_or_pre_comp_const_index[0] = 1; 8522 const_or_pre_comp_const_index[1] = 0; 8523 8524 const_or_pre_comp_const_index[2] = 3; 8525 const_or_pre_comp_const_index[3] = 2; 8526 8527 const_or_pre_comp_const_index[4] = 5; 8528 const_or_pre_comp_const_index[5] = 4; 8529 } 8530 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8531 in2, in1, in_out, 8532 tmp1, tmp2, tmp3, 8533 w_xtmp1, w_xtmp2, w_xtmp3, 8534 tmp4, tmp5, 8535 tmp6); 8536 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8537 in2, in1, in_out, 8538 tmp1, tmp2, tmp3, 8539 w_xtmp1, w_xtmp2, w_xtmp3, 8540 tmp4, tmp5, 8541 tmp6); 8542 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8543 in2, in1, in_out, 8544 tmp1, tmp2, tmp3, 8545 w_xtmp1, w_xtmp2, w_xtmp3, 8546 tmp4, tmp5, 8547 tmp6); 8548 movl(tmp1, in2); 8549 andl(tmp1, 0x00000007); 8550 negl(tmp1); 8551 addl(tmp1, in2); 8552 addl(tmp1, in1); 8553 8554 BIND(L_wordByWord); 8555 cmpl(in1, tmp1); 8556 jcc(Assembler::greaterEqual, L_byteByByteProlog); 8557 crc32(in_out, Address(in1,0), 4); 8558 addl(in1, 4); 8559 jmp(L_wordByWord); 8560 8561 BIND(L_byteByByteProlog); 8562 andl(in2, 0x00000007); 8563 movl(tmp2, 1); 8564 8565 BIND(L_byteByByte); 8566 cmpl(tmp2, in2); 8567 jccb(Assembler::greater, L_exit); 8568 movb(tmp1, Address(in1, 0)); 8569 crc32(in_out, tmp1, 1); 8570 incl(in1); 8571 incl(tmp2); 8572 jmp(L_byteByByte); 8573 8574 BIND(L_exit); 8575 } 8576 #endif // LP64 8577 #undef BIND 8578 #undef BLOCK_COMMENT 8579 8580 // Compress char[] array to byte[]. 8581 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 8582 // Return the array length if every element in array can be encoded, 8583 // otherwise, the index of first non-latin1 (> 0xff) character. 8584 // @IntrinsicCandidate 8585 // public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 8586 // for (int i = 0; i < len; i++) { 8587 // char c = src[srcOff]; 8588 // if (c > 0xff) { 8589 // return i; // return index of non-latin1 char 8590 // } 8591 // dst[dstOff] = (byte)c; 8592 // srcOff++; 8593 // dstOff++; 8594 // } 8595 // return len; 8596 // } 8597 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 8598 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 8599 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 8600 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 8601 Label copy_chars_loop, done, reset_sp, copy_tail; 8602 8603 // rsi: src 8604 // rdi: dst 8605 // rdx: len 8606 // rcx: tmp5 8607 // rax: result 8608 8609 // rsi holds start addr of source char[] to be compressed 8610 // rdi holds start addr of destination byte[] 8611 // rdx holds length 8612 8613 assert(len != result, ""); 8614 8615 // save length for return 8616 movl(result, len); 8617 8618 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 8619 VM_Version::supports_avx512vlbw() && 8620 VM_Version::supports_bmi2()) { 8621 8622 Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail; 8623 8624 // alignment 8625 Label post_alignment; 8626 8627 // if length of the string is less than 32, handle it the old fashioned way 8628 testl(len, -32); 8629 jcc(Assembler::zero, below_threshold); 8630 8631 // First check whether a character is compressible ( <= 0xFF). 8632 // Create mask to test for Unicode chars inside zmm vector 8633 movl(tmp5, 0x00FF); 8634 evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit); 8635 8636 testl(len, -64); 8637 jccb(Assembler::zero, post_alignment); 8638 8639 movl(tmp5, dst); 8640 andl(tmp5, (32 - 1)); 8641 negl(tmp5); 8642 andl(tmp5, (32 - 1)); 8643 8644 // bail out when there is nothing to be done 8645 testl(tmp5, 0xFFFFFFFF); 8646 jccb(Assembler::zero, post_alignment); 8647 8648 // ~(~0 << len), where len is the # of remaining elements to process 8649 movl(len, 0xFFFFFFFF); 8650 shlxl(len, len, tmp5); 8651 notl(len); 8652 kmovdl(mask2, len); 8653 movl(len, result); 8654 8655 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8656 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8657 ktestd(mask1, mask2); 8658 jcc(Assembler::carryClear, copy_tail); 8659 8660 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8661 8662 addptr(src, tmp5); 8663 addptr(src, tmp5); 8664 addptr(dst, tmp5); 8665 subl(len, tmp5); 8666 8667 bind(post_alignment); 8668 // end of alignment 8669 8670 movl(tmp5, len); 8671 andl(tmp5, (32 - 1)); // tail count (in chars) 8672 andl(len, ~(32 - 1)); // vector count (in chars) 8673 jccb(Assembler::zero, copy_loop_tail); 8674 8675 lea(src, Address(src, len, Address::times_2)); 8676 lea(dst, Address(dst, len, Address::times_1)); 8677 negptr(len); 8678 8679 bind(copy_32_loop); 8680 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 8681 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 8682 kortestdl(mask1, mask1); 8683 jccb(Assembler::carryClear, reset_for_copy_tail); 8684 8685 // All elements in current processed chunk are valid candidates for 8686 // compression. Write a truncated byte elements to the memory. 8687 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 8688 addptr(len, 32); 8689 jccb(Assembler::notZero, copy_32_loop); 8690 8691 bind(copy_loop_tail); 8692 // bail out when there is nothing to be done 8693 testl(tmp5, 0xFFFFFFFF); 8694 jcc(Assembler::zero, done); 8695 8696 movl(len, tmp5); 8697 8698 // ~(~0 << len), where len is the # of remaining elements to process 8699 movl(tmp5, 0xFFFFFFFF); 8700 shlxl(tmp5, tmp5, len); 8701 notl(tmp5); 8702 8703 kmovdl(mask2, tmp5); 8704 8705 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8706 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8707 ktestd(mask1, mask2); 8708 jcc(Assembler::carryClear, copy_tail); 8709 8710 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8711 jmp(done); 8712 8713 bind(reset_for_copy_tail); 8714 lea(src, Address(src, tmp5, Address::times_2)); 8715 lea(dst, Address(dst, tmp5, Address::times_1)); 8716 subptr(len, tmp5); 8717 jmp(copy_chars_loop); 8718 8719 bind(below_threshold); 8720 } 8721 8722 if (UseSSE42Intrinsics) { 8723 Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail; 8724 8725 // vectored compression 8726 testl(len, 0xfffffff8); 8727 jcc(Assembler::zero, copy_tail); 8728 8729 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 8730 movdl(tmp1Reg, tmp5); 8731 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8732 8733 andl(len, 0xfffffff0); 8734 jccb(Assembler::zero, copy_16); 8735 8736 // compress 16 chars per iter 8737 pxor(tmp4Reg, tmp4Reg); 8738 8739 lea(src, Address(src, len, Address::times_2)); 8740 lea(dst, Address(dst, len, Address::times_1)); 8741 negptr(len); 8742 8743 bind(copy_32_loop); 8744 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 8745 por(tmp4Reg, tmp2Reg); 8746 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 8747 por(tmp4Reg, tmp3Reg); 8748 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 8749 jccb(Assembler::notZero, reset_for_copy_tail); 8750 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 8751 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 8752 addptr(len, 16); 8753 jccb(Assembler::notZero, copy_32_loop); 8754 8755 // compress next vector of 8 chars (if any) 8756 bind(copy_16); 8757 // len = 0 8758 testl(result, 0x00000008); // check if there's a block of 8 chars to compress 8759 jccb(Assembler::zero, copy_tail_sse); 8760 8761 pxor(tmp3Reg, tmp3Reg); 8762 8763 movdqu(tmp2Reg, Address(src, 0)); 8764 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 8765 jccb(Assembler::notZero, reset_for_copy_tail); 8766 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 8767 movq(Address(dst, 0), tmp2Reg); 8768 addptr(src, 16); 8769 addptr(dst, 8); 8770 jmpb(copy_tail_sse); 8771 8772 bind(reset_for_copy_tail); 8773 movl(tmp5, result); 8774 andl(tmp5, 0x0000000f); 8775 lea(src, Address(src, tmp5, Address::times_2)); 8776 lea(dst, Address(dst, tmp5, Address::times_1)); 8777 subptr(len, tmp5); 8778 jmpb(copy_chars_loop); 8779 8780 bind(copy_tail_sse); 8781 movl(len, result); 8782 andl(len, 0x00000007); // tail count (in chars) 8783 } 8784 // compress 1 char per iter 8785 bind(copy_tail); 8786 testl(len, len); 8787 jccb(Assembler::zero, done); 8788 lea(src, Address(src, len, Address::times_2)); 8789 lea(dst, Address(dst, len, Address::times_1)); 8790 negptr(len); 8791 8792 bind(copy_chars_loop); 8793 load_unsigned_short(tmp5, Address(src, len, Address::times_2)); 8794 testl(tmp5, 0xff00); // check if Unicode char 8795 jccb(Assembler::notZero, reset_sp); 8796 movb(Address(dst, len, Address::times_1), tmp5); // ASCII char; compress to 1 byte 8797 increment(len); 8798 jccb(Assembler::notZero, copy_chars_loop); 8799 8800 // add len then return (len will be zero if compress succeeded, otherwise negative) 8801 bind(reset_sp); 8802 addl(result, len); 8803 8804 bind(done); 8805 } 8806 8807 // Inflate byte[] array to char[]. 8808 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 8809 // @IntrinsicCandidate 8810 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 8811 // for (int i = 0; i < len; i++) { 8812 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 8813 // } 8814 // } 8815 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 8816 XMMRegister tmp1, Register tmp2, KRegister mask) { 8817 Label copy_chars_loop, done, below_threshold, avx3_threshold; 8818 // rsi: src 8819 // rdi: dst 8820 // rdx: len 8821 // rcx: tmp2 8822 8823 // rsi holds start addr of source byte[] to be inflated 8824 // rdi holds start addr of destination char[] 8825 // rdx holds length 8826 assert_different_registers(src, dst, len, tmp2); 8827 movl(tmp2, len); 8828 if ((UseAVX > 2) && // AVX512 8829 VM_Version::supports_avx512vlbw() && 8830 VM_Version::supports_bmi2()) { 8831 8832 Label copy_32_loop, copy_tail; 8833 Register tmp3_aliased = len; 8834 8835 // if length of the string is less than 16, handle it in an old fashioned way 8836 testl(len, -16); 8837 jcc(Assembler::zero, below_threshold); 8838 8839 testl(len, -1 * AVX3Threshold); 8840 jcc(Assembler::zero, avx3_threshold); 8841 8842 // In order to use only one arithmetic operation for the main loop we use 8843 // this pre-calculation 8844 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 8845 andl(len, -32); // vector count 8846 jccb(Assembler::zero, copy_tail); 8847 8848 lea(src, Address(src, len, Address::times_1)); 8849 lea(dst, Address(dst, len, Address::times_2)); 8850 negptr(len); 8851 8852 8853 // inflate 32 chars per iter 8854 bind(copy_32_loop); 8855 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 8856 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 8857 addptr(len, 32); 8858 jcc(Assembler::notZero, copy_32_loop); 8859 8860 bind(copy_tail); 8861 // bail out when there is nothing to be done 8862 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 8863 jcc(Assembler::zero, done); 8864 8865 // ~(~0 << length), where length is the # of remaining elements to process 8866 movl(tmp3_aliased, -1); 8867 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 8868 notl(tmp3_aliased); 8869 kmovdl(mask, tmp3_aliased); 8870 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 8871 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 8872 8873 jmp(done); 8874 bind(avx3_threshold); 8875 } 8876 if (UseSSE42Intrinsics) { 8877 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 8878 8879 if (UseAVX > 1) { 8880 andl(tmp2, (16 - 1)); 8881 andl(len, -16); 8882 jccb(Assembler::zero, copy_new_tail); 8883 } else { 8884 andl(tmp2, 0x00000007); // tail count (in chars) 8885 andl(len, 0xfffffff8); // vector count (in chars) 8886 jccb(Assembler::zero, copy_tail); 8887 } 8888 8889 // vectored inflation 8890 lea(src, Address(src, len, Address::times_1)); 8891 lea(dst, Address(dst, len, Address::times_2)); 8892 negptr(len); 8893 8894 if (UseAVX > 1) { 8895 bind(copy_16_loop); 8896 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 8897 vmovdqu(Address(dst, len, Address::times_2), tmp1); 8898 addptr(len, 16); 8899 jcc(Assembler::notZero, copy_16_loop); 8900 8901 bind(below_threshold); 8902 bind(copy_new_tail); 8903 movl(len, tmp2); 8904 andl(tmp2, 0x00000007); 8905 andl(len, 0xFFFFFFF8); 8906 jccb(Assembler::zero, copy_tail); 8907 8908 pmovzxbw(tmp1, Address(src, 0)); 8909 movdqu(Address(dst, 0), tmp1); 8910 addptr(src, 8); 8911 addptr(dst, 2 * 8); 8912 8913 jmp(copy_tail, true); 8914 } 8915 8916 // inflate 8 chars per iter 8917 bind(copy_8_loop); 8918 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 8919 movdqu(Address(dst, len, Address::times_2), tmp1); 8920 addptr(len, 8); 8921 jcc(Assembler::notZero, copy_8_loop); 8922 8923 bind(copy_tail); 8924 movl(len, tmp2); 8925 8926 cmpl(len, 4); 8927 jccb(Assembler::less, copy_bytes); 8928 8929 movdl(tmp1, Address(src, 0)); // load 4 byte chars 8930 pmovzxbw(tmp1, tmp1); 8931 movq(Address(dst, 0), tmp1); 8932 subptr(len, 4); 8933 addptr(src, 4); 8934 addptr(dst, 8); 8935 8936 bind(copy_bytes); 8937 } else { 8938 bind(below_threshold); 8939 } 8940 8941 testl(len, len); 8942 jccb(Assembler::zero, done); 8943 lea(src, Address(src, len, Address::times_1)); 8944 lea(dst, Address(dst, len, Address::times_2)); 8945 negptr(len); 8946 8947 // inflate 1 char per iter 8948 bind(copy_chars_loop); 8949 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 8950 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 8951 increment(len); 8952 jcc(Assembler::notZero, copy_chars_loop); 8953 8954 bind(done); 8955 } 8956 8957 8958 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 8959 switch(type) { 8960 case T_BYTE: 8961 case T_BOOLEAN: 8962 evmovdqub(dst, kmask, src, merge, vector_len); 8963 break; 8964 case T_CHAR: 8965 case T_SHORT: 8966 evmovdquw(dst, kmask, src, merge, vector_len); 8967 break; 8968 case T_INT: 8969 case T_FLOAT: 8970 evmovdqul(dst, kmask, src, merge, vector_len); 8971 break; 8972 case T_LONG: 8973 case T_DOUBLE: 8974 evmovdquq(dst, kmask, src, merge, vector_len); 8975 break; 8976 default: 8977 fatal("Unexpected type argument %s", type2name(type)); 8978 break; 8979 } 8980 } 8981 8982 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 8983 switch(type) { 8984 case T_BYTE: 8985 case T_BOOLEAN: 8986 evmovdqub(dst, kmask, src, merge, vector_len); 8987 break; 8988 case T_CHAR: 8989 case T_SHORT: 8990 evmovdquw(dst, kmask, src, merge, vector_len); 8991 break; 8992 case T_INT: 8993 case T_FLOAT: 8994 evmovdqul(dst, kmask, src, merge, vector_len); 8995 break; 8996 case T_LONG: 8997 case T_DOUBLE: 8998 evmovdquq(dst, kmask, src, merge, vector_len); 8999 break; 9000 default: 9001 fatal("Unexpected type argument %s", type2name(type)); 9002 break; 9003 } 9004 } 9005 9006 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 9007 switch(masklen) { 9008 case 2: 9009 knotbl(dst, src); 9010 movl(rtmp, 3); 9011 kmovbl(ktmp, rtmp); 9012 kandbl(dst, ktmp, dst); 9013 break; 9014 case 4: 9015 knotbl(dst, src); 9016 movl(rtmp, 15); 9017 kmovbl(ktmp, rtmp); 9018 kandbl(dst, ktmp, dst); 9019 break; 9020 case 8: 9021 knotbl(dst, src); 9022 break; 9023 case 16: 9024 knotwl(dst, src); 9025 break; 9026 case 32: 9027 knotdl(dst, src); 9028 break; 9029 case 64: 9030 knotql(dst, src); 9031 break; 9032 default: 9033 fatal("Unexpected vector length %d", masklen); 9034 break; 9035 } 9036 } 9037 9038 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9039 switch(type) { 9040 case T_BOOLEAN: 9041 case T_BYTE: 9042 kandbl(dst, src1, src2); 9043 break; 9044 case T_CHAR: 9045 case T_SHORT: 9046 kandwl(dst, src1, src2); 9047 break; 9048 case T_INT: 9049 case T_FLOAT: 9050 kanddl(dst, src1, src2); 9051 break; 9052 case T_LONG: 9053 case T_DOUBLE: 9054 kandql(dst, src1, src2); 9055 break; 9056 default: 9057 fatal("Unexpected type argument %s", type2name(type)); 9058 break; 9059 } 9060 } 9061 9062 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9063 switch(type) { 9064 case T_BOOLEAN: 9065 case T_BYTE: 9066 korbl(dst, src1, src2); 9067 break; 9068 case T_CHAR: 9069 case T_SHORT: 9070 korwl(dst, src1, src2); 9071 break; 9072 case T_INT: 9073 case T_FLOAT: 9074 kordl(dst, src1, src2); 9075 break; 9076 case T_LONG: 9077 case T_DOUBLE: 9078 korql(dst, src1, src2); 9079 break; 9080 default: 9081 fatal("Unexpected type argument %s", type2name(type)); 9082 break; 9083 } 9084 } 9085 9086 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9087 switch(type) { 9088 case T_BOOLEAN: 9089 case T_BYTE: 9090 kxorbl(dst, src1, src2); 9091 break; 9092 case T_CHAR: 9093 case T_SHORT: 9094 kxorwl(dst, src1, src2); 9095 break; 9096 case T_INT: 9097 case T_FLOAT: 9098 kxordl(dst, src1, src2); 9099 break; 9100 case T_LONG: 9101 case T_DOUBLE: 9102 kxorql(dst, src1, src2); 9103 break; 9104 default: 9105 fatal("Unexpected type argument %s", type2name(type)); 9106 break; 9107 } 9108 } 9109 9110 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9111 switch(type) { 9112 case T_BOOLEAN: 9113 case T_BYTE: 9114 evpermb(dst, mask, nds, src, merge, vector_len); break; 9115 case T_CHAR: 9116 case T_SHORT: 9117 evpermw(dst, mask, nds, src, merge, vector_len); break; 9118 case T_INT: 9119 case T_FLOAT: 9120 evpermd(dst, mask, nds, src, merge, vector_len); break; 9121 case T_LONG: 9122 case T_DOUBLE: 9123 evpermq(dst, mask, nds, src, merge, vector_len); break; 9124 default: 9125 fatal("Unexpected type argument %s", type2name(type)); break; 9126 } 9127 } 9128 9129 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9130 switch(type) { 9131 case T_BOOLEAN: 9132 case T_BYTE: 9133 evpermb(dst, mask, nds, src, merge, vector_len); break; 9134 case T_CHAR: 9135 case T_SHORT: 9136 evpermw(dst, mask, nds, src, merge, vector_len); break; 9137 case T_INT: 9138 case T_FLOAT: 9139 evpermd(dst, mask, nds, src, merge, vector_len); break; 9140 case T_LONG: 9141 case T_DOUBLE: 9142 evpermq(dst, mask, nds, src, merge, vector_len); break; 9143 default: 9144 fatal("Unexpected type argument %s", type2name(type)); break; 9145 } 9146 } 9147 9148 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9149 switch(type) { 9150 case T_BYTE: 9151 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9152 case T_SHORT: 9153 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9154 case T_INT: 9155 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9156 case T_LONG: 9157 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9158 default: 9159 fatal("Unexpected type argument %s", type2name(type)); break; 9160 } 9161 } 9162 9163 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9164 switch(type) { 9165 case T_BYTE: 9166 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9167 case T_SHORT: 9168 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9169 case T_INT: 9170 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9171 case T_LONG: 9172 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9173 default: 9174 fatal("Unexpected type argument %s", type2name(type)); break; 9175 } 9176 } 9177 9178 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9179 switch(type) { 9180 case T_BYTE: 9181 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9182 case T_SHORT: 9183 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9184 case T_INT: 9185 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9186 case T_LONG: 9187 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9188 default: 9189 fatal("Unexpected type argument %s", type2name(type)); break; 9190 } 9191 } 9192 9193 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9194 switch(type) { 9195 case T_BYTE: 9196 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9197 case T_SHORT: 9198 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9199 case T_INT: 9200 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9201 case T_LONG: 9202 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9203 default: 9204 fatal("Unexpected type argument %s", type2name(type)); break; 9205 } 9206 } 9207 9208 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9209 switch(type) { 9210 case T_INT: 9211 evpxord(dst, mask, nds, src, merge, vector_len); break; 9212 case T_LONG: 9213 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9214 default: 9215 fatal("Unexpected type argument %s", type2name(type)); break; 9216 } 9217 } 9218 9219 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9220 switch(type) { 9221 case T_INT: 9222 evpxord(dst, mask, nds, src, merge, vector_len); break; 9223 case T_LONG: 9224 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9225 default: 9226 fatal("Unexpected type argument %s", type2name(type)); break; 9227 } 9228 } 9229 9230 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9231 switch(type) { 9232 case T_INT: 9233 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9234 case T_LONG: 9235 evporq(dst, mask, nds, src, merge, vector_len); break; 9236 default: 9237 fatal("Unexpected type argument %s", type2name(type)); break; 9238 } 9239 } 9240 9241 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9242 switch(type) { 9243 case T_INT: 9244 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9245 case T_LONG: 9246 evporq(dst, mask, nds, src, merge, vector_len); break; 9247 default: 9248 fatal("Unexpected type argument %s", type2name(type)); break; 9249 } 9250 } 9251 9252 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9253 switch(type) { 9254 case T_INT: 9255 evpandd(dst, mask, nds, src, merge, vector_len); break; 9256 case T_LONG: 9257 evpandq(dst, mask, nds, src, merge, vector_len); break; 9258 default: 9259 fatal("Unexpected type argument %s", type2name(type)); break; 9260 } 9261 } 9262 9263 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9264 switch(type) { 9265 case T_INT: 9266 evpandd(dst, mask, nds, src, merge, vector_len); break; 9267 case T_LONG: 9268 evpandq(dst, mask, nds, src, merge, vector_len); break; 9269 default: 9270 fatal("Unexpected type argument %s", type2name(type)); break; 9271 } 9272 } 9273 9274 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 9275 switch(masklen) { 9276 case 8: 9277 kortestbl(src1, src2); 9278 break; 9279 case 16: 9280 kortestwl(src1, src2); 9281 break; 9282 case 32: 9283 kortestdl(src1, src2); 9284 break; 9285 case 64: 9286 kortestql(src1, src2); 9287 break; 9288 default: 9289 fatal("Unexpected mask length %d", masklen); 9290 break; 9291 } 9292 } 9293 9294 9295 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 9296 switch(masklen) { 9297 case 8: 9298 ktestbl(src1, src2); 9299 break; 9300 case 16: 9301 ktestwl(src1, src2); 9302 break; 9303 case 32: 9304 ktestdl(src1, src2); 9305 break; 9306 case 64: 9307 ktestql(src1, src2); 9308 break; 9309 default: 9310 fatal("Unexpected mask length %d", masklen); 9311 break; 9312 } 9313 } 9314 9315 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9316 switch(type) { 9317 case T_INT: 9318 evprold(dst, mask, src, shift, merge, vlen_enc); break; 9319 case T_LONG: 9320 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 9321 default: 9322 fatal("Unexpected type argument %s", type2name(type)); break; 9323 break; 9324 } 9325 } 9326 9327 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9328 switch(type) { 9329 case T_INT: 9330 evprord(dst, mask, src, shift, merge, vlen_enc); break; 9331 case T_LONG: 9332 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 9333 default: 9334 fatal("Unexpected type argument %s", type2name(type)); break; 9335 } 9336 } 9337 9338 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9339 switch(type) { 9340 case T_INT: 9341 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 9342 case T_LONG: 9343 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 9344 default: 9345 fatal("Unexpected type argument %s", type2name(type)); break; 9346 } 9347 } 9348 9349 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9350 switch(type) { 9351 case T_INT: 9352 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 9353 case T_LONG: 9354 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 9355 default: 9356 fatal("Unexpected type argument %s", type2name(type)); break; 9357 } 9358 } 9359 9360 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9361 assert(rscratch != noreg || always_reachable(src), "missing"); 9362 9363 if (reachable(src)) { 9364 evpandq(dst, nds, as_Address(src), vector_len); 9365 } else { 9366 lea(rscratch, src); 9367 evpandq(dst, nds, Address(rscratch, 0), vector_len); 9368 } 9369 } 9370 9371 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 9372 assert(rscratch != noreg || always_reachable(src), "missing"); 9373 9374 if (reachable(src)) { 9375 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 9376 } else { 9377 lea(rscratch, src); 9378 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 9379 } 9380 } 9381 9382 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9383 assert(rscratch != noreg || always_reachable(src), "missing"); 9384 9385 if (reachable(src)) { 9386 evporq(dst, nds, as_Address(src), vector_len); 9387 } else { 9388 lea(rscratch, src); 9389 evporq(dst, nds, Address(rscratch, 0), vector_len); 9390 } 9391 } 9392 9393 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9394 assert(rscratch != noreg || always_reachable(src), "missing"); 9395 9396 if (reachable(src)) { 9397 vpshufb(dst, nds, as_Address(src), vector_len); 9398 } else { 9399 lea(rscratch, src); 9400 vpshufb(dst, nds, Address(rscratch, 0), vector_len); 9401 } 9402 } 9403 9404 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 9405 assert(rscratch != noreg || always_reachable(src3), "missing"); 9406 9407 if (reachable(src3)) { 9408 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 9409 } else { 9410 lea(rscratch, src3); 9411 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 9412 } 9413 } 9414 9415 #if COMPILER2_OR_JVMCI 9416 9417 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 9418 Register length, Register temp, int vec_enc) { 9419 // Computing mask for predicated vector store. 9420 movptr(temp, -1); 9421 bzhiq(temp, temp, length); 9422 kmov(mask, temp); 9423 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 9424 } 9425 9426 // Set memory operation for length "less than" 64 bytes. 9427 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 9428 XMMRegister xmm, KRegister mask, Register length, 9429 Register temp, bool use64byteVector) { 9430 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9431 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9432 if (!use64byteVector) { 9433 fill32(dst, disp, xmm); 9434 subptr(length, 32 >> shift); 9435 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 9436 } else { 9437 assert(MaxVectorSize == 64, "vector length != 64"); 9438 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 9439 } 9440 } 9441 9442 9443 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 9444 XMMRegister xmm, KRegister mask, Register length, 9445 Register temp) { 9446 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9447 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9448 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 9449 } 9450 9451 9452 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 9453 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9454 vmovdqu(dst, xmm); 9455 } 9456 9457 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 9458 fill32(Address(dst, disp), xmm); 9459 } 9460 9461 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 9462 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9463 if (!use64byteVector) { 9464 fill32(dst, xmm); 9465 fill32(dst.plus_disp(32), xmm); 9466 } else { 9467 evmovdquq(dst, xmm, Assembler::AVX_512bit); 9468 } 9469 } 9470 9471 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 9472 fill64(Address(dst, disp), xmm, use64byteVector); 9473 } 9474 9475 #ifdef _LP64 9476 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 9477 Register count, Register rtmp, XMMRegister xtmp) { 9478 Label L_exit; 9479 Label L_fill_start; 9480 Label L_fill_64_bytes; 9481 Label L_fill_96_bytes; 9482 Label L_fill_128_bytes; 9483 Label L_fill_128_bytes_loop; 9484 Label L_fill_128_loop_header; 9485 Label L_fill_128_bytes_loop_header; 9486 Label L_fill_128_bytes_loop_pre_header; 9487 Label L_fill_zmm_sequence; 9488 9489 int shift = -1; 9490 int avx3threshold = VM_Version::avx3_threshold(); 9491 switch(type) { 9492 case T_BYTE: shift = 0; 9493 break; 9494 case T_SHORT: shift = 1; 9495 break; 9496 case T_INT: shift = 2; 9497 break; 9498 /* Uncomment when LONG fill stubs are supported. 9499 case T_LONG: shift = 3; 9500 break; 9501 */ 9502 default: 9503 fatal("Unhandled type: %s\n", type2name(type)); 9504 } 9505 9506 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 9507 9508 if (MaxVectorSize == 64) { 9509 cmpq(count, avx3threshold >> shift); 9510 jcc(Assembler::greater, L_fill_zmm_sequence); 9511 } 9512 9513 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 9514 9515 bind(L_fill_start); 9516 9517 cmpq(count, 32 >> shift); 9518 jccb(Assembler::greater, L_fill_64_bytes); 9519 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 9520 jmp(L_exit); 9521 9522 bind(L_fill_64_bytes); 9523 cmpq(count, 64 >> shift); 9524 jccb(Assembler::greater, L_fill_96_bytes); 9525 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 9526 jmp(L_exit); 9527 9528 bind(L_fill_96_bytes); 9529 cmpq(count, 96 >> shift); 9530 jccb(Assembler::greater, L_fill_128_bytes); 9531 fill64(to, 0, xtmp); 9532 subq(count, 64 >> shift); 9533 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 9534 jmp(L_exit); 9535 9536 bind(L_fill_128_bytes); 9537 cmpq(count, 128 >> shift); 9538 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 9539 fill64(to, 0, xtmp); 9540 fill32(to, 64, xtmp); 9541 subq(count, 96 >> shift); 9542 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 9543 jmp(L_exit); 9544 9545 bind(L_fill_128_bytes_loop_pre_header); 9546 { 9547 mov(rtmp, to); 9548 andq(rtmp, 31); 9549 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 9550 negq(rtmp); 9551 addq(rtmp, 32); 9552 mov64(r8, -1L); 9553 bzhiq(r8, r8, rtmp); 9554 kmovql(k2, r8); 9555 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 9556 addq(to, rtmp); 9557 shrq(rtmp, shift); 9558 subq(count, rtmp); 9559 } 9560 9561 cmpq(count, 128 >> shift); 9562 jcc(Assembler::less, L_fill_start); 9563 9564 bind(L_fill_128_bytes_loop_header); 9565 subq(count, 128 >> shift); 9566 9567 align32(); 9568 bind(L_fill_128_bytes_loop); 9569 fill64(to, 0, xtmp); 9570 fill64(to, 64, xtmp); 9571 addq(to, 128); 9572 subq(count, 128 >> shift); 9573 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 9574 9575 addq(count, 128 >> shift); 9576 jcc(Assembler::zero, L_exit); 9577 jmp(L_fill_start); 9578 } 9579 9580 if (MaxVectorSize == 64) { 9581 // Sequence using 64 byte ZMM register. 9582 Label L_fill_128_bytes_zmm; 9583 Label L_fill_192_bytes_zmm; 9584 Label L_fill_192_bytes_loop_zmm; 9585 Label L_fill_192_bytes_loop_header_zmm; 9586 Label L_fill_192_bytes_loop_pre_header_zmm; 9587 Label L_fill_start_zmm_sequence; 9588 9589 bind(L_fill_zmm_sequence); 9590 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 9591 9592 bind(L_fill_start_zmm_sequence); 9593 cmpq(count, 64 >> shift); 9594 jccb(Assembler::greater, L_fill_128_bytes_zmm); 9595 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 9596 jmp(L_exit); 9597 9598 bind(L_fill_128_bytes_zmm); 9599 cmpq(count, 128 >> shift); 9600 jccb(Assembler::greater, L_fill_192_bytes_zmm); 9601 fill64(to, 0, xtmp, true); 9602 subq(count, 64 >> shift); 9603 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 9604 jmp(L_exit); 9605 9606 bind(L_fill_192_bytes_zmm); 9607 cmpq(count, 192 >> shift); 9608 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 9609 fill64(to, 0, xtmp, true); 9610 fill64(to, 64, xtmp, true); 9611 subq(count, 128 >> shift); 9612 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 9613 jmp(L_exit); 9614 9615 bind(L_fill_192_bytes_loop_pre_header_zmm); 9616 { 9617 movq(rtmp, to); 9618 andq(rtmp, 63); 9619 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 9620 negq(rtmp); 9621 addq(rtmp, 64); 9622 mov64(r8, -1L); 9623 bzhiq(r8, r8, rtmp); 9624 kmovql(k2, r8); 9625 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 9626 addq(to, rtmp); 9627 shrq(rtmp, shift); 9628 subq(count, rtmp); 9629 } 9630 9631 cmpq(count, 192 >> shift); 9632 jcc(Assembler::less, L_fill_start_zmm_sequence); 9633 9634 bind(L_fill_192_bytes_loop_header_zmm); 9635 subq(count, 192 >> shift); 9636 9637 align32(); 9638 bind(L_fill_192_bytes_loop_zmm); 9639 fill64(to, 0, xtmp, true); 9640 fill64(to, 64, xtmp, true); 9641 fill64(to, 128, xtmp, true); 9642 addq(to, 192); 9643 subq(count, 192 >> shift); 9644 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 9645 9646 addq(count, 192 >> shift); 9647 jcc(Assembler::zero, L_exit); 9648 jmp(L_fill_start_zmm_sequence); 9649 } 9650 bind(L_exit); 9651 } 9652 #endif 9653 #endif //COMPILER2_OR_JVMCI 9654 9655 9656 #ifdef _LP64 9657 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 9658 Label done; 9659 cvttss2sil(dst, src); 9660 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9661 cmpl(dst, 0x80000000); // float_sign_flip 9662 jccb(Assembler::notEqual, done); 9663 subptr(rsp, 8); 9664 movflt(Address(rsp, 0), src); 9665 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 9666 pop(dst); 9667 bind(done); 9668 } 9669 9670 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 9671 Label done; 9672 cvttsd2sil(dst, src); 9673 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9674 cmpl(dst, 0x80000000); // float_sign_flip 9675 jccb(Assembler::notEqual, done); 9676 subptr(rsp, 8); 9677 movdbl(Address(rsp, 0), src); 9678 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 9679 pop(dst); 9680 bind(done); 9681 } 9682 9683 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 9684 Label done; 9685 cvttss2siq(dst, src); 9686 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9687 jccb(Assembler::notEqual, done); 9688 subptr(rsp, 8); 9689 movflt(Address(rsp, 0), src); 9690 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 9691 pop(dst); 9692 bind(done); 9693 } 9694 9695 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9696 // Following code is line by line assembly translation rounding algorithm. 9697 // Please refer to java.lang.Math.round(float) algorithm for details. 9698 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 9699 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 9700 const int32_t FloatConsts_EXP_BIAS = 127; 9701 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 9702 const int32_t MINUS_32 = 0xFFFFFFE0; 9703 Label L_special_case, L_block1, L_exit; 9704 movl(rtmp, FloatConsts_EXP_BIT_MASK); 9705 movdl(dst, src); 9706 andl(dst, rtmp); 9707 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 9708 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 9709 subl(rtmp, dst); 9710 movl(rcx, rtmp); 9711 movl(dst, MINUS_32); 9712 testl(rtmp, dst); 9713 jccb(Assembler::notEqual, L_special_case); 9714 movdl(dst, src); 9715 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 9716 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 9717 movdl(rtmp, src); 9718 testl(rtmp, rtmp); 9719 jccb(Assembler::greaterEqual, L_block1); 9720 negl(dst); 9721 bind(L_block1); 9722 sarl(dst); 9723 addl(dst, 0x1); 9724 sarl(dst, 0x1); 9725 jmp(L_exit); 9726 bind(L_special_case); 9727 convert_f2i(dst, src); 9728 bind(L_exit); 9729 } 9730 9731 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9732 // Following code is line by line assembly translation rounding algorithm. 9733 // Please refer to java.lang.Math.round(double) algorithm for details. 9734 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 9735 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 9736 const int64_t DoubleConsts_EXP_BIAS = 1023; 9737 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 9738 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 9739 Label L_special_case, L_block1, L_exit; 9740 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 9741 movq(dst, src); 9742 andq(dst, rtmp); 9743 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 9744 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 9745 subq(rtmp, dst); 9746 movq(rcx, rtmp); 9747 mov64(dst, MINUS_64); 9748 testq(rtmp, dst); 9749 jccb(Assembler::notEqual, L_special_case); 9750 movq(dst, src); 9751 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 9752 andq(dst, rtmp); 9753 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 9754 orq(dst, rtmp); 9755 movq(rtmp, src); 9756 testq(rtmp, rtmp); 9757 jccb(Assembler::greaterEqual, L_block1); 9758 negq(dst); 9759 bind(L_block1); 9760 sarq(dst); 9761 addq(dst, 0x1); 9762 sarq(dst, 0x1); 9763 jmp(L_exit); 9764 bind(L_special_case); 9765 convert_d2l(dst, src); 9766 bind(L_exit); 9767 } 9768 9769 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 9770 Label done; 9771 cvttsd2siq(dst, src); 9772 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9773 jccb(Assembler::notEqual, done); 9774 subptr(rsp, 8); 9775 movdbl(Address(rsp, 0), src); 9776 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 9777 pop(dst); 9778 bind(done); 9779 } 9780 9781 void MacroAssembler::cache_wb(Address line) 9782 { 9783 // 64 bit cpus always support clflush 9784 assert(VM_Version::supports_clflush(), "clflush should be available"); 9785 bool optimized = VM_Version::supports_clflushopt(); 9786 bool no_evict = VM_Version::supports_clwb(); 9787 9788 // prefer clwb (writeback without evict) otherwise 9789 // prefer clflushopt (potentially parallel writeback with evict) 9790 // otherwise fallback on clflush (serial writeback with evict) 9791 9792 if (optimized) { 9793 if (no_evict) { 9794 clwb(line); 9795 } else { 9796 clflushopt(line); 9797 } 9798 } else { 9799 // no need for fence when using CLFLUSH 9800 clflush(line); 9801 } 9802 } 9803 9804 void MacroAssembler::cache_wbsync(bool is_pre) 9805 { 9806 assert(VM_Version::supports_clflush(), "clflush should be available"); 9807 bool optimized = VM_Version::supports_clflushopt(); 9808 bool no_evict = VM_Version::supports_clwb(); 9809 9810 // pick the correct implementation 9811 9812 if (!is_pre && (optimized || no_evict)) { 9813 // need an sfence for post flush when using clflushopt or clwb 9814 // otherwise no no need for any synchroniaztion 9815 9816 sfence(); 9817 } 9818 } 9819 9820 #endif // _LP64 9821 9822 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 9823 switch (cond) { 9824 // Note some conditions are synonyms for others 9825 case Assembler::zero: return Assembler::notZero; 9826 case Assembler::notZero: return Assembler::zero; 9827 case Assembler::less: return Assembler::greaterEqual; 9828 case Assembler::lessEqual: return Assembler::greater; 9829 case Assembler::greater: return Assembler::lessEqual; 9830 case Assembler::greaterEqual: return Assembler::less; 9831 case Assembler::below: return Assembler::aboveEqual; 9832 case Assembler::belowEqual: return Assembler::above; 9833 case Assembler::above: return Assembler::belowEqual; 9834 case Assembler::aboveEqual: return Assembler::below; 9835 case Assembler::overflow: return Assembler::noOverflow; 9836 case Assembler::noOverflow: return Assembler::overflow; 9837 case Assembler::negative: return Assembler::positive; 9838 case Assembler::positive: return Assembler::negative; 9839 case Assembler::parity: return Assembler::noParity; 9840 case Assembler::noParity: return Assembler::parity; 9841 } 9842 ShouldNotReachHere(); return Assembler::overflow; 9843 } 9844 9845 SkipIfEqual::SkipIfEqual( 9846 MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) { 9847 _masm = masm; 9848 _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch); 9849 _masm->jcc(Assembler::equal, _label); 9850 } 9851 9852 SkipIfEqual::~SkipIfEqual() { 9853 _masm->bind(_label); 9854 } 9855 9856 // 32-bit Windows has its own fast-path implementation 9857 // of get_thread 9858 #if !defined(WIN32) || defined(_LP64) 9859 9860 // This is simply a call to Thread::current() 9861 void MacroAssembler::get_thread(Register thread) { 9862 if (thread != rax) { 9863 push(rax); 9864 } 9865 LP64_ONLY(push(rdi);) 9866 LP64_ONLY(push(rsi);) 9867 push(rdx); 9868 push(rcx); 9869 #ifdef _LP64 9870 push(r8); 9871 push(r9); 9872 push(r10); 9873 push(r11); 9874 #endif 9875 9876 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 9877 9878 #ifdef _LP64 9879 pop(r11); 9880 pop(r10); 9881 pop(r9); 9882 pop(r8); 9883 #endif 9884 pop(rcx); 9885 pop(rdx); 9886 LP64_ONLY(pop(rsi);) 9887 LP64_ONLY(pop(rdi);) 9888 if (thread != rax) { 9889 mov(thread, rax); 9890 pop(rax); 9891 } 9892 } 9893 9894 9895 #endif // !WIN32 || _LP64 9896 9897 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 9898 Label L_stack_ok; 9899 if (bias == 0) { 9900 testptr(sp, 2 * wordSize - 1); 9901 } else { 9902 // lea(tmp, Address(rsp, bias); 9903 mov(tmp, sp); 9904 addptr(tmp, bias); 9905 testptr(tmp, 2 * wordSize - 1); 9906 } 9907 jcc(Assembler::equal, L_stack_ok); 9908 block_comment(msg); 9909 stop(msg); 9910 bind(L_stack_ok); 9911 } 9912 9913 // Implements lightweight-locking. 9914 // 9915 // obj: the object to be locked 9916 // reg_rax: rax 9917 // thread: the thread which attempts to lock obj 9918 // tmp: a temporary register 9919 void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 9920 assert(reg_rax == rax, ""); 9921 assert_different_registers(obj, reg_rax, thread, tmp); 9922 9923 Label push; 9924 const Register top = tmp; 9925 9926 // Preload the markWord. It is important that this is the first 9927 // instruction emitted as it is part of C1's null check semantics. 9928 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 9929 9930 // Load top. 9931 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9932 9933 // Check if the lock-stack is full. 9934 cmpl(top, LockStack::end_offset()); 9935 jcc(Assembler::greaterEqual, slow); 9936 9937 // Check for recursion. 9938 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 9939 jcc(Assembler::equal, push); 9940 9941 // Check header for monitor (0b10). 9942 testptr(reg_rax, markWord::monitor_value); 9943 jcc(Assembler::notZero, slow); 9944 9945 // Try to lock. Transition lock bits 0b01 => 0b00 9946 movptr(tmp, reg_rax); 9947 andptr(tmp, ~(int32_t)markWord::unlocked_value); 9948 orptr(reg_rax, markWord::unlocked_value); 9949 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9950 jcc(Assembler::notEqual, slow); 9951 9952 // Restore top, CAS clobbers register. 9953 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9954 9955 bind(push); 9956 // After successful lock, push object on lock-stack. 9957 movptr(Address(thread, top), obj); 9958 incrementl(top, oopSize); 9959 movl(Address(thread, JavaThread::lock_stack_top_offset()), top); 9960 } 9961 9962 // Implements lightweight-unlocking. 9963 // 9964 // obj: the object to be unlocked 9965 // reg_rax: rax 9966 // thread: the thread 9967 // tmp: a temporary register 9968 // 9969 // x86_32 Note: reg_rax and thread may alias each other due to limited register 9970 // availiability. 9971 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 9972 assert(reg_rax == rax, ""); 9973 assert_different_registers(obj, reg_rax, tmp); 9974 LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);) 9975 9976 Label unlocked, push_and_slow; 9977 const Register top = tmp; 9978 9979 // Check if obj is top of lock-stack. 9980 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9981 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 9982 jcc(Assembler::notEqual, slow); 9983 9984 // Pop lock-stack. 9985 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);) 9986 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 9987 9988 // Check if recursive. 9989 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize)); 9990 jcc(Assembler::equal, unlocked); 9991 9992 // Not recursive. Check header for monitor (0b10). 9993 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 9994 testptr(reg_rax, markWord::monitor_value); 9995 jcc(Assembler::notZero, push_and_slow); 9996 9997 #ifdef ASSERT 9998 // Check header not unlocked (0b01). 9999 Label not_unlocked; 10000 testptr(reg_rax, markWord::unlocked_value); 10001 jcc(Assembler::zero, not_unlocked); 10002 stop("lightweight_unlock already unlocked"); 10003 bind(not_unlocked); 10004 #endif 10005 10006 // Try to unlock. Transition lock bits 0b00 => 0b01 10007 movptr(tmp, reg_rax); 10008 orptr(tmp, markWord::unlocked_value); 10009 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10010 jcc(Assembler::equal, unlocked); 10011 10012 bind(push_and_slow); 10013 // Restore lock-stack and handle the unlock in runtime. 10014 if (thread == reg_rax) { 10015 // On x86_32 we may lose the thread. 10016 get_thread(thread); 10017 } 10018 #ifdef ASSERT 10019 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10020 movptr(Address(thread, top), obj); 10021 #endif 10022 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10023 jmp(slow); 10024 10025 bind(unlocked); 10026 }