1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "compiler/compiler_globals.hpp" 29 #include "compiler/disassembler.hpp" 30 #include "crc32c.h" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/collectedHeap.inline.hpp" 34 #include "gc/shared/tlab_globals.hpp" 35 #include "interpreter/bytecodeHistogram.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "jvm.h" 38 #include "memory/resourceArea.hpp" 39 #include "memory/universe.hpp" 40 #include "oops/accessDecorators.hpp" 41 #include "oops/compressedKlass.inline.hpp" 42 #include "oops/compressedOops.inline.hpp" 43 #include "oops/klass.inline.hpp" 44 #include "prims/methodHandles.hpp" 45 #include "runtime/continuation.hpp" 46 #include "runtime/interfaceSupport.inline.hpp" 47 #include "runtime/javaThread.hpp" 48 #include "runtime/jniHandles.hpp" 49 #include "runtime/objectMonitor.hpp" 50 #include "runtime/os.hpp" 51 #include "runtime/safepoint.hpp" 52 #include "runtime/safepointMechanism.hpp" 53 #include "runtime/sharedRuntime.hpp" 54 #include "runtime/stubRoutines.hpp" 55 #include "utilities/checkedCast.hpp" 56 #include "utilities/macros.hpp" 57 58 #ifdef PRODUCT 59 #define BLOCK_COMMENT(str) /* nothing */ 60 #define STOP(error) stop(error) 61 #else 62 #define BLOCK_COMMENT(str) block_comment(str) 63 #define STOP(error) block_comment(error); stop(error) 64 #endif 65 66 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 67 68 #ifdef ASSERT 69 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 70 #endif 71 72 static const Assembler::Condition reverse[] = { 73 Assembler::noOverflow /* overflow = 0x0 */ , 74 Assembler::overflow /* noOverflow = 0x1 */ , 75 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 76 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 77 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 78 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 79 Assembler::above /* belowEqual = 0x6 */ , 80 Assembler::belowEqual /* above = 0x7 */ , 81 Assembler::positive /* negative = 0x8 */ , 82 Assembler::negative /* positive = 0x9 */ , 83 Assembler::noParity /* parity = 0xa */ , 84 Assembler::parity /* noParity = 0xb */ , 85 Assembler::greaterEqual /* less = 0xc */ , 86 Assembler::less /* greaterEqual = 0xd */ , 87 Assembler::greater /* lessEqual = 0xe */ , 88 Assembler::lessEqual /* greater = 0xf, */ 89 90 }; 91 92 93 // Implementation of MacroAssembler 94 95 // First all the versions that have distinct versions depending on 32/64 bit 96 // Unless the difference is trivial (1 line or so). 97 98 #ifndef _LP64 99 100 // 32bit versions 101 102 Address MacroAssembler::as_Address(AddressLiteral adr) { 103 return Address(adr.target(), adr.rspec()); 104 } 105 106 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 107 assert(rscratch == noreg, ""); 108 return Address::make_array(adr); 109 } 110 111 void MacroAssembler::call_VM_leaf_base(address entry_point, 112 int number_of_arguments) { 113 call(RuntimeAddress(entry_point)); 114 increment(rsp, number_of_arguments * wordSize); 115 } 116 117 void MacroAssembler::cmpklass(Address src1, Metadata* obj) { 118 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 119 } 120 121 122 void MacroAssembler::cmpklass(Register src1, Metadata* obj) { 123 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 124 } 125 126 void MacroAssembler::cmpoop(Address src1, jobject obj) { 127 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 128 } 129 130 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) { 131 assert(rscratch == noreg, "redundant"); 132 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 133 } 134 135 void MacroAssembler::extend_sign(Register hi, Register lo) { 136 // According to Intel Doc. AP-526, "Integer Divide", p.18. 137 if (VM_Version::is_P6() && hi == rdx && lo == rax) { 138 cdql(); 139 } else { 140 movl(hi, lo); 141 sarl(hi, 31); 142 } 143 } 144 145 void MacroAssembler::jC2(Register tmp, Label& L) { 146 // set parity bit if FPU flag C2 is set (via rax) 147 save_rax(tmp); 148 fwait(); fnstsw_ax(); 149 sahf(); 150 restore_rax(tmp); 151 // branch 152 jcc(Assembler::parity, L); 153 } 154 155 void MacroAssembler::jnC2(Register tmp, Label& L) { 156 // set parity bit if FPU flag C2 is set (via rax) 157 save_rax(tmp); 158 fwait(); fnstsw_ax(); 159 sahf(); 160 restore_rax(tmp); 161 // branch 162 jcc(Assembler::noParity, L); 163 } 164 165 // 32bit can do a case table jump in one instruction but we no longer allow the base 166 // to be installed in the Address class 167 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 168 assert(rscratch == noreg, "not needed"); 169 jmp(as_Address(entry, noreg)); 170 } 171 172 // Note: y_lo will be destroyed 173 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 174 // Long compare for Java (semantics as described in JVM spec.) 175 Label high, low, done; 176 177 cmpl(x_hi, y_hi); 178 jcc(Assembler::less, low); 179 jcc(Assembler::greater, high); 180 // x_hi is the return register 181 xorl(x_hi, x_hi); 182 cmpl(x_lo, y_lo); 183 jcc(Assembler::below, low); 184 jcc(Assembler::equal, done); 185 186 bind(high); 187 xorl(x_hi, x_hi); 188 increment(x_hi); 189 jmp(done); 190 191 bind(low); 192 xorl(x_hi, x_hi); 193 decrementl(x_hi); 194 195 bind(done); 196 } 197 198 void MacroAssembler::lea(Register dst, AddressLiteral src) { 199 mov_literal32(dst, (int32_t)src.target(), src.rspec()); 200 } 201 202 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 203 assert(rscratch == noreg, "not needed"); 204 205 // leal(dst, as_Address(adr)); 206 // see note in movl as to why we must use a move 207 mov_literal32(dst, (int32_t)adr.target(), adr.rspec()); 208 } 209 210 void MacroAssembler::leave() { 211 mov(rsp, rbp); 212 pop(rbp); 213 } 214 215 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { 216 // Multiplication of two Java long values stored on the stack 217 // as illustrated below. Result is in rdx:rax. 218 // 219 // rsp ---> [ ?? ] \ \ 220 // .... | y_rsp_offset | 221 // [ y_lo ] / (in bytes) | x_rsp_offset 222 // [ y_hi ] | (in bytes) 223 // .... | 224 // [ x_lo ] / 225 // [ x_hi ] 226 // .... 227 // 228 // Basic idea: lo(result) = lo(x_lo * y_lo) 229 // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) 230 Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); 231 Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); 232 Label quick; 233 // load x_hi, y_hi and check if quick 234 // multiplication is possible 235 movl(rbx, x_hi); 236 movl(rcx, y_hi); 237 movl(rax, rbx); 238 orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 239 jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply 240 // do full multiplication 241 // 1st step 242 mull(y_lo); // x_hi * y_lo 243 movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, 244 // 2nd step 245 movl(rax, x_lo); 246 mull(rcx); // x_lo * y_hi 247 addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, 248 // 3rd step 249 bind(quick); // note: rbx, = 0 if quick multiply! 250 movl(rax, x_lo); 251 mull(y_lo); // x_lo * y_lo 252 addl(rdx, rbx); // correct hi(x_lo * y_lo) 253 } 254 255 void MacroAssembler::lneg(Register hi, Register lo) { 256 negl(lo); 257 adcl(hi, 0); 258 negl(hi); 259 } 260 261 void MacroAssembler::lshl(Register hi, Register lo) { 262 // Java shift left long support (semantics as described in JVM spec., p.305) 263 // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) 264 // shift value is in rcx ! 265 assert(hi != rcx, "must not use rcx"); 266 assert(lo != rcx, "must not use rcx"); 267 const Register s = rcx; // shift count 268 const int n = BitsPerWord; 269 Label L; 270 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 271 cmpl(s, n); // if (s < n) 272 jcc(Assembler::less, L); // else (s >= n) 273 movl(hi, lo); // x := x << n 274 xorl(lo, lo); 275 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 276 bind(L); // s (mod n) < n 277 shldl(hi, lo); // x := x << s 278 shll(lo); 279 } 280 281 282 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { 283 // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) 284 // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) 285 assert(hi != rcx, "must not use rcx"); 286 assert(lo != rcx, "must not use rcx"); 287 const Register s = rcx; // shift count 288 const int n = BitsPerWord; 289 Label L; 290 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 291 cmpl(s, n); // if (s < n) 292 jcc(Assembler::less, L); // else (s >= n) 293 movl(lo, hi); // x := x >> n 294 if (sign_extension) sarl(hi, 31); 295 else xorl(hi, hi); 296 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 297 bind(L); // s (mod n) < n 298 shrdl(lo, hi); // x := x >> s 299 if (sign_extension) sarl(hi); 300 else shrl(hi); 301 } 302 303 void MacroAssembler::movoop(Register dst, jobject obj) { 304 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 305 } 306 307 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 308 assert(rscratch == noreg, "redundant"); 309 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 310 } 311 312 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 313 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 314 } 315 316 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 317 assert(rscratch == noreg, "redundant"); 318 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 319 } 320 321 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 322 if (src.is_lval()) { 323 mov_literal32(dst, (intptr_t)src.target(), src.rspec()); 324 } else { 325 movl(dst, as_Address(src)); 326 } 327 } 328 329 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 330 assert(rscratch == noreg, "redundant"); 331 movl(as_Address(dst, noreg), src); 332 } 333 334 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 335 movl(dst, as_Address(src, noreg)); 336 } 337 338 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 339 assert(rscratch == noreg, "redundant"); 340 movl(dst, src); 341 } 342 343 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 344 assert(rscratch == noreg, "redundant"); 345 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); 346 } 347 348 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 349 assert(rscratch == noreg, "redundant"); 350 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); 351 } 352 353 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 354 assert(rscratch == noreg, "redundant"); 355 if (src.is_lval()) { 356 push_literal32((int32_t)src.target(), src.rspec()); 357 } else { 358 pushl(as_Address(src)); 359 } 360 } 361 362 static void pass_arg0(MacroAssembler* masm, Register arg) { 363 masm->push(arg); 364 } 365 366 static void pass_arg1(MacroAssembler* masm, Register arg) { 367 masm->push(arg); 368 } 369 370 static void pass_arg2(MacroAssembler* masm, Register arg) { 371 masm->push(arg); 372 } 373 374 static void pass_arg3(MacroAssembler* masm, Register arg) { 375 masm->push(arg); 376 } 377 378 #ifndef PRODUCT 379 extern "C" void findpc(intptr_t x); 380 #endif 381 382 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { 383 // In order to get locks to work, we need to fake a in_VM state 384 JavaThread* thread = JavaThread::current(); 385 JavaThreadState saved_state = thread->thread_state(); 386 thread->set_thread_state(_thread_in_vm); 387 if (ShowMessageBoxOnError) { 388 JavaThread* thread = JavaThread::current(); 389 JavaThreadState saved_state = thread->thread_state(); 390 thread->set_thread_state(_thread_in_vm); 391 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 392 ttyLocker ttyl; 393 BytecodeCounter::print(); 394 } 395 // To see where a verify_oop failed, get $ebx+40/X for this frame. 396 // This is the value of eip which points to where verify_oop will return. 397 if (os::message_box(msg, "Execution stopped, print registers?")) { 398 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); 399 BREAKPOINT; 400 } 401 } 402 fatal("DEBUG MESSAGE: %s", msg); 403 } 404 405 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { 406 ttyLocker ttyl; 407 DebuggingContext debugging{}; 408 tty->print_cr("eip = 0x%08x", eip); 409 #ifndef PRODUCT 410 if ((WizardMode || Verbose) && PrintMiscellaneous) { 411 tty->cr(); 412 findpc(eip); 413 tty->cr(); 414 } 415 #endif 416 #define PRINT_REG(rax) \ 417 { tty->print("%s = ", #rax); os::print_location(tty, rax); } 418 PRINT_REG(rax); 419 PRINT_REG(rbx); 420 PRINT_REG(rcx); 421 PRINT_REG(rdx); 422 PRINT_REG(rdi); 423 PRINT_REG(rsi); 424 PRINT_REG(rbp); 425 PRINT_REG(rsp); 426 #undef PRINT_REG 427 // Print some words near top of staack. 428 int* dump_sp = (int*) rsp; 429 for (int col1 = 0; col1 < 8; col1++) { 430 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 431 os::print_location(tty, *dump_sp++); 432 } 433 for (int row = 0; row < 16; row++) { 434 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 435 for (int col = 0; col < 8; col++) { 436 tty->print(" 0x%08x", *dump_sp++); 437 } 438 tty->cr(); 439 } 440 // Print some instructions around pc: 441 Disassembler::decode((address)eip-64, (address)eip); 442 tty->print_cr("--------"); 443 Disassembler::decode((address)eip, (address)eip+32); 444 } 445 446 void MacroAssembler::stop(const char* msg) { 447 // push address of message 448 ExternalAddress message((address)msg); 449 pushptr(message.addr(), noreg); 450 { Label L; call(L, relocInfo::none); bind(L); } // push eip 451 pusha(); // push registers 452 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); 453 hlt(); 454 } 455 456 void MacroAssembler::warn(const char* msg) { 457 push_CPU_state(); 458 459 // push address of message 460 ExternalAddress message((address)msg); 461 pushptr(message.addr(), noreg); 462 463 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 464 addl(rsp, wordSize); // discard argument 465 pop_CPU_state(); 466 } 467 468 void MacroAssembler::print_state() { 469 { Label L; call(L, relocInfo::none); bind(L); } // push eip 470 pusha(); // push registers 471 472 push_CPU_state(); 473 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); 474 pop_CPU_state(); 475 476 popa(); 477 addl(rsp, wordSize); 478 } 479 480 #else // _LP64 481 482 // 64 bit versions 483 484 Address MacroAssembler::as_Address(AddressLiteral adr) { 485 // amd64 always does this as a pc-rel 486 // we can be absolute or disp based on the instruction type 487 // jmp/call are displacements others are absolute 488 assert(!adr.is_lval(), "must be rval"); 489 assert(reachable(adr), "must be"); 490 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 491 492 } 493 494 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 495 AddressLiteral base = adr.base(); 496 lea(rscratch, base); 497 Address index = adr.index(); 498 assert(index._disp == 0, "must not have disp"); // maybe it can? 499 Address array(rscratch, index._index, index._scale, index._disp); 500 return array; 501 } 502 503 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 504 Label L, E; 505 506 #ifdef _WIN64 507 // Windows always allocates space for it's register args 508 assert(num_args <= 4, "only register arguments supported"); 509 subq(rsp, frame::arg_reg_save_area_bytes); 510 #endif 511 512 // Align stack if necessary 513 testl(rsp, 15); 514 jcc(Assembler::zero, L); 515 516 subq(rsp, 8); 517 call(RuntimeAddress(entry_point)); 518 addq(rsp, 8); 519 jmp(E); 520 521 bind(L); 522 call(RuntimeAddress(entry_point)); 523 524 bind(E); 525 526 #ifdef _WIN64 527 // restore stack pointer 528 addq(rsp, frame::arg_reg_save_area_bytes); 529 #endif 530 531 } 532 533 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 534 assert(!src2.is_lval(), "should use cmpptr"); 535 assert(rscratch != noreg || always_reachable(src2), "missing"); 536 537 if (reachable(src2)) { 538 cmpq(src1, as_Address(src2)); 539 } else { 540 lea(rscratch, src2); 541 Assembler::cmpq(src1, Address(rscratch, 0)); 542 } 543 } 544 545 int MacroAssembler::corrected_idivq(Register reg) { 546 // Full implementation of Java ldiv and lrem; checks for special 547 // case as described in JVM spec., p.243 & p.271. The function 548 // returns the (pc) offset of the idivl instruction - may be needed 549 // for implicit exceptions. 550 // 551 // normal case special case 552 // 553 // input : rax: dividend min_long 554 // reg: divisor (may not be eax/edx) -1 555 // 556 // output: rax: quotient (= rax idiv reg) min_long 557 // rdx: remainder (= rax irem reg) 0 558 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 559 static const int64_t min_long = 0x8000000000000000; 560 Label normal_case, special_case; 561 562 // check for special case 563 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 564 jcc(Assembler::notEqual, normal_case); 565 xorl(rdx, rdx); // prepare rdx for possible special case (where 566 // remainder = 0) 567 cmpq(reg, -1); 568 jcc(Assembler::equal, special_case); 569 570 // handle normal case 571 bind(normal_case); 572 cdqq(); 573 int idivq_offset = offset(); 574 idivq(reg); 575 576 // normal and special case exit 577 bind(special_case); 578 579 return idivq_offset; 580 } 581 582 void MacroAssembler::decrementq(Register reg, int value) { 583 if (value == min_jint) { subq(reg, value); return; } 584 if (value < 0) { incrementq(reg, -value); return; } 585 if (value == 0) { ; return; } 586 if (value == 1 && UseIncDec) { decq(reg) ; return; } 587 /* else */ { subq(reg, value) ; return; } 588 } 589 590 void MacroAssembler::decrementq(Address dst, int value) { 591 if (value == min_jint) { subq(dst, value); return; } 592 if (value < 0) { incrementq(dst, -value); return; } 593 if (value == 0) { ; return; } 594 if (value == 1 && UseIncDec) { decq(dst) ; return; } 595 /* else */ { subq(dst, value) ; return; } 596 } 597 598 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 599 assert(rscratch != noreg || always_reachable(dst), "missing"); 600 601 if (reachable(dst)) { 602 incrementq(as_Address(dst)); 603 } else { 604 lea(rscratch, dst); 605 incrementq(Address(rscratch, 0)); 606 } 607 } 608 609 void MacroAssembler::incrementq(Register reg, int value) { 610 if (value == min_jint) { addq(reg, value); return; } 611 if (value < 0) { decrementq(reg, -value); return; } 612 if (value == 0) { ; return; } 613 if (value == 1 && UseIncDec) { incq(reg) ; return; } 614 /* else */ { addq(reg, value) ; return; } 615 } 616 617 void MacroAssembler::incrementq(Address dst, int value) { 618 if (value == min_jint) { addq(dst, value); return; } 619 if (value < 0) { decrementq(dst, -value); return; } 620 if (value == 0) { ; return; } 621 if (value == 1 && UseIncDec) { incq(dst) ; return; } 622 /* else */ { addq(dst, value) ; return; } 623 } 624 625 // 32bit can do a case table jump in one instruction but we no longer allow the base 626 // to be installed in the Address class 627 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 628 lea(rscratch, entry.base()); 629 Address dispatch = entry.index(); 630 assert(dispatch._base == noreg, "must be"); 631 dispatch._base = rscratch; 632 jmp(dispatch); 633 } 634 635 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 636 ShouldNotReachHere(); // 64bit doesn't use two regs 637 cmpq(x_lo, y_lo); 638 } 639 640 void MacroAssembler::lea(Register dst, AddressLiteral src) { 641 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 642 } 643 644 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 645 lea(rscratch, adr); 646 movptr(dst, rscratch); 647 } 648 649 void MacroAssembler::leave() { 650 // %%% is this really better? Why not on 32bit too? 651 emit_int8((unsigned char)0xC9); // LEAVE 652 } 653 654 void MacroAssembler::lneg(Register hi, Register lo) { 655 ShouldNotReachHere(); // 64bit doesn't use two regs 656 negq(lo); 657 } 658 659 void MacroAssembler::movoop(Register dst, jobject obj) { 660 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 661 } 662 663 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 664 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 665 movq(dst, rscratch); 666 } 667 668 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 669 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 670 } 671 672 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 673 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 674 movq(dst, rscratch); 675 } 676 677 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 678 if (src.is_lval()) { 679 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 680 } else { 681 if (reachable(src)) { 682 movq(dst, as_Address(src)); 683 } else { 684 lea(dst, src); 685 movq(dst, Address(dst, 0)); 686 } 687 } 688 } 689 690 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 691 movq(as_Address(dst, rscratch), src); 692 } 693 694 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 695 movq(dst, as_Address(src, dst /*rscratch*/)); 696 } 697 698 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 699 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 700 if (is_simm32(src)) { 701 movptr(dst, checked_cast<int32_t>(src)); 702 } else { 703 mov64(rscratch, src); 704 movq(dst, rscratch); 705 } 706 } 707 708 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 709 movoop(rscratch, obj); 710 push(rscratch); 711 } 712 713 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 714 mov_metadata(rscratch, obj); 715 push(rscratch); 716 } 717 718 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 719 lea(rscratch, src); 720 if (src.is_lval()) { 721 push(rscratch); 722 } else { 723 pushq(Address(rscratch, 0)); 724 } 725 } 726 727 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 728 reset_last_Java_frame(r15_thread, clear_fp); 729 } 730 731 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 732 Register last_java_fp, 733 address last_java_pc, 734 Register rscratch) { 735 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch); 736 } 737 738 static void pass_arg0(MacroAssembler* masm, Register arg) { 739 if (c_rarg0 != arg ) { 740 masm->mov(c_rarg0, arg); 741 } 742 } 743 744 static void pass_arg1(MacroAssembler* masm, Register arg) { 745 if (c_rarg1 != arg ) { 746 masm->mov(c_rarg1, arg); 747 } 748 } 749 750 static void pass_arg2(MacroAssembler* masm, Register arg) { 751 if (c_rarg2 != arg ) { 752 masm->mov(c_rarg2, arg); 753 } 754 } 755 756 static void pass_arg3(MacroAssembler* masm, Register arg) { 757 if (c_rarg3 != arg ) { 758 masm->mov(c_rarg3, arg); 759 } 760 } 761 762 void MacroAssembler::stop(const char* msg) { 763 if (ShowMessageBoxOnError) { 764 address rip = pc(); 765 pusha(); // get regs on stack 766 lea(c_rarg1, InternalAddress(rip)); 767 movq(c_rarg2, rsp); // pass pointer to regs array 768 } 769 lea(c_rarg0, ExternalAddress((address) msg)); 770 andq(rsp, -16); // align stack as required by ABI 771 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 772 hlt(); 773 } 774 775 void MacroAssembler::warn(const char* msg) { 776 push(rbp); 777 movq(rbp, rsp); 778 andq(rsp, -16); // align stack as required by push_CPU_state and call 779 push_CPU_state(); // keeps alignment at 16 bytes 780 781 lea(c_rarg0, ExternalAddress((address) msg)); 782 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 783 784 pop_CPU_state(); 785 mov(rsp, rbp); 786 pop(rbp); 787 } 788 789 void MacroAssembler::print_state() { 790 address rip = pc(); 791 pusha(); // get regs on stack 792 push(rbp); 793 movq(rbp, rsp); 794 andq(rsp, -16); // align stack as required by push_CPU_state and call 795 push_CPU_state(); // keeps alignment at 16 bytes 796 797 lea(c_rarg0, InternalAddress(rip)); 798 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 799 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 800 801 pop_CPU_state(); 802 mov(rsp, rbp); 803 pop(rbp); 804 popa(); 805 } 806 807 #ifndef PRODUCT 808 extern "C" void findpc(intptr_t x); 809 #endif 810 811 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 812 // In order to get locks to work, we need to fake a in_VM state 813 if (ShowMessageBoxOnError) { 814 JavaThread* thread = JavaThread::current(); 815 JavaThreadState saved_state = thread->thread_state(); 816 thread->set_thread_state(_thread_in_vm); 817 #ifndef PRODUCT 818 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 819 ttyLocker ttyl; 820 BytecodeCounter::print(); 821 } 822 #endif 823 // To see where a verify_oop failed, get $ebx+40/X for this frame. 824 // XXX correct this offset for amd64 825 // This is the value of eip which points to where verify_oop will return. 826 if (os::message_box(msg, "Execution stopped, print registers?")) { 827 print_state64(pc, regs); 828 BREAKPOINT; 829 } 830 } 831 fatal("DEBUG MESSAGE: %s", msg); 832 } 833 834 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 835 ttyLocker ttyl; 836 DebuggingContext debugging{}; 837 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 838 #ifndef PRODUCT 839 tty->cr(); 840 findpc(pc); 841 tty->cr(); 842 #endif 843 #define PRINT_REG(rax, value) \ 844 { tty->print("%s = ", #rax); os::print_location(tty, value); } 845 PRINT_REG(rax, regs[15]); 846 PRINT_REG(rbx, regs[12]); 847 PRINT_REG(rcx, regs[14]); 848 PRINT_REG(rdx, regs[13]); 849 PRINT_REG(rdi, regs[8]); 850 PRINT_REG(rsi, regs[9]); 851 PRINT_REG(rbp, regs[10]); 852 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 853 PRINT_REG(rsp, (intptr_t)(®s[16])); 854 PRINT_REG(r8 , regs[7]); 855 PRINT_REG(r9 , regs[6]); 856 PRINT_REG(r10, regs[5]); 857 PRINT_REG(r11, regs[4]); 858 PRINT_REG(r12, regs[3]); 859 PRINT_REG(r13, regs[2]); 860 PRINT_REG(r14, regs[1]); 861 PRINT_REG(r15, regs[0]); 862 #undef PRINT_REG 863 // Print some words near the top of the stack. 864 int64_t* rsp = ®s[16]; 865 int64_t* dump_sp = rsp; 866 for (int col1 = 0; col1 < 8; col1++) { 867 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 868 os::print_location(tty, *dump_sp++); 869 } 870 for (int row = 0; row < 25; row++) { 871 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 872 for (int col = 0; col < 4; col++) { 873 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 874 } 875 tty->cr(); 876 } 877 // Print some instructions around pc: 878 Disassembler::decode((address)pc-64, (address)pc); 879 tty->print_cr("--------"); 880 Disassembler::decode((address)pc, (address)pc+32); 881 } 882 883 // The java_calling_convention describes stack locations as ideal slots on 884 // a frame with no abi restrictions. Since we must observe abi restrictions 885 // (like the placement of the register window) the slots must be biased by 886 // the following value. 887 static int reg2offset_in(VMReg r) { 888 // Account for saved rbp and return address 889 // This should really be in_preserve_stack_slots 890 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 891 } 892 893 static int reg2offset_out(VMReg r) { 894 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 895 } 896 897 // A long move 898 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 899 900 // The calling conventions assures us that each VMregpair is either 901 // all really one physical register or adjacent stack slots. 902 903 if (src.is_single_phys_reg() ) { 904 if (dst.is_single_phys_reg()) { 905 if (dst.first() != src.first()) { 906 mov(dst.first()->as_Register(), src.first()->as_Register()); 907 } 908 } else { 909 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 910 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 911 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 912 } 913 } else if (dst.is_single_phys_reg()) { 914 assert(src.is_single_reg(), "not a stack pair"); 915 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 916 } else { 917 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 918 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 919 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 920 } 921 } 922 923 // A double move 924 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 925 926 // The calling conventions assures us that each VMregpair is either 927 // all really one physical register or adjacent stack slots. 928 929 if (src.is_single_phys_reg() ) { 930 if (dst.is_single_phys_reg()) { 931 // In theory these overlap but the ordering is such that this is likely a nop 932 if ( src.first() != dst.first()) { 933 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 934 } 935 } else { 936 assert(dst.is_single_reg(), "not a stack pair"); 937 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 938 } 939 } else if (dst.is_single_phys_reg()) { 940 assert(src.is_single_reg(), "not a stack pair"); 941 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 942 } else { 943 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 944 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 945 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 946 } 947 } 948 949 950 // A float arg may have to do float reg int reg conversion 951 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 952 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 953 954 // The calling conventions assures us that each VMregpair is either 955 // all really one physical register or adjacent stack slots. 956 957 if (src.first()->is_stack()) { 958 if (dst.first()->is_stack()) { 959 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 960 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 961 } else { 962 // stack to reg 963 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 964 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 965 } 966 } else if (dst.first()->is_stack()) { 967 // reg to stack 968 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 969 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 970 } else { 971 // reg to reg 972 // In theory these overlap but the ordering is such that this is likely a nop 973 if ( src.first() != dst.first()) { 974 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 975 } 976 } 977 } 978 979 // On 64 bit we will store integer like items to the stack as 980 // 64 bits items (x86_32/64 abi) even though java would only store 981 // 32bits for a parameter. On 32bit it will simply be 32 bits 982 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 983 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 984 if (src.first()->is_stack()) { 985 if (dst.first()->is_stack()) { 986 // stack to stack 987 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 988 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 989 } else { 990 // stack to reg 991 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 992 } 993 } else if (dst.first()->is_stack()) { 994 // reg to stack 995 // Do we really have to sign extend??? 996 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 997 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 998 } else { 999 // Do we really have to sign extend??? 1000 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 1001 if (dst.first() != src.first()) { 1002 movq(dst.first()->as_Register(), src.first()->as_Register()); 1003 } 1004 } 1005 } 1006 1007 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 1008 if (src.first()->is_stack()) { 1009 if (dst.first()->is_stack()) { 1010 // stack to stack 1011 movq(rax, Address(rbp, reg2offset_in(src.first()))); 1012 movq(Address(rsp, reg2offset_out(dst.first())), rax); 1013 } else { 1014 // stack to reg 1015 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 1016 } 1017 } else if (dst.first()->is_stack()) { 1018 // reg to stack 1019 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 1020 } else { 1021 if (dst.first() != src.first()) { 1022 movq(dst.first()->as_Register(), src.first()->as_Register()); 1023 } 1024 } 1025 } 1026 1027 // An oop arg. Must pass a handle not the oop itself 1028 void MacroAssembler::object_move(OopMap* map, 1029 int oop_handle_offset, 1030 int framesize_in_slots, 1031 VMRegPair src, 1032 VMRegPair dst, 1033 bool is_receiver, 1034 int* receiver_offset) { 1035 1036 // must pass a handle. First figure out the location we use as a handle 1037 1038 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 1039 1040 // See if oop is null if it is we need no handle 1041 1042 if (src.first()->is_stack()) { 1043 1044 // Oop is already on the stack as an argument 1045 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 1046 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 1047 if (is_receiver) { 1048 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 1049 } 1050 1051 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 1052 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 1053 // conditionally move a null 1054 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 1055 } else { 1056 1057 // Oop is in a register we must store it to the space we reserve 1058 // on the stack for oop_handles and pass a handle if oop is non-null 1059 1060 const Register rOop = src.first()->as_Register(); 1061 int oop_slot; 1062 if (rOop == j_rarg0) 1063 oop_slot = 0; 1064 else if (rOop == j_rarg1) 1065 oop_slot = 1; 1066 else if (rOop == j_rarg2) 1067 oop_slot = 2; 1068 else if (rOop == j_rarg3) 1069 oop_slot = 3; 1070 else if (rOop == j_rarg4) 1071 oop_slot = 4; 1072 else { 1073 assert(rOop == j_rarg5, "wrong register"); 1074 oop_slot = 5; 1075 } 1076 1077 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 1078 int offset = oop_slot*VMRegImpl::stack_slot_size; 1079 1080 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 1081 // Store oop in handle area, may be null 1082 movptr(Address(rsp, offset), rOop); 1083 if (is_receiver) { 1084 *receiver_offset = offset; 1085 } 1086 1087 cmpptr(rOop, NULL_WORD); 1088 lea(rHandle, Address(rsp, offset)); 1089 // conditionally move a null from the handle area where it was just stored 1090 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 1091 } 1092 1093 // If arg is on the stack then place it otherwise it is already in correct reg. 1094 if (dst.first()->is_stack()) { 1095 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 1096 } 1097 } 1098 1099 #endif // _LP64 1100 1101 // Now versions that are common to 32/64 bit 1102 1103 void MacroAssembler::addptr(Register dst, int32_t imm32) { 1104 LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); 1105 } 1106 1107 void MacroAssembler::addptr(Register dst, Register src) { 1108 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1109 } 1110 1111 void MacroAssembler::addptr(Address dst, Register src) { 1112 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1113 } 1114 1115 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1116 assert(rscratch != noreg || always_reachable(src), "missing"); 1117 1118 if (reachable(src)) { 1119 Assembler::addsd(dst, as_Address(src)); 1120 } else { 1121 lea(rscratch, src); 1122 Assembler::addsd(dst, Address(rscratch, 0)); 1123 } 1124 } 1125 1126 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1127 assert(rscratch != noreg || always_reachable(src), "missing"); 1128 1129 if (reachable(src)) { 1130 addss(dst, as_Address(src)); 1131 } else { 1132 lea(rscratch, src); 1133 addss(dst, Address(rscratch, 0)); 1134 } 1135 } 1136 1137 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1138 assert(rscratch != noreg || always_reachable(src), "missing"); 1139 1140 if (reachable(src)) { 1141 Assembler::addpd(dst, as_Address(src)); 1142 } else { 1143 lea(rscratch, src); 1144 Assembler::addpd(dst, Address(rscratch, 0)); 1145 } 1146 } 1147 1148 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 1149 // Stub code is generated once and never copied. 1150 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 1151 void MacroAssembler::align64() { 1152 align(64, (unsigned long long) pc()); 1153 } 1154 1155 void MacroAssembler::align32() { 1156 align(32, (unsigned long long) pc()); 1157 } 1158 1159 void MacroAssembler::align(int modulus) { 1160 // 8273459: Ensure alignment is possible with current segment alignment 1161 assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 1162 align(modulus, offset()); 1163 } 1164 1165 void MacroAssembler::align(int modulus, int target) { 1166 if (target % modulus != 0) { 1167 nop(modulus - (target % modulus)); 1168 } 1169 } 1170 1171 void MacroAssembler::push_f(XMMRegister r) { 1172 subptr(rsp, wordSize); 1173 movflt(Address(rsp, 0), r); 1174 } 1175 1176 void MacroAssembler::pop_f(XMMRegister r) { 1177 movflt(r, Address(rsp, 0)); 1178 addptr(rsp, wordSize); 1179 } 1180 1181 void MacroAssembler::push_d(XMMRegister r) { 1182 subptr(rsp, 2 * wordSize); 1183 movdbl(Address(rsp, 0), r); 1184 } 1185 1186 void MacroAssembler::pop_d(XMMRegister r) { 1187 movdbl(r, Address(rsp, 0)); 1188 addptr(rsp, 2 * Interpreter::stackElementSize); 1189 } 1190 1191 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1192 // Used in sign-masking with aligned address. 1193 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1194 assert(rscratch != noreg || always_reachable(src), "missing"); 1195 1196 if (reachable(src)) { 1197 Assembler::andpd(dst, as_Address(src)); 1198 } else { 1199 lea(rscratch, src); 1200 Assembler::andpd(dst, Address(rscratch, 0)); 1201 } 1202 } 1203 1204 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 1205 // Used in sign-masking with aligned address. 1206 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1207 assert(rscratch != noreg || always_reachable(src), "missing"); 1208 1209 if (reachable(src)) { 1210 Assembler::andps(dst, as_Address(src)); 1211 } else { 1212 lea(rscratch, src); 1213 Assembler::andps(dst, Address(rscratch, 0)); 1214 } 1215 } 1216 1217 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1218 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1219 } 1220 1221 #ifdef _LP64 1222 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 1223 assert(rscratch != noreg || always_reachable(src), "missing"); 1224 1225 if (reachable(src)) { 1226 andq(dst, as_Address(src)); 1227 } else { 1228 lea(rscratch, src); 1229 andq(dst, Address(rscratch, 0)); 1230 } 1231 } 1232 #endif 1233 1234 void MacroAssembler::atomic_incl(Address counter_addr) { 1235 lock(); 1236 incrementl(counter_addr); 1237 } 1238 1239 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 1240 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1241 1242 if (reachable(counter_addr)) { 1243 atomic_incl(as_Address(counter_addr)); 1244 } else { 1245 lea(rscratch, counter_addr); 1246 atomic_incl(Address(rscratch, 0)); 1247 } 1248 } 1249 1250 #ifdef _LP64 1251 void MacroAssembler::atomic_incq(Address counter_addr) { 1252 lock(); 1253 incrementq(counter_addr); 1254 } 1255 1256 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 1257 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1258 1259 if (reachable(counter_addr)) { 1260 atomic_incq(as_Address(counter_addr)); 1261 } else { 1262 lea(rscratch, counter_addr); 1263 atomic_incq(Address(rscratch, 0)); 1264 } 1265 } 1266 #endif 1267 1268 // Writes to stack successive pages until offset reached to check for 1269 // stack overflow + shadow pages. This clobbers tmp. 1270 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 1271 movptr(tmp, rsp); 1272 // Bang stack for total size given plus shadow page size. 1273 // Bang one page at a time because large size can bang beyond yellow and 1274 // red zones. 1275 Label loop; 1276 bind(loop); 1277 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 1278 subptr(tmp, (int)os::vm_page_size()); 1279 subl(size, (int)os::vm_page_size()); 1280 jcc(Assembler::greater, loop); 1281 1282 // Bang down shadow pages too. 1283 // At this point, (tmp-0) is the last address touched, so don't 1284 // touch it again. (It was touched as (tmp-pagesize) but then tmp 1285 // was post-decremented.) Skip this address by starting at i=1, and 1286 // touch a few more pages below. N.B. It is important to touch all 1287 // the way down including all pages in the shadow zone. 1288 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 1289 // this could be any sized move but this is can be a debugging crumb 1290 // so the bigger the better. 1291 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 1292 } 1293 } 1294 1295 void MacroAssembler::reserved_stack_check() { 1296 // testing if reserved zone needs to be enabled 1297 Label no_reserved_zone_enabling; 1298 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 1299 NOT_LP64(get_thread(rsi);) 1300 1301 cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); 1302 jcc(Assembler::below, no_reserved_zone_enabling); 1303 1304 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); 1305 jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); 1306 should_not_reach_here(); 1307 1308 bind(no_reserved_zone_enabling); 1309 } 1310 1311 void MacroAssembler::c2bool(Register x) { 1312 // implements x == 0 ? 0 : 1 1313 // note: must only look at least-significant byte of x 1314 // since C-style booleans are stored in one byte 1315 // only! (was bug) 1316 andl(x, 0xFF); 1317 setb(Assembler::notZero, x); 1318 } 1319 1320 // Wouldn't need if AddressLiteral version had new name 1321 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 1322 Assembler::call(L, rtype); 1323 } 1324 1325 void MacroAssembler::call(Register entry) { 1326 Assembler::call(entry); 1327 } 1328 1329 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 1330 assert(rscratch != noreg || always_reachable(entry), "missing"); 1331 1332 if (reachable(entry)) { 1333 Assembler::call_literal(entry.target(), entry.rspec()); 1334 } else { 1335 lea(rscratch, entry); 1336 Assembler::call(rscratch); 1337 } 1338 } 1339 1340 void MacroAssembler::ic_call(address entry, jint method_index) { 1341 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1342 #ifdef _LP64 1343 // Needs full 64-bit immediate for later patching. 1344 mov64(rax, (intptr_t)Universe::non_oop_word()); 1345 #else 1346 movptr(rax, (intptr_t)Universe::non_oop_word()); 1347 #endif 1348 call(AddressLiteral(entry, rh)); 1349 } 1350 1351 void MacroAssembler::emit_static_call_stub() { 1352 // Static stub relocation also tags the Method* in the code-stream. 1353 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 1354 // This is recognized as unresolved by relocs/nativeinst/ic code. 1355 jump(RuntimeAddress(pc())); 1356 } 1357 1358 // Implementation of call_VM versions 1359 1360 void MacroAssembler::call_VM(Register oop_result, 1361 address entry_point, 1362 bool check_exceptions) { 1363 Label C, E; 1364 call(C, relocInfo::none); 1365 jmp(E); 1366 1367 bind(C); 1368 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1369 ret(0); 1370 1371 bind(E); 1372 } 1373 1374 void MacroAssembler::call_VM(Register oop_result, 1375 address entry_point, 1376 Register arg_1, 1377 bool check_exceptions) { 1378 Label C, E; 1379 call(C, relocInfo::none); 1380 jmp(E); 1381 1382 bind(C); 1383 pass_arg1(this, arg_1); 1384 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1385 ret(0); 1386 1387 bind(E); 1388 } 1389 1390 void MacroAssembler::call_VM(Register oop_result, 1391 address entry_point, 1392 Register arg_1, 1393 Register arg_2, 1394 bool check_exceptions) { 1395 Label C, E; 1396 call(C, relocInfo::none); 1397 jmp(E); 1398 1399 bind(C); 1400 1401 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1402 1403 pass_arg2(this, arg_2); 1404 pass_arg1(this, arg_1); 1405 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1406 ret(0); 1407 1408 bind(E); 1409 } 1410 1411 void MacroAssembler::call_VM(Register oop_result, 1412 address entry_point, 1413 Register arg_1, 1414 Register arg_2, 1415 Register arg_3, 1416 bool check_exceptions) { 1417 Label C, E; 1418 call(C, relocInfo::none); 1419 jmp(E); 1420 1421 bind(C); 1422 1423 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1424 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1425 pass_arg3(this, arg_3); 1426 pass_arg2(this, arg_2); 1427 pass_arg1(this, arg_1); 1428 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1429 ret(0); 1430 1431 bind(E); 1432 } 1433 1434 void MacroAssembler::call_VM(Register oop_result, 1435 Register last_java_sp, 1436 address entry_point, 1437 int number_of_arguments, 1438 bool check_exceptions) { 1439 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1440 call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1441 } 1442 1443 void MacroAssembler::call_VM(Register oop_result, 1444 Register last_java_sp, 1445 address entry_point, 1446 Register arg_1, 1447 bool check_exceptions) { 1448 pass_arg1(this, arg_1); 1449 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1450 } 1451 1452 void MacroAssembler::call_VM(Register oop_result, 1453 Register last_java_sp, 1454 address entry_point, 1455 Register arg_1, 1456 Register arg_2, 1457 bool check_exceptions) { 1458 1459 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1460 pass_arg2(this, arg_2); 1461 pass_arg1(this, arg_1); 1462 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1463 } 1464 1465 void MacroAssembler::call_VM(Register oop_result, 1466 Register last_java_sp, 1467 address entry_point, 1468 Register arg_1, 1469 Register arg_2, 1470 Register arg_3, 1471 bool check_exceptions) { 1472 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1473 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1474 pass_arg3(this, arg_3); 1475 pass_arg2(this, arg_2); 1476 pass_arg1(this, arg_1); 1477 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1478 } 1479 1480 void MacroAssembler::super_call_VM(Register oop_result, 1481 Register last_java_sp, 1482 address entry_point, 1483 int number_of_arguments, 1484 bool check_exceptions) { 1485 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1486 MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1487 } 1488 1489 void MacroAssembler::super_call_VM(Register oop_result, 1490 Register last_java_sp, 1491 address entry_point, 1492 Register arg_1, 1493 bool check_exceptions) { 1494 pass_arg1(this, arg_1); 1495 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1496 } 1497 1498 void MacroAssembler::super_call_VM(Register oop_result, 1499 Register last_java_sp, 1500 address entry_point, 1501 Register arg_1, 1502 Register arg_2, 1503 bool check_exceptions) { 1504 1505 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1506 pass_arg2(this, arg_2); 1507 pass_arg1(this, arg_1); 1508 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1509 } 1510 1511 void MacroAssembler::super_call_VM(Register oop_result, 1512 Register last_java_sp, 1513 address entry_point, 1514 Register arg_1, 1515 Register arg_2, 1516 Register arg_3, 1517 bool check_exceptions) { 1518 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1519 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1520 pass_arg3(this, arg_3); 1521 pass_arg2(this, arg_2); 1522 pass_arg1(this, arg_1); 1523 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1524 } 1525 1526 void MacroAssembler::call_VM_base(Register oop_result, 1527 Register java_thread, 1528 Register last_java_sp, 1529 address entry_point, 1530 int number_of_arguments, 1531 bool check_exceptions) { 1532 // determine java_thread register 1533 if (!java_thread->is_valid()) { 1534 #ifdef _LP64 1535 java_thread = r15_thread; 1536 #else 1537 java_thread = rdi; 1538 get_thread(java_thread); 1539 #endif // LP64 1540 } 1541 // determine last_java_sp register 1542 if (!last_java_sp->is_valid()) { 1543 last_java_sp = rsp; 1544 } 1545 // debugging support 1546 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1547 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); 1548 #ifdef ASSERT 1549 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1550 // r12 is the heapbase. 1551 LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) 1552 #endif // ASSERT 1553 1554 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1555 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1556 1557 // push java thread (becomes first argument of C function) 1558 1559 NOT_LP64(push(java_thread); number_of_arguments++); 1560 LP64_ONLY(mov(c_rarg0, r15_thread)); 1561 1562 // set last Java frame before call 1563 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1564 1565 // Only interpreter should have to set fp 1566 set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1); 1567 1568 // do the call, remove parameters 1569 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1570 1571 // restore the thread (cannot use the pushed argument since arguments 1572 // may be overwritten by C code generated by an optimizing compiler); 1573 // however can use the register value directly if it is callee saved. 1574 if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { 1575 // rdi & rsi (also r15) are callee saved -> nothing to do 1576 #ifdef ASSERT 1577 guarantee(java_thread != rax, "change this code"); 1578 push(rax); 1579 { Label L; 1580 get_thread(rax); 1581 cmpptr(java_thread, rax); 1582 jcc(Assembler::equal, L); 1583 STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); 1584 bind(L); 1585 } 1586 pop(rax); 1587 #endif 1588 } else { 1589 get_thread(java_thread); 1590 } 1591 // reset last Java frame 1592 // Only interpreter should have to clear fp 1593 reset_last_Java_frame(java_thread, true); 1594 1595 // C++ interp handles this in the interpreter 1596 check_and_handle_popframe(java_thread); 1597 check_and_handle_earlyret(java_thread); 1598 1599 if (check_exceptions) { 1600 // check for pending exceptions (java_thread is set upon return) 1601 cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD); 1602 #ifndef _LP64 1603 jump_cc(Assembler::notEqual, 1604 RuntimeAddress(StubRoutines::forward_exception_entry())); 1605 #else 1606 // This used to conditionally jump to forward_exception however it is 1607 // possible if we relocate that the branch will not reach. So we must jump 1608 // around so we can always reach 1609 1610 Label ok; 1611 jcc(Assembler::equal, ok); 1612 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1613 bind(ok); 1614 #endif // LP64 1615 } 1616 1617 // get oop result if there is one and reset the value in the thread 1618 if (oop_result->is_valid()) { 1619 get_vm_result(oop_result, java_thread); 1620 } 1621 } 1622 1623 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1624 1625 // Calculate the value for last_Java_sp 1626 // somewhat subtle. call_VM does an intermediate call 1627 // which places a return address on the stack just under the 1628 // stack pointer as the user finished with it. This allows 1629 // use to retrieve last_Java_pc from last_Java_sp[-1]. 1630 // On 32bit we then have to push additional args on the stack to accomplish 1631 // the actual requested call. On 64bit call_VM only can use register args 1632 // so the only extra space is the return address that call_VM created. 1633 // This hopefully explains the calculations here. 1634 1635 #ifdef _LP64 1636 // We've pushed one address, correct last_Java_sp 1637 lea(rax, Address(rsp, wordSize)); 1638 #else 1639 lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); 1640 #endif // LP64 1641 1642 call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); 1643 1644 } 1645 1646 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1647 void MacroAssembler::call_VM_leaf0(address entry_point) { 1648 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1649 } 1650 1651 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1652 call_VM_leaf_base(entry_point, number_of_arguments); 1653 } 1654 1655 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1656 pass_arg0(this, arg_0); 1657 call_VM_leaf(entry_point, 1); 1658 } 1659 1660 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1661 1662 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1663 pass_arg1(this, arg_1); 1664 pass_arg0(this, arg_0); 1665 call_VM_leaf(entry_point, 2); 1666 } 1667 1668 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1669 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1670 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1671 pass_arg2(this, arg_2); 1672 pass_arg1(this, arg_1); 1673 pass_arg0(this, arg_0); 1674 call_VM_leaf(entry_point, 3); 1675 } 1676 1677 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1678 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1679 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1680 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1681 pass_arg3(this, arg_3); 1682 pass_arg2(this, arg_2); 1683 pass_arg1(this, arg_1); 1684 pass_arg0(this, arg_0); 1685 call_VM_leaf(entry_point, 3); 1686 } 1687 1688 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1689 pass_arg0(this, arg_0); 1690 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1691 } 1692 1693 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1694 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1695 pass_arg1(this, arg_1); 1696 pass_arg0(this, arg_0); 1697 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1698 } 1699 1700 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1701 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1702 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1703 pass_arg2(this, arg_2); 1704 pass_arg1(this, arg_1); 1705 pass_arg0(this, arg_0); 1706 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1707 } 1708 1709 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1710 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1711 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1712 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1713 pass_arg3(this, arg_3); 1714 pass_arg2(this, arg_2); 1715 pass_arg1(this, arg_1); 1716 pass_arg0(this, arg_0); 1717 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1718 } 1719 1720 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1721 movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1722 movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); 1723 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1724 } 1725 1726 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1727 movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1728 movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); 1729 } 1730 1731 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { 1732 } 1733 1734 void MacroAssembler::check_and_handle_popframe(Register java_thread) { 1735 } 1736 1737 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1738 assert(rscratch != noreg || always_reachable(src1), "missing"); 1739 1740 if (reachable(src1)) { 1741 cmpl(as_Address(src1), imm); 1742 } else { 1743 lea(rscratch, src1); 1744 cmpl(Address(rscratch, 0), imm); 1745 } 1746 } 1747 1748 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1749 assert(!src2.is_lval(), "use cmpptr"); 1750 assert(rscratch != noreg || always_reachable(src2), "missing"); 1751 1752 if (reachable(src2)) { 1753 cmpl(src1, as_Address(src2)); 1754 } else { 1755 lea(rscratch, src2); 1756 cmpl(src1, Address(rscratch, 0)); 1757 } 1758 } 1759 1760 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1761 Assembler::cmpl(src1, imm); 1762 } 1763 1764 void MacroAssembler::cmp32(Register src1, Address src2) { 1765 Assembler::cmpl(src1, src2); 1766 } 1767 1768 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1769 ucomisd(opr1, opr2); 1770 1771 Label L; 1772 if (unordered_is_less) { 1773 movl(dst, -1); 1774 jcc(Assembler::parity, L); 1775 jcc(Assembler::below , L); 1776 movl(dst, 0); 1777 jcc(Assembler::equal , L); 1778 increment(dst); 1779 } else { // unordered is greater 1780 movl(dst, 1); 1781 jcc(Assembler::parity, L); 1782 jcc(Assembler::above , L); 1783 movl(dst, 0); 1784 jcc(Assembler::equal , L); 1785 decrementl(dst); 1786 } 1787 bind(L); 1788 } 1789 1790 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1791 ucomiss(opr1, opr2); 1792 1793 Label L; 1794 if (unordered_is_less) { 1795 movl(dst, -1); 1796 jcc(Assembler::parity, L); 1797 jcc(Assembler::below , L); 1798 movl(dst, 0); 1799 jcc(Assembler::equal , L); 1800 increment(dst); 1801 } else { // unordered is greater 1802 movl(dst, 1); 1803 jcc(Assembler::parity, L); 1804 jcc(Assembler::above , L); 1805 movl(dst, 0); 1806 jcc(Assembler::equal , L); 1807 decrementl(dst); 1808 } 1809 bind(L); 1810 } 1811 1812 1813 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1814 assert(rscratch != noreg || always_reachable(src1), "missing"); 1815 1816 if (reachable(src1)) { 1817 cmpb(as_Address(src1), imm); 1818 } else { 1819 lea(rscratch, src1); 1820 cmpb(Address(rscratch, 0), imm); 1821 } 1822 } 1823 1824 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1825 #ifdef _LP64 1826 assert(rscratch != noreg || always_reachable(src2), "missing"); 1827 1828 if (src2.is_lval()) { 1829 movptr(rscratch, src2); 1830 Assembler::cmpq(src1, rscratch); 1831 } else if (reachable(src2)) { 1832 cmpq(src1, as_Address(src2)); 1833 } else { 1834 lea(rscratch, src2); 1835 Assembler::cmpq(src1, Address(rscratch, 0)); 1836 } 1837 #else 1838 assert(rscratch == noreg, "not needed"); 1839 if (src2.is_lval()) { 1840 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1841 } else { 1842 cmpl(src1, as_Address(src2)); 1843 } 1844 #endif // _LP64 1845 } 1846 1847 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1848 assert(src2.is_lval(), "not a mem-mem compare"); 1849 #ifdef _LP64 1850 // moves src2's literal address 1851 movptr(rscratch, src2); 1852 Assembler::cmpq(src1, rscratch); 1853 #else 1854 assert(rscratch == noreg, "not needed"); 1855 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1856 #endif // _LP64 1857 } 1858 1859 void MacroAssembler::cmpoop(Register src1, Register src2) { 1860 cmpptr(src1, src2); 1861 } 1862 1863 void MacroAssembler::cmpoop(Register src1, Address src2) { 1864 cmpptr(src1, src2); 1865 } 1866 1867 #ifdef _LP64 1868 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1869 movoop(rscratch, src2); 1870 cmpptr(src1, rscratch); 1871 } 1872 #endif 1873 1874 void MacroAssembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 1875 if ((UseAVX > 0) && (dst != src)) { 1876 xorpd(dst, dst); 1877 } 1878 Assembler::cvtss2sd(dst, src); 1879 } 1880 1881 void MacroAssembler::cvtss2sd(XMMRegister dst, Address src) { 1882 if (UseAVX > 0) { 1883 xorpd(dst, dst); 1884 } 1885 Assembler::cvtss2sd(dst, src); 1886 } 1887 1888 void MacroAssembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 1889 if ((UseAVX > 0) && (dst != src)) { 1890 xorps(dst, dst); 1891 } 1892 Assembler::cvtsd2ss(dst, src); 1893 } 1894 1895 void MacroAssembler::cvtsd2ss(XMMRegister dst, Address src) { 1896 if (UseAVX > 0) { 1897 xorps(dst, dst); 1898 } 1899 Assembler::cvtsd2ss(dst, src); 1900 } 1901 1902 void MacroAssembler::cvtsi2sdl(XMMRegister dst, Register src) { 1903 if (UseAVX > 0) { 1904 xorpd(dst, dst); 1905 } 1906 Assembler::cvtsi2sdl(dst, src); 1907 } 1908 1909 void MacroAssembler::cvtsi2sdl(XMMRegister dst, Address src) { 1910 if (UseAVX > 0) { 1911 xorpd(dst, dst); 1912 } 1913 Assembler::cvtsi2sdl(dst, src); 1914 } 1915 1916 void MacroAssembler::cvtsi2ssl(XMMRegister dst, Register src) { 1917 if (UseAVX > 0) { 1918 xorps(dst, dst); 1919 } 1920 Assembler::cvtsi2ssl(dst, src); 1921 } 1922 1923 void MacroAssembler::cvtsi2ssl(XMMRegister dst, Address src) { 1924 if (UseAVX > 0) { 1925 xorps(dst, dst); 1926 } 1927 Assembler::cvtsi2ssl(dst, src); 1928 } 1929 1930 #ifdef _LP64 1931 void MacroAssembler::cvtsi2sdq(XMMRegister dst, Register src) { 1932 if (UseAVX > 0) { 1933 xorpd(dst, dst); 1934 } 1935 Assembler::cvtsi2sdq(dst, src); 1936 } 1937 1938 void MacroAssembler::cvtsi2sdq(XMMRegister dst, Address src) { 1939 if (UseAVX > 0) { 1940 xorpd(dst, dst); 1941 } 1942 Assembler::cvtsi2sdq(dst, src); 1943 } 1944 1945 void MacroAssembler::cvtsi2ssq(XMMRegister dst, Register src) { 1946 if (UseAVX > 0) { 1947 xorps(dst, dst); 1948 } 1949 Assembler::cvtsi2ssq(dst, src); 1950 } 1951 1952 void MacroAssembler::cvtsi2ssq(XMMRegister dst, Address src) { 1953 if (UseAVX > 0) { 1954 xorps(dst, dst); 1955 } 1956 Assembler::cvtsi2ssq(dst, src); 1957 } 1958 #endif // _LP64 1959 1960 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1961 assert(rscratch != noreg || always_reachable(adr), "missing"); 1962 1963 if (reachable(adr)) { 1964 lock(); 1965 cmpxchgptr(reg, as_Address(adr)); 1966 } else { 1967 lea(rscratch, adr); 1968 lock(); 1969 cmpxchgptr(reg, Address(rscratch, 0)); 1970 } 1971 } 1972 1973 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1974 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); 1975 } 1976 1977 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1978 assert(rscratch != noreg || always_reachable(src), "missing"); 1979 1980 if (reachable(src)) { 1981 Assembler::comisd(dst, as_Address(src)); 1982 } else { 1983 lea(rscratch, src); 1984 Assembler::comisd(dst, Address(rscratch, 0)); 1985 } 1986 } 1987 1988 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1989 assert(rscratch != noreg || always_reachable(src), "missing"); 1990 1991 if (reachable(src)) { 1992 Assembler::comiss(dst, as_Address(src)); 1993 } else { 1994 lea(rscratch, src); 1995 Assembler::comiss(dst, Address(rscratch, 0)); 1996 } 1997 } 1998 1999 2000 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 2001 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 2002 2003 Condition negated_cond = negate_condition(cond); 2004 Label L; 2005 jcc(negated_cond, L); 2006 pushf(); // Preserve flags 2007 atomic_incl(counter_addr, rscratch); 2008 popf(); 2009 bind(L); 2010 } 2011 2012 int MacroAssembler::corrected_idivl(Register reg) { 2013 // Full implementation of Java idiv and irem; checks for 2014 // special case as described in JVM spec., p.243 & p.271. 2015 // The function returns the (pc) offset of the idivl 2016 // instruction - may be needed for implicit exceptions. 2017 // 2018 // normal case special case 2019 // 2020 // input : rax,: dividend min_int 2021 // reg: divisor (may not be rax,/rdx) -1 2022 // 2023 // output: rax,: quotient (= rax, idiv reg) min_int 2024 // rdx: remainder (= rax, irem reg) 0 2025 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 2026 const int min_int = 0x80000000; 2027 Label normal_case, special_case; 2028 2029 // check for special case 2030 cmpl(rax, min_int); 2031 jcc(Assembler::notEqual, normal_case); 2032 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 2033 cmpl(reg, -1); 2034 jcc(Assembler::equal, special_case); 2035 2036 // handle normal case 2037 bind(normal_case); 2038 cdql(); 2039 int idivl_offset = offset(); 2040 idivl(reg); 2041 2042 // normal and special case exit 2043 bind(special_case); 2044 2045 return idivl_offset; 2046 } 2047 2048 2049 2050 void MacroAssembler::decrementl(Register reg, int value) { 2051 if (value == min_jint) {subl(reg, value) ; return; } 2052 if (value < 0) { incrementl(reg, -value); return; } 2053 if (value == 0) { ; return; } 2054 if (value == 1 && UseIncDec) { decl(reg) ; return; } 2055 /* else */ { subl(reg, value) ; return; } 2056 } 2057 2058 void MacroAssembler::decrementl(Address dst, int value) { 2059 if (value == min_jint) {subl(dst, value) ; return; } 2060 if (value < 0) { incrementl(dst, -value); return; } 2061 if (value == 0) { ; return; } 2062 if (value == 1 && UseIncDec) { decl(dst) ; return; } 2063 /* else */ { subl(dst, value) ; return; } 2064 } 2065 2066 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 2067 assert(shift_value > 0, "illegal shift value"); 2068 Label _is_positive; 2069 testl (reg, reg); 2070 jcc (Assembler::positive, _is_positive); 2071 int offset = (1 << shift_value) - 1 ; 2072 2073 if (offset == 1) { 2074 incrementl(reg); 2075 } else { 2076 addl(reg, offset); 2077 } 2078 2079 bind (_is_positive); 2080 sarl(reg, shift_value); 2081 } 2082 2083 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2084 assert(rscratch != noreg || always_reachable(src), "missing"); 2085 2086 if (reachable(src)) { 2087 Assembler::divsd(dst, as_Address(src)); 2088 } else { 2089 lea(rscratch, src); 2090 Assembler::divsd(dst, Address(rscratch, 0)); 2091 } 2092 } 2093 2094 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2095 assert(rscratch != noreg || always_reachable(src), "missing"); 2096 2097 if (reachable(src)) { 2098 Assembler::divss(dst, as_Address(src)); 2099 } else { 2100 lea(rscratch, src); 2101 Assembler::divss(dst, Address(rscratch, 0)); 2102 } 2103 } 2104 2105 void MacroAssembler::enter() { 2106 push(rbp); 2107 mov(rbp, rsp); 2108 } 2109 2110 void MacroAssembler::post_call_nop() { 2111 if (!Continuations::enabled()) { 2112 return; 2113 } 2114 InstructionMark im(this); 2115 relocate(post_call_nop_Relocation::spec()); 2116 InlineSkippedInstructionsCounter skipCounter(this); 2117 emit_int8((uint8_t)0x0f); 2118 emit_int8((uint8_t)0x1f); 2119 emit_int8((uint8_t)0x84); 2120 emit_int8((uint8_t)0x00); 2121 emit_int32(0x00); 2122 } 2123 2124 // A 5 byte nop that is safe for patching (see patch_verified_entry) 2125 void MacroAssembler::fat_nop() { 2126 if (UseAddressNop) { 2127 addr_nop_5(); 2128 } else { 2129 emit_int8((uint8_t)0x26); // es: 2130 emit_int8((uint8_t)0x2e); // cs: 2131 emit_int8((uint8_t)0x64); // fs: 2132 emit_int8((uint8_t)0x65); // gs: 2133 emit_int8((uint8_t)0x90); 2134 } 2135 } 2136 2137 #ifndef _LP64 2138 void MacroAssembler::fcmp(Register tmp) { 2139 fcmp(tmp, 1, true, true); 2140 } 2141 2142 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { 2143 assert(!pop_right || pop_left, "usage error"); 2144 if (VM_Version::supports_cmov()) { 2145 assert(tmp == noreg, "unneeded temp"); 2146 if (pop_left) { 2147 fucomip(index); 2148 } else { 2149 fucomi(index); 2150 } 2151 if (pop_right) { 2152 fpop(); 2153 } 2154 } else { 2155 assert(tmp != noreg, "need temp"); 2156 if (pop_left) { 2157 if (pop_right) { 2158 fcompp(); 2159 } else { 2160 fcomp(index); 2161 } 2162 } else { 2163 fcom(index); 2164 } 2165 // convert FPU condition into eflags condition via rax, 2166 save_rax(tmp); 2167 fwait(); fnstsw_ax(); 2168 sahf(); 2169 restore_rax(tmp); 2170 } 2171 // condition codes set as follows: 2172 // 2173 // CF (corresponds to C0) if x < y 2174 // PF (corresponds to C2) if unordered 2175 // ZF (corresponds to C3) if x = y 2176 } 2177 2178 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { 2179 fcmp2int(dst, unordered_is_less, 1, true, true); 2180 } 2181 2182 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { 2183 fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); 2184 Label L; 2185 if (unordered_is_less) { 2186 movl(dst, -1); 2187 jcc(Assembler::parity, L); 2188 jcc(Assembler::below , L); 2189 movl(dst, 0); 2190 jcc(Assembler::equal , L); 2191 increment(dst); 2192 } else { // unordered is greater 2193 movl(dst, 1); 2194 jcc(Assembler::parity, L); 2195 jcc(Assembler::above , L); 2196 movl(dst, 0); 2197 jcc(Assembler::equal , L); 2198 decrementl(dst); 2199 } 2200 bind(L); 2201 } 2202 2203 void MacroAssembler::fld_d(AddressLiteral src) { 2204 fld_d(as_Address(src)); 2205 } 2206 2207 void MacroAssembler::fld_s(AddressLiteral src) { 2208 fld_s(as_Address(src)); 2209 } 2210 2211 void MacroAssembler::fldcw(AddressLiteral src) { 2212 fldcw(as_Address(src)); 2213 } 2214 2215 void MacroAssembler::fpop() { 2216 ffree(); 2217 fincstp(); 2218 } 2219 2220 void MacroAssembler::fremr(Register tmp) { 2221 save_rax(tmp); 2222 { Label L; 2223 bind(L); 2224 fprem(); 2225 fwait(); fnstsw_ax(); 2226 sahf(); 2227 jcc(Assembler::parity, L); 2228 } 2229 restore_rax(tmp); 2230 // Result is in ST0. 2231 // Note: fxch & fpop to get rid of ST1 2232 // (otherwise FPU stack could overflow eventually) 2233 fxch(1); 2234 fpop(); 2235 } 2236 2237 void MacroAssembler::empty_FPU_stack() { 2238 if (VM_Version::supports_mmx()) { 2239 emms(); 2240 } else { 2241 for (int i = 8; i-- > 0; ) ffree(i); 2242 } 2243 } 2244 #endif // !LP64 2245 2246 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2247 assert(rscratch != noreg || always_reachable(src), "missing"); 2248 if (reachable(src)) { 2249 Assembler::mulpd(dst, as_Address(src)); 2250 } else { 2251 lea(rscratch, src); 2252 Assembler::mulpd(dst, Address(rscratch, 0)); 2253 } 2254 } 2255 2256 void MacroAssembler::load_float(Address src) { 2257 #ifdef _LP64 2258 movflt(xmm0, src); 2259 #else 2260 if (UseSSE >= 1) { 2261 movflt(xmm0, src); 2262 } else { 2263 fld_s(src); 2264 } 2265 #endif // LP64 2266 } 2267 2268 void MacroAssembler::store_float(Address dst) { 2269 #ifdef _LP64 2270 movflt(dst, xmm0); 2271 #else 2272 if (UseSSE >= 1) { 2273 movflt(dst, xmm0); 2274 } else { 2275 fstp_s(dst); 2276 } 2277 #endif // LP64 2278 } 2279 2280 void MacroAssembler::load_double(Address src) { 2281 #ifdef _LP64 2282 movdbl(xmm0, src); 2283 #else 2284 if (UseSSE >= 2) { 2285 movdbl(xmm0, src); 2286 } else { 2287 fld_d(src); 2288 } 2289 #endif // LP64 2290 } 2291 2292 void MacroAssembler::store_double(Address dst) { 2293 #ifdef _LP64 2294 movdbl(dst, xmm0); 2295 #else 2296 if (UseSSE >= 2) { 2297 movdbl(dst, xmm0); 2298 } else { 2299 fstp_d(dst); 2300 } 2301 #endif // LP64 2302 } 2303 2304 // dst = c = a * b + c 2305 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2306 Assembler::vfmadd231sd(c, a, b); 2307 if (dst != c) { 2308 movdbl(dst, c); 2309 } 2310 } 2311 2312 // dst = c = a * b + c 2313 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2314 Assembler::vfmadd231ss(c, a, b); 2315 if (dst != c) { 2316 movflt(dst, c); 2317 } 2318 } 2319 2320 // dst = c = a * b + c 2321 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2322 Assembler::vfmadd231pd(c, a, b, vector_len); 2323 if (dst != c) { 2324 vmovdqu(dst, c); 2325 } 2326 } 2327 2328 // dst = c = a * b + c 2329 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2330 Assembler::vfmadd231ps(c, a, b, vector_len); 2331 if (dst != c) { 2332 vmovdqu(dst, c); 2333 } 2334 } 2335 2336 // dst = c = a * b + c 2337 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2338 Assembler::vfmadd231pd(c, a, b, vector_len); 2339 if (dst != c) { 2340 vmovdqu(dst, c); 2341 } 2342 } 2343 2344 // dst = c = a * b + c 2345 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2346 Assembler::vfmadd231ps(c, a, b, vector_len); 2347 if (dst != c) { 2348 vmovdqu(dst, c); 2349 } 2350 } 2351 2352 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 2353 assert(rscratch != noreg || always_reachable(dst), "missing"); 2354 2355 if (reachable(dst)) { 2356 incrementl(as_Address(dst)); 2357 } else { 2358 lea(rscratch, dst); 2359 incrementl(Address(rscratch, 0)); 2360 } 2361 } 2362 2363 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 2364 incrementl(as_Address(dst, rscratch)); 2365 } 2366 2367 void MacroAssembler::incrementl(Register reg, int value) { 2368 if (value == min_jint) {addl(reg, value) ; return; } 2369 if (value < 0) { decrementl(reg, -value); return; } 2370 if (value == 0) { ; return; } 2371 if (value == 1 && UseIncDec) { incl(reg) ; return; } 2372 /* else */ { addl(reg, value) ; return; } 2373 } 2374 2375 void MacroAssembler::incrementl(Address dst, int value) { 2376 if (value == min_jint) {addl(dst, value) ; return; } 2377 if (value < 0) { decrementl(dst, -value); return; } 2378 if (value == 0) { ; return; } 2379 if (value == 1 && UseIncDec) { incl(dst) ; return; } 2380 /* else */ { addl(dst, value) ; return; } 2381 } 2382 2383 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 2384 assert(rscratch != noreg || always_reachable(dst), "missing"); 2385 2386 if (reachable(dst)) { 2387 jmp_literal(dst.target(), dst.rspec()); 2388 } else { 2389 lea(rscratch, dst); 2390 jmp(rscratch); 2391 } 2392 } 2393 2394 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 2395 assert(rscratch != noreg || always_reachable(dst), "missing"); 2396 2397 if (reachable(dst)) { 2398 InstructionMark im(this); 2399 relocate(dst.reloc()); 2400 const int short_size = 2; 2401 const int long_size = 6; 2402 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 2403 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 2404 // 0111 tttn #8-bit disp 2405 emit_int8(0x70 | cc); 2406 emit_int8((offs - short_size) & 0xFF); 2407 } else { 2408 // 0000 1111 1000 tttn #32-bit disp 2409 emit_int8(0x0F); 2410 emit_int8((unsigned char)(0x80 | cc)); 2411 emit_int32(offs - long_size); 2412 } 2413 } else { 2414 #ifdef ASSERT 2415 warning("reversing conditional branch"); 2416 #endif /* ASSERT */ 2417 Label skip; 2418 jccb(reverse[cc], skip); 2419 lea(rscratch, dst); 2420 Assembler::jmp(rscratch); 2421 bind(skip); 2422 } 2423 } 2424 2425 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 2426 assert(rscratch != noreg || always_reachable(src), "missing"); 2427 2428 if (reachable(src)) { 2429 Assembler::ldmxcsr(as_Address(src)); 2430 } else { 2431 lea(rscratch, src); 2432 Assembler::ldmxcsr(Address(rscratch, 0)); 2433 } 2434 } 2435 2436 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2437 int off; 2438 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2439 off = offset(); 2440 movsbl(dst, src); // movsxb 2441 } else { 2442 off = load_unsigned_byte(dst, src); 2443 shll(dst, 24); 2444 sarl(dst, 24); 2445 } 2446 return off; 2447 } 2448 2449 // Note: load_signed_short used to be called load_signed_word. 2450 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 2451 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 2452 // The term "word" in HotSpot means a 32- or 64-bit machine word. 2453 int MacroAssembler::load_signed_short(Register dst, Address src) { 2454 int off; 2455 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2456 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 2457 // version but this is what 64bit has always done. This seems to imply 2458 // that users are only using 32bits worth. 2459 off = offset(); 2460 movswl(dst, src); // movsxw 2461 } else { 2462 off = load_unsigned_short(dst, src); 2463 shll(dst, 16); 2464 sarl(dst, 16); 2465 } 2466 return off; 2467 } 2468 2469 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2470 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2471 // and "3.9 Partial Register Penalties", p. 22). 2472 int off; 2473 if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { 2474 off = offset(); 2475 movzbl(dst, src); // movzxb 2476 } else { 2477 xorl(dst, dst); 2478 off = offset(); 2479 movb(dst, src); 2480 } 2481 return off; 2482 } 2483 2484 // Note: load_unsigned_short used to be called load_unsigned_word. 2485 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2486 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2487 // and "3.9 Partial Register Penalties", p. 22). 2488 int off; 2489 if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { 2490 off = offset(); 2491 movzwl(dst, src); // movzxw 2492 } else { 2493 xorl(dst, dst); 2494 off = offset(); 2495 movw(dst, src); 2496 } 2497 return off; 2498 } 2499 2500 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 2501 switch (size_in_bytes) { 2502 #ifndef _LP64 2503 case 8: 2504 assert(dst2 != noreg, "second dest register required"); 2505 movl(dst, src); 2506 movl(dst2, src.plus_disp(BytesPerInt)); 2507 break; 2508 #else 2509 case 8: movq(dst, src); break; 2510 #endif 2511 case 4: movl(dst, src); break; 2512 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2513 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2514 default: ShouldNotReachHere(); 2515 } 2516 } 2517 2518 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 2519 switch (size_in_bytes) { 2520 #ifndef _LP64 2521 case 8: 2522 assert(src2 != noreg, "second source register required"); 2523 movl(dst, src); 2524 movl(dst.plus_disp(BytesPerInt), src2); 2525 break; 2526 #else 2527 case 8: movq(dst, src); break; 2528 #endif 2529 case 4: movl(dst, src); break; 2530 case 2: movw(dst, src); break; 2531 case 1: movb(dst, src); break; 2532 default: ShouldNotReachHere(); 2533 } 2534 } 2535 2536 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 2537 assert(rscratch != noreg || always_reachable(dst), "missing"); 2538 2539 if (reachable(dst)) { 2540 movl(as_Address(dst), src); 2541 } else { 2542 lea(rscratch, dst); 2543 movl(Address(rscratch, 0), src); 2544 } 2545 } 2546 2547 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 2548 if (reachable(src)) { 2549 movl(dst, as_Address(src)); 2550 } else { 2551 lea(dst, src); 2552 movl(dst, Address(dst, 0)); 2553 } 2554 } 2555 2556 // C++ bool manipulation 2557 2558 void MacroAssembler::movbool(Register dst, Address src) { 2559 if(sizeof(bool) == 1) 2560 movb(dst, src); 2561 else if(sizeof(bool) == 2) 2562 movw(dst, src); 2563 else if(sizeof(bool) == 4) 2564 movl(dst, src); 2565 else 2566 // unsupported 2567 ShouldNotReachHere(); 2568 } 2569 2570 void MacroAssembler::movbool(Address dst, bool boolconst) { 2571 if(sizeof(bool) == 1) 2572 movb(dst, (int) boolconst); 2573 else if(sizeof(bool) == 2) 2574 movw(dst, (int) boolconst); 2575 else if(sizeof(bool) == 4) 2576 movl(dst, (int) boolconst); 2577 else 2578 // unsupported 2579 ShouldNotReachHere(); 2580 } 2581 2582 void MacroAssembler::movbool(Address dst, Register src) { 2583 if(sizeof(bool) == 1) 2584 movb(dst, src); 2585 else if(sizeof(bool) == 2) 2586 movw(dst, src); 2587 else if(sizeof(bool) == 4) 2588 movl(dst, src); 2589 else 2590 // unsupported 2591 ShouldNotReachHere(); 2592 } 2593 2594 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2595 assert(rscratch != noreg || always_reachable(src), "missing"); 2596 2597 if (reachable(src)) { 2598 movdl(dst, as_Address(src)); 2599 } else { 2600 lea(rscratch, src); 2601 movdl(dst, Address(rscratch, 0)); 2602 } 2603 } 2604 2605 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 2606 assert(rscratch != noreg || always_reachable(src), "missing"); 2607 2608 if (reachable(src)) { 2609 movq(dst, as_Address(src)); 2610 } else { 2611 lea(rscratch, src); 2612 movq(dst, Address(rscratch, 0)); 2613 } 2614 } 2615 2616 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2617 assert(rscratch != noreg || always_reachable(src), "missing"); 2618 2619 if (reachable(src)) { 2620 if (UseXmmLoadAndClearUpper) { 2621 movsd (dst, as_Address(src)); 2622 } else { 2623 movlpd(dst, as_Address(src)); 2624 } 2625 } else { 2626 lea(rscratch, src); 2627 if (UseXmmLoadAndClearUpper) { 2628 movsd (dst, Address(rscratch, 0)); 2629 } else { 2630 movlpd(dst, Address(rscratch, 0)); 2631 } 2632 } 2633 } 2634 2635 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 2636 assert(rscratch != noreg || always_reachable(src), "missing"); 2637 2638 if (reachable(src)) { 2639 movss(dst, as_Address(src)); 2640 } else { 2641 lea(rscratch, src); 2642 movss(dst, Address(rscratch, 0)); 2643 } 2644 } 2645 2646 void MacroAssembler::movptr(Register dst, Register src) { 2647 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2648 } 2649 2650 void MacroAssembler::movptr(Register dst, Address src) { 2651 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2652 } 2653 2654 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 2655 void MacroAssembler::movptr(Register dst, intptr_t src) { 2656 #ifdef _LP64 2657 if (is_simm32(src)) { 2658 movq(dst, checked_cast<int32_t>(src)); 2659 } else { 2660 mov64(dst, src); 2661 } 2662 #else 2663 movl(dst, src); 2664 #endif 2665 } 2666 2667 void MacroAssembler::movptr(Address dst, Register src) { 2668 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2669 } 2670 2671 void MacroAssembler::movptr(Address dst, int32_t src) { 2672 LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); 2673 } 2674 2675 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 2676 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2677 Assembler::movdqu(dst, src); 2678 } 2679 2680 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 2681 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2682 Assembler::movdqu(dst, src); 2683 } 2684 2685 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 2686 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2687 Assembler::movdqu(dst, src); 2688 } 2689 2690 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2691 assert(rscratch != noreg || always_reachable(src), "missing"); 2692 2693 if (reachable(src)) { 2694 movdqu(dst, as_Address(src)); 2695 } else { 2696 lea(rscratch, src); 2697 movdqu(dst, Address(rscratch, 0)); 2698 } 2699 } 2700 2701 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2702 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2703 Assembler::vmovdqu(dst, src); 2704 } 2705 2706 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2707 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2708 Assembler::vmovdqu(dst, src); 2709 } 2710 2711 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2712 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2713 Assembler::vmovdqu(dst, src); 2714 } 2715 2716 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2717 assert(rscratch != noreg || always_reachable(src), "missing"); 2718 2719 if (reachable(src)) { 2720 vmovdqu(dst, as_Address(src)); 2721 } 2722 else { 2723 lea(rscratch, src); 2724 vmovdqu(dst, Address(rscratch, 0)); 2725 } 2726 } 2727 2728 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2729 assert(rscratch != noreg || always_reachable(src), "missing"); 2730 2731 if (vector_len == AVX_512bit) { 2732 evmovdquq(dst, src, AVX_512bit, rscratch); 2733 } else if (vector_len == AVX_256bit) { 2734 vmovdqu(dst, src, rscratch); 2735 } else { 2736 movdqu(dst, src, rscratch); 2737 } 2738 } 2739 2740 void MacroAssembler::kmov(KRegister dst, Address src) { 2741 if (VM_Version::supports_avx512bw()) { 2742 kmovql(dst, src); 2743 } else { 2744 assert(VM_Version::supports_evex(), ""); 2745 kmovwl(dst, src); 2746 } 2747 } 2748 2749 void MacroAssembler::kmov(Address dst, KRegister src) { 2750 if (VM_Version::supports_avx512bw()) { 2751 kmovql(dst, src); 2752 } else { 2753 assert(VM_Version::supports_evex(), ""); 2754 kmovwl(dst, src); 2755 } 2756 } 2757 2758 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2759 if (VM_Version::supports_avx512bw()) { 2760 kmovql(dst, src); 2761 } else { 2762 assert(VM_Version::supports_evex(), ""); 2763 kmovwl(dst, src); 2764 } 2765 } 2766 2767 void MacroAssembler::kmov(Register dst, KRegister src) { 2768 if (VM_Version::supports_avx512bw()) { 2769 kmovql(dst, src); 2770 } else { 2771 assert(VM_Version::supports_evex(), ""); 2772 kmovwl(dst, src); 2773 } 2774 } 2775 2776 void MacroAssembler::kmov(KRegister dst, Register src) { 2777 if (VM_Version::supports_avx512bw()) { 2778 kmovql(dst, src); 2779 } else { 2780 assert(VM_Version::supports_evex(), ""); 2781 kmovwl(dst, src); 2782 } 2783 } 2784 2785 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2786 assert(rscratch != noreg || always_reachable(src), "missing"); 2787 2788 if (reachable(src)) { 2789 kmovql(dst, as_Address(src)); 2790 } else { 2791 lea(rscratch, src); 2792 kmovql(dst, Address(rscratch, 0)); 2793 } 2794 } 2795 2796 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2797 assert(rscratch != noreg || always_reachable(src), "missing"); 2798 2799 if (reachable(src)) { 2800 kmovwl(dst, as_Address(src)); 2801 } else { 2802 lea(rscratch, src); 2803 kmovwl(dst, Address(rscratch, 0)); 2804 } 2805 } 2806 2807 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2808 int vector_len, Register rscratch) { 2809 assert(rscratch != noreg || always_reachable(src), "missing"); 2810 2811 if (reachable(src)) { 2812 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2813 } else { 2814 lea(rscratch, src); 2815 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2816 } 2817 } 2818 2819 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2820 int vector_len, Register rscratch) { 2821 assert(rscratch != noreg || always_reachable(src), "missing"); 2822 2823 if (reachable(src)) { 2824 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2825 } else { 2826 lea(rscratch, src); 2827 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2828 } 2829 } 2830 2831 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2832 assert(rscratch != noreg || always_reachable(src), "missing"); 2833 2834 if (reachable(src)) { 2835 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2836 } else { 2837 lea(rscratch, src); 2838 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2839 } 2840 } 2841 2842 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2843 assert(rscratch != noreg || always_reachable(src), "missing"); 2844 2845 if (reachable(src)) { 2846 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2847 } else { 2848 lea(rscratch, src); 2849 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2850 } 2851 } 2852 2853 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2854 assert(rscratch != noreg || always_reachable(src), "missing"); 2855 2856 if (reachable(src)) { 2857 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2858 } else { 2859 lea(rscratch, src); 2860 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2861 } 2862 } 2863 2864 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2865 assert(rscratch != noreg || always_reachable(src), "missing"); 2866 2867 if (reachable(src)) { 2868 Assembler::movdqa(dst, as_Address(src)); 2869 } else { 2870 lea(rscratch, src); 2871 Assembler::movdqa(dst, Address(rscratch, 0)); 2872 } 2873 } 2874 2875 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2876 assert(rscratch != noreg || always_reachable(src), "missing"); 2877 2878 if (reachable(src)) { 2879 Assembler::movsd(dst, as_Address(src)); 2880 } else { 2881 lea(rscratch, src); 2882 Assembler::movsd(dst, Address(rscratch, 0)); 2883 } 2884 } 2885 2886 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2887 assert(rscratch != noreg || always_reachable(src), "missing"); 2888 2889 if (reachable(src)) { 2890 Assembler::movss(dst, as_Address(src)); 2891 } else { 2892 lea(rscratch, src); 2893 Assembler::movss(dst, Address(rscratch, 0)); 2894 } 2895 } 2896 2897 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2898 assert(rscratch != noreg || always_reachable(src), "missing"); 2899 2900 if (reachable(src)) { 2901 Assembler::movddup(dst, as_Address(src)); 2902 } else { 2903 lea(rscratch, src); 2904 Assembler::movddup(dst, Address(rscratch, 0)); 2905 } 2906 } 2907 2908 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2909 assert(rscratch != noreg || always_reachable(src), "missing"); 2910 2911 if (reachable(src)) { 2912 Assembler::vmovddup(dst, as_Address(src), vector_len); 2913 } else { 2914 lea(rscratch, src); 2915 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2916 } 2917 } 2918 2919 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2920 assert(rscratch != noreg || always_reachable(src), "missing"); 2921 2922 if (reachable(src)) { 2923 Assembler::mulsd(dst, as_Address(src)); 2924 } else { 2925 lea(rscratch, src); 2926 Assembler::mulsd(dst, Address(rscratch, 0)); 2927 } 2928 } 2929 2930 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2931 assert(rscratch != noreg || always_reachable(src), "missing"); 2932 2933 if (reachable(src)) { 2934 Assembler::mulss(dst, as_Address(src)); 2935 } else { 2936 lea(rscratch, src); 2937 Assembler::mulss(dst, Address(rscratch, 0)); 2938 } 2939 } 2940 2941 void MacroAssembler::null_check(Register reg, int offset) { 2942 if (needs_explicit_null_check(offset)) { 2943 // provoke OS null exception if reg is null by 2944 // accessing M[reg] w/o changing any (non-CC) registers 2945 // NOTE: cmpl is plenty here to provoke a segv 2946 cmpptr(rax, Address(reg, 0)); 2947 // Note: should probably use testl(rax, Address(reg, 0)); 2948 // may be shorter code (however, this version of 2949 // testl needs to be implemented first) 2950 } else { 2951 // nothing to do, (later) access of M[reg + offset] 2952 // will provoke OS null exception if reg is null 2953 } 2954 } 2955 2956 void MacroAssembler::os_breakpoint() { 2957 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2958 // (e.g., MSVC can't call ps() otherwise) 2959 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2960 } 2961 2962 void MacroAssembler::unimplemented(const char* what) { 2963 const char* buf = nullptr; 2964 { 2965 ResourceMark rm; 2966 stringStream ss; 2967 ss.print("unimplemented: %s", what); 2968 buf = code_string(ss.as_string()); 2969 } 2970 stop(buf); 2971 } 2972 2973 #ifdef _LP64 2974 #define XSTATE_BV 0x200 2975 #endif 2976 2977 void MacroAssembler::pop_CPU_state() { 2978 pop_FPU_state(); 2979 pop_IU_state(); 2980 } 2981 2982 void MacroAssembler::pop_FPU_state() { 2983 #ifndef _LP64 2984 frstor(Address(rsp, 0)); 2985 #else 2986 fxrstor(Address(rsp, 0)); 2987 #endif 2988 addptr(rsp, FPUStateSizeInWords * wordSize); 2989 } 2990 2991 void MacroAssembler::pop_IU_state() { 2992 popa(); 2993 LP64_ONLY(addq(rsp, 8)); 2994 popf(); 2995 } 2996 2997 // Save Integer and Float state 2998 // Warning: Stack must be 16 byte aligned (64bit) 2999 void MacroAssembler::push_CPU_state() { 3000 push_IU_state(); 3001 push_FPU_state(); 3002 } 3003 3004 void MacroAssembler::push_FPU_state() { 3005 subptr(rsp, FPUStateSizeInWords * wordSize); 3006 #ifndef _LP64 3007 fnsave(Address(rsp, 0)); 3008 fwait(); 3009 #else 3010 fxsave(Address(rsp, 0)); 3011 #endif // LP64 3012 } 3013 3014 void MacroAssembler::push_IU_state() { 3015 // Push flags first because pusha kills them 3016 pushf(); 3017 // Make sure rsp stays 16-byte aligned 3018 LP64_ONLY(subq(rsp, 8)); 3019 pusha(); 3020 } 3021 3022 void MacroAssembler::push_cont_fastpath() { 3023 if (!Continuations::enabled()) return; 3024 3025 #ifndef _LP64 3026 Register rthread = rax; 3027 Register rrealsp = rbx; 3028 push(rthread); 3029 push(rrealsp); 3030 3031 get_thread(rthread); 3032 3033 // The code below wants the original RSP. 3034 // Move it back after the pushes above. 3035 movptr(rrealsp, rsp); 3036 addptr(rrealsp, 2*wordSize); 3037 #else 3038 Register rthread = r15_thread; 3039 Register rrealsp = rsp; 3040 #endif 3041 3042 Label done; 3043 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3044 jccb(Assembler::belowEqual, done); 3045 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp); 3046 bind(done); 3047 3048 #ifndef _LP64 3049 pop(rrealsp); 3050 pop(rthread); 3051 #endif 3052 } 3053 3054 void MacroAssembler::pop_cont_fastpath() { 3055 if (!Continuations::enabled()) return; 3056 3057 #ifndef _LP64 3058 Register rthread = rax; 3059 Register rrealsp = rbx; 3060 push(rthread); 3061 push(rrealsp); 3062 3063 get_thread(rthread); 3064 3065 // The code below wants the original RSP. 3066 // Move it back after the pushes above. 3067 movptr(rrealsp, rsp); 3068 addptr(rrealsp, 2*wordSize); 3069 #else 3070 Register rthread = r15_thread; 3071 Register rrealsp = rsp; 3072 #endif 3073 3074 Label done; 3075 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3076 jccb(Assembler::below, done); 3077 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0); 3078 bind(done); 3079 3080 #ifndef _LP64 3081 pop(rrealsp); 3082 pop(rthread); 3083 #endif 3084 } 3085 3086 void MacroAssembler::inc_held_monitor_count() { 3087 #ifndef _LP64 3088 Register thread = rax; 3089 push(thread); 3090 get_thread(thread); 3091 incrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3092 pop(thread); 3093 #else // LP64 3094 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3095 #endif 3096 } 3097 3098 void MacroAssembler::dec_held_monitor_count() { 3099 #ifndef _LP64 3100 Register thread = rax; 3101 push(thread); 3102 get_thread(thread); 3103 decrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3104 pop(thread); 3105 #else // LP64 3106 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3107 #endif 3108 } 3109 3110 #ifdef ASSERT 3111 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 3112 #ifdef _LP64 3113 Label no_cont; 3114 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 3115 testl(cont, cont); 3116 jcc(Assembler::zero, no_cont); 3117 stop(name); 3118 bind(no_cont); 3119 #else 3120 Unimplemented(); 3121 #endif 3122 } 3123 #endif 3124 3125 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register 3126 if (!java_thread->is_valid()) { 3127 java_thread = rdi; 3128 get_thread(java_thread); 3129 } 3130 // we must set sp to zero to clear frame 3131 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 3132 // must clear fp, so that compiled frames are not confused; it is 3133 // possible that we need it only for debugging 3134 if (clear_fp) { 3135 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 3136 } 3137 // Always clear the pc because it could have been set by make_walkable() 3138 movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 3139 vzeroupper(); 3140 } 3141 3142 void MacroAssembler::restore_rax(Register tmp) { 3143 if (tmp == noreg) pop(rax); 3144 else if (tmp != rax) mov(rax, tmp); 3145 } 3146 3147 void MacroAssembler::round_to(Register reg, int modulus) { 3148 addptr(reg, modulus - 1); 3149 andptr(reg, -modulus); 3150 } 3151 3152 void MacroAssembler::save_rax(Register tmp) { 3153 if (tmp == noreg) push(rax); 3154 else if (tmp != rax) mov(tmp, rax); 3155 } 3156 3157 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) { 3158 if (at_return) { 3159 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 3160 // we may safely use rsp instead to perform the stack watermark check. 3161 cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset())); 3162 jcc(Assembler::above, slow_path); 3163 return; 3164 } 3165 testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 3166 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 3167 } 3168 3169 // Calls to C land 3170 // 3171 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 3172 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 3173 // has to be reset to 0. This is required to allow proper stack traversal. 3174 void MacroAssembler::set_last_Java_frame(Register java_thread, 3175 Register last_java_sp, 3176 Register last_java_fp, 3177 address last_java_pc, 3178 Register rscratch) { 3179 vzeroupper(); 3180 // determine java_thread register 3181 if (!java_thread->is_valid()) { 3182 java_thread = rdi; 3183 get_thread(java_thread); 3184 } 3185 // determine last_java_sp register 3186 if (!last_java_sp->is_valid()) { 3187 last_java_sp = rsp; 3188 } 3189 // last_java_fp is optional 3190 if (last_java_fp->is_valid()) { 3191 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 3192 } 3193 // last_java_pc is optional 3194 if (last_java_pc != nullptr) { 3195 Address java_pc(java_thread, 3196 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 3197 lea(java_pc, InternalAddress(last_java_pc), rscratch); 3198 } 3199 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 3200 } 3201 3202 void MacroAssembler::shlptr(Register dst, int imm8) { 3203 LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); 3204 } 3205 3206 void MacroAssembler::shrptr(Register dst, int imm8) { 3207 LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); 3208 } 3209 3210 void MacroAssembler::sign_extend_byte(Register reg) { 3211 if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { 3212 movsbl(reg, reg); // movsxb 3213 } else { 3214 shll(reg, 24); 3215 sarl(reg, 24); 3216 } 3217 } 3218 3219 void MacroAssembler::sign_extend_short(Register reg) { 3220 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 3221 movswl(reg, reg); // movsxw 3222 } else { 3223 shll(reg, 16); 3224 sarl(reg, 16); 3225 } 3226 } 3227 3228 void MacroAssembler::testl(Address dst, int32_t imm32) { 3229 if (imm32 >= 0 && is8bit(imm32)) { 3230 testb(dst, imm32); 3231 } else { 3232 Assembler::testl(dst, imm32); 3233 } 3234 } 3235 3236 void MacroAssembler::testl(Register dst, int32_t imm32) { 3237 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 3238 testb(dst, imm32); 3239 } else { 3240 Assembler::testl(dst, imm32); 3241 } 3242 } 3243 3244 void MacroAssembler::testl(Register dst, AddressLiteral src) { 3245 assert(always_reachable(src), "Address should be reachable"); 3246 testl(dst, as_Address(src)); 3247 } 3248 3249 #ifdef _LP64 3250 3251 void MacroAssembler::testq(Address dst, int32_t imm32) { 3252 if (imm32 >= 0) { 3253 testl(dst, imm32); 3254 } else { 3255 Assembler::testq(dst, imm32); 3256 } 3257 } 3258 3259 void MacroAssembler::testq(Register dst, int32_t imm32) { 3260 if (imm32 >= 0) { 3261 testl(dst, imm32); 3262 } else { 3263 Assembler::testq(dst, imm32); 3264 } 3265 } 3266 3267 #endif 3268 3269 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 3270 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3271 Assembler::pcmpeqb(dst, src); 3272 } 3273 3274 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 3275 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3276 Assembler::pcmpeqw(dst, src); 3277 } 3278 3279 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 3280 assert((dst->encoding() < 16),"XMM register should be 0-15"); 3281 Assembler::pcmpestri(dst, src, imm8); 3282 } 3283 3284 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 3285 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3286 Assembler::pcmpestri(dst, src, imm8); 3287 } 3288 3289 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 3290 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3291 Assembler::pmovzxbw(dst, src); 3292 } 3293 3294 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 3295 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3296 Assembler::pmovzxbw(dst, src); 3297 } 3298 3299 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 3300 assert((src->encoding() < 16),"XMM register should be 0-15"); 3301 Assembler::pmovmskb(dst, src); 3302 } 3303 3304 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 3305 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3306 Assembler::ptest(dst, src); 3307 } 3308 3309 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3310 assert(rscratch != noreg || always_reachable(src), "missing"); 3311 3312 if (reachable(src)) { 3313 Assembler::sqrtss(dst, as_Address(src)); 3314 } else { 3315 lea(rscratch, src); 3316 Assembler::sqrtss(dst, Address(rscratch, 0)); 3317 } 3318 } 3319 3320 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3321 assert(rscratch != noreg || always_reachable(src), "missing"); 3322 3323 if (reachable(src)) { 3324 Assembler::subsd(dst, as_Address(src)); 3325 } else { 3326 lea(rscratch, src); 3327 Assembler::subsd(dst, Address(rscratch, 0)); 3328 } 3329 } 3330 3331 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 3332 assert(rscratch != noreg || always_reachable(src), "missing"); 3333 3334 if (reachable(src)) { 3335 Assembler::roundsd(dst, as_Address(src), rmode); 3336 } else { 3337 lea(rscratch, src); 3338 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 3339 } 3340 } 3341 3342 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3343 assert(rscratch != noreg || always_reachable(src), "missing"); 3344 3345 if (reachable(src)) { 3346 Assembler::subss(dst, as_Address(src)); 3347 } else { 3348 lea(rscratch, src); 3349 Assembler::subss(dst, Address(rscratch, 0)); 3350 } 3351 } 3352 3353 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3354 assert(rscratch != noreg || always_reachable(src), "missing"); 3355 3356 if (reachable(src)) { 3357 Assembler::ucomisd(dst, as_Address(src)); 3358 } else { 3359 lea(rscratch, src); 3360 Assembler::ucomisd(dst, Address(rscratch, 0)); 3361 } 3362 } 3363 3364 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3365 assert(rscratch != noreg || always_reachable(src), "missing"); 3366 3367 if (reachable(src)) { 3368 Assembler::ucomiss(dst, as_Address(src)); 3369 } else { 3370 lea(rscratch, src); 3371 Assembler::ucomiss(dst, Address(rscratch, 0)); 3372 } 3373 } 3374 3375 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3376 assert(rscratch != noreg || always_reachable(src), "missing"); 3377 3378 // Used in sign-bit flipping with aligned address. 3379 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3380 if (reachable(src)) { 3381 Assembler::xorpd(dst, as_Address(src)); 3382 } else { 3383 lea(rscratch, src); 3384 Assembler::xorpd(dst, Address(rscratch, 0)); 3385 } 3386 } 3387 3388 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3389 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3390 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3391 } 3392 else { 3393 Assembler::xorpd(dst, src); 3394 } 3395 } 3396 3397 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3398 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3399 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3400 } else { 3401 Assembler::xorps(dst, src); 3402 } 3403 } 3404 3405 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 3406 assert(rscratch != noreg || always_reachable(src), "missing"); 3407 3408 // Used in sign-bit flipping with aligned address. 3409 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3410 if (reachable(src)) { 3411 Assembler::xorps(dst, as_Address(src)); 3412 } else { 3413 lea(rscratch, src); 3414 Assembler::xorps(dst, Address(rscratch, 0)); 3415 } 3416 } 3417 3418 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 3419 assert(rscratch != noreg || always_reachable(src), "missing"); 3420 3421 // Used in sign-bit flipping with aligned address. 3422 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3423 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3424 if (reachable(src)) { 3425 Assembler::pshufb(dst, as_Address(src)); 3426 } else { 3427 lea(rscratch, src); 3428 Assembler::pshufb(dst, Address(rscratch, 0)); 3429 } 3430 } 3431 3432 // AVX 3-operands instructions 3433 3434 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3435 assert(rscratch != noreg || always_reachable(src), "missing"); 3436 3437 if (reachable(src)) { 3438 vaddsd(dst, nds, as_Address(src)); 3439 } else { 3440 lea(rscratch, src); 3441 vaddsd(dst, nds, Address(rscratch, 0)); 3442 } 3443 } 3444 3445 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3446 assert(rscratch != noreg || always_reachable(src), "missing"); 3447 3448 if (reachable(src)) { 3449 vaddss(dst, nds, as_Address(src)); 3450 } else { 3451 lea(rscratch, src); 3452 vaddss(dst, nds, Address(rscratch, 0)); 3453 } 3454 } 3455 3456 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3457 assert(UseAVX > 0, "requires some form of AVX"); 3458 assert(rscratch != noreg || always_reachable(src), "missing"); 3459 3460 if (reachable(src)) { 3461 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 3462 } else { 3463 lea(rscratch, src); 3464 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 3465 } 3466 } 3467 3468 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3469 assert(UseAVX > 0, "requires some form of AVX"); 3470 assert(rscratch != noreg || always_reachable(src), "missing"); 3471 3472 if (reachable(src)) { 3473 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 3474 } else { 3475 lea(rscratch, src); 3476 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 3477 } 3478 } 3479 3480 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3481 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3482 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3483 3484 vandps(dst, nds, negate_field, vector_len, rscratch); 3485 } 3486 3487 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3488 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3489 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3490 3491 vandpd(dst, nds, negate_field, vector_len, rscratch); 3492 } 3493 3494 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3495 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3496 Assembler::vpaddb(dst, nds, src, vector_len); 3497 } 3498 3499 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3500 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3501 Assembler::vpaddb(dst, nds, src, vector_len); 3502 } 3503 3504 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3505 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3506 Assembler::vpaddw(dst, nds, src, vector_len); 3507 } 3508 3509 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3510 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3511 Assembler::vpaddw(dst, nds, src, vector_len); 3512 } 3513 3514 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3515 assert(rscratch != noreg || always_reachable(src), "missing"); 3516 3517 if (reachable(src)) { 3518 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3519 } else { 3520 lea(rscratch, src); 3521 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 3522 } 3523 } 3524 3525 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3526 assert(rscratch != noreg || always_reachable(src), "missing"); 3527 3528 if (reachable(src)) { 3529 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 3530 } else { 3531 lea(rscratch, src); 3532 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 3533 } 3534 } 3535 3536 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3537 assert(rscratch != noreg || always_reachable(src), "missing"); 3538 3539 if (reachable(src)) { 3540 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 3541 } else { 3542 lea(rscratch, src); 3543 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 3544 } 3545 } 3546 3547 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3548 assert(rscratch != noreg || always_reachable(src), "missing"); 3549 3550 if (reachable(src)) { 3551 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 3552 } else { 3553 lea(rscratch, src); 3554 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 3555 } 3556 } 3557 3558 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3559 assert(rscratch != noreg || always_reachable(src), "missing"); 3560 3561 if (reachable(src)) { 3562 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 3563 } else { 3564 lea(rscratch, src); 3565 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 3566 } 3567 } 3568 3569 // Vector float blend 3570 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3571 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3572 // WARN: Allow dst == (src1|src2), mask == scratch 3573 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3574 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst; 3575 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3576 if (blend_emulation && scratch_available && dst_available) { 3577 if (compute_mask) { 3578 vpsrad(scratch, mask, 32, vector_len); 3579 mask = scratch; 3580 } 3581 if (dst == src1) { 3582 vpandn(dst, mask, src1, vector_len); // if mask == 0, src1 3583 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3584 } else { 3585 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3586 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1 3587 } 3588 vpor(dst, dst, scratch, vector_len); 3589 } else { 3590 Assembler::vblendvps(dst, src1, src2, mask, vector_len); 3591 } 3592 } 3593 3594 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3595 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3596 // WARN: Allow dst == (src1|src2), mask == scratch 3597 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3598 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask); 3599 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3600 if (blend_emulation && scratch_available && dst_available) { 3601 if (compute_mask) { 3602 vpxor(scratch, scratch, scratch, vector_len); 3603 vpcmpgtq(scratch, scratch, mask, vector_len); 3604 mask = scratch; 3605 } 3606 if (dst == src1) { 3607 vpandn(dst, mask, src1, vector_len); // if mask == 0, src 3608 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3609 } else { 3610 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3611 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src 3612 } 3613 vpor(dst, dst, scratch, vector_len); 3614 } else { 3615 Assembler::vblendvpd(dst, src1, src2, mask, vector_len); 3616 } 3617 } 3618 3619 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3620 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3621 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3622 } 3623 3624 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3625 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3626 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3627 } 3628 3629 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3630 assert(rscratch != noreg || always_reachable(src), "missing"); 3631 3632 if (reachable(src)) { 3633 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 3634 } else { 3635 lea(rscratch, src); 3636 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 3637 } 3638 } 3639 3640 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3641 int comparison, bool is_signed, int vector_len, Register rscratch) { 3642 assert(rscratch != noreg || always_reachable(src), "missing"); 3643 3644 if (reachable(src)) { 3645 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3646 } else { 3647 lea(rscratch, src); 3648 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3649 } 3650 } 3651 3652 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3653 int comparison, bool is_signed, int vector_len, Register rscratch) { 3654 assert(rscratch != noreg || always_reachable(src), "missing"); 3655 3656 if (reachable(src)) { 3657 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3658 } else { 3659 lea(rscratch, src); 3660 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3661 } 3662 } 3663 3664 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3665 int comparison, bool is_signed, int vector_len, Register rscratch) { 3666 assert(rscratch != noreg || always_reachable(src), "missing"); 3667 3668 if (reachable(src)) { 3669 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3670 } else { 3671 lea(rscratch, src); 3672 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3673 } 3674 } 3675 3676 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3677 int comparison, bool is_signed, int vector_len, Register rscratch) { 3678 assert(rscratch != noreg || always_reachable(src), "missing"); 3679 3680 if (reachable(src)) { 3681 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3682 } else { 3683 lea(rscratch, src); 3684 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3685 } 3686 } 3687 3688 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3689 if (width == Assembler::Q) { 3690 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3691 } else { 3692 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3693 } 3694 } 3695 3696 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3697 int eq_cond_enc = 0x29; 3698 int gt_cond_enc = 0x37; 3699 if (width != Assembler::Q) { 3700 eq_cond_enc = 0x74 + width; 3701 gt_cond_enc = 0x64 + width; 3702 } 3703 switch (cond) { 3704 case eq: 3705 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3706 break; 3707 case neq: 3708 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3709 vallones(xtmp, vector_len); 3710 vpxor(dst, xtmp, dst, vector_len); 3711 break; 3712 case le: 3713 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3714 vallones(xtmp, vector_len); 3715 vpxor(dst, xtmp, dst, vector_len); 3716 break; 3717 case nlt: 3718 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3719 vallones(xtmp, vector_len); 3720 vpxor(dst, xtmp, dst, vector_len); 3721 break; 3722 case lt: 3723 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3724 break; 3725 case nle: 3726 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3727 break; 3728 default: 3729 assert(false, "Should not reach here"); 3730 } 3731 } 3732 3733 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3734 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3735 Assembler::vpmovzxbw(dst, src, vector_len); 3736 } 3737 3738 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3739 assert((src->encoding() < 16),"XMM register should be 0-15"); 3740 Assembler::vpmovmskb(dst, src, vector_len); 3741 } 3742 3743 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3744 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3745 Assembler::vpmullw(dst, nds, src, vector_len); 3746 } 3747 3748 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3749 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3750 Assembler::vpmullw(dst, nds, src, vector_len); 3751 } 3752 3753 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3754 assert((UseAVX > 0), "AVX support is needed"); 3755 assert(rscratch != noreg || always_reachable(src), "missing"); 3756 3757 if (reachable(src)) { 3758 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3759 } else { 3760 lea(rscratch, src); 3761 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3762 } 3763 } 3764 3765 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3766 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3767 Assembler::vpsubb(dst, nds, src, vector_len); 3768 } 3769 3770 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3771 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3772 Assembler::vpsubb(dst, nds, src, vector_len); 3773 } 3774 3775 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3776 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3777 Assembler::vpsubw(dst, nds, src, vector_len); 3778 } 3779 3780 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3781 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3782 Assembler::vpsubw(dst, nds, src, vector_len); 3783 } 3784 3785 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3786 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3787 Assembler::vpsraw(dst, nds, shift, vector_len); 3788 } 3789 3790 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3791 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3792 Assembler::vpsraw(dst, nds, shift, vector_len); 3793 } 3794 3795 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3796 assert(UseAVX > 2,""); 3797 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3798 vector_len = 2; 3799 } 3800 Assembler::evpsraq(dst, nds, shift, vector_len); 3801 } 3802 3803 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3804 assert(UseAVX > 2,""); 3805 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3806 vector_len = 2; 3807 } 3808 Assembler::evpsraq(dst, nds, shift, vector_len); 3809 } 3810 3811 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3812 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3813 Assembler::vpsrlw(dst, nds, shift, vector_len); 3814 } 3815 3816 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3817 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3818 Assembler::vpsrlw(dst, nds, shift, vector_len); 3819 } 3820 3821 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3822 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3823 Assembler::vpsllw(dst, nds, shift, vector_len); 3824 } 3825 3826 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3827 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3828 Assembler::vpsllw(dst, nds, shift, vector_len); 3829 } 3830 3831 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3832 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3833 Assembler::vptest(dst, src); 3834 } 3835 3836 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3837 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3838 Assembler::punpcklbw(dst, src); 3839 } 3840 3841 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3842 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3843 Assembler::pshufd(dst, src, mode); 3844 } 3845 3846 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3847 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3848 Assembler::pshuflw(dst, src, mode); 3849 } 3850 3851 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3852 assert(rscratch != noreg || always_reachable(src), "missing"); 3853 3854 if (reachable(src)) { 3855 vandpd(dst, nds, as_Address(src), vector_len); 3856 } else { 3857 lea(rscratch, src); 3858 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3859 } 3860 } 3861 3862 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3863 assert(rscratch != noreg || always_reachable(src), "missing"); 3864 3865 if (reachable(src)) { 3866 vandps(dst, nds, as_Address(src), vector_len); 3867 } else { 3868 lea(rscratch, src); 3869 vandps(dst, nds, Address(rscratch, 0), vector_len); 3870 } 3871 } 3872 3873 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3874 bool merge, int vector_len, Register rscratch) { 3875 assert(rscratch != noreg || always_reachable(src), "missing"); 3876 3877 if (reachable(src)) { 3878 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3879 } else { 3880 lea(rscratch, src); 3881 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3882 } 3883 } 3884 3885 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3886 assert(rscratch != noreg || always_reachable(src), "missing"); 3887 3888 if (reachable(src)) { 3889 vdivsd(dst, nds, as_Address(src)); 3890 } else { 3891 lea(rscratch, src); 3892 vdivsd(dst, nds, Address(rscratch, 0)); 3893 } 3894 } 3895 3896 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3897 assert(rscratch != noreg || always_reachable(src), "missing"); 3898 3899 if (reachable(src)) { 3900 vdivss(dst, nds, as_Address(src)); 3901 } else { 3902 lea(rscratch, src); 3903 vdivss(dst, nds, Address(rscratch, 0)); 3904 } 3905 } 3906 3907 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3908 assert(rscratch != noreg || always_reachable(src), "missing"); 3909 3910 if (reachable(src)) { 3911 vmulsd(dst, nds, as_Address(src)); 3912 } else { 3913 lea(rscratch, src); 3914 vmulsd(dst, nds, Address(rscratch, 0)); 3915 } 3916 } 3917 3918 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3919 assert(rscratch != noreg || always_reachable(src), "missing"); 3920 3921 if (reachable(src)) { 3922 vmulss(dst, nds, as_Address(src)); 3923 } else { 3924 lea(rscratch, src); 3925 vmulss(dst, nds, Address(rscratch, 0)); 3926 } 3927 } 3928 3929 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3930 assert(rscratch != noreg || always_reachable(src), "missing"); 3931 3932 if (reachable(src)) { 3933 vsubsd(dst, nds, as_Address(src)); 3934 } else { 3935 lea(rscratch, src); 3936 vsubsd(dst, nds, Address(rscratch, 0)); 3937 } 3938 } 3939 3940 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3941 assert(rscratch != noreg || always_reachable(src), "missing"); 3942 3943 if (reachable(src)) { 3944 vsubss(dst, nds, as_Address(src)); 3945 } else { 3946 lea(rscratch, src); 3947 vsubss(dst, nds, Address(rscratch, 0)); 3948 } 3949 } 3950 3951 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3952 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3953 assert(rscratch != noreg || always_reachable(src), "missing"); 3954 3955 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3956 } 3957 3958 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3959 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3960 assert(rscratch != noreg || always_reachable(src), "missing"); 3961 3962 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3963 } 3964 3965 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3966 assert(rscratch != noreg || always_reachable(src), "missing"); 3967 3968 if (reachable(src)) { 3969 vxorpd(dst, nds, as_Address(src), vector_len); 3970 } else { 3971 lea(rscratch, src); 3972 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3973 } 3974 } 3975 3976 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3977 assert(rscratch != noreg || always_reachable(src), "missing"); 3978 3979 if (reachable(src)) { 3980 vxorps(dst, nds, as_Address(src), vector_len); 3981 } else { 3982 lea(rscratch, src); 3983 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3984 } 3985 } 3986 3987 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3988 assert(rscratch != noreg || always_reachable(src), "missing"); 3989 3990 if (UseAVX > 1 || (vector_len < 1)) { 3991 if (reachable(src)) { 3992 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3993 } else { 3994 lea(rscratch, src); 3995 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3996 } 3997 } else { 3998 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3999 } 4000 } 4001 4002 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 4003 assert(rscratch != noreg || always_reachable(src), "missing"); 4004 4005 if (reachable(src)) { 4006 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 4007 } else { 4008 lea(rscratch, src); 4009 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 4010 } 4011 } 4012 4013 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 4014 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 4015 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 4016 // The inverted mask is sign-extended 4017 andptr(possibly_non_local, inverted_mask); 4018 } 4019 4020 void MacroAssembler::resolve_jobject(Register value, 4021 Register thread, 4022 Register tmp) { 4023 assert_different_registers(value, thread, tmp); 4024 Label done, tagged, weak_tagged; 4025 testptr(value, value); 4026 jcc(Assembler::zero, done); // Use null as-is. 4027 testptr(value, JNIHandles::tag_mask); // Test for tag. 4028 jcc(Assembler::notZero, tagged); 4029 4030 // Resolve local handle 4031 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread); 4032 verify_oop(value); 4033 jmp(done); 4034 4035 bind(tagged); 4036 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 4037 jcc(Assembler::notZero, weak_tagged); 4038 4039 // Resolve global handle 4040 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4041 verify_oop(value); 4042 jmp(done); 4043 4044 bind(weak_tagged); 4045 // Resolve jweak. 4046 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 4047 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread); 4048 verify_oop(value); 4049 4050 bind(done); 4051 } 4052 4053 void MacroAssembler::resolve_global_jobject(Register value, 4054 Register thread, 4055 Register tmp) { 4056 assert_different_registers(value, thread, tmp); 4057 Label done; 4058 4059 testptr(value, value); 4060 jcc(Assembler::zero, done); // Use null as-is. 4061 4062 #ifdef ASSERT 4063 { 4064 Label valid_global_tag; 4065 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 4066 jcc(Assembler::notZero, valid_global_tag); 4067 stop("non global jobject using resolve_global_jobject"); 4068 bind(valid_global_tag); 4069 } 4070 #endif 4071 4072 // Resolve global handle 4073 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4074 verify_oop(value); 4075 4076 bind(done); 4077 } 4078 4079 void MacroAssembler::subptr(Register dst, int32_t imm32) { 4080 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 4081 } 4082 4083 // Force generation of a 4 byte immediate value even if it fits into 8bit 4084 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 4085 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 4086 } 4087 4088 void MacroAssembler::subptr(Register dst, Register src) { 4089 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 4090 } 4091 4092 // C++ bool manipulation 4093 void MacroAssembler::testbool(Register dst) { 4094 if(sizeof(bool) == 1) 4095 testb(dst, 0xff); 4096 else if(sizeof(bool) == 2) { 4097 // testw implementation needed for two byte bools 4098 ShouldNotReachHere(); 4099 } else if(sizeof(bool) == 4) 4100 testl(dst, dst); 4101 else 4102 // unsupported 4103 ShouldNotReachHere(); 4104 } 4105 4106 void MacroAssembler::testptr(Register dst, Register src) { 4107 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); 4108 } 4109 4110 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 4111 void MacroAssembler::tlab_allocate(Register thread, Register obj, 4112 Register var_size_in_bytes, 4113 int con_size_in_bytes, 4114 Register t1, 4115 Register t2, 4116 Label& slow_case) { 4117 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 4118 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 4119 } 4120 4121 RegSet MacroAssembler::call_clobbered_gp_registers() { 4122 RegSet regs; 4123 #ifdef _LP64 4124 regs += RegSet::of(rax, rcx, rdx); 4125 #ifndef WINDOWS 4126 regs += RegSet::of(rsi, rdi); 4127 #endif 4128 regs += RegSet::range(r8, r11); 4129 #else 4130 regs += RegSet::of(rax, rcx, rdx); 4131 #endif 4132 return regs; 4133 } 4134 4135 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 4136 int num_xmm_registers = XMMRegister::available_xmm_registers(); 4137 #if defined(WINDOWS) && defined(_LP64) 4138 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 4139 if (num_xmm_registers > 16) { 4140 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 4141 } 4142 return result; 4143 #else 4144 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 4145 #endif 4146 } 4147 4148 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor 4149 4150 #ifndef _LP64 4151 static bool use_x87_registers() { return UseSSE < 2; } 4152 #endif 4153 static bool use_xmm_registers() { return UseSSE >= 1; } 4154 4155 // C1 only ever uses the first double/float of the XMM register. 4156 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); } 4157 4158 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4159 if (UseSSE == 1) { 4160 masm->movflt(Address(rsp, offset), reg); 4161 } else { 4162 masm->movdbl(Address(rsp, offset), reg); 4163 } 4164 } 4165 4166 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4167 if (UseSSE == 1) { 4168 masm->movflt(reg, Address(rsp, offset)); 4169 } else { 4170 masm->movdbl(reg, Address(rsp, offset)); 4171 } 4172 } 4173 4174 int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu, 4175 int& gp_area_size, int& fp_area_size, int& xmm_area_size) { 4176 4177 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 4178 StackAlignmentInBytes); 4179 #ifdef _LP64 4180 fp_area_size = 0; 4181 #else 4182 fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0; 4183 #endif 4184 xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0; 4185 4186 return gp_area_size + fp_area_size + xmm_area_size; 4187 } 4188 4189 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 4190 block_comment("push_call_clobbered_registers start"); 4191 // Regular registers 4192 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 4193 4194 int gp_area_size; 4195 int fp_area_size; 4196 int xmm_area_size; 4197 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 4198 gp_area_size, fp_area_size, xmm_area_size); 4199 subptr(rsp, total_save_size); 4200 4201 push_set(gp_registers_to_push, 0); 4202 4203 #ifndef _LP64 4204 if (save_fpu && use_x87_registers()) { 4205 fnsave(Address(rsp, gp_area_size)); 4206 fwait(); 4207 } 4208 #endif 4209 if (save_fpu && use_xmm_registers()) { 4210 push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4211 } 4212 4213 block_comment("push_call_clobbered_registers end"); 4214 } 4215 4216 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 4217 block_comment("pop_call_clobbered_registers start"); 4218 4219 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 4220 4221 int gp_area_size; 4222 int fp_area_size; 4223 int xmm_area_size; 4224 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 4225 gp_area_size, fp_area_size, xmm_area_size); 4226 4227 if (restore_fpu && use_xmm_registers()) { 4228 pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4229 } 4230 #ifndef _LP64 4231 if (restore_fpu && use_x87_registers()) { 4232 frstor(Address(rsp, gp_area_size)); 4233 } 4234 #endif 4235 4236 pop_set(gp_registers_to_pop, 0); 4237 4238 addptr(rsp, total_save_size); 4239 4240 vzeroupper(); 4241 4242 block_comment("pop_call_clobbered_registers end"); 4243 } 4244 4245 void MacroAssembler::push_set(XMMRegSet set, int offset) { 4246 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 4247 int spill_offset = offset; 4248 4249 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 4250 save_xmm_register(this, spill_offset, *it); 4251 spill_offset += xmm_save_size(); 4252 } 4253 } 4254 4255 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 4256 int restore_size = set.size() * xmm_save_size(); 4257 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 4258 4259 int restore_offset = offset + restore_size - xmm_save_size(); 4260 4261 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 4262 restore_xmm_register(this, restore_offset, *it); 4263 restore_offset -= xmm_save_size(); 4264 } 4265 } 4266 4267 void MacroAssembler::push_set(RegSet set, int offset) { 4268 int spill_offset; 4269 if (offset == -1) { 4270 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4271 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4272 subptr(rsp, aligned_size); 4273 spill_offset = 0; 4274 } else { 4275 spill_offset = offset; 4276 } 4277 4278 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 4279 movptr(Address(rsp, spill_offset), *it); 4280 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4281 } 4282 } 4283 4284 void MacroAssembler::pop_set(RegSet set, int offset) { 4285 4286 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4287 int restore_size = set.size() * gp_reg_size; 4288 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 4289 4290 int restore_offset; 4291 if (offset == -1) { 4292 restore_offset = restore_size - gp_reg_size; 4293 } else { 4294 restore_offset = offset + restore_size - gp_reg_size; 4295 } 4296 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 4297 movptr(*it, Address(rsp, restore_offset)); 4298 restore_offset -= gp_reg_size; 4299 } 4300 4301 if (offset == -1) { 4302 addptr(rsp, aligned_size); 4303 } 4304 } 4305 4306 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 4307 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 4308 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 4309 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 4310 Label done; 4311 4312 testptr(length_in_bytes, length_in_bytes); 4313 jcc(Assembler::zero, done); 4314 4315 // initialize topmost word, divide index by 2, check if odd and test if zero 4316 // note: for the remaining code to work, index must be a multiple of BytesPerWord 4317 #ifdef ASSERT 4318 { 4319 Label L; 4320 testptr(length_in_bytes, BytesPerWord - 1); 4321 jcc(Assembler::zero, L); 4322 stop("length must be a multiple of BytesPerWord"); 4323 bind(L); 4324 } 4325 #endif 4326 Register index = length_in_bytes; 4327 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 4328 if (UseIncDec) { 4329 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 4330 } else { 4331 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 4332 shrptr(index, 1); 4333 } 4334 #ifndef _LP64 4335 // index could have not been a multiple of 8 (i.e., bit 2 was set) 4336 { 4337 Label even; 4338 // note: if index was a multiple of 8, then it cannot 4339 // be 0 now otherwise it must have been 0 before 4340 // => if it is even, we don't need to check for 0 again 4341 jcc(Assembler::carryClear, even); 4342 // clear topmost word (no jump would be needed if conditional assignment worked here) 4343 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); 4344 // index could be 0 now, must check again 4345 jcc(Assembler::zero, done); 4346 bind(even); 4347 } 4348 #endif // !_LP64 4349 // initialize remaining object fields: index is a multiple of 2 now 4350 { 4351 Label loop; 4352 bind(loop); 4353 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 4354 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) 4355 decrement(index); 4356 jcc(Assembler::notZero, loop); 4357 } 4358 4359 bind(done); 4360 } 4361 4362 // Look up the method for a megamorphic invokeinterface call. 4363 // The target method is determined by <intf_klass, itable_index>. 4364 // The receiver klass is in recv_klass. 4365 // On success, the result will be in method_result, and execution falls through. 4366 // On failure, execution transfers to the given label. 4367 void MacroAssembler::lookup_interface_method(Register recv_klass, 4368 Register intf_klass, 4369 RegisterOrConstant itable_index, 4370 Register method_result, 4371 Register scan_temp, 4372 Label& L_no_such_interface, 4373 bool return_method) { 4374 assert_different_registers(recv_klass, intf_klass, scan_temp); 4375 assert_different_registers(method_result, intf_klass, scan_temp); 4376 assert(recv_klass != method_result || !return_method, 4377 "recv_klass can be destroyed when method isn't needed"); 4378 4379 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 4380 "caller must use same register for non-constant itable index as for method"); 4381 4382 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 4383 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4384 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4385 int scan_step = itableOffsetEntry::size() * wordSize; 4386 int vte_size = vtableEntry::size_in_bytes(); 4387 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4388 assert(vte_size == wordSize, "else adjust times_vte_scale"); 4389 4390 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4391 4392 // %%% Could store the aligned, prescaled offset in the klassoop. 4393 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 4394 4395 if (return_method) { 4396 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 4397 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4398 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 4399 } 4400 4401 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 4402 // if (scan->interface() == intf) { 4403 // result = (klass + scan->offset() + itable_index); 4404 // } 4405 // } 4406 Label search, found_method; 4407 4408 for (int peel = 1; peel >= 0; peel--) { 4409 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 4410 cmpptr(intf_klass, method_result); 4411 4412 if (peel) { 4413 jccb(Assembler::equal, found_method); 4414 } else { 4415 jccb(Assembler::notEqual, search); 4416 // (invert the test to fall through to found_method...) 4417 } 4418 4419 if (!peel) break; 4420 4421 bind(search); 4422 4423 // Check that the previous entry is non-null. A null entry means that 4424 // the receiver class doesn't implement the interface, and wasn't the 4425 // same as when the caller was compiled. 4426 testptr(method_result, method_result); 4427 jcc(Assembler::zero, L_no_such_interface); 4428 addptr(scan_temp, scan_step); 4429 } 4430 4431 bind(found_method); 4432 4433 if (return_method) { 4434 // Got a hit. 4435 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 4436 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 4437 } 4438 } 4439 4440 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 4441 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder 4442 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 4443 // The target method is determined by <holder_klass, itable_index>. 4444 // The receiver klass is in recv_klass. 4445 // On success, the result will be in method_result, and execution falls through. 4446 // On failure, execution transfers to the given label. 4447 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 4448 Register holder_klass, 4449 Register resolved_klass, 4450 Register method_result, 4451 Register scan_temp, 4452 Register temp_reg2, 4453 Register receiver, 4454 int itable_index, 4455 Label& L_no_such_interface) { 4456 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 4457 Register temp_itbl_klass = method_result; 4458 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 4459 4460 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4461 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4462 int scan_step = itableOffsetEntry::size() * wordSize; 4463 int vte_size = vtableEntry::size_in_bytes(); 4464 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 4465 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 4466 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4467 assert(vte_size == wordSize, "adjust times_vte_scale"); 4468 4469 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 4470 4471 // temp_itbl_klass = recv_klass.itable[0] 4472 // scan_temp = &recv_klass.itable[0] + step 4473 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4474 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 4475 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 4476 xorptr(temp_reg, temp_reg); 4477 4478 // Initial checks: 4479 // - if (holder_klass != resolved_klass), go to "scan for resolved" 4480 // - if (itable[0] == 0), no such interface 4481 // - if (itable[0] == holder_klass), shortcut to "holder found" 4482 cmpptr(holder_klass, resolved_klass); 4483 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 4484 testptr(temp_itbl_klass, temp_itbl_klass); 4485 jccb(Assembler::zero, L_no_such_interface); 4486 cmpptr(holder_klass, temp_itbl_klass); 4487 jccb(Assembler::equal, L_holder_found); 4488 4489 // Loop: Look for holder_klass record in itable 4490 // do { 4491 // tmp = itable[index]; 4492 // index += step; 4493 // if (tmp == holder_klass) { 4494 // goto L_holder_found; // Found! 4495 // } 4496 // } while (tmp != 0); 4497 // goto L_no_such_interface // Not found. 4498 Label L_scan_holder; 4499 bind(L_scan_holder); 4500 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4501 addptr(scan_temp, scan_step); 4502 cmpptr(holder_klass, temp_itbl_klass); 4503 jccb(Assembler::equal, L_holder_found); 4504 testptr(temp_itbl_klass, temp_itbl_klass); 4505 jccb(Assembler::notZero, L_scan_holder); 4506 4507 jmpb(L_no_such_interface); 4508 4509 // Loop: Look for resolved_class record in itable 4510 // do { 4511 // tmp = itable[index]; 4512 // index += step; 4513 // if (tmp == holder_klass) { 4514 // // Also check if we have met a holder klass 4515 // holder_tmp = itable[index-step-ioffset]; 4516 // } 4517 // if (tmp == resolved_klass) { 4518 // goto L_resolved_found; // Found! 4519 // } 4520 // } while (tmp != 0); 4521 // goto L_no_such_interface // Not found. 4522 // 4523 Label L_loop_scan_resolved; 4524 bind(L_loop_scan_resolved); 4525 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4526 addptr(scan_temp, scan_step); 4527 bind(L_loop_scan_resolved_entry); 4528 cmpptr(holder_klass, temp_itbl_klass); 4529 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4530 cmpptr(resolved_klass, temp_itbl_klass); 4531 jccb(Assembler::equal, L_resolved_found); 4532 testptr(temp_itbl_klass, temp_itbl_klass); 4533 jccb(Assembler::notZero, L_loop_scan_resolved); 4534 4535 jmpb(L_no_such_interface); 4536 4537 Label L_ready; 4538 4539 // See if we already have a holder klass. If not, go and scan for it. 4540 bind(L_resolved_found); 4541 testptr(temp_reg, temp_reg); 4542 jccb(Assembler::zero, L_scan_holder); 4543 jmpb(L_ready); 4544 4545 bind(L_holder_found); 4546 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4547 4548 // Finally, temp_reg contains holder_klass vtable offset 4549 bind(L_ready); 4550 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4551 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 4552 load_klass(scan_temp, receiver, noreg); 4553 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4554 } else { 4555 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4556 } 4557 } 4558 4559 4560 // virtual method calling 4561 void MacroAssembler::lookup_virtual_method(Register recv_klass, 4562 RegisterOrConstant vtable_index, 4563 Register method_result) { 4564 const ByteSize base = Klass::vtable_start_offset(); 4565 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 4566 Address vtable_entry_addr(recv_klass, 4567 vtable_index, Address::times_ptr, 4568 base + vtableEntry::method_offset()); 4569 movptr(method_result, vtable_entry_addr); 4570 } 4571 4572 4573 void MacroAssembler::check_klass_subtype(Register sub_klass, 4574 Register super_klass, 4575 Register temp_reg, 4576 Label& L_success) { 4577 Label L_failure; 4578 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 4579 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 4580 bind(L_failure); 4581 } 4582 4583 4584 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 4585 Register super_klass, 4586 Register temp_reg, 4587 Label* L_success, 4588 Label* L_failure, 4589 Label* L_slow_path, 4590 RegisterOrConstant super_check_offset) { 4591 assert_different_registers(sub_klass, super_klass, temp_reg); 4592 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 4593 if (super_check_offset.is_register()) { 4594 assert_different_registers(sub_klass, super_klass, 4595 super_check_offset.as_register()); 4596 } else if (must_load_sco) { 4597 assert(temp_reg != noreg, "supply either a temp or a register offset"); 4598 } 4599 4600 Label L_fallthrough; 4601 int label_nulls = 0; 4602 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4603 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4604 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 4605 assert(label_nulls <= 1, "at most one null in the batch"); 4606 4607 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4608 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 4609 Address super_check_offset_addr(super_klass, sco_offset); 4610 4611 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 4612 // range of a jccb. If this routine grows larger, reconsider at 4613 // least some of these. 4614 #define local_jcc(assembler_cond, label) \ 4615 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 4616 else jcc( assembler_cond, label) /*omit semi*/ 4617 4618 // Hacked jmp, which may only be used just before L_fallthrough. 4619 #define final_jmp(label) \ 4620 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 4621 else jmp(label) /*omit semi*/ 4622 4623 // If the pointers are equal, we are done (e.g., String[] elements). 4624 // This self-check enables sharing of secondary supertype arrays among 4625 // non-primary types such as array-of-interface. Otherwise, each such 4626 // type would need its own customized SSA. 4627 // We move this check to the front of the fast path because many 4628 // type checks are in fact trivially successful in this manner, 4629 // so we get a nicely predicted branch right at the start of the check. 4630 cmpptr(sub_klass, super_klass); 4631 local_jcc(Assembler::equal, *L_success); 4632 4633 // Check the supertype display: 4634 if (must_load_sco) { 4635 // Positive movl does right thing on LP64. 4636 movl(temp_reg, super_check_offset_addr); 4637 super_check_offset = RegisterOrConstant(temp_reg); 4638 } 4639 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 4640 cmpptr(super_klass, super_check_addr); // load displayed supertype 4641 4642 // This check has worked decisively for primary supers. 4643 // Secondary supers are sought in the super_cache ('super_cache_addr'). 4644 // (Secondary supers are interfaces and very deeply nested subtypes.) 4645 // This works in the same check above because of a tricky aliasing 4646 // between the super_cache and the primary super display elements. 4647 // (The 'super_check_addr' can address either, as the case requires.) 4648 // Note that the cache is updated below if it does not help us find 4649 // what we need immediately. 4650 // So if it was a primary super, we can just fail immediately. 4651 // Otherwise, it's the slow path for us (no success at this point). 4652 4653 if (super_check_offset.is_register()) { 4654 local_jcc(Assembler::equal, *L_success); 4655 cmpl(super_check_offset.as_register(), sc_offset); 4656 if (L_failure == &L_fallthrough) { 4657 local_jcc(Assembler::equal, *L_slow_path); 4658 } else { 4659 local_jcc(Assembler::notEqual, *L_failure); 4660 final_jmp(*L_slow_path); 4661 } 4662 } else if (super_check_offset.as_constant() == sc_offset) { 4663 // Need a slow path; fast failure is impossible. 4664 if (L_slow_path == &L_fallthrough) { 4665 local_jcc(Assembler::equal, *L_success); 4666 } else { 4667 local_jcc(Assembler::notEqual, *L_slow_path); 4668 final_jmp(*L_success); 4669 } 4670 } else { 4671 // No slow path; it's a fast decision. 4672 if (L_failure == &L_fallthrough) { 4673 local_jcc(Assembler::equal, *L_success); 4674 } else { 4675 local_jcc(Assembler::notEqual, *L_failure); 4676 final_jmp(*L_success); 4677 } 4678 } 4679 4680 bind(L_fallthrough); 4681 4682 #undef local_jcc 4683 #undef final_jmp 4684 } 4685 4686 4687 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4688 Register super_klass, 4689 Register temp_reg, 4690 Register temp2_reg, 4691 Label* L_success, 4692 Label* L_failure, 4693 bool set_cond_codes) { 4694 assert_different_registers(sub_klass, super_klass, temp_reg); 4695 if (temp2_reg != noreg) 4696 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 4697 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 4698 4699 Label L_fallthrough; 4700 int label_nulls = 0; 4701 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4702 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4703 assert(label_nulls <= 1, "at most one null in the batch"); 4704 4705 // a couple of useful fields in sub_klass: 4706 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 4707 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4708 Address secondary_supers_addr(sub_klass, ss_offset); 4709 Address super_cache_addr( sub_klass, sc_offset); 4710 4711 // Do a linear scan of the secondary super-klass chain. 4712 // This code is rarely used, so simplicity is a virtue here. 4713 // The repne_scan instruction uses fixed registers, which we must spill. 4714 // Don't worry too much about pre-existing connections with the input regs. 4715 4716 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 4717 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 4718 4719 // Get super_klass value into rax (even if it was in rdi or rcx). 4720 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 4721 if (super_klass != rax) { 4722 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 4723 mov(rax, super_klass); 4724 } 4725 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 4726 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 4727 4728 #ifndef PRODUCT 4729 uint* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4730 ExternalAddress pst_counter_addr((address) pst_counter); 4731 NOT_LP64( incrementl(pst_counter_addr) ); 4732 LP64_ONLY( lea(rcx, pst_counter_addr) ); 4733 LP64_ONLY( incrementl(Address(rcx, 0)) ); 4734 #endif //PRODUCT 4735 4736 // We will consult the secondary-super array. 4737 movptr(rdi, secondary_supers_addr); 4738 // Load the array length. (Positive movl does right thing on LP64.) 4739 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4740 // Skip to start of data. 4741 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4742 4743 // Scan RCX words at [RDI] for an occurrence of RAX. 4744 // Set NZ/Z based on last compare. 4745 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4746 // not change flags (only scas instruction which is repeated sets flags). 4747 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4748 4749 testptr(rax,rax); // Set Z = 0 4750 repne_scan(); 4751 4752 // Unspill the temp. registers: 4753 if (pushed_rdi) pop(rdi); 4754 if (pushed_rcx) pop(rcx); 4755 if (pushed_rax) pop(rax); 4756 4757 if (set_cond_codes) { 4758 // Special hack for the AD files: rdi is guaranteed non-zero. 4759 assert(!pushed_rdi, "rdi must be left non-null"); 4760 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4761 } 4762 4763 if (L_failure == &L_fallthrough) 4764 jccb(Assembler::notEqual, *L_failure); 4765 else jcc(Assembler::notEqual, *L_failure); 4766 4767 // Success. Cache the super we found and proceed in triumph. 4768 movptr(super_cache_addr, super_klass); 4769 4770 if (L_success != &L_fallthrough) { 4771 jmp(*L_success); 4772 } 4773 4774 #undef IS_A_TEMP 4775 4776 bind(L_fallthrough); 4777 } 4778 4779 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { 4780 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 4781 4782 Label L_fallthrough; 4783 if (L_fast_path == nullptr) { 4784 L_fast_path = &L_fallthrough; 4785 } else if (L_slow_path == nullptr) { 4786 L_slow_path = &L_fallthrough; 4787 } 4788 4789 // Fast path check: class is fully initialized 4790 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 4791 jcc(Assembler::equal, *L_fast_path); 4792 4793 // Fast path check: current thread is initializer thread 4794 cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); 4795 if (L_slow_path == &L_fallthrough) { 4796 jcc(Assembler::equal, *L_fast_path); 4797 bind(*L_slow_path); 4798 } else if (L_fast_path == &L_fallthrough) { 4799 jcc(Assembler::notEqual, *L_slow_path); 4800 bind(*L_fast_path); 4801 } else { 4802 Unimplemented(); 4803 } 4804 } 4805 4806 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 4807 if (VM_Version::supports_cmov()) { 4808 cmovl(cc, dst, src); 4809 } else { 4810 Label L; 4811 jccb(negate_condition(cc), L); 4812 movl(dst, src); 4813 bind(L); 4814 } 4815 } 4816 4817 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 4818 if (VM_Version::supports_cmov()) { 4819 cmovl(cc, dst, src); 4820 } else { 4821 Label L; 4822 jccb(negate_condition(cc), L); 4823 movl(dst, src); 4824 bind(L); 4825 } 4826 } 4827 4828 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 4829 if (!VerifyOops) return; 4830 4831 BLOCK_COMMENT("verify_oop {"); 4832 #ifdef _LP64 4833 push(rscratch1); 4834 #endif 4835 push(rax); // save rax 4836 push(reg); // pass register argument 4837 4838 // Pass register number to verify_oop_subroutine 4839 const char* b = nullptr; 4840 { 4841 ResourceMark rm; 4842 stringStream ss; 4843 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 4844 b = code_string(ss.as_string()); 4845 } 4846 ExternalAddress buffer((address) b); 4847 pushptr(buffer.addr(), rscratch1); 4848 4849 // call indirectly to solve generation ordering problem 4850 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4851 call(rax); 4852 // Caller pops the arguments (oop, message) and restores rax, r10 4853 BLOCK_COMMENT("} verify_oop"); 4854 } 4855 4856 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 4857 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 4858 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 4859 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 4860 vpternlogd(dst, 0xFF, dst, dst, vector_len); 4861 } else if (VM_Version::supports_avx()) { 4862 vpcmpeqd(dst, dst, dst, vector_len); 4863 } else { 4864 assert(VM_Version::supports_sse2(), ""); 4865 pcmpeqd(dst, dst); 4866 } 4867 } 4868 4869 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 4870 int extra_slot_offset) { 4871 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 4872 int stackElementSize = Interpreter::stackElementSize; 4873 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 4874 #ifdef ASSERT 4875 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 4876 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 4877 #endif 4878 Register scale_reg = noreg; 4879 Address::ScaleFactor scale_factor = Address::no_scale; 4880 if (arg_slot.is_constant()) { 4881 offset += arg_slot.as_constant() * stackElementSize; 4882 } else { 4883 scale_reg = arg_slot.as_register(); 4884 scale_factor = Address::times(stackElementSize); 4885 } 4886 offset += wordSize; // return PC is on stack 4887 return Address(rsp, scale_reg, scale_factor, offset); 4888 } 4889 4890 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 4891 if (!VerifyOops) return; 4892 4893 #ifdef _LP64 4894 push(rscratch1); 4895 #endif 4896 push(rax); // save rax, 4897 // addr may contain rsp so we will have to adjust it based on the push 4898 // we just did (and on 64 bit we do two pushes) 4899 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 4900 // stores rax into addr which is backwards of what was intended. 4901 if (addr.uses(rsp)) { 4902 lea(rax, addr); 4903 pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); 4904 } else { 4905 pushptr(addr); 4906 } 4907 4908 // Pass register number to verify_oop_subroutine 4909 const char* b = nullptr; 4910 { 4911 ResourceMark rm; 4912 stringStream ss; 4913 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 4914 b = code_string(ss.as_string()); 4915 } 4916 ExternalAddress buffer((address) b); 4917 pushptr(buffer.addr(), rscratch1); 4918 4919 // call indirectly to solve generation ordering problem 4920 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4921 call(rax); 4922 // Caller pops the arguments (addr, message) and restores rax, r10. 4923 } 4924 4925 void MacroAssembler::verify_tlab() { 4926 #ifdef ASSERT 4927 if (UseTLAB && VerifyOops) { 4928 Label next, ok; 4929 Register t1 = rsi; 4930 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); 4931 4932 push(t1); 4933 NOT_LP64(push(thread_reg)); 4934 NOT_LP64(get_thread(thread_reg)); 4935 4936 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4937 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); 4938 jcc(Assembler::aboveEqual, next); 4939 STOP("assert(top >= start)"); 4940 should_not_reach_here(); 4941 4942 bind(next); 4943 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); 4944 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4945 jcc(Assembler::aboveEqual, ok); 4946 STOP("assert(top <= end)"); 4947 should_not_reach_here(); 4948 4949 bind(ok); 4950 NOT_LP64(pop(thread_reg)); 4951 pop(t1); 4952 } 4953 #endif 4954 } 4955 4956 class ControlWord { 4957 public: 4958 int32_t _value; 4959 4960 int rounding_control() const { return (_value >> 10) & 3 ; } 4961 int precision_control() const { return (_value >> 8) & 3 ; } 4962 bool precision() const { return ((_value >> 5) & 1) != 0; } 4963 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4964 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4965 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4966 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4967 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4968 4969 void print() const { 4970 // rounding control 4971 const char* rc; 4972 switch (rounding_control()) { 4973 case 0: rc = "round near"; break; 4974 case 1: rc = "round down"; break; 4975 case 2: rc = "round up "; break; 4976 case 3: rc = "chop "; break; 4977 default: 4978 rc = nullptr; // silence compiler warnings 4979 fatal("Unknown rounding control: %d", rounding_control()); 4980 }; 4981 // precision control 4982 const char* pc; 4983 switch (precision_control()) { 4984 case 0: pc = "24 bits "; break; 4985 case 1: pc = "reserved"; break; 4986 case 2: pc = "53 bits "; break; 4987 case 3: pc = "64 bits "; break; 4988 default: 4989 pc = nullptr; // silence compiler warnings 4990 fatal("Unknown precision control: %d", precision_control()); 4991 }; 4992 // flags 4993 char f[9]; 4994 f[0] = ' '; 4995 f[1] = ' '; 4996 f[2] = (precision ()) ? 'P' : 'p'; 4997 f[3] = (underflow ()) ? 'U' : 'u'; 4998 f[4] = (overflow ()) ? 'O' : 'o'; 4999 f[5] = (zero_divide ()) ? 'Z' : 'z'; 5000 f[6] = (denormalized()) ? 'D' : 'd'; 5001 f[7] = (invalid ()) ? 'I' : 'i'; 5002 f[8] = '\x0'; 5003 // output 5004 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 5005 } 5006 5007 }; 5008 5009 class StatusWord { 5010 public: 5011 int32_t _value; 5012 5013 bool busy() const { return ((_value >> 15) & 1) != 0; } 5014 bool C3() const { return ((_value >> 14) & 1) != 0; } 5015 bool C2() const { return ((_value >> 10) & 1) != 0; } 5016 bool C1() const { return ((_value >> 9) & 1) != 0; } 5017 bool C0() const { return ((_value >> 8) & 1) != 0; } 5018 int top() const { return (_value >> 11) & 7 ; } 5019 bool error_status() const { return ((_value >> 7) & 1) != 0; } 5020 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 5021 bool precision() const { return ((_value >> 5) & 1) != 0; } 5022 bool underflow() const { return ((_value >> 4) & 1) != 0; } 5023 bool overflow() const { return ((_value >> 3) & 1) != 0; } 5024 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 5025 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 5026 bool invalid() const { return ((_value >> 0) & 1) != 0; } 5027 5028 void print() const { 5029 // condition codes 5030 char c[5]; 5031 c[0] = (C3()) ? '3' : '-'; 5032 c[1] = (C2()) ? '2' : '-'; 5033 c[2] = (C1()) ? '1' : '-'; 5034 c[3] = (C0()) ? '0' : '-'; 5035 c[4] = '\x0'; 5036 // flags 5037 char f[9]; 5038 f[0] = (error_status()) ? 'E' : '-'; 5039 f[1] = (stack_fault ()) ? 'S' : '-'; 5040 f[2] = (precision ()) ? 'P' : '-'; 5041 f[3] = (underflow ()) ? 'U' : '-'; 5042 f[4] = (overflow ()) ? 'O' : '-'; 5043 f[5] = (zero_divide ()) ? 'Z' : '-'; 5044 f[6] = (denormalized()) ? 'D' : '-'; 5045 f[7] = (invalid ()) ? 'I' : '-'; 5046 f[8] = '\x0'; 5047 // output 5048 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 5049 } 5050 5051 }; 5052 5053 class TagWord { 5054 public: 5055 int32_t _value; 5056 5057 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 5058 5059 void print() const { 5060 printf("%04x", _value & 0xFFFF); 5061 } 5062 5063 }; 5064 5065 class FPU_Register { 5066 public: 5067 int32_t _m0; 5068 int32_t _m1; 5069 int16_t _ex; 5070 5071 bool is_indefinite() const { 5072 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 5073 } 5074 5075 void print() const { 5076 char sign = (_ex < 0) ? '-' : '+'; 5077 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 5078 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 5079 }; 5080 5081 }; 5082 5083 class FPU_State { 5084 public: 5085 enum { 5086 register_size = 10, 5087 number_of_registers = 8, 5088 register_mask = 7 5089 }; 5090 5091 ControlWord _control_word; 5092 StatusWord _status_word; 5093 TagWord _tag_word; 5094 int32_t _error_offset; 5095 int32_t _error_selector; 5096 int32_t _data_offset; 5097 int32_t _data_selector; 5098 int8_t _register[register_size * number_of_registers]; 5099 5100 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 5101 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 5102 5103 const char* tag_as_string(int tag) const { 5104 switch (tag) { 5105 case 0: return "valid"; 5106 case 1: return "zero"; 5107 case 2: return "special"; 5108 case 3: return "empty"; 5109 } 5110 ShouldNotReachHere(); 5111 return nullptr; 5112 } 5113 5114 void print() const { 5115 // print computation registers 5116 { int t = _status_word.top(); 5117 for (int i = 0; i < number_of_registers; i++) { 5118 int j = (i - t) & register_mask; 5119 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 5120 st(j)->print(); 5121 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 5122 } 5123 } 5124 printf("\n"); 5125 // print control registers 5126 printf("ctrl = "); _control_word.print(); printf("\n"); 5127 printf("stat = "); _status_word .print(); printf("\n"); 5128 printf("tags = "); _tag_word .print(); printf("\n"); 5129 } 5130 5131 }; 5132 5133 class Flag_Register { 5134 public: 5135 int32_t _value; 5136 5137 bool overflow() const { return ((_value >> 11) & 1) != 0; } 5138 bool direction() const { return ((_value >> 10) & 1) != 0; } 5139 bool sign() const { return ((_value >> 7) & 1) != 0; } 5140 bool zero() const { return ((_value >> 6) & 1) != 0; } 5141 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 5142 bool parity() const { return ((_value >> 2) & 1) != 0; } 5143 bool carry() const { return ((_value >> 0) & 1) != 0; } 5144 5145 void print() const { 5146 // flags 5147 char f[8]; 5148 f[0] = (overflow ()) ? 'O' : '-'; 5149 f[1] = (direction ()) ? 'D' : '-'; 5150 f[2] = (sign ()) ? 'S' : '-'; 5151 f[3] = (zero ()) ? 'Z' : '-'; 5152 f[4] = (auxiliary_carry()) ? 'A' : '-'; 5153 f[5] = (parity ()) ? 'P' : '-'; 5154 f[6] = (carry ()) ? 'C' : '-'; 5155 f[7] = '\x0'; 5156 // output 5157 printf("%08x flags = %s", _value, f); 5158 } 5159 5160 }; 5161 5162 class IU_Register { 5163 public: 5164 int32_t _value; 5165 5166 void print() const { 5167 printf("%08x %11d", _value, _value); 5168 } 5169 5170 }; 5171 5172 class IU_State { 5173 public: 5174 Flag_Register _eflags; 5175 IU_Register _rdi; 5176 IU_Register _rsi; 5177 IU_Register _rbp; 5178 IU_Register _rsp; 5179 IU_Register _rbx; 5180 IU_Register _rdx; 5181 IU_Register _rcx; 5182 IU_Register _rax; 5183 5184 void print() const { 5185 // computation registers 5186 printf("rax, = "); _rax.print(); printf("\n"); 5187 printf("rbx, = "); _rbx.print(); printf("\n"); 5188 printf("rcx = "); _rcx.print(); printf("\n"); 5189 printf("rdx = "); _rdx.print(); printf("\n"); 5190 printf("rdi = "); _rdi.print(); printf("\n"); 5191 printf("rsi = "); _rsi.print(); printf("\n"); 5192 printf("rbp, = "); _rbp.print(); printf("\n"); 5193 printf("rsp = "); _rsp.print(); printf("\n"); 5194 printf("\n"); 5195 // control registers 5196 printf("flgs = "); _eflags.print(); printf("\n"); 5197 } 5198 }; 5199 5200 5201 class CPU_State { 5202 public: 5203 FPU_State _fpu_state; 5204 IU_State _iu_state; 5205 5206 void print() const { 5207 printf("--------------------------------------------------\n"); 5208 _iu_state .print(); 5209 printf("\n"); 5210 _fpu_state.print(); 5211 printf("--------------------------------------------------\n"); 5212 } 5213 5214 }; 5215 5216 5217 static void _print_CPU_state(CPU_State* state) { 5218 state->print(); 5219 }; 5220 5221 5222 void MacroAssembler::print_CPU_state() { 5223 push_CPU_state(); 5224 push(rsp); // pass CPU state 5225 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5226 addptr(rsp, wordSize); // discard argument 5227 pop_CPU_state(); 5228 } 5229 5230 5231 #ifndef _LP64 5232 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { 5233 static int counter = 0; 5234 FPU_State* fs = &state->_fpu_state; 5235 counter++; 5236 // For leaf calls, only verify that the top few elements remain empty. 5237 // We only need 1 empty at the top for C2 code. 5238 if( stack_depth < 0 ) { 5239 if( fs->tag_for_st(7) != 3 ) { 5240 printf("FPR7 not empty\n"); 5241 state->print(); 5242 assert(false, "error"); 5243 return false; 5244 } 5245 return true; // All other stack states do not matter 5246 } 5247 5248 assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(), 5249 "bad FPU control word"); 5250 5251 // compute stack depth 5252 int i = 0; 5253 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; 5254 int d = i; 5255 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; 5256 // verify findings 5257 if (i != FPU_State::number_of_registers) { 5258 // stack not contiguous 5259 printf("%s: stack not contiguous at ST%d\n", s, i); 5260 state->print(); 5261 assert(false, "error"); 5262 return false; 5263 } 5264 // check if computed stack depth corresponds to expected stack depth 5265 if (stack_depth < 0) { 5266 // expected stack depth is -stack_depth or less 5267 if (d > -stack_depth) { 5268 // too many elements on the stack 5269 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); 5270 state->print(); 5271 assert(false, "error"); 5272 return false; 5273 } 5274 } else { 5275 // expected stack depth is stack_depth 5276 if (d != stack_depth) { 5277 // wrong stack depth 5278 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); 5279 state->print(); 5280 assert(false, "error"); 5281 return false; 5282 } 5283 } 5284 // everything is cool 5285 return true; 5286 } 5287 5288 void MacroAssembler::verify_FPU(int stack_depth, const char* s) { 5289 if (!VerifyFPU) return; 5290 push_CPU_state(); 5291 push(rsp); // pass CPU state 5292 ExternalAddress msg((address) s); 5293 // pass message string s 5294 pushptr(msg.addr(), noreg); 5295 push(stack_depth); // pass stack depth 5296 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); 5297 addptr(rsp, 3 * wordSize); // discard arguments 5298 // check for error 5299 { Label L; 5300 testl(rax, rax); 5301 jcc(Assembler::notZero, L); 5302 int3(); // break if error condition 5303 bind(L); 5304 } 5305 pop_CPU_state(); 5306 } 5307 #endif // _LP64 5308 5309 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5310 // Either restore the MXCSR register after returning from the JNI Call 5311 // or verify that it wasn't changed (with -Xcheck:jni flag). 5312 if (VM_Version::supports_sse()) { 5313 if (RestoreMXCSROnJNICalls) { 5314 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5315 } else if (CheckJNICalls) { 5316 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5317 } 5318 } 5319 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5320 vzeroupper(); 5321 5322 #ifndef _LP64 5323 // Either restore the x87 floating pointer control word after returning 5324 // from the JNI call or verify that it wasn't changed. 5325 if (CheckJNICalls) { 5326 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); 5327 } 5328 #endif // _LP64 5329 } 5330 5331 // ((OopHandle)result).resolve(); 5332 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5333 assert_different_registers(result, tmp); 5334 5335 // Only 64 bit platforms support GCs that require a tmp register 5336 // Only IN_HEAP loads require a thread_tmp register 5337 // OopHandle::resolve is an indirection like jobject. 5338 access_load_at(T_OBJECT, IN_NATIVE, 5339 result, Address(result, 0), tmp, /*tmp_thread*/noreg); 5340 } 5341 5342 // ((WeakHandle)result).resolve(); 5343 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5344 assert_different_registers(rresult, rtmp); 5345 Label resolved; 5346 5347 // A null weak handle resolves to null. 5348 cmpptr(rresult, 0); 5349 jcc(Assembler::equal, resolved); 5350 5351 // Only 64 bit platforms support GCs that require a tmp register 5352 // Only IN_HEAP loads require a thread_tmp register 5353 // WeakHandle::resolve is an indirection like jweak. 5354 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5355 rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); 5356 bind(resolved); 5357 } 5358 5359 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5360 // get mirror 5361 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5362 load_method_holder(mirror, method); 5363 movptr(mirror, Address(mirror, mirror_offset)); 5364 resolve_oop_handle(mirror, tmp); 5365 } 5366 5367 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5368 load_method_holder(rresult, rmethod); 5369 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5370 } 5371 5372 void MacroAssembler::load_method_holder(Register holder, Register method) { 5373 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5374 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5375 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5376 } 5377 5378 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5379 assert_different_registers(src, tmp); 5380 assert_different_registers(dst, tmp); 5381 #ifdef _LP64 5382 if (UseCompressedClassPointers) { 5383 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5384 decode_klass_not_null(dst, tmp); 5385 } else 5386 #endif 5387 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5388 } 5389 5390 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 5391 assert_different_registers(src, tmp); 5392 assert_different_registers(dst, tmp); 5393 #ifdef _LP64 5394 if (UseCompressedClassPointers) { 5395 encode_klass_not_null(src, tmp); 5396 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5397 } else 5398 #endif 5399 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5400 } 5401 5402 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 5403 Register tmp1, Register thread_tmp) { 5404 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5405 decorators = AccessInternal::decorator_fixup(decorators, type); 5406 bool as_raw = (decorators & AS_RAW) != 0; 5407 if (as_raw) { 5408 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5409 } else { 5410 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5411 } 5412 } 5413 5414 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 5415 Register tmp1, Register tmp2, Register tmp3) { 5416 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5417 decorators = AccessInternal::decorator_fixup(decorators, type); 5418 bool as_raw = (decorators & AS_RAW) != 0; 5419 if (as_raw) { 5420 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5421 } else { 5422 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5423 } 5424 } 5425 5426 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5427 Register thread_tmp, DecoratorSet decorators) { 5428 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); 5429 } 5430 5431 // Doesn't do verification, generates fixed size code 5432 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5433 Register thread_tmp, DecoratorSet decorators) { 5434 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); 5435 } 5436 5437 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5438 Register tmp2, Register tmp3, DecoratorSet decorators) { 5439 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5440 } 5441 5442 // Used for storing nulls. 5443 void MacroAssembler::store_heap_oop_null(Address dst) { 5444 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5445 } 5446 5447 #ifdef _LP64 5448 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5449 if (UseCompressedClassPointers) { 5450 // Store to klass gap in destination 5451 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 5452 } 5453 } 5454 5455 #ifdef ASSERT 5456 void MacroAssembler::verify_heapbase(const char* msg) { 5457 assert (UseCompressedOops, "should be compressed"); 5458 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5459 if (CheckCompressedOops) { 5460 Label ok; 5461 ExternalAddress src2(CompressedOops::ptrs_base_addr()); 5462 const bool is_src2_reachable = reachable(src2); 5463 if (!is_src2_reachable) { 5464 push(rscratch1); // cmpptr trashes rscratch1 5465 } 5466 cmpptr(r12_heapbase, src2, rscratch1); 5467 jcc(Assembler::equal, ok); 5468 STOP(msg); 5469 bind(ok); 5470 if (!is_src2_reachable) { 5471 pop(rscratch1); 5472 } 5473 } 5474 } 5475 #endif 5476 5477 // Algorithm must match oop.inline.hpp encode_heap_oop. 5478 void MacroAssembler::encode_heap_oop(Register r) { 5479 #ifdef ASSERT 5480 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5481 #endif 5482 verify_oop_msg(r, "broken oop in encode_heap_oop"); 5483 if (CompressedOops::base() == nullptr) { 5484 if (CompressedOops::shift() != 0) { 5485 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5486 shrq(r, LogMinObjAlignmentInBytes); 5487 } 5488 return; 5489 } 5490 testq(r, r); 5491 cmovq(Assembler::equal, r, r12_heapbase); 5492 subq(r, r12_heapbase); 5493 shrq(r, LogMinObjAlignmentInBytes); 5494 } 5495 5496 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5497 #ifdef ASSERT 5498 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5499 if (CheckCompressedOops) { 5500 Label ok; 5501 testq(r, r); 5502 jcc(Assembler::notEqual, ok); 5503 STOP("null oop passed to encode_heap_oop_not_null"); 5504 bind(ok); 5505 } 5506 #endif 5507 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5508 if (CompressedOops::base() != nullptr) { 5509 subq(r, r12_heapbase); 5510 } 5511 if (CompressedOops::shift() != 0) { 5512 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5513 shrq(r, LogMinObjAlignmentInBytes); 5514 } 5515 } 5516 5517 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5518 #ifdef ASSERT 5519 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5520 if (CheckCompressedOops) { 5521 Label ok; 5522 testq(src, src); 5523 jcc(Assembler::notEqual, ok); 5524 STOP("null oop passed to encode_heap_oop_not_null2"); 5525 bind(ok); 5526 } 5527 #endif 5528 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5529 if (dst != src) { 5530 movq(dst, src); 5531 } 5532 if (CompressedOops::base() != nullptr) { 5533 subq(dst, r12_heapbase); 5534 } 5535 if (CompressedOops::shift() != 0) { 5536 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5537 shrq(dst, LogMinObjAlignmentInBytes); 5538 } 5539 } 5540 5541 void MacroAssembler::decode_heap_oop(Register r) { 5542 #ifdef ASSERT 5543 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5544 #endif 5545 if (CompressedOops::base() == nullptr) { 5546 if (CompressedOops::shift() != 0) { 5547 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5548 shlq(r, LogMinObjAlignmentInBytes); 5549 } 5550 } else { 5551 Label done; 5552 shlq(r, LogMinObjAlignmentInBytes); 5553 jccb(Assembler::equal, done); 5554 addq(r, r12_heapbase); 5555 bind(done); 5556 } 5557 verify_oop_msg(r, "broken oop in decode_heap_oop"); 5558 } 5559 5560 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5561 // Note: it will change flags 5562 assert (UseCompressedOops, "should only be used for compressed headers"); 5563 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5564 // Cannot assert, unverified entry point counts instructions (see .ad file) 5565 // vtableStubs also counts instructions in pd_code_size_limit. 5566 // Also do not verify_oop as this is called by verify_oop. 5567 if (CompressedOops::shift() != 0) { 5568 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5569 shlq(r, LogMinObjAlignmentInBytes); 5570 if (CompressedOops::base() != nullptr) { 5571 addq(r, r12_heapbase); 5572 } 5573 } else { 5574 assert (CompressedOops::base() == nullptr, "sanity"); 5575 } 5576 } 5577 5578 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5579 // Note: it will change flags 5580 assert (UseCompressedOops, "should only be used for compressed headers"); 5581 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5582 // Cannot assert, unverified entry point counts instructions (see .ad file) 5583 // vtableStubs also counts instructions in pd_code_size_limit. 5584 // Also do not verify_oop as this is called by verify_oop. 5585 if (CompressedOops::shift() != 0) { 5586 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5587 if (LogMinObjAlignmentInBytes == Address::times_8) { 5588 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 5589 } else { 5590 if (dst != src) { 5591 movq(dst, src); 5592 } 5593 shlq(dst, LogMinObjAlignmentInBytes); 5594 if (CompressedOops::base() != nullptr) { 5595 addq(dst, r12_heapbase); 5596 } 5597 } 5598 } else { 5599 assert (CompressedOops::base() == nullptr, "sanity"); 5600 if (dst != src) { 5601 movq(dst, src); 5602 } 5603 } 5604 } 5605 5606 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 5607 assert_different_registers(r, tmp); 5608 if (CompressedKlassPointers::base() != nullptr) { 5609 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5610 subq(r, tmp); 5611 } 5612 if (CompressedKlassPointers::shift() != 0) { 5613 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5614 shrq(r, LogKlassAlignmentInBytes); 5615 } 5616 } 5617 5618 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 5619 assert_different_registers(src, dst); 5620 if (CompressedKlassPointers::base() != nullptr) { 5621 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 5622 addq(dst, src); 5623 } else { 5624 movptr(dst, src); 5625 } 5626 if (CompressedKlassPointers::shift() != 0) { 5627 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5628 shrq(dst, LogKlassAlignmentInBytes); 5629 } 5630 } 5631 5632 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 5633 assert_different_registers(r, tmp); 5634 // Note: it will change flags 5635 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 5636 // Cannot assert, unverified entry point counts instructions (see .ad file) 5637 // vtableStubs also counts instructions in pd_code_size_limit. 5638 // Also do not verify_oop as this is called by verify_oop. 5639 if (CompressedKlassPointers::shift() != 0) { 5640 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5641 shlq(r, LogKlassAlignmentInBytes); 5642 } 5643 if (CompressedKlassPointers::base() != nullptr) { 5644 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5645 addq(r, tmp); 5646 } 5647 } 5648 5649 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 5650 assert_different_registers(src, dst); 5651 // Note: it will change flags 5652 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5653 // Cannot assert, unverified entry point counts instructions (see .ad file) 5654 // vtableStubs also counts instructions in pd_code_size_limit. 5655 // Also do not verify_oop as this is called by verify_oop. 5656 5657 if (CompressedKlassPointers::base() == nullptr && 5658 CompressedKlassPointers::shift() == 0) { 5659 // The best case scenario is that there is no base or shift. Then it is already 5660 // a pointer that needs nothing but a register rename. 5661 movl(dst, src); 5662 } else { 5663 if (CompressedKlassPointers::base() != nullptr) { 5664 mov64(dst, (int64_t)CompressedKlassPointers::base()); 5665 } else { 5666 xorq(dst, dst); 5667 } 5668 if (CompressedKlassPointers::shift() != 0) { 5669 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5670 assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); 5671 leaq(dst, Address(dst, src, Address::times_8, 0)); 5672 } else { 5673 addq(dst, src); 5674 } 5675 } 5676 } 5677 5678 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5679 assert (UseCompressedOops, "should only be used for compressed headers"); 5680 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5681 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5682 int oop_index = oop_recorder()->find_index(obj); 5683 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5684 mov_narrow_oop(dst, oop_index, rspec); 5685 } 5686 5687 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 5688 assert (UseCompressedOops, "should only be used for compressed headers"); 5689 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5690 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5691 int oop_index = oop_recorder()->find_index(obj); 5692 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5693 mov_narrow_oop(dst, oop_index, rspec); 5694 } 5695 5696 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5697 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5698 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5699 int klass_index = oop_recorder()->find_index(k); 5700 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5701 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5702 } 5703 5704 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 5705 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5706 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5707 int klass_index = oop_recorder()->find_index(k); 5708 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5709 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5710 } 5711 5712 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 5713 assert (UseCompressedOops, "should only be used for compressed headers"); 5714 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5715 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5716 int oop_index = oop_recorder()->find_index(obj); 5717 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5718 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5719 } 5720 5721 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 5722 assert (UseCompressedOops, "should only be used for compressed headers"); 5723 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5724 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5725 int oop_index = oop_recorder()->find_index(obj); 5726 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5727 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5728 } 5729 5730 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 5731 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5732 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5733 int klass_index = oop_recorder()->find_index(k); 5734 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5735 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5736 } 5737 5738 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 5739 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5740 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5741 int klass_index = oop_recorder()->find_index(k); 5742 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5743 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5744 } 5745 5746 void MacroAssembler::reinit_heapbase() { 5747 if (UseCompressedOops) { 5748 if (Universe::heap() != nullptr) { 5749 if (CompressedOops::base() == nullptr) { 5750 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 5751 } else { 5752 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base()); 5753 } 5754 } else { 5755 movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr())); 5756 } 5757 } 5758 } 5759 5760 #endif // _LP64 5761 5762 #if COMPILER2_OR_JVMCI 5763 5764 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 5765 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5766 // cnt - number of qwords (8-byte words). 5767 // base - start address, qword aligned. 5768 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 5769 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 5770 if (use64byteVector) { 5771 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 5772 } else if (MaxVectorSize >= 32) { 5773 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 5774 } else { 5775 pxor(xtmp, xtmp); 5776 } 5777 jmp(L_zero_64_bytes); 5778 5779 BIND(L_loop); 5780 if (MaxVectorSize >= 32) { 5781 fill64(base, 0, xtmp, use64byteVector); 5782 } else { 5783 movdqu(Address(base, 0), xtmp); 5784 movdqu(Address(base, 16), xtmp); 5785 movdqu(Address(base, 32), xtmp); 5786 movdqu(Address(base, 48), xtmp); 5787 } 5788 addptr(base, 64); 5789 5790 BIND(L_zero_64_bytes); 5791 subptr(cnt, 8); 5792 jccb(Assembler::greaterEqual, L_loop); 5793 5794 // Copy trailing 64 bytes 5795 if (use64byteVector) { 5796 addptr(cnt, 8); 5797 jccb(Assembler::equal, L_end); 5798 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 5799 jmp(L_end); 5800 } else { 5801 addptr(cnt, 4); 5802 jccb(Assembler::less, L_tail); 5803 if (MaxVectorSize >= 32) { 5804 vmovdqu(Address(base, 0), xtmp); 5805 } else { 5806 movdqu(Address(base, 0), xtmp); 5807 movdqu(Address(base, 16), xtmp); 5808 } 5809 } 5810 addptr(base, 32); 5811 subptr(cnt, 4); 5812 5813 BIND(L_tail); 5814 addptr(cnt, 4); 5815 jccb(Assembler::lessEqual, L_end); 5816 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 5817 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 5818 } else { 5819 decrement(cnt); 5820 5821 BIND(L_sloop); 5822 movq(Address(base, 0), xtmp); 5823 addptr(base, 8); 5824 decrement(cnt); 5825 jccb(Assembler::greaterEqual, L_sloop); 5826 } 5827 BIND(L_end); 5828 } 5829 5830 // Clearing constant sized memory using YMM/ZMM registers. 5831 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5832 assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), ""); 5833 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 5834 5835 int vector64_count = (cnt & (~0x7)) >> 3; 5836 cnt = cnt & 0x7; 5837 const int fill64_per_loop = 4; 5838 const int max_unrolled_fill64 = 8; 5839 5840 // 64 byte initialization loop. 5841 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 5842 int start64 = 0; 5843 if (vector64_count > max_unrolled_fill64) { 5844 Label LOOP; 5845 Register index = rtmp; 5846 5847 start64 = vector64_count - (vector64_count % fill64_per_loop); 5848 5849 movl(index, 0); 5850 BIND(LOOP); 5851 for (int i = 0; i < fill64_per_loop; i++) { 5852 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 5853 } 5854 addl(index, fill64_per_loop * 64); 5855 cmpl(index, start64 * 64); 5856 jccb(Assembler::less, LOOP); 5857 } 5858 for (int i = start64; i < vector64_count; i++) { 5859 fill64(base, i * 64, xtmp, use64byteVector); 5860 } 5861 5862 // Clear remaining 64 byte tail. 5863 int disp = vector64_count * 64; 5864 if (cnt) { 5865 switch (cnt) { 5866 case 1: 5867 movq(Address(base, disp), xtmp); 5868 break; 5869 case 2: 5870 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 5871 break; 5872 case 3: 5873 movl(rtmp, 0x7); 5874 kmovwl(mask, rtmp); 5875 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 5876 break; 5877 case 4: 5878 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5879 break; 5880 case 5: 5881 if (use64byteVector) { 5882 movl(rtmp, 0x1F); 5883 kmovwl(mask, rtmp); 5884 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5885 } else { 5886 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5887 movq(Address(base, disp + 32), xtmp); 5888 } 5889 break; 5890 case 6: 5891 if (use64byteVector) { 5892 movl(rtmp, 0x3F); 5893 kmovwl(mask, rtmp); 5894 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5895 } else { 5896 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5897 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 5898 } 5899 break; 5900 case 7: 5901 if (use64byteVector) { 5902 movl(rtmp, 0x7F); 5903 kmovwl(mask, rtmp); 5904 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5905 } else { 5906 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5907 movl(rtmp, 0x7); 5908 kmovwl(mask, rtmp); 5909 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 5910 } 5911 break; 5912 default: 5913 fatal("Unexpected length : %d\n",cnt); 5914 break; 5915 } 5916 } 5917 } 5918 5919 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 5920 bool is_large, KRegister mask) { 5921 // cnt - number of qwords (8-byte words). 5922 // base - start address, qword aligned. 5923 // is_large - if optimizers know cnt is larger than InitArrayShortSize 5924 assert(base==rdi, "base register must be edi for rep stos"); 5925 assert(tmp==rax, "tmp register must be eax for rep stos"); 5926 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 5927 assert(InitArrayShortSize % BytesPerLong == 0, 5928 "InitArrayShortSize should be the multiple of BytesPerLong"); 5929 5930 Label DONE; 5931 if (!is_large || !UseXMMForObjInit) { 5932 xorptr(tmp, tmp); 5933 } 5934 5935 if (!is_large) { 5936 Label LOOP, LONG; 5937 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 5938 jccb(Assembler::greater, LONG); 5939 5940 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5941 5942 decrement(cnt); 5943 jccb(Assembler::negative, DONE); // Zero length 5944 5945 // Use individual pointer-sized stores for small counts: 5946 BIND(LOOP); 5947 movptr(Address(base, cnt, Address::times_ptr), tmp); 5948 decrement(cnt); 5949 jccb(Assembler::greaterEqual, LOOP); 5950 jmpb(DONE); 5951 5952 BIND(LONG); 5953 } 5954 5955 // Use longer rep-prefixed ops for non-small counts: 5956 if (UseFastStosb) { 5957 shlptr(cnt, 3); // convert to number of bytes 5958 rep_stosb(); 5959 } else if (UseXMMForObjInit) { 5960 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 5961 } else { 5962 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5963 rep_stos(); 5964 } 5965 5966 BIND(DONE); 5967 } 5968 5969 #endif //COMPILER2_OR_JVMCI 5970 5971 5972 void MacroAssembler::generate_fill(BasicType t, bool aligned, 5973 Register to, Register value, Register count, 5974 Register rtmp, XMMRegister xtmp) { 5975 ShortBranchVerifier sbv(this); 5976 assert_different_registers(to, value, count, rtmp); 5977 Label L_exit; 5978 Label L_fill_2_bytes, L_fill_4_bytes; 5979 5980 #if defined(COMPILER2) && defined(_LP64) 5981 if(MaxVectorSize >=32 && 5982 VM_Version::supports_avx512vlbw() && 5983 VM_Version::supports_bmi2()) { 5984 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 5985 return; 5986 } 5987 #endif 5988 5989 int shift = -1; 5990 switch (t) { 5991 case T_BYTE: 5992 shift = 2; 5993 break; 5994 case T_SHORT: 5995 shift = 1; 5996 break; 5997 case T_INT: 5998 shift = 0; 5999 break; 6000 default: ShouldNotReachHere(); 6001 } 6002 6003 if (t == T_BYTE) { 6004 andl(value, 0xff); 6005 movl(rtmp, value); 6006 shll(rtmp, 8); 6007 orl(value, rtmp); 6008 } 6009 if (t == T_SHORT) { 6010 andl(value, 0xffff); 6011 } 6012 if (t == T_BYTE || t == T_SHORT) { 6013 movl(rtmp, value); 6014 shll(rtmp, 16); 6015 orl(value, rtmp); 6016 } 6017 6018 cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 6019 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 6020 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 6021 Label L_skip_align2; 6022 // align source address at 4 bytes address boundary 6023 if (t == T_BYTE) { 6024 Label L_skip_align1; 6025 // One byte misalignment happens only for byte arrays 6026 testptr(to, 1); 6027 jccb(Assembler::zero, L_skip_align1); 6028 movb(Address(to, 0), value); 6029 increment(to); 6030 decrement(count); 6031 BIND(L_skip_align1); 6032 } 6033 // Two bytes misalignment happens only for byte and short (char) arrays 6034 testptr(to, 2); 6035 jccb(Assembler::zero, L_skip_align2); 6036 movw(Address(to, 0), value); 6037 addptr(to, 2); 6038 subl(count, 1<<(shift-1)); 6039 BIND(L_skip_align2); 6040 } 6041 if (UseSSE < 2) { 6042 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6043 // Fill 32-byte chunks 6044 subl(count, 8 << shift); 6045 jcc(Assembler::less, L_check_fill_8_bytes); 6046 align(16); 6047 6048 BIND(L_fill_32_bytes_loop); 6049 6050 for (int i = 0; i < 32; i += 4) { 6051 movl(Address(to, i), value); 6052 } 6053 6054 addptr(to, 32); 6055 subl(count, 8 << shift); 6056 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6057 BIND(L_check_fill_8_bytes); 6058 addl(count, 8 << shift); 6059 jccb(Assembler::zero, L_exit); 6060 jmpb(L_fill_8_bytes); 6061 6062 // 6063 // length is too short, just fill qwords 6064 // 6065 BIND(L_fill_8_bytes_loop); 6066 movl(Address(to, 0), value); 6067 movl(Address(to, 4), value); 6068 addptr(to, 8); 6069 BIND(L_fill_8_bytes); 6070 subl(count, 1 << (shift + 1)); 6071 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6072 // fall through to fill 4 bytes 6073 } else { 6074 Label L_fill_32_bytes; 6075 if (!UseUnalignedLoadStores) { 6076 // align to 8 bytes, we know we are 4 byte aligned to start 6077 testptr(to, 4); 6078 jccb(Assembler::zero, L_fill_32_bytes); 6079 movl(Address(to, 0), value); 6080 addptr(to, 4); 6081 subl(count, 1<<shift); 6082 } 6083 BIND(L_fill_32_bytes); 6084 { 6085 assert( UseSSE >= 2, "supported cpu only" ); 6086 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6087 movdl(xtmp, value); 6088 if (UseAVX >= 2 && UseUnalignedLoadStores) { 6089 Label L_check_fill_32_bytes; 6090 if (UseAVX > 2) { 6091 // Fill 64-byte chunks 6092 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 6093 6094 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 6095 cmpl(count, VM_Version::avx3_threshold()); 6096 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 6097 6098 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 6099 6100 subl(count, 16 << shift); 6101 jccb(Assembler::less, L_check_fill_32_bytes); 6102 align(16); 6103 6104 BIND(L_fill_64_bytes_loop_avx3); 6105 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 6106 addptr(to, 64); 6107 subl(count, 16 << shift); 6108 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 6109 jmpb(L_check_fill_32_bytes); 6110 6111 BIND(L_check_fill_64_bytes_avx2); 6112 } 6113 // Fill 64-byte chunks 6114 Label L_fill_64_bytes_loop; 6115 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 6116 6117 subl(count, 16 << shift); 6118 jcc(Assembler::less, L_check_fill_32_bytes); 6119 align(16); 6120 6121 BIND(L_fill_64_bytes_loop); 6122 vmovdqu(Address(to, 0), xtmp); 6123 vmovdqu(Address(to, 32), xtmp); 6124 addptr(to, 64); 6125 subl(count, 16 << shift); 6126 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 6127 6128 BIND(L_check_fill_32_bytes); 6129 addl(count, 8 << shift); 6130 jccb(Assembler::less, L_check_fill_8_bytes); 6131 vmovdqu(Address(to, 0), xtmp); 6132 addptr(to, 32); 6133 subl(count, 8 << shift); 6134 6135 BIND(L_check_fill_8_bytes); 6136 // clean upper bits of YMM registers 6137 movdl(xtmp, value); 6138 pshufd(xtmp, xtmp, 0); 6139 } else { 6140 // Fill 32-byte chunks 6141 pshufd(xtmp, xtmp, 0); 6142 6143 subl(count, 8 << shift); 6144 jcc(Assembler::less, L_check_fill_8_bytes); 6145 align(16); 6146 6147 BIND(L_fill_32_bytes_loop); 6148 6149 if (UseUnalignedLoadStores) { 6150 movdqu(Address(to, 0), xtmp); 6151 movdqu(Address(to, 16), xtmp); 6152 } else { 6153 movq(Address(to, 0), xtmp); 6154 movq(Address(to, 8), xtmp); 6155 movq(Address(to, 16), xtmp); 6156 movq(Address(to, 24), xtmp); 6157 } 6158 6159 addptr(to, 32); 6160 subl(count, 8 << shift); 6161 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6162 6163 BIND(L_check_fill_8_bytes); 6164 } 6165 addl(count, 8 << shift); 6166 jccb(Assembler::zero, L_exit); 6167 jmpb(L_fill_8_bytes); 6168 6169 // 6170 // length is too short, just fill qwords 6171 // 6172 BIND(L_fill_8_bytes_loop); 6173 movq(Address(to, 0), xtmp); 6174 addptr(to, 8); 6175 BIND(L_fill_8_bytes); 6176 subl(count, 1 << (shift + 1)); 6177 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6178 } 6179 } 6180 // fill trailing 4 bytes 6181 BIND(L_fill_4_bytes); 6182 testl(count, 1<<shift); 6183 jccb(Assembler::zero, L_fill_2_bytes); 6184 movl(Address(to, 0), value); 6185 if (t == T_BYTE || t == T_SHORT) { 6186 Label L_fill_byte; 6187 addptr(to, 4); 6188 BIND(L_fill_2_bytes); 6189 // fill trailing 2 bytes 6190 testl(count, 1<<(shift-1)); 6191 jccb(Assembler::zero, L_fill_byte); 6192 movw(Address(to, 0), value); 6193 if (t == T_BYTE) { 6194 addptr(to, 2); 6195 BIND(L_fill_byte); 6196 // fill trailing byte 6197 testl(count, 1); 6198 jccb(Assembler::zero, L_exit); 6199 movb(Address(to, 0), value); 6200 } else { 6201 BIND(L_fill_byte); 6202 } 6203 } else { 6204 BIND(L_fill_2_bytes); 6205 } 6206 BIND(L_exit); 6207 } 6208 6209 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 6210 switch(type) { 6211 case T_BYTE: 6212 case T_BOOLEAN: 6213 evpbroadcastb(dst, src, vector_len); 6214 break; 6215 case T_SHORT: 6216 case T_CHAR: 6217 evpbroadcastw(dst, src, vector_len); 6218 break; 6219 case T_INT: 6220 case T_FLOAT: 6221 evpbroadcastd(dst, src, vector_len); 6222 break; 6223 case T_LONG: 6224 case T_DOUBLE: 6225 evpbroadcastq(dst, src, vector_len); 6226 break; 6227 default: 6228 fatal("Unhandled type : %s", type2name(type)); 6229 break; 6230 } 6231 } 6232 6233 // encode char[] to byte[] in ISO_8859_1 or ASCII 6234 //@IntrinsicCandidate 6235 //private static int implEncodeISOArray(byte[] sa, int sp, 6236 //byte[] da, int dp, int len) { 6237 // int i = 0; 6238 // for (; i < len; i++) { 6239 // char c = StringUTF16.getChar(sa, sp++); 6240 // if (c > '\u00FF') 6241 // break; 6242 // da[dp++] = (byte)c; 6243 // } 6244 // return i; 6245 //} 6246 // 6247 //@IntrinsicCandidate 6248 //private static int implEncodeAsciiArray(char[] sa, int sp, 6249 // byte[] da, int dp, int len) { 6250 // int i = 0; 6251 // for (; i < len; i++) { 6252 // char c = sa[sp++]; 6253 // if (c >= '\u0080') 6254 // break; 6255 // da[dp++] = (byte)c; 6256 // } 6257 // return i; 6258 //} 6259 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6260 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6261 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6262 Register tmp5, Register result, bool ascii) { 6263 6264 // rsi: src 6265 // rdi: dst 6266 // rdx: len 6267 // rcx: tmp5 6268 // rax: result 6269 ShortBranchVerifier sbv(this); 6270 assert_different_registers(src, dst, len, tmp5, result); 6271 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6272 6273 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6274 int short_mask = ascii ? 0xff80 : 0xff00; 6275 6276 // set result 6277 xorl(result, result); 6278 // check for zero length 6279 testl(len, len); 6280 jcc(Assembler::zero, L_done); 6281 6282 movl(result, len); 6283 6284 // Setup pointers 6285 lea(src, Address(src, len, Address::times_2)); // char[] 6286 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6287 negptr(len); 6288 6289 if (UseSSE42Intrinsics || UseAVX >= 2) { 6290 Label L_copy_8_chars, L_copy_8_chars_exit; 6291 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6292 6293 if (UseAVX >= 2) { 6294 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6295 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6296 movdl(tmp1Reg, tmp5); 6297 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6298 jmp(L_chars_32_check); 6299 6300 bind(L_copy_32_chars); 6301 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6302 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6303 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6304 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6305 jccb(Assembler::notZero, L_copy_32_chars_exit); 6306 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6307 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6308 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6309 6310 bind(L_chars_32_check); 6311 addptr(len, 32); 6312 jcc(Assembler::lessEqual, L_copy_32_chars); 6313 6314 bind(L_copy_32_chars_exit); 6315 subptr(len, 16); 6316 jccb(Assembler::greater, L_copy_16_chars_exit); 6317 6318 } else if (UseSSE42Intrinsics) { 6319 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6320 movdl(tmp1Reg, tmp5); 6321 pshufd(tmp1Reg, tmp1Reg, 0); 6322 jmpb(L_chars_16_check); 6323 } 6324 6325 bind(L_copy_16_chars); 6326 if (UseAVX >= 2) { 6327 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6328 vptest(tmp2Reg, tmp1Reg); 6329 jcc(Assembler::notZero, L_copy_16_chars_exit); 6330 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6331 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6332 } else { 6333 if (UseAVX > 0) { 6334 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6335 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6336 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6337 } else { 6338 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6339 por(tmp2Reg, tmp3Reg); 6340 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6341 por(tmp2Reg, tmp4Reg); 6342 } 6343 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6344 jccb(Assembler::notZero, L_copy_16_chars_exit); 6345 packuswb(tmp3Reg, tmp4Reg); 6346 } 6347 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 6348 6349 bind(L_chars_16_check); 6350 addptr(len, 16); 6351 jcc(Assembler::lessEqual, L_copy_16_chars); 6352 6353 bind(L_copy_16_chars_exit); 6354 if (UseAVX >= 2) { 6355 // clean upper bits of YMM registers 6356 vpxor(tmp2Reg, tmp2Reg); 6357 vpxor(tmp3Reg, tmp3Reg); 6358 vpxor(tmp4Reg, tmp4Reg); 6359 movdl(tmp1Reg, tmp5); 6360 pshufd(tmp1Reg, tmp1Reg, 0); 6361 } 6362 subptr(len, 8); 6363 jccb(Assembler::greater, L_copy_8_chars_exit); 6364 6365 bind(L_copy_8_chars); 6366 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 6367 ptest(tmp3Reg, tmp1Reg); 6368 jccb(Assembler::notZero, L_copy_8_chars_exit); 6369 packuswb(tmp3Reg, tmp1Reg); 6370 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 6371 addptr(len, 8); 6372 jccb(Assembler::lessEqual, L_copy_8_chars); 6373 6374 bind(L_copy_8_chars_exit); 6375 subptr(len, 8); 6376 jccb(Assembler::zero, L_done); 6377 } 6378 6379 bind(L_copy_1_char); 6380 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 6381 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 6382 jccb(Assembler::notZero, L_copy_1_char_exit); 6383 movb(Address(dst, len, Address::times_1, 0), tmp5); 6384 addptr(len, 1); 6385 jccb(Assembler::less, L_copy_1_char); 6386 6387 bind(L_copy_1_char_exit); 6388 addptr(result, len); // len is negative count of not processed elements 6389 6390 bind(L_done); 6391 } 6392 6393 #ifdef _LP64 6394 /** 6395 * Helper for multiply_to_len(). 6396 */ 6397 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 6398 addq(dest_lo, src1); 6399 adcq(dest_hi, 0); 6400 addq(dest_lo, src2); 6401 adcq(dest_hi, 0); 6402 } 6403 6404 /** 6405 * Multiply 64 bit by 64 bit first loop. 6406 */ 6407 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 6408 Register y, Register y_idx, Register z, 6409 Register carry, Register product, 6410 Register idx, Register kdx) { 6411 // 6412 // jlong carry, x[], y[], z[]; 6413 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6414 // huge_128 product = y[idx] * x[xstart] + carry; 6415 // z[kdx] = (jlong)product; 6416 // carry = (jlong)(product >>> 64); 6417 // } 6418 // z[xstart] = carry; 6419 // 6420 6421 Label L_first_loop, L_first_loop_exit; 6422 Label L_one_x, L_one_y, L_multiply; 6423 6424 decrementl(xstart); 6425 jcc(Assembler::negative, L_one_x); 6426 6427 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6428 rorq(x_xstart, 32); // convert big-endian to little-endian 6429 6430 bind(L_first_loop); 6431 decrementl(idx); 6432 jcc(Assembler::negative, L_first_loop_exit); 6433 decrementl(idx); 6434 jcc(Assembler::negative, L_one_y); 6435 movq(y_idx, Address(y, idx, Address::times_4, 0)); 6436 rorq(y_idx, 32); // convert big-endian to little-endian 6437 bind(L_multiply); 6438 movq(product, x_xstart); 6439 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 6440 addq(product, carry); 6441 adcq(rdx, 0); 6442 subl(kdx, 2); 6443 movl(Address(z, kdx, Address::times_4, 4), product); 6444 shrq(product, 32); 6445 movl(Address(z, kdx, Address::times_4, 0), product); 6446 movq(carry, rdx); 6447 jmp(L_first_loop); 6448 6449 bind(L_one_y); 6450 movl(y_idx, Address(y, 0)); 6451 jmp(L_multiply); 6452 6453 bind(L_one_x); 6454 movl(x_xstart, Address(x, 0)); 6455 jmp(L_first_loop); 6456 6457 bind(L_first_loop_exit); 6458 } 6459 6460 /** 6461 * Multiply 64 bit by 64 bit and add 128 bit. 6462 */ 6463 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 6464 Register yz_idx, Register idx, 6465 Register carry, Register product, int offset) { 6466 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 6467 // z[kdx] = (jlong)product; 6468 6469 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 6470 rorq(yz_idx, 32); // convert big-endian to little-endian 6471 movq(product, x_xstart); 6472 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6473 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 6474 rorq(yz_idx, 32); // convert big-endian to little-endian 6475 6476 add2_with_carry(rdx, product, carry, yz_idx); 6477 6478 movl(Address(z, idx, Address::times_4, offset+4), product); 6479 shrq(product, 32); 6480 movl(Address(z, idx, Address::times_4, offset), product); 6481 6482 } 6483 6484 /** 6485 * Multiply 128 bit by 128 bit. Unrolled inner loop. 6486 */ 6487 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 6488 Register yz_idx, Register idx, Register jdx, 6489 Register carry, Register product, 6490 Register carry2) { 6491 // jlong carry, x[], y[], z[]; 6492 // int kdx = ystart+1; 6493 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6494 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 6495 // z[kdx+idx+1] = (jlong)product; 6496 // jlong carry2 = (jlong)(product >>> 64); 6497 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 6498 // z[kdx+idx] = (jlong)product; 6499 // carry = (jlong)(product >>> 64); 6500 // } 6501 // idx += 2; 6502 // if (idx > 0) { 6503 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 6504 // z[kdx+idx] = (jlong)product; 6505 // carry = (jlong)(product >>> 64); 6506 // } 6507 // 6508 6509 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6510 6511 movl(jdx, idx); 6512 andl(jdx, 0xFFFFFFFC); 6513 shrl(jdx, 2); 6514 6515 bind(L_third_loop); 6516 subl(jdx, 1); 6517 jcc(Assembler::negative, L_third_loop_exit); 6518 subl(idx, 4); 6519 6520 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 6521 movq(carry2, rdx); 6522 6523 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 6524 movq(carry, rdx); 6525 jmp(L_third_loop); 6526 6527 bind (L_third_loop_exit); 6528 6529 andl (idx, 0x3); 6530 jcc(Assembler::zero, L_post_third_loop_done); 6531 6532 Label L_check_1; 6533 subl(idx, 2); 6534 jcc(Assembler::negative, L_check_1); 6535 6536 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 6537 movq(carry, rdx); 6538 6539 bind (L_check_1); 6540 addl (idx, 0x2); 6541 andl (idx, 0x1); 6542 subl(idx, 1); 6543 jcc(Assembler::negative, L_post_third_loop_done); 6544 6545 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 6546 movq(product, x_xstart); 6547 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6548 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 6549 6550 add2_with_carry(rdx, product, yz_idx, carry); 6551 6552 movl(Address(z, idx, Address::times_4, 0), product); 6553 shrq(product, 32); 6554 6555 shlq(rdx, 32); 6556 orq(product, rdx); 6557 movq(carry, product); 6558 6559 bind(L_post_third_loop_done); 6560 } 6561 6562 /** 6563 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 6564 * 6565 */ 6566 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 6567 Register carry, Register carry2, 6568 Register idx, Register jdx, 6569 Register yz_idx1, Register yz_idx2, 6570 Register tmp, Register tmp3, Register tmp4) { 6571 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 6572 6573 // jlong carry, x[], y[], z[]; 6574 // int kdx = ystart+1; 6575 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6576 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 6577 // jlong carry2 = (jlong)(tmp3 >>> 64); 6578 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 6579 // carry = (jlong)(tmp4 >>> 64); 6580 // z[kdx+idx+1] = (jlong)tmp3; 6581 // z[kdx+idx] = (jlong)tmp4; 6582 // } 6583 // idx += 2; 6584 // if (idx > 0) { 6585 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 6586 // z[kdx+idx] = (jlong)yz_idx1; 6587 // carry = (jlong)(yz_idx1 >>> 64); 6588 // } 6589 // 6590 6591 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6592 6593 movl(jdx, idx); 6594 andl(jdx, 0xFFFFFFFC); 6595 shrl(jdx, 2); 6596 6597 bind(L_third_loop); 6598 subl(jdx, 1); 6599 jcc(Assembler::negative, L_third_loop_exit); 6600 subl(idx, 4); 6601 6602 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 6603 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 6604 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 6605 rorxq(yz_idx2, yz_idx2, 32); 6606 6607 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6608 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 6609 6610 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 6611 rorxq(yz_idx1, yz_idx1, 32); 6612 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6613 rorxq(yz_idx2, yz_idx2, 32); 6614 6615 if (VM_Version::supports_adx()) { 6616 adcxq(tmp3, carry); 6617 adoxq(tmp3, yz_idx1); 6618 6619 adcxq(tmp4, tmp); 6620 adoxq(tmp4, yz_idx2); 6621 6622 movl(carry, 0); // does not affect flags 6623 adcxq(carry2, carry); 6624 adoxq(carry2, carry); 6625 } else { 6626 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 6627 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 6628 } 6629 movq(carry, carry2); 6630 6631 movl(Address(z, idx, Address::times_4, 12), tmp3); 6632 shrq(tmp3, 32); 6633 movl(Address(z, idx, Address::times_4, 8), tmp3); 6634 6635 movl(Address(z, idx, Address::times_4, 4), tmp4); 6636 shrq(tmp4, 32); 6637 movl(Address(z, idx, Address::times_4, 0), tmp4); 6638 6639 jmp(L_third_loop); 6640 6641 bind (L_third_loop_exit); 6642 6643 andl (idx, 0x3); 6644 jcc(Assembler::zero, L_post_third_loop_done); 6645 6646 Label L_check_1; 6647 subl(idx, 2); 6648 jcc(Assembler::negative, L_check_1); 6649 6650 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 6651 rorxq(yz_idx1, yz_idx1, 32); 6652 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6653 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6654 rorxq(yz_idx2, yz_idx2, 32); 6655 6656 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 6657 6658 movl(Address(z, idx, Address::times_4, 4), tmp3); 6659 shrq(tmp3, 32); 6660 movl(Address(z, idx, Address::times_4, 0), tmp3); 6661 movq(carry, tmp4); 6662 6663 bind (L_check_1); 6664 addl (idx, 0x2); 6665 andl (idx, 0x1); 6666 subl(idx, 1); 6667 jcc(Assembler::negative, L_post_third_loop_done); 6668 movl(tmp4, Address(y, idx, Address::times_4, 0)); 6669 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 6670 movl(tmp4, Address(z, idx, Address::times_4, 0)); 6671 6672 add2_with_carry(carry2, tmp3, tmp4, carry); 6673 6674 movl(Address(z, idx, Address::times_4, 0), tmp3); 6675 shrq(tmp3, 32); 6676 6677 shlq(carry2, 32); 6678 orq(tmp3, carry2); 6679 movq(carry, tmp3); 6680 6681 bind(L_post_third_loop_done); 6682 } 6683 6684 /** 6685 * Code for BigInteger::multiplyToLen() intrinsic. 6686 * 6687 * rdi: x 6688 * rax: xlen 6689 * rsi: y 6690 * rcx: ylen 6691 * r8: z 6692 * r11: zlen 6693 * r12: tmp1 6694 * r13: tmp2 6695 * r14: tmp3 6696 * r15: tmp4 6697 * rbx: tmp5 6698 * 6699 */ 6700 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, 6701 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 6702 ShortBranchVerifier sbv(this); 6703 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 6704 6705 push(tmp1); 6706 push(tmp2); 6707 push(tmp3); 6708 push(tmp4); 6709 push(tmp5); 6710 6711 push(xlen); 6712 push(zlen); 6713 6714 const Register idx = tmp1; 6715 const Register kdx = tmp2; 6716 const Register xstart = tmp3; 6717 6718 const Register y_idx = tmp4; 6719 const Register carry = tmp5; 6720 const Register product = xlen; 6721 const Register x_xstart = zlen; // reuse register 6722 6723 // First Loop. 6724 // 6725 // final static long LONG_MASK = 0xffffffffL; 6726 // int xstart = xlen - 1; 6727 // int ystart = ylen - 1; 6728 // long carry = 0; 6729 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6730 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 6731 // z[kdx] = (int)product; 6732 // carry = product >>> 32; 6733 // } 6734 // z[xstart] = (int)carry; 6735 // 6736 6737 movl(idx, ylen); // idx = ylen; 6738 movl(kdx, zlen); // kdx = xlen+ylen; 6739 xorq(carry, carry); // carry = 0; 6740 6741 Label L_done; 6742 6743 movl(xstart, xlen); 6744 decrementl(xstart); 6745 jcc(Assembler::negative, L_done); 6746 6747 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 6748 6749 Label L_second_loop; 6750 testl(kdx, kdx); 6751 jcc(Assembler::zero, L_second_loop); 6752 6753 Label L_carry; 6754 subl(kdx, 1); 6755 jcc(Assembler::zero, L_carry); 6756 6757 movl(Address(z, kdx, Address::times_4, 0), carry); 6758 shrq(carry, 32); 6759 subl(kdx, 1); 6760 6761 bind(L_carry); 6762 movl(Address(z, kdx, Address::times_4, 0), carry); 6763 6764 // Second and third (nested) loops. 6765 // 6766 // for (int i = xstart-1; i >= 0; i--) { // Second loop 6767 // carry = 0; 6768 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 6769 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 6770 // (z[k] & LONG_MASK) + carry; 6771 // z[k] = (int)product; 6772 // carry = product >>> 32; 6773 // } 6774 // z[i] = (int)carry; 6775 // } 6776 // 6777 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 6778 6779 const Register jdx = tmp1; 6780 6781 bind(L_second_loop); 6782 xorl(carry, carry); // carry = 0; 6783 movl(jdx, ylen); // j = ystart+1 6784 6785 subl(xstart, 1); // i = xstart-1; 6786 jcc(Assembler::negative, L_done); 6787 6788 push (z); 6789 6790 Label L_last_x; 6791 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 6792 subl(xstart, 1); // i = xstart-1; 6793 jcc(Assembler::negative, L_last_x); 6794 6795 if (UseBMI2Instructions) { 6796 movq(rdx, Address(x, xstart, Address::times_4, 0)); 6797 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 6798 } else { 6799 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6800 rorq(x_xstart, 32); // convert big-endian to little-endian 6801 } 6802 6803 Label L_third_loop_prologue; 6804 bind(L_third_loop_prologue); 6805 6806 push (x); 6807 push (xstart); 6808 push (ylen); 6809 6810 6811 if (UseBMI2Instructions) { 6812 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 6813 } else { // !UseBMI2Instructions 6814 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 6815 } 6816 6817 pop(ylen); 6818 pop(xlen); 6819 pop(x); 6820 pop(z); 6821 6822 movl(tmp3, xlen); 6823 addl(tmp3, 1); 6824 movl(Address(z, tmp3, Address::times_4, 0), carry); 6825 subl(tmp3, 1); 6826 jccb(Assembler::negative, L_done); 6827 6828 shrq(carry, 32); 6829 movl(Address(z, tmp3, Address::times_4, 0), carry); 6830 jmp(L_second_loop); 6831 6832 // Next infrequent code is moved outside loops. 6833 bind(L_last_x); 6834 if (UseBMI2Instructions) { 6835 movl(rdx, Address(x, 0)); 6836 } else { 6837 movl(x_xstart, Address(x, 0)); 6838 } 6839 jmp(L_third_loop_prologue); 6840 6841 bind(L_done); 6842 6843 pop(zlen); 6844 pop(xlen); 6845 6846 pop(tmp5); 6847 pop(tmp4); 6848 pop(tmp3); 6849 pop(tmp2); 6850 pop(tmp1); 6851 } 6852 6853 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 6854 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 6855 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 6856 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 6857 Label VECTOR8_TAIL, VECTOR4_TAIL; 6858 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 6859 Label SAME_TILL_END, DONE; 6860 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 6861 6862 //scale is in rcx in both Win64 and Unix 6863 ShortBranchVerifier sbv(this); 6864 6865 shlq(length); 6866 xorq(result, result); 6867 6868 if ((AVX3Threshold == 0) && (UseAVX > 2) && 6869 VM_Version::supports_avx512vlbw()) { 6870 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 6871 6872 cmpq(length, 64); 6873 jcc(Assembler::less, VECTOR32_TAIL); 6874 6875 movq(tmp1, length); 6876 andq(tmp1, 0x3F); // tail count 6877 andq(length, ~(0x3F)); //vector count 6878 6879 bind(VECTOR64_LOOP); 6880 // AVX512 code to compare 64 byte vectors. 6881 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 6882 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 6883 kortestql(k7, k7); 6884 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 6885 addq(result, 64); 6886 subq(length, 64); 6887 jccb(Assembler::notZero, VECTOR64_LOOP); 6888 6889 //bind(VECTOR64_TAIL); 6890 testq(tmp1, tmp1); 6891 jcc(Assembler::zero, SAME_TILL_END); 6892 6893 //bind(VECTOR64_TAIL); 6894 // AVX512 code to compare up to 63 byte vectors. 6895 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 6896 shlxq(tmp2, tmp2, tmp1); 6897 notq(tmp2); 6898 kmovql(k3, tmp2); 6899 6900 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 6901 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 6902 6903 ktestql(k7, k3); 6904 jcc(Assembler::below, SAME_TILL_END); // not mismatch 6905 6906 bind(VECTOR64_NOT_EQUAL); 6907 kmovql(tmp1, k7); 6908 notq(tmp1); 6909 tzcntq(tmp1, tmp1); 6910 addq(result, tmp1); 6911 shrq(result); 6912 jmp(DONE); 6913 bind(VECTOR32_TAIL); 6914 } 6915 6916 cmpq(length, 8); 6917 jcc(Assembler::equal, VECTOR8_LOOP); 6918 jcc(Assembler::less, VECTOR4_TAIL); 6919 6920 if (UseAVX >= 2) { 6921 Label VECTOR16_TAIL, VECTOR32_LOOP; 6922 6923 cmpq(length, 16); 6924 jcc(Assembler::equal, VECTOR16_LOOP); 6925 jcc(Assembler::less, VECTOR8_LOOP); 6926 6927 cmpq(length, 32); 6928 jccb(Assembler::less, VECTOR16_TAIL); 6929 6930 subq(length, 32); 6931 bind(VECTOR32_LOOP); 6932 vmovdqu(rymm0, Address(obja, result)); 6933 vmovdqu(rymm1, Address(objb, result)); 6934 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 6935 vptest(rymm2, rymm2); 6936 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 6937 addq(result, 32); 6938 subq(length, 32); 6939 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 6940 addq(length, 32); 6941 jcc(Assembler::equal, SAME_TILL_END); 6942 //falling through if less than 32 bytes left //close the branch here. 6943 6944 bind(VECTOR16_TAIL); 6945 cmpq(length, 16); 6946 jccb(Assembler::less, VECTOR8_TAIL); 6947 bind(VECTOR16_LOOP); 6948 movdqu(rymm0, Address(obja, result)); 6949 movdqu(rymm1, Address(objb, result)); 6950 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 6951 ptest(rymm2, rymm2); 6952 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6953 addq(result, 16); 6954 subq(length, 16); 6955 jcc(Assembler::equal, SAME_TILL_END); 6956 //falling through if less than 16 bytes left 6957 } else {//regular intrinsics 6958 6959 cmpq(length, 16); 6960 jccb(Assembler::less, VECTOR8_TAIL); 6961 6962 subq(length, 16); 6963 bind(VECTOR16_LOOP); 6964 movdqu(rymm0, Address(obja, result)); 6965 movdqu(rymm1, Address(objb, result)); 6966 pxor(rymm0, rymm1); 6967 ptest(rymm0, rymm0); 6968 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6969 addq(result, 16); 6970 subq(length, 16); 6971 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 6972 addq(length, 16); 6973 jcc(Assembler::equal, SAME_TILL_END); 6974 //falling through if less than 16 bytes left 6975 } 6976 6977 bind(VECTOR8_TAIL); 6978 cmpq(length, 8); 6979 jccb(Assembler::less, VECTOR4_TAIL); 6980 bind(VECTOR8_LOOP); 6981 movq(tmp1, Address(obja, result)); 6982 movq(tmp2, Address(objb, result)); 6983 xorq(tmp1, tmp2); 6984 testq(tmp1, tmp1); 6985 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 6986 addq(result, 8); 6987 subq(length, 8); 6988 jcc(Assembler::equal, SAME_TILL_END); 6989 //falling through if less than 8 bytes left 6990 6991 bind(VECTOR4_TAIL); 6992 cmpq(length, 4); 6993 jccb(Assembler::less, BYTES_TAIL); 6994 bind(VECTOR4_LOOP); 6995 movl(tmp1, Address(obja, result)); 6996 xorl(tmp1, Address(objb, result)); 6997 testl(tmp1, tmp1); 6998 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 6999 addq(result, 4); 7000 subq(length, 4); 7001 jcc(Assembler::equal, SAME_TILL_END); 7002 //falling through if less than 4 bytes left 7003 7004 bind(BYTES_TAIL); 7005 bind(BYTES_LOOP); 7006 load_unsigned_byte(tmp1, Address(obja, result)); 7007 load_unsigned_byte(tmp2, Address(objb, result)); 7008 xorl(tmp1, tmp2); 7009 testl(tmp1, tmp1); 7010 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7011 decq(length); 7012 jcc(Assembler::zero, SAME_TILL_END); 7013 incq(result); 7014 load_unsigned_byte(tmp1, Address(obja, result)); 7015 load_unsigned_byte(tmp2, Address(objb, result)); 7016 xorl(tmp1, tmp2); 7017 testl(tmp1, tmp1); 7018 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7019 decq(length); 7020 jcc(Assembler::zero, SAME_TILL_END); 7021 incq(result); 7022 load_unsigned_byte(tmp1, Address(obja, result)); 7023 load_unsigned_byte(tmp2, Address(objb, result)); 7024 xorl(tmp1, tmp2); 7025 testl(tmp1, tmp1); 7026 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7027 jmp(SAME_TILL_END); 7028 7029 if (UseAVX >= 2) { 7030 bind(VECTOR32_NOT_EQUAL); 7031 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 7032 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 7033 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 7034 vpmovmskb(tmp1, rymm0); 7035 bsfq(tmp1, tmp1); 7036 addq(result, tmp1); 7037 shrq(result); 7038 jmp(DONE); 7039 } 7040 7041 bind(VECTOR16_NOT_EQUAL); 7042 if (UseAVX >= 2) { 7043 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 7044 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 7045 pxor(rymm0, rymm2); 7046 } else { 7047 pcmpeqb(rymm2, rymm2); 7048 pxor(rymm0, rymm1); 7049 pcmpeqb(rymm0, rymm1); 7050 pxor(rymm0, rymm2); 7051 } 7052 pmovmskb(tmp1, rymm0); 7053 bsfq(tmp1, tmp1); 7054 addq(result, tmp1); 7055 shrq(result); 7056 jmpb(DONE); 7057 7058 bind(VECTOR8_NOT_EQUAL); 7059 bind(VECTOR4_NOT_EQUAL); 7060 bsfq(tmp1, tmp1); 7061 shrq(tmp1, 3); 7062 addq(result, tmp1); 7063 bind(BYTES_NOT_EQUAL); 7064 shrq(result); 7065 jmpb(DONE); 7066 7067 bind(SAME_TILL_END); 7068 mov64(result, -1); 7069 7070 bind(DONE); 7071 } 7072 7073 //Helper functions for square_to_len() 7074 7075 /** 7076 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 7077 * Preserves x and z and modifies rest of the registers. 7078 */ 7079 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7080 // Perform square and right shift by 1 7081 // Handle odd xlen case first, then for even xlen do the following 7082 // jlong carry = 0; 7083 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 7084 // huge_128 product = x[j:j+1] * x[j:j+1]; 7085 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 7086 // z[i+2:i+3] = (jlong)(product >>> 1); 7087 // carry = (jlong)product; 7088 // } 7089 7090 xorq(tmp5, tmp5); // carry 7091 xorq(rdxReg, rdxReg); 7092 xorl(tmp1, tmp1); // index for x 7093 xorl(tmp4, tmp4); // index for z 7094 7095 Label L_first_loop, L_first_loop_exit; 7096 7097 testl(xlen, 1); 7098 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 7099 7100 // Square and right shift by 1 the odd element using 32 bit multiply 7101 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 7102 imulq(raxReg, raxReg); 7103 shrq(raxReg, 1); 7104 adcq(tmp5, 0); 7105 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 7106 incrementl(tmp1); 7107 addl(tmp4, 2); 7108 7109 // Square and right shift by 1 the rest using 64 bit multiply 7110 bind(L_first_loop); 7111 cmpptr(tmp1, xlen); 7112 jccb(Assembler::equal, L_first_loop_exit); 7113 7114 // Square 7115 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 7116 rorq(raxReg, 32); // convert big-endian to little-endian 7117 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 7118 7119 // Right shift by 1 and save carry 7120 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 7121 rcrq(rdxReg, 1); 7122 rcrq(raxReg, 1); 7123 adcq(tmp5, 0); 7124 7125 // Store result in z 7126 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 7127 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 7128 7129 // Update indices for x and z 7130 addl(tmp1, 2); 7131 addl(tmp4, 4); 7132 jmp(L_first_loop); 7133 7134 bind(L_first_loop_exit); 7135 } 7136 7137 7138 /** 7139 * Perform the following multiply add operation using BMI2 instructions 7140 * carry:sum = sum + op1*op2 + carry 7141 * op2 should be in rdx 7142 * op2 is preserved, all other registers are modified 7143 */ 7144 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 7145 // assert op2 is rdx 7146 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 7147 addq(sum, carry); 7148 adcq(tmp2, 0); 7149 addq(sum, op1); 7150 adcq(tmp2, 0); 7151 movq(carry, tmp2); 7152 } 7153 7154 /** 7155 * Perform the following multiply add operation: 7156 * carry:sum = sum + op1*op2 + carry 7157 * Preserves op1, op2 and modifies rest of registers 7158 */ 7159 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 7160 // rdx:rax = op1 * op2 7161 movq(raxReg, op2); 7162 mulq(op1); 7163 7164 // rdx:rax = sum + carry + rdx:rax 7165 addq(sum, carry); 7166 adcq(rdxReg, 0); 7167 addq(sum, raxReg); 7168 adcq(rdxReg, 0); 7169 7170 // carry:sum = rdx:sum 7171 movq(carry, rdxReg); 7172 } 7173 7174 /** 7175 * Add 64 bit long carry into z[] with carry propagation. 7176 * Preserves z and carry register values and modifies rest of registers. 7177 * 7178 */ 7179 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 7180 Label L_fourth_loop, L_fourth_loop_exit; 7181 7182 movl(tmp1, 1); 7183 subl(zlen, 2); 7184 addq(Address(z, zlen, Address::times_4, 0), carry); 7185 7186 bind(L_fourth_loop); 7187 jccb(Assembler::carryClear, L_fourth_loop_exit); 7188 subl(zlen, 2); 7189 jccb(Assembler::negative, L_fourth_loop_exit); 7190 addq(Address(z, zlen, Address::times_4, 0), tmp1); 7191 jmp(L_fourth_loop); 7192 bind(L_fourth_loop_exit); 7193 } 7194 7195 /** 7196 * Shift z[] left by 1 bit. 7197 * Preserves x, len, z and zlen registers and modifies rest of the registers. 7198 * 7199 */ 7200 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 7201 7202 Label L_fifth_loop, L_fifth_loop_exit; 7203 7204 // Fifth loop 7205 // Perform primitiveLeftShift(z, zlen, 1) 7206 7207 const Register prev_carry = tmp1; 7208 const Register new_carry = tmp4; 7209 const Register value = tmp2; 7210 const Register zidx = tmp3; 7211 7212 // int zidx, carry; 7213 // long value; 7214 // carry = 0; 7215 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 7216 // (carry:value) = (z[i] << 1) | carry ; 7217 // z[i] = value; 7218 // } 7219 7220 movl(zidx, zlen); 7221 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 7222 7223 bind(L_fifth_loop); 7224 decl(zidx); // Use decl to preserve carry flag 7225 decl(zidx); 7226 jccb(Assembler::negative, L_fifth_loop_exit); 7227 7228 if (UseBMI2Instructions) { 7229 movq(value, Address(z, zidx, Address::times_4, 0)); 7230 rclq(value, 1); 7231 rorxq(value, value, 32); 7232 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7233 } 7234 else { 7235 // clear new_carry 7236 xorl(new_carry, new_carry); 7237 7238 // Shift z[i] by 1, or in previous carry and save new carry 7239 movq(value, Address(z, zidx, Address::times_4, 0)); 7240 shlq(value, 1); 7241 adcl(new_carry, 0); 7242 7243 orq(value, prev_carry); 7244 rorq(value, 0x20); 7245 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7246 7247 // Set previous carry = new carry 7248 movl(prev_carry, new_carry); 7249 } 7250 jmp(L_fifth_loop); 7251 7252 bind(L_fifth_loop_exit); 7253 } 7254 7255 7256 /** 7257 * Code for BigInteger::squareToLen() intrinsic 7258 * 7259 * rdi: x 7260 * rsi: len 7261 * r8: z 7262 * rcx: zlen 7263 * r12: tmp1 7264 * r13: tmp2 7265 * r14: tmp3 7266 * r15: tmp4 7267 * rbx: tmp5 7268 * 7269 */ 7270 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7271 7272 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7273 push(tmp1); 7274 push(tmp2); 7275 push(tmp3); 7276 push(tmp4); 7277 push(tmp5); 7278 7279 // First loop 7280 // Store the squares, right shifted one bit (i.e., divided by 2). 7281 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7282 7283 // Add in off-diagonal sums. 7284 // 7285 // Second, third (nested) and fourth loops. 7286 // zlen +=2; 7287 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7288 // carry = 0; 7289 // long op2 = x[xidx:xidx+1]; 7290 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7291 // k -= 2; 7292 // long op1 = x[j:j+1]; 7293 // long sum = z[k:k+1]; 7294 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7295 // z[k:k+1] = sum; 7296 // } 7297 // add_one_64(z, k, carry, tmp_regs); 7298 // } 7299 7300 const Register carry = tmp5; 7301 const Register sum = tmp3; 7302 const Register op1 = tmp4; 7303 Register op2 = tmp2; 7304 7305 push(zlen); 7306 push(len); 7307 addl(zlen,2); 7308 bind(L_second_loop); 7309 xorq(carry, carry); 7310 subl(zlen, 4); 7311 subl(len, 2); 7312 push(zlen); 7313 push(len); 7314 cmpl(len, 0); 7315 jccb(Assembler::lessEqual, L_second_loop_exit); 7316 7317 // Multiply an array by one 64 bit long. 7318 if (UseBMI2Instructions) { 7319 op2 = rdxReg; 7320 movq(op2, Address(x, len, Address::times_4, 0)); 7321 rorxq(op2, op2, 32); 7322 } 7323 else { 7324 movq(op2, Address(x, len, Address::times_4, 0)); 7325 rorq(op2, 32); 7326 } 7327 7328 bind(L_third_loop); 7329 decrementl(len); 7330 jccb(Assembler::negative, L_third_loop_exit); 7331 decrementl(len); 7332 jccb(Assembler::negative, L_last_x); 7333 7334 movq(op1, Address(x, len, Address::times_4, 0)); 7335 rorq(op1, 32); 7336 7337 bind(L_multiply); 7338 subl(zlen, 2); 7339 movq(sum, Address(z, zlen, Address::times_4, 0)); 7340 7341 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7342 if (UseBMI2Instructions) { 7343 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 7344 } 7345 else { 7346 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7347 } 7348 7349 movq(Address(z, zlen, Address::times_4, 0), sum); 7350 7351 jmp(L_third_loop); 7352 bind(L_third_loop_exit); 7353 7354 // Fourth loop 7355 // Add 64 bit long carry into z with carry propagation. 7356 // Uses offsetted zlen. 7357 add_one_64(z, zlen, carry, tmp1); 7358 7359 pop(len); 7360 pop(zlen); 7361 jmp(L_second_loop); 7362 7363 // Next infrequent code is moved outside loops. 7364 bind(L_last_x); 7365 movl(op1, Address(x, 0)); 7366 jmp(L_multiply); 7367 7368 bind(L_second_loop_exit); 7369 pop(len); 7370 pop(zlen); 7371 pop(len); 7372 pop(zlen); 7373 7374 // Fifth loop 7375 // Shift z left 1 bit. 7376 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 7377 7378 // z[zlen-1] |= x[len-1] & 1; 7379 movl(tmp3, Address(x, len, Address::times_4, -4)); 7380 andl(tmp3, 1); 7381 orl(Address(z, zlen, Address::times_4, -4), tmp3); 7382 7383 pop(tmp5); 7384 pop(tmp4); 7385 pop(tmp3); 7386 pop(tmp2); 7387 pop(tmp1); 7388 } 7389 7390 /** 7391 * Helper function for mul_add() 7392 * Multiply the in[] by int k and add to out[] starting at offset offs using 7393 * 128 bit by 32 bit multiply and return the carry in tmp5. 7394 * Only quad int aligned length of in[] is operated on in this function. 7395 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 7396 * This function preserves out, in and k registers. 7397 * len and offset point to the appropriate index in "in" & "out" correspondingly 7398 * tmp5 has the carry. 7399 * other registers are temporary and are modified. 7400 * 7401 */ 7402 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 7403 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 7404 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7405 7406 Label L_first_loop, L_first_loop_exit; 7407 7408 movl(tmp1, len); 7409 shrl(tmp1, 2); 7410 7411 bind(L_first_loop); 7412 subl(tmp1, 1); 7413 jccb(Assembler::negative, L_first_loop_exit); 7414 7415 subl(len, 4); 7416 subl(offset, 4); 7417 7418 Register op2 = tmp2; 7419 const Register sum = tmp3; 7420 const Register op1 = tmp4; 7421 const Register carry = tmp5; 7422 7423 if (UseBMI2Instructions) { 7424 op2 = rdxReg; 7425 } 7426 7427 movq(op1, Address(in, len, Address::times_4, 8)); 7428 rorq(op1, 32); 7429 movq(sum, Address(out, offset, Address::times_4, 8)); 7430 rorq(sum, 32); 7431 if (UseBMI2Instructions) { 7432 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7433 } 7434 else { 7435 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7436 } 7437 // Store back in big endian from little endian 7438 rorq(sum, 0x20); 7439 movq(Address(out, offset, Address::times_4, 8), sum); 7440 7441 movq(op1, Address(in, len, Address::times_4, 0)); 7442 rorq(op1, 32); 7443 movq(sum, Address(out, offset, Address::times_4, 0)); 7444 rorq(sum, 32); 7445 if (UseBMI2Instructions) { 7446 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7447 } 7448 else { 7449 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7450 } 7451 // Store back in big endian from little endian 7452 rorq(sum, 0x20); 7453 movq(Address(out, offset, Address::times_4, 0), sum); 7454 7455 jmp(L_first_loop); 7456 bind(L_first_loop_exit); 7457 } 7458 7459 /** 7460 * Code for BigInteger::mulAdd() intrinsic 7461 * 7462 * rdi: out 7463 * rsi: in 7464 * r11: offs (out.length - offset) 7465 * rcx: len 7466 * r8: k 7467 * r12: tmp1 7468 * r13: tmp2 7469 * r14: tmp3 7470 * r15: tmp4 7471 * rbx: tmp5 7472 * Multiply the in[] by word k and add to out[], return the carry in rax 7473 */ 7474 void MacroAssembler::mul_add(Register out, Register in, Register offs, 7475 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 7476 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7477 7478 Label L_carry, L_last_in, L_done; 7479 7480 // carry = 0; 7481 // for (int j=len-1; j >= 0; j--) { 7482 // long product = (in[j] & LONG_MASK) * kLong + 7483 // (out[offs] & LONG_MASK) + carry; 7484 // out[offs--] = (int)product; 7485 // carry = product >>> 32; 7486 // } 7487 // 7488 push(tmp1); 7489 push(tmp2); 7490 push(tmp3); 7491 push(tmp4); 7492 push(tmp5); 7493 7494 Register op2 = tmp2; 7495 const Register sum = tmp3; 7496 const Register op1 = tmp4; 7497 const Register carry = tmp5; 7498 7499 if (UseBMI2Instructions) { 7500 op2 = rdxReg; 7501 movl(op2, k); 7502 } 7503 else { 7504 movl(op2, k); 7505 } 7506 7507 xorq(carry, carry); 7508 7509 //First loop 7510 7511 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 7512 //The carry is in tmp5 7513 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 7514 7515 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 7516 decrementl(len); 7517 jccb(Assembler::negative, L_carry); 7518 decrementl(len); 7519 jccb(Assembler::negative, L_last_in); 7520 7521 movq(op1, Address(in, len, Address::times_4, 0)); 7522 rorq(op1, 32); 7523 7524 subl(offs, 2); 7525 movq(sum, Address(out, offs, Address::times_4, 0)); 7526 rorq(sum, 32); 7527 7528 if (UseBMI2Instructions) { 7529 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7530 } 7531 else { 7532 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7533 } 7534 7535 // Store back in big endian from little endian 7536 rorq(sum, 0x20); 7537 movq(Address(out, offs, Address::times_4, 0), sum); 7538 7539 testl(len, len); 7540 jccb(Assembler::zero, L_carry); 7541 7542 //Multiply the last in[] entry, if any 7543 bind(L_last_in); 7544 movl(op1, Address(in, 0)); 7545 movl(sum, Address(out, offs, Address::times_4, -4)); 7546 7547 movl(raxReg, k); 7548 mull(op1); //tmp4 * eax -> edx:eax 7549 addl(sum, carry); 7550 adcl(rdxReg, 0); 7551 addl(sum, raxReg); 7552 adcl(rdxReg, 0); 7553 movl(carry, rdxReg); 7554 7555 movl(Address(out, offs, Address::times_4, -4), sum); 7556 7557 bind(L_carry); 7558 //return tmp5/carry as carry in rax 7559 movl(rax, carry); 7560 7561 bind(L_done); 7562 pop(tmp5); 7563 pop(tmp4); 7564 pop(tmp3); 7565 pop(tmp2); 7566 pop(tmp1); 7567 } 7568 #endif 7569 7570 /** 7571 * Emits code to update CRC-32 with a byte value according to constants in table 7572 * 7573 * @param [in,out]crc Register containing the crc. 7574 * @param [in]val Register containing the byte to fold into the CRC. 7575 * @param [in]table Register containing the table of crc constants. 7576 * 7577 * uint32_t crc; 7578 * val = crc_table[(val ^ crc) & 0xFF]; 7579 * crc = val ^ (crc >> 8); 7580 * 7581 */ 7582 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 7583 xorl(val, crc); 7584 andl(val, 0xFF); 7585 shrl(crc, 8); // unsigned shift 7586 xorl(crc, Address(table, val, Address::times_4, 0)); 7587 } 7588 7589 /** 7590 * Fold 128-bit data chunk 7591 */ 7592 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 7593 if (UseAVX > 0) { 7594 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 7595 vpclmulldq(xcrc, xK, xcrc); // [63:0] 7596 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 7597 pxor(xcrc, xtmp); 7598 } else { 7599 movdqa(xtmp, xcrc); 7600 pclmulhdq(xtmp, xK); // [123:64] 7601 pclmulldq(xcrc, xK); // [63:0] 7602 pxor(xcrc, xtmp); 7603 movdqu(xtmp, Address(buf, offset)); 7604 pxor(xcrc, xtmp); 7605 } 7606 } 7607 7608 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 7609 if (UseAVX > 0) { 7610 vpclmulhdq(xtmp, xK, xcrc); 7611 vpclmulldq(xcrc, xK, xcrc); 7612 pxor(xcrc, xbuf); 7613 pxor(xcrc, xtmp); 7614 } else { 7615 movdqa(xtmp, xcrc); 7616 pclmulhdq(xtmp, xK); 7617 pclmulldq(xcrc, xK); 7618 pxor(xcrc, xbuf); 7619 pxor(xcrc, xtmp); 7620 } 7621 } 7622 7623 /** 7624 * 8-bit folds to compute 32-bit CRC 7625 * 7626 * uint64_t xcrc; 7627 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 7628 */ 7629 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 7630 movdl(tmp, xcrc); 7631 andl(tmp, 0xFF); 7632 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 7633 psrldq(xcrc, 1); // unsigned shift one byte 7634 pxor(xcrc, xtmp); 7635 } 7636 7637 /** 7638 * uint32_t crc; 7639 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 7640 */ 7641 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 7642 movl(tmp, crc); 7643 andl(tmp, 0xFF); 7644 shrl(crc, 8); 7645 xorl(crc, Address(table, tmp, Address::times_4, 0)); 7646 } 7647 7648 /** 7649 * @param crc register containing existing CRC (32-bit) 7650 * @param buf register pointing to input byte buffer (byte*) 7651 * @param len register containing number of bytes 7652 * @param table register that will contain address of CRC table 7653 * @param tmp scratch register 7654 */ 7655 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 7656 assert_different_registers(crc, buf, len, table, tmp, rax); 7657 7658 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7659 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7660 7661 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7662 // context for the registers used, where all instructions below are using 128-bit mode 7663 // On EVEX without VL and BW, these instructions will all be AVX. 7664 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 7665 notl(crc); // ~crc 7666 cmpl(len, 16); 7667 jcc(Assembler::less, L_tail); 7668 7669 // Align buffer to 16 bytes 7670 movl(tmp, buf); 7671 andl(tmp, 0xF); 7672 jccb(Assembler::zero, L_aligned); 7673 subl(tmp, 16); 7674 addl(len, tmp); 7675 7676 align(4); 7677 BIND(L_align_loop); 7678 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7679 update_byte_crc32(crc, rax, table); 7680 increment(buf); 7681 incrementl(tmp); 7682 jccb(Assembler::less, L_align_loop); 7683 7684 BIND(L_aligned); 7685 movl(tmp, len); // save 7686 shrl(len, 4); 7687 jcc(Assembler::zero, L_tail_restore); 7688 7689 // Fold crc into first bytes of vector 7690 movdqa(xmm1, Address(buf, 0)); 7691 movdl(rax, xmm1); 7692 xorl(crc, rax); 7693 if (VM_Version::supports_sse4_1()) { 7694 pinsrd(xmm1, crc, 0); 7695 } else { 7696 pinsrw(xmm1, crc, 0); 7697 shrl(crc, 16); 7698 pinsrw(xmm1, crc, 1); 7699 } 7700 addptr(buf, 16); 7701 subl(len, 4); // len > 0 7702 jcc(Assembler::less, L_fold_tail); 7703 7704 movdqa(xmm2, Address(buf, 0)); 7705 movdqa(xmm3, Address(buf, 16)); 7706 movdqa(xmm4, Address(buf, 32)); 7707 addptr(buf, 48); 7708 subl(len, 3); 7709 jcc(Assembler::lessEqual, L_fold_512b); 7710 7711 // Fold total 512 bits of polynomial on each iteration, 7712 // 128 bits per each of 4 parallel streams. 7713 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 7714 7715 align32(); 7716 BIND(L_fold_512b_loop); 7717 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7718 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 7719 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 7720 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 7721 addptr(buf, 64); 7722 subl(len, 4); 7723 jcc(Assembler::greater, L_fold_512b_loop); 7724 7725 // Fold 512 bits to 128 bits. 7726 BIND(L_fold_512b); 7727 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7728 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 7729 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 7730 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 7731 7732 // Fold the rest of 128 bits data chunks 7733 BIND(L_fold_tail); 7734 addl(len, 3); 7735 jccb(Assembler::lessEqual, L_fold_128b); 7736 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7737 7738 BIND(L_fold_tail_loop); 7739 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7740 addptr(buf, 16); 7741 decrementl(len); 7742 jccb(Assembler::greater, L_fold_tail_loop); 7743 7744 // Fold 128 bits in xmm1 down into 32 bits in crc register. 7745 BIND(L_fold_128b); 7746 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 7747 if (UseAVX > 0) { 7748 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 7749 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 7750 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 7751 } else { 7752 movdqa(xmm2, xmm0); 7753 pclmulqdq(xmm2, xmm1, 0x1); 7754 movdqa(xmm3, xmm0); 7755 pand(xmm3, xmm2); 7756 pclmulqdq(xmm0, xmm3, 0x1); 7757 } 7758 psrldq(xmm1, 8); 7759 psrldq(xmm2, 4); 7760 pxor(xmm0, xmm1); 7761 pxor(xmm0, xmm2); 7762 7763 // 8 8-bit folds to compute 32-bit CRC. 7764 for (int j = 0; j < 4; j++) { 7765 fold_8bit_crc32(xmm0, table, xmm1, rax); 7766 } 7767 movdl(crc, xmm0); // mov 32 bits to general register 7768 for (int j = 0; j < 4; j++) { 7769 fold_8bit_crc32(crc, table, rax); 7770 } 7771 7772 BIND(L_tail_restore); 7773 movl(len, tmp); // restore 7774 BIND(L_tail); 7775 andl(len, 0xf); 7776 jccb(Assembler::zero, L_exit); 7777 7778 // Fold the rest of bytes 7779 align(4); 7780 BIND(L_tail_loop); 7781 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7782 update_byte_crc32(crc, rax, table); 7783 increment(buf); 7784 decrementl(len); 7785 jccb(Assembler::greater, L_tail_loop); 7786 7787 BIND(L_exit); 7788 notl(crc); // ~c 7789 } 7790 7791 #ifdef _LP64 7792 // Helper function for AVX 512 CRC32 7793 // Fold 512-bit data chunks 7794 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 7795 Register pos, int offset) { 7796 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 7797 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 7798 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 7799 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 7800 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 7801 } 7802 7803 // Helper function for AVX 512 CRC32 7804 // Compute CRC32 for < 256B buffers 7805 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 7806 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 7807 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 7808 7809 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 7810 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 7811 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 7812 7813 // check if there is enough buffer to be able to fold 16B at a time 7814 cmpl(len, 32); 7815 jcc(Assembler::less, L_less_than_32); 7816 7817 // if there is, load the constants 7818 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 7819 movdl(xmm0, crc); // get the initial crc value 7820 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7821 pxor(xmm7, xmm0); 7822 7823 // update the buffer pointer 7824 addl(pos, 16); 7825 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 7826 subl(len, 32); 7827 jmp(L_16B_reduction_loop); 7828 7829 bind(L_less_than_32); 7830 //mov initial crc to the return value. this is necessary for zero - length buffers. 7831 movl(rax, crc); 7832 testl(len, len); 7833 jcc(Assembler::equal, L_cleanup); 7834 7835 movdl(xmm0, crc); //get the initial crc value 7836 7837 cmpl(len, 16); 7838 jcc(Assembler::equal, L_exact_16_left); 7839 jcc(Assembler::less, L_less_than_16_left); 7840 7841 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7842 pxor(xmm7, xmm0); //xor the initial crc value 7843 addl(pos, 16); 7844 subl(len, 16); 7845 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 7846 jmp(L_get_last_two_xmms); 7847 7848 bind(L_less_than_16_left); 7849 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 7850 pxor(xmm1, xmm1); 7851 movptr(tmp1, rsp); 7852 movdqu(Address(tmp1, 0 * 16), xmm1); 7853 7854 cmpl(len, 4); 7855 jcc(Assembler::less, L_only_less_than_4); 7856 7857 //backup the counter value 7858 movl(tmp2, len); 7859 cmpl(len, 8); 7860 jcc(Assembler::less, L_less_than_8_left); 7861 7862 //load 8 Bytes 7863 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 7864 movq(Address(tmp1, 0 * 16), rax); 7865 addptr(tmp1, 8); 7866 subl(len, 8); 7867 addl(pos, 8); 7868 7869 bind(L_less_than_8_left); 7870 cmpl(len, 4); 7871 jcc(Assembler::less, L_less_than_4_left); 7872 7873 //load 4 Bytes 7874 movl(rax, Address(buf, pos, Address::times_1, 0)); 7875 movl(Address(tmp1, 0 * 16), rax); 7876 addptr(tmp1, 4); 7877 subl(len, 4); 7878 addl(pos, 4); 7879 7880 bind(L_less_than_4_left); 7881 cmpl(len, 2); 7882 jcc(Assembler::less, L_less_than_2_left); 7883 7884 // load 2 Bytes 7885 movw(rax, Address(buf, pos, Address::times_1, 0)); 7886 movl(Address(tmp1, 0 * 16), rax); 7887 addptr(tmp1, 2); 7888 subl(len, 2); 7889 addl(pos, 2); 7890 7891 bind(L_less_than_2_left); 7892 cmpl(len, 1); 7893 jcc(Assembler::less, L_zero_left); 7894 7895 // load 1 Byte 7896 movb(rax, Address(buf, pos, Address::times_1, 0)); 7897 movb(Address(tmp1, 0 * 16), rax); 7898 7899 bind(L_zero_left); 7900 movdqu(xmm7, Address(rsp, 0)); 7901 pxor(xmm7, xmm0); //xor the initial crc value 7902 7903 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7904 movdqu(xmm0, Address(rax, tmp2)); 7905 pshufb(xmm7, xmm0); 7906 jmp(L_128_done); 7907 7908 bind(L_exact_16_left); 7909 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 7910 pxor(xmm7, xmm0); //xor the initial crc value 7911 jmp(L_128_done); 7912 7913 bind(L_only_less_than_4); 7914 cmpl(len, 3); 7915 jcc(Assembler::less, L_only_less_than_3); 7916 7917 // load 3 Bytes 7918 movb(rax, Address(buf, pos, Address::times_1, 0)); 7919 movb(Address(tmp1, 0), rax); 7920 7921 movb(rax, Address(buf, pos, Address::times_1, 1)); 7922 movb(Address(tmp1, 1), rax); 7923 7924 movb(rax, Address(buf, pos, Address::times_1, 2)); 7925 movb(Address(tmp1, 2), rax); 7926 7927 movdqu(xmm7, Address(rsp, 0)); 7928 pxor(xmm7, xmm0); //xor the initial crc value 7929 7930 pslldq(xmm7, 0x5); 7931 jmp(L_barrett); 7932 bind(L_only_less_than_3); 7933 cmpl(len, 2); 7934 jcc(Assembler::less, L_only_less_than_2); 7935 7936 // load 2 Bytes 7937 movb(rax, Address(buf, pos, Address::times_1, 0)); 7938 movb(Address(tmp1, 0), rax); 7939 7940 movb(rax, Address(buf, pos, Address::times_1, 1)); 7941 movb(Address(tmp1, 1), rax); 7942 7943 movdqu(xmm7, Address(rsp, 0)); 7944 pxor(xmm7, xmm0); //xor the initial crc value 7945 7946 pslldq(xmm7, 0x6); 7947 jmp(L_barrett); 7948 7949 bind(L_only_less_than_2); 7950 //load 1 Byte 7951 movb(rax, Address(buf, pos, Address::times_1, 0)); 7952 movb(Address(tmp1, 0), rax); 7953 7954 movdqu(xmm7, Address(rsp, 0)); 7955 pxor(xmm7, xmm0); //xor the initial crc value 7956 7957 pslldq(xmm7, 0x7); 7958 } 7959 7960 /** 7961 * Compute CRC32 using AVX512 instructions 7962 * param crc register containing existing CRC (32-bit) 7963 * param buf register pointing to input byte buffer (byte*) 7964 * param len register containing number of bytes 7965 * param table address of crc or crc32c table 7966 * param tmp1 scratch register 7967 * param tmp2 scratch register 7968 * return rax result register 7969 * 7970 * This routine is identical for crc32c with the exception of the precomputed constant 7971 * table which will be passed as the table argument. The calculation steps are 7972 * the same for both variants. 7973 */ 7974 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 7975 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 7976 7977 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7978 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7979 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 7980 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 7981 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 7982 7983 const Register pos = r12; 7984 push(r12); 7985 subptr(rsp, 16 * 2 + 8); 7986 7987 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7988 // context for the registers used, where all instructions below are using 128-bit mode 7989 // On EVEX without VL and BW, these instructions will all be AVX. 7990 movl(pos, 0); 7991 7992 // check if smaller than 256B 7993 cmpl(len, 256); 7994 jcc(Assembler::less, L_less_than_256); 7995 7996 // load the initial crc value 7997 movdl(xmm10, crc); 7998 7999 // receive the initial 64B data, xor the initial crc value 8000 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 8001 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 8002 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 8003 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 8004 8005 subl(len, 256); 8006 cmpl(len, 256); 8007 jcc(Assembler::less, L_fold_128_B_loop); 8008 8009 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 8010 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 8011 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 8012 subl(len, 256); 8013 8014 bind(L_fold_256_B_loop); 8015 addl(pos, 256); 8016 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 8017 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 8018 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 8019 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 8020 8021 subl(len, 256); 8022 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 8023 8024 // Fold 256 into 128 8025 addl(pos, 256); 8026 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 8027 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 8028 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 8029 8030 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 8031 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 8032 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 8033 8034 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 8035 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 8036 8037 addl(len, 128); 8038 jmp(L_fold_128_B_register); 8039 8040 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 8041 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 8042 8043 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 8044 bind(L_fold_128_B_loop); 8045 addl(pos, 128); 8046 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 8047 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 8048 8049 subl(len, 128); 8050 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 8051 8052 addl(pos, 128); 8053 8054 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 8055 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 8056 bind(L_fold_128_B_register); 8057 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 8058 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 8059 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 8060 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 8061 // save last that has no multiplicand 8062 vextracti64x2(xmm7, xmm4, 3); 8063 8064 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 8065 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 8066 // Needed later in reduction loop 8067 movdqu(xmm10, Address(table, 1 * 16)); 8068 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 8069 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 8070 8071 // Swap 1,0,3,2 - 01 00 11 10 8072 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 8073 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 8074 vextracti128(xmm5, xmm8, 1); 8075 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 8076 8077 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 8078 // instead of a cmp instruction, we use the negative flag with the jl instruction 8079 addl(len, 128 - 16); 8080 jcc(Assembler::less, L_final_reduction_for_128); 8081 8082 bind(L_16B_reduction_loop); 8083 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8084 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8085 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8086 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 8087 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8088 addl(pos, 16); 8089 subl(len, 16); 8090 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 8091 8092 bind(L_final_reduction_for_128); 8093 addl(len, 16); 8094 jcc(Assembler::equal, L_128_done); 8095 8096 bind(L_get_last_two_xmms); 8097 movdqu(xmm2, xmm7); 8098 addl(pos, len); 8099 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 8100 subl(pos, len); 8101 8102 // get rid of the extra data that was loaded before 8103 // load the shift constant 8104 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8105 movdqu(xmm0, Address(rax, len)); 8106 addl(rax, len); 8107 8108 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8109 //Change mask to 512 8110 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 8111 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 8112 8113 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 8114 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8115 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8116 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8117 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 8118 8119 bind(L_128_done); 8120 // compute crc of a 128-bit value 8121 movdqu(xmm10, Address(table, 3 * 16)); 8122 movdqu(xmm0, xmm7); 8123 8124 // 64b fold 8125 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 8126 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 8127 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8128 8129 // 32b fold 8130 movdqu(xmm0, xmm7); 8131 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 8132 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8133 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8134 jmp(L_barrett); 8135 8136 bind(L_less_than_256); 8137 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 8138 8139 //barrett reduction 8140 bind(L_barrett); 8141 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 8142 movdqu(xmm1, xmm7); 8143 movdqu(xmm2, xmm7); 8144 movdqu(xmm10, Address(table, 4 * 16)); 8145 8146 pclmulqdq(xmm7, xmm10, 0x0); 8147 pxor(xmm7, xmm2); 8148 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 8149 movdqu(xmm2, xmm7); 8150 pclmulqdq(xmm7, xmm10, 0x10); 8151 pxor(xmm7, xmm2); 8152 pxor(xmm7, xmm1); 8153 pextrd(crc, xmm7, 2); 8154 8155 bind(L_cleanup); 8156 addptr(rsp, 16 * 2 + 8); 8157 pop(r12); 8158 } 8159 8160 // S. Gueron / Information Processing Letters 112 (2012) 184 8161 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 8162 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 8163 // Output: the 64-bit carry-less product of B * CONST 8164 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 8165 Register tmp1, Register tmp2, Register tmp3) { 8166 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8167 if (n > 0) { 8168 addq(tmp3, n * 256 * 8); 8169 } 8170 // Q1 = TABLEExt[n][B & 0xFF]; 8171 movl(tmp1, in); 8172 andl(tmp1, 0x000000FF); 8173 shll(tmp1, 3); 8174 addq(tmp1, tmp3); 8175 movq(tmp1, Address(tmp1, 0)); 8176 8177 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8178 movl(tmp2, in); 8179 shrl(tmp2, 8); 8180 andl(tmp2, 0x000000FF); 8181 shll(tmp2, 3); 8182 addq(tmp2, tmp3); 8183 movq(tmp2, Address(tmp2, 0)); 8184 8185 shlq(tmp2, 8); 8186 xorq(tmp1, tmp2); 8187 8188 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8189 movl(tmp2, in); 8190 shrl(tmp2, 16); 8191 andl(tmp2, 0x000000FF); 8192 shll(tmp2, 3); 8193 addq(tmp2, tmp3); 8194 movq(tmp2, Address(tmp2, 0)); 8195 8196 shlq(tmp2, 16); 8197 xorq(tmp1, tmp2); 8198 8199 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8200 shrl(in, 24); 8201 andl(in, 0x000000FF); 8202 shll(in, 3); 8203 addq(in, tmp3); 8204 movq(in, Address(in, 0)); 8205 8206 shlq(in, 24); 8207 xorq(in, tmp1); 8208 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8209 } 8210 8211 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8212 Register in_out, 8213 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8214 XMMRegister w_xtmp2, 8215 Register tmp1, 8216 Register n_tmp2, Register n_tmp3) { 8217 if (is_pclmulqdq_supported) { 8218 movdl(w_xtmp1, in_out); // modified blindly 8219 8220 movl(tmp1, const_or_pre_comp_const_index); 8221 movdl(w_xtmp2, tmp1); 8222 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8223 8224 movdq(in_out, w_xtmp1); 8225 } else { 8226 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 8227 } 8228 } 8229 8230 // Recombination Alternative 2: No bit-reflections 8231 // T1 = (CRC_A * U1) << 1 8232 // T2 = (CRC_B * U2) << 1 8233 // C1 = T1 >> 32 8234 // C2 = T2 >> 32 8235 // T1 = T1 & 0xFFFFFFFF 8236 // T2 = T2 & 0xFFFFFFFF 8237 // T1 = CRC32(0, T1) 8238 // T2 = CRC32(0, T2) 8239 // C1 = C1 ^ T1 8240 // C2 = C2 ^ T2 8241 // CRC = C1 ^ C2 ^ CRC_C 8242 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8243 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8244 Register tmp1, Register tmp2, 8245 Register n_tmp3) { 8246 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8247 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8248 shlq(in_out, 1); 8249 movl(tmp1, in_out); 8250 shrq(in_out, 32); 8251 xorl(tmp2, tmp2); 8252 crc32(tmp2, tmp1, 4); 8253 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8254 shlq(in1, 1); 8255 movl(tmp1, in1); 8256 shrq(in1, 32); 8257 xorl(tmp2, tmp2); 8258 crc32(tmp2, tmp1, 4); 8259 xorl(in1, tmp2); 8260 xorl(in_out, in1); 8261 xorl(in_out, in2); 8262 } 8263 8264 // Set N to predefined value 8265 // Subtract from a length of a buffer 8266 // execute in a loop: 8267 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8268 // for i = 1 to N do 8269 // CRC_A = CRC32(CRC_A, A[i]) 8270 // CRC_B = CRC32(CRC_B, B[i]) 8271 // CRC_C = CRC32(CRC_C, C[i]) 8272 // end for 8273 // Recombine 8274 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8275 Register in_out1, Register in_out2, Register in_out3, 8276 Register tmp1, Register tmp2, Register tmp3, 8277 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8278 Register tmp4, Register tmp5, 8279 Register n_tmp6) { 8280 Label L_processPartitions; 8281 Label L_processPartition; 8282 Label L_exit; 8283 8284 bind(L_processPartitions); 8285 cmpl(in_out1, 3 * size); 8286 jcc(Assembler::less, L_exit); 8287 xorl(tmp1, tmp1); 8288 xorl(tmp2, tmp2); 8289 movq(tmp3, in_out2); 8290 addq(tmp3, size); 8291 8292 bind(L_processPartition); 8293 crc32(in_out3, Address(in_out2, 0), 8); 8294 crc32(tmp1, Address(in_out2, size), 8); 8295 crc32(tmp2, Address(in_out2, size * 2), 8); 8296 addq(in_out2, 8); 8297 cmpq(in_out2, tmp3); 8298 jcc(Assembler::less, L_processPartition); 8299 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8300 w_xtmp1, w_xtmp2, w_xtmp3, 8301 tmp4, tmp5, 8302 n_tmp6); 8303 addq(in_out2, 2 * size); 8304 subl(in_out1, 3 * size); 8305 jmp(L_processPartitions); 8306 8307 bind(L_exit); 8308 } 8309 #else 8310 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, 8311 Register tmp1, Register tmp2, Register tmp3, 8312 XMMRegister xtmp1, XMMRegister xtmp2) { 8313 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8314 if (n > 0) { 8315 addl(tmp3, n * 256 * 8); 8316 } 8317 // Q1 = TABLEExt[n][B & 0xFF]; 8318 movl(tmp1, in_out); 8319 andl(tmp1, 0x000000FF); 8320 shll(tmp1, 3); 8321 addl(tmp1, tmp3); 8322 movq(xtmp1, Address(tmp1, 0)); 8323 8324 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8325 movl(tmp2, in_out); 8326 shrl(tmp2, 8); 8327 andl(tmp2, 0x000000FF); 8328 shll(tmp2, 3); 8329 addl(tmp2, tmp3); 8330 movq(xtmp2, Address(tmp2, 0)); 8331 8332 psllq(xtmp2, 8); 8333 pxor(xtmp1, xtmp2); 8334 8335 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8336 movl(tmp2, in_out); 8337 shrl(tmp2, 16); 8338 andl(tmp2, 0x000000FF); 8339 shll(tmp2, 3); 8340 addl(tmp2, tmp3); 8341 movq(xtmp2, Address(tmp2, 0)); 8342 8343 psllq(xtmp2, 16); 8344 pxor(xtmp1, xtmp2); 8345 8346 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8347 shrl(in_out, 24); 8348 andl(in_out, 0x000000FF); 8349 shll(in_out, 3); 8350 addl(in_out, tmp3); 8351 movq(xtmp2, Address(in_out, 0)); 8352 8353 psllq(xtmp2, 24); 8354 pxor(xtmp1, xtmp2); // Result in CXMM 8355 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8356 } 8357 8358 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8359 Register in_out, 8360 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8361 XMMRegister w_xtmp2, 8362 Register tmp1, 8363 Register n_tmp2, Register n_tmp3) { 8364 if (is_pclmulqdq_supported) { 8365 movdl(w_xtmp1, in_out); 8366 8367 movl(tmp1, const_or_pre_comp_const_index); 8368 movdl(w_xtmp2, tmp1); 8369 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8370 // Keep result in XMM since GPR is 32 bit in length 8371 } else { 8372 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); 8373 } 8374 } 8375 8376 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8377 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8378 Register tmp1, Register tmp2, 8379 Register n_tmp3) { 8380 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8381 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8382 8383 psllq(w_xtmp1, 1); 8384 movdl(tmp1, w_xtmp1); 8385 psrlq(w_xtmp1, 32); 8386 movdl(in_out, w_xtmp1); 8387 8388 xorl(tmp2, tmp2); 8389 crc32(tmp2, tmp1, 4); 8390 xorl(in_out, tmp2); 8391 8392 psllq(w_xtmp2, 1); 8393 movdl(tmp1, w_xtmp2); 8394 psrlq(w_xtmp2, 32); 8395 movdl(in1, w_xtmp2); 8396 8397 xorl(tmp2, tmp2); 8398 crc32(tmp2, tmp1, 4); 8399 xorl(in1, tmp2); 8400 xorl(in_out, in1); 8401 xorl(in_out, in2); 8402 } 8403 8404 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8405 Register in_out1, Register in_out2, Register in_out3, 8406 Register tmp1, Register tmp2, Register tmp3, 8407 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8408 Register tmp4, Register tmp5, 8409 Register n_tmp6) { 8410 Label L_processPartitions; 8411 Label L_processPartition; 8412 Label L_exit; 8413 8414 bind(L_processPartitions); 8415 cmpl(in_out1, 3 * size); 8416 jcc(Assembler::less, L_exit); 8417 xorl(tmp1, tmp1); 8418 xorl(tmp2, tmp2); 8419 movl(tmp3, in_out2); 8420 addl(tmp3, size); 8421 8422 bind(L_processPartition); 8423 crc32(in_out3, Address(in_out2, 0), 4); 8424 crc32(tmp1, Address(in_out2, size), 4); 8425 crc32(tmp2, Address(in_out2, size*2), 4); 8426 crc32(in_out3, Address(in_out2, 0+4), 4); 8427 crc32(tmp1, Address(in_out2, size+4), 4); 8428 crc32(tmp2, Address(in_out2, size*2+4), 4); 8429 addl(in_out2, 8); 8430 cmpl(in_out2, tmp3); 8431 jcc(Assembler::less, L_processPartition); 8432 8433 push(tmp3); 8434 push(in_out1); 8435 push(in_out2); 8436 tmp4 = tmp3; 8437 tmp5 = in_out1; 8438 n_tmp6 = in_out2; 8439 8440 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8441 w_xtmp1, w_xtmp2, w_xtmp3, 8442 tmp4, tmp5, 8443 n_tmp6); 8444 8445 pop(in_out2); 8446 pop(in_out1); 8447 pop(tmp3); 8448 8449 addl(in_out2, 2 * size); 8450 subl(in_out1, 3 * size); 8451 jmp(L_processPartitions); 8452 8453 bind(L_exit); 8454 } 8455 #endif //LP64 8456 8457 #ifdef _LP64 8458 // Algorithm 2: Pipelined usage of the CRC32 instruction. 8459 // Input: A buffer I of L bytes. 8460 // Output: the CRC32C value of the buffer. 8461 // Notations: 8462 // Write L = 24N + r, with N = floor (L/24). 8463 // r = L mod 24 (0 <= r < 24). 8464 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 8465 // N quadwords, and R consists of r bytes. 8466 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 8467 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 8468 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 8469 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 8470 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8471 Register tmp1, Register tmp2, Register tmp3, 8472 Register tmp4, Register tmp5, Register tmp6, 8473 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8474 bool is_pclmulqdq_supported) { 8475 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8476 Label L_wordByWord; 8477 Label L_byteByByteProlog; 8478 Label L_byteByByte; 8479 Label L_exit; 8480 8481 if (is_pclmulqdq_supported ) { 8482 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8483 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); 8484 8485 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8486 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8487 8488 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8489 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8490 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 8491 } else { 8492 const_or_pre_comp_const_index[0] = 1; 8493 const_or_pre_comp_const_index[1] = 0; 8494 8495 const_or_pre_comp_const_index[2] = 3; 8496 const_or_pre_comp_const_index[3] = 2; 8497 8498 const_or_pre_comp_const_index[4] = 5; 8499 const_or_pre_comp_const_index[5] = 4; 8500 } 8501 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8502 in2, in1, in_out, 8503 tmp1, tmp2, tmp3, 8504 w_xtmp1, w_xtmp2, w_xtmp3, 8505 tmp4, tmp5, 8506 tmp6); 8507 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8508 in2, in1, in_out, 8509 tmp1, tmp2, tmp3, 8510 w_xtmp1, w_xtmp2, w_xtmp3, 8511 tmp4, tmp5, 8512 tmp6); 8513 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8514 in2, in1, in_out, 8515 tmp1, tmp2, tmp3, 8516 w_xtmp1, w_xtmp2, w_xtmp3, 8517 tmp4, tmp5, 8518 tmp6); 8519 movl(tmp1, in2); 8520 andl(tmp1, 0x00000007); 8521 negl(tmp1); 8522 addl(tmp1, in2); 8523 addq(tmp1, in1); 8524 8525 cmpq(in1, tmp1); 8526 jccb(Assembler::greaterEqual, L_byteByByteProlog); 8527 align(16); 8528 BIND(L_wordByWord); 8529 crc32(in_out, Address(in1, 0), 8); 8530 addq(in1, 8); 8531 cmpq(in1, tmp1); 8532 jcc(Assembler::less, L_wordByWord); 8533 8534 BIND(L_byteByByteProlog); 8535 andl(in2, 0x00000007); 8536 movl(tmp2, 1); 8537 8538 cmpl(tmp2, in2); 8539 jccb(Assembler::greater, L_exit); 8540 BIND(L_byteByByte); 8541 crc32(in_out, Address(in1, 0), 1); 8542 incq(in1); 8543 incl(tmp2); 8544 cmpl(tmp2, in2); 8545 jcc(Assembler::lessEqual, L_byteByByte); 8546 8547 BIND(L_exit); 8548 } 8549 #else 8550 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8551 Register tmp1, Register tmp2, Register tmp3, 8552 Register tmp4, Register tmp5, Register tmp6, 8553 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8554 bool is_pclmulqdq_supported) { 8555 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8556 Label L_wordByWord; 8557 Label L_byteByByteProlog; 8558 Label L_byteByByte; 8559 Label L_exit; 8560 8561 if (is_pclmulqdq_supported) { 8562 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8563 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); 8564 8565 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8566 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8567 8568 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8569 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8570 } else { 8571 const_or_pre_comp_const_index[0] = 1; 8572 const_or_pre_comp_const_index[1] = 0; 8573 8574 const_or_pre_comp_const_index[2] = 3; 8575 const_or_pre_comp_const_index[3] = 2; 8576 8577 const_or_pre_comp_const_index[4] = 5; 8578 const_or_pre_comp_const_index[5] = 4; 8579 } 8580 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8581 in2, in1, in_out, 8582 tmp1, tmp2, tmp3, 8583 w_xtmp1, w_xtmp2, w_xtmp3, 8584 tmp4, tmp5, 8585 tmp6); 8586 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8587 in2, in1, in_out, 8588 tmp1, tmp2, tmp3, 8589 w_xtmp1, w_xtmp2, w_xtmp3, 8590 tmp4, tmp5, 8591 tmp6); 8592 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8593 in2, in1, in_out, 8594 tmp1, tmp2, tmp3, 8595 w_xtmp1, w_xtmp2, w_xtmp3, 8596 tmp4, tmp5, 8597 tmp6); 8598 movl(tmp1, in2); 8599 andl(tmp1, 0x00000007); 8600 negl(tmp1); 8601 addl(tmp1, in2); 8602 addl(tmp1, in1); 8603 8604 BIND(L_wordByWord); 8605 cmpl(in1, tmp1); 8606 jcc(Assembler::greaterEqual, L_byteByByteProlog); 8607 crc32(in_out, Address(in1,0), 4); 8608 addl(in1, 4); 8609 jmp(L_wordByWord); 8610 8611 BIND(L_byteByByteProlog); 8612 andl(in2, 0x00000007); 8613 movl(tmp2, 1); 8614 8615 BIND(L_byteByByte); 8616 cmpl(tmp2, in2); 8617 jccb(Assembler::greater, L_exit); 8618 movb(tmp1, Address(in1, 0)); 8619 crc32(in_out, tmp1, 1); 8620 incl(in1); 8621 incl(tmp2); 8622 jmp(L_byteByByte); 8623 8624 BIND(L_exit); 8625 } 8626 #endif // LP64 8627 #undef BIND 8628 #undef BLOCK_COMMENT 8629 8630 // Compress char[] array to byte[]. 8631 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 8632 // Return the array length if every element in array can be encoded, 8633 // otherwise, the index of first non-latin1 (> 0xff) character. 8634 // @IntrinsicCandidate 8635 // public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 8636 // for (int i = 0; i < len; i++) { 8637 // char c = src[srcOff]; 8638 // if (c > 0xff) { 8639 // return i; // return index of non-latin1 char 8640 // } 8641 // dst[dstOff] = (byte)c; 8642 // srcOff++; 8643 // dstOff++; 8644 // } 8645 // return len; 8646 // } 8647 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 8648 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 8649 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 8650 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 8651 Label copy_chars_loop, done, reset_sp, copy_tail; 8652 8653 // rsi: src 8654 // rdi: dst 8655 // rdx: len 8656 // rcx: tmp5 8657 // rax: result 8658 8659 // rsi holds start addr of source char[] to be compressed 8660 // rdi holds start addr of destination byte[] 8661 // rdx holds length 8662 8663 assert(len != result, ""); 8664 8665 // save length for return 8666 movl(result, len); 8667 8668 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 8669 VM_Version::supports_avx512vlbw() && 8670 VM_Version::supports_bmi2()) { 8671 8672 Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail; 8673 8674 // alignment 8675 Label post_alignment; 8676 8677 // if length of the string is less than 32, handle it the old fashioned way 8678 testl(len, -32); 8679 jcc(Assembler::zero, below_threshold); 8680 8681 // First check whether a character is compressible ( <= 0xFF). 8682 // Create mask to test for Unicode chars inside zmm vector 8683 movl(tmp5, 0x00FF); 8684 evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit); 8685 8686 testl(len, -64); 8687 jccb(Assembler::zero, post_alignment); 8688 8689 movl(tmp5, dst); 8690 andl(tmp5, (32 - 1)); 8691 negl(tmp5); 8692 andl(tmp5, (32 - 1)); 8693 8694 // bail out when there is nothing to be done 8695 testl(tmp5, 0xFFFFFFFF); 8696 jccb(Assembler::zero, post_alignment); 8697 8698 // ~(~0 << len), where len is the # of remaining elements to process 8699 movl(len, 0xFFFFFFFF); 8700 shlxl(len, len, tmp5); 8701 notl(len); 8702 kmovdl(mask2, len); 8703 movl(len, result); 8704 8705 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8706 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8707 ktestd(mask1, mask2); 8708 jcc(Assembler::carryClear, copy_tail); 8709 8710 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8711 8712 addptr(src, tmp5); 8713 addptr(src, tmp5); 8714 addptr(dst, tmp5); 8715 subl(len, tmp5); 8716 8717 bind(post_alignment); 8718 // end of alignment 8719 8720 movl(tmp5, len); 8721 andl(tmp5, (32 - 1)); // tail count (in chars) 8722 andl(len, ~(32 - 1)); // vector count (in chars) 8723 jccb(Assembler::zero, copy_loop_tail); 8724 8725 lea(src, Address(src, len, Address::times_2)); 8726 lea(dst, Address(dst, len, Address::times_1)); 8727 negptr(len); 8728 8729 bind(copy_32_loop); 8730 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 8731 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 8732 kortestdl(mask1, mask1); 8733 jccb(Assembler::carryClear, reset_for_copy_tail); 8734 8735 // All elements in current processed chunk are valid candidates for 8736 // compression. Write a truncated byte elements to the memory. 8737 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 8738 addptr(len, 32); 8739 jccb(Assembler::notZero, copy_32_loop); 8740 8741 bind(copy_loop_tail); 8742 // bail out when there is nothing to be done 8743 testl(tmp5, 0xFFFFFFFF); 8744 jcc(Assembler::zero, done); 8745 8746 movl(len, tmp5); 8747 8748 // ~(~0 << len), where len is the # of remaining elements to process 8749 movl(tmp5, 0xFFFFFFFF); 8750 shlxl(tmp5, tmp5, len); 8751 notl(tmp5); 8752 8753 kmovdl(mask2, tmp5); 8754 8755 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8756 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8757 ktestd(mask1, mask2); 8758 jcc(Assembler::carryClear, copy_tail); 8759 8760 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8761 jmp(done); 8762 8763 bind(reset_for_copy_tail); 8764 lea(src, Address(src, tmp5, Address::times_2)); 8765 lea(dst, Address(dst, tmp5, Address::times_1)); 8766 subptr(len, tmp5); 8767 jmp(copy_chars_loop); 8768 8769 bind(below_threshold); 8770 } 8771 8772 if (UseSSE42Intrinsics) { 8773 Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail; 8774 8775 // vectored compression 8776 testl(len, 0xfffffff8); 8777 jcc(Assembler::zero, copy_tail); 8778 8779 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 8780 movdl(tmp1Reg, tmp5); 8781 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8782 8783 andl(len, 0xfffffff0); 8784 jccb(Assembler::zero, copy_16); 8785 8786 // compress 16 chars per iter 8787 pxor(tmp4Reg, tmp4Reg); 8788 8789 lea(src, Address(src, len, Address::times_2)); 8790 lea(dst, Address(dst, len, Address::times_1)); 8791 negptr(len); 8792 8793 bind(copy_32_loop); 8794 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 8795 por(tmp4Reg, tmp2Reg); 8796 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 8797 por(tmp4Reg, tmp3Reg); 8798 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 8799 jccb(Assembler::notZero, reset_for_copy_tail); 8800 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 8801 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 8802 addptr(len, 16); 8803 jccb(Assembler::notZero, copy_32_loop); 8804 8805 // compress next vector of 8 chars (if any) 8806 bind(copy_16); 8807 // len = 0 8808 testl(result, 0x00000008); // check if there's a block of 8 chars to compress 8809 jccb(Assembler::zero, copy_tail_sse); 8810 8811 pxor(tmp3Reg, tmp3Reg); 8812 8813 movdqu(tmp2Reg, Address(src, 0)); 8814 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 8815 jccb(Assembler::notZero, reset_for_copy_tail); 8816 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 8817 movq(Address(dst, 0), tmp2Reg); 8818 addptr(src, 16); 8819 addptr(dst, 8); 8820 jmpb(copy_tail_sse); 8821 8822 bind(reset_for_copy_tail); 8823 movl(tmp5, result); 8824 andl(tmp5, 0x0000000f); 8825 lea(src, Address(src, tmp5, Address::times_2)); 8826 lea(dst, Address(dst, tmp5, Address::times_1)); 8827 subptr(len, tmp5); 8828 jmpb(copy_chars_loop); 8829 8830 bind(copy_tail_sse); 8831 movl(len, result); 8832 andl(len, 0x00000007); // tail count (in chars) 8833 } 8834 // compress 1 char per iter 8835 bind(copy_tail); 8836 testl(len, len); 8837 jccb(Assembler::zero, done); 8838 lea(src, Address(src, len, Address::times_2)); 8839 lea(dst, Address(dst, len, Address::times_1)); 8840 negptr(len); 8841 8842 bind(copy_chars_loop); 8843 load_unsigned_short(tmp5, Address(src, len, Address::times_2)); 8844 testl(tmp5, 0xff00); // check if Unicode char 8845 jccb(Assembler::notZero, reset_sp); 8846 movb(Address(dst, len, Address::times_1), tmp5); // ASCII char; compress to 1 byte 8847 increment(len); 8848 jccb(Assembler::notZero, copy_chars_loop); 8849 8850 // add len then return (len will be zero if compress succeeded, otherwise negative) 8851 bind(reset_sp); 8852 addl(result, len); 8853 8854 bind(done); 8855 } 8856 8857 // Inflate byte[] array to char[]. 8858 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 8859 // @IntrinsicCandidate 8860 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 8861 // for (int i = 0; i < len; i++) { 8862 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 8863 // } 8864 // } 8865 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 8866 XMMRegister tmp1, Register tmp2, KRegister mask) { 8867 Label copy_chars_loop, done, below_threshold, avx3_threshold; 8868 // rsi: src 8869 // rdi: dst 8870 // rdx: len 8871 // rcx: tmp2 8872 8873 // rsi holds start addr of source byte[] to be inflated 8874 // rdi holds start addr of destination char[] 8875 // rdx holds length 8876 assert_different_registers(src, dst, len, tmp2); 8877 movl(tmp2, len); 8878 if ((UseAVX > 2) && // AVX512 8879 VM_Version::supports_avx512vlbw() && 8880 VM_Version::supports_bmi2()) { 8881 8882 Label copy_32_loop, copy_tail; 8883 Register tmp3_aliased = len; 8884 8885 // if length of the string is less than 16, handle it in an old fashioned way 8886 testl(len, -16); 8887 jcc(Assembler::zero, below_threshold); 8888 8889 testl(len, -1 * AVX3Threshold); 8890 jcc(Assembler::zero, avx3_threshold); 8891 8892 // In order to use only one arithmetic operation for the main loop we use 8893 // this pre-calculation 8894 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 8895 andl(len, -32); // vector count 8896 jccb(Assembler::zero, copy_tail); 8897 8898 lea(src, Address(src, len, Address::times_1)); 8899 lea(dst, Address(dst, len, Address::times_2)); 8900 negptr(len); 8901 8902 8903 // inflate 32 chars per iter 8904 bind(copy_32_loop); 8905 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 8906 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 8907 addptr(len, 32); 8908 jcc(Assembler::notZero, copy_32_loop); 8909 8910 bind(copy_tail); 8911 // bail out when there is nothing to be done 8912 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 8913 jcc(Assembler::zero, done); 8914 8915 // ~(~0 << length), where length is the # of remaining elements to process 8916 movl(tmp3_aliased, -1); 8917 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 8918 notl(tmp3_aliased); 8919 kmovdl(mask, tmp3_aliased); 8920 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 8921 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 8922 8923 jmp(done); 8924 bind(avx3_threshold); 8925 } 8926 if (UseSSE42Intrinsics) { 8927 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 8928 8929 if (UseAVX > 1) { 8930 andl(tmp2, (16 - 1)); 8931 andl(len, -16); 8932 jccb(Assembler::zero, copy_new_tail); 8933 } else { 8934 andl(tmp2, 0x00000007); // tail count (in chars) 8935 andl(len, 0xfffffff8); // vector count (in chars) 8936 jccb(Assembler::zero, copy_tail); 8937 } 8938 8939 // vectored inflation 8940 lea(src, Address(src, len, Address::times_1)); 8941 lea(dst, Address(dst, len, Address::times_2)); 8942 negptr(len); 8943 8944 if (UseAVX > 1) { 8945 bind(copy_16_loop); 8946 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 8947 vmovdqu(Address(dst, len, Address::times_2), tmp1); 8948 addptr(len, 16); 8949 jcc(Assembler::notZero, copy_16_loop); 8950 8951 bind(below_threshold); 8952 bind(copy_new_tail); 8953 movl(len, tmp2); 8954 andl(tmp2, 0x00000007); 8955 andl(len, 0xFFFFFFF8); 8956 jccb(Assembler::zero, copy_tail); 8957 8958 pmovzxbw(tmp1, Address(src, 0)); 8959 movdqu(Address(dst, 0), tmp1); 8960 addptr(src, 8); 8961 addptr(dst, 2 * 8); 8962 8963 jmp(copy_tail, true); 8964 } 8965 8966 // inflate 8 chars per iter 8967 bind(copy_8_loop); 8968 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 8969 movdqu(Address(dst, len, Address::times_2), tmp1); 8970 addptr(len, 8); 8971 jcc(Assembler::notZero, copy_8_loop); 8972 8973 bind(copy_tail); 8974 movl(len, tmp2); 8975 8976 cmpl(len, 4); 8977 jccb(Assembler::less, copy_bytes); 8978 8979 movdl(tmp1, Address(src, 0)); // load 4 byte chars 8980 pmovzxbw(tmp1, tmp1); 8981 movq(Address(dst, 0), tmp1); 8982 subptr(len, 4); 8983 addptr(src, 4); 8984 addptr(dst, 8); 8985 8986 bind(copy_bytes); 8987 } else { 8988 bind(below_threshold); 8989 } 8990 8991 testl(len, len); 8992 jccb(Assembler::zero, done); 8993 lea(src, Address(src, len, Address::times_1)); 8994 lea(dst, Address(dst, len, Address::times_2)); 8995 negptr(len); 8996 8997 // inflate 1 char per iter 8998 bind(copy_chars_loop); 8999 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 9000 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 9001 increment(len); 9002 jcc(Assembler::notZero, copy_chars_loop); 9003 9004 bind(done); 9005 } 9006 9007 9008 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 9009 switch(type) { 9010 case T_BYTE: 9011 case T_BOOLEAN: 9012 evmovdqub(dst, kmask, src, merge, vector_len); 9013 break; 9014 case T_CHAR: 9015 case T_SHORT: 9016 evmovdquw(dst, kmask, src, merge, vector_len); 9017 break; 9018 case T_INT: 9019 case T_FLOAT: 9020 evmovdqul(dst, kmask, src, merge, vector_len); 9021 break; 9022 case T_LONG: 9023 case T_DOUBLE: 9024 evmovdquq(dst, kmask, src, merge, vector_len); 9025 break; 9026 default: 9027 fatal("Unexpected type argument %s", type2name(type)); 9028 break; 9029 } 9030 } 9031 9032 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 9033 switch(type) { 9034 case T_BYTE: 9035 case T_BOOLEAN: 9036 evmovdqub(dst, kmask, src, merge, vector_len); 9037 break; 9038 case T_CHAR: 9039 case T_SHORT: 9040 evmovdquw(dst, kmask, src, merge, vector_len); 9041 break; 9042 case T_INT: 9043 case T_FLOAT: 9044 evmovdqul(dst, kmask, src, merge, vector_len); 9045 break; 9046 case T_LONG: 9047 case T_DOUBLE: 9048 evmovdquq(dst, kmask, src, merge, vector_len); 9049 break; 9050 default: 9051 fatal("Unexpected type argument %s", type2name(type)); 9052 break; 9053 } 9054 } 9055 9056 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 9057 switch(masklen) { 9058 case 2: 9059 knotbl(dst, src); 9060 movl(rtmp, 3); 9061 kmovbl(ktmp, rtmp); 9062 kandbl(dst, ktmp, dst); 9063 break; 9064 case 4: 9065 knotbl(dst, src); 9066 movl(rtmp, 15); 9067 kmovbl(ktmp, rtmp); 9068 kandbl(dst, ktmp, dst); 9069 break; 9070 case 8: 9071 knotbl(dst, src); 9072 break; 9073 case 16: 9074 knotwl(dst, src); 9075 break; 9076 case 32: 9077 knotdl(dst, src); 9078 break; 9079 case 64: 9080 knotql(dst, src); 9081 break; 9082 default: 9083 fatal("Unexpected vector length %d", masklen); 9084 break; 9085 } 9086 } 9087 9088 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9089 switch(type) { 9090 case T_BOOLEAN: 9091 case T_BYTE: 9092 kandbl(dst, src1, src2); 9093 break; 9094 case T_CHAR: 9095 case T_SHORT: 9096 kandwl(dst, src1, src2); 9097 break; 9098 case T_INT: 9099 case T_FLOAT: 9100 kanddl(dst, src1, src2); 9101 break; 9102 case T_LONG: 9103 case T_DOUBLE: 9104 kandql(dst, src1, src2); 9105 break; 9106 default: 9107 fatal("Unexpected type argument %s", type2name(type)); 9108 break; 9109 } 9110 } 9111 9112 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9113 switch(type) { 9114 case T_BOOLEAN: 9115 case T_BYTE: 9116 korbl(dst, src1, src2); 9117 break; 9118 case T_CHAR: 9119 case T_SHORT: 9120 korwl(dst, src1, src2); 9121 break; 9122 case T_INT: 9123 case T_FLOAT: 9124 kordl(dst, src1, src2); 9125 break; 9126 case T_LONG: 9127 case T_DOUBLE: 9128 korql(dst, src1, src2); 9129 break; 9130 default: 9131 fatal("Unexpected type argument %s", type2name(type)); 9132 break; 9133 } 9134 } 9135 9136 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9137 switch(type) { 9138 case T_BOOLEAN: 9139 case T_BYTE: 9140 kxorbl(dst, src1, src2); 9141 break; 9142 case T_CHAR: 9143 case T_SHORT: 9144 kxorwl(dst, src1, src2); 9145 break; 9146 case T_INT: 9147 case T_FLOAT: 9148 kxordl(dst, src1, src2); 9149 break; 9150 case T_LONG: 9151 case T_DOUBLE: 9152 kxorql(dst, src1, src2); 9153 break; 9154 default: 9155 fatal("Unexpected type argument %s", type2name(type)); 9156 break; 9157 } 9158 } 9159 9160 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9161 switch(type) { 9162 case T_BOOLEAN: 9163 case T_BYTE: 9164 evpermb(dst, mask, nds, src, merge, vector_len); break; 9165 case T_CHAR: 9166 case T_SHORT: 9167 evpermw(dst, mask, nds, src, merge, vector_len); break; 9168 case T_INT: 9169 case T_FLOAT: 9170 evpermd(dst, mask, nds, src, merge, vector_len); break; 9171 case T_LONG: 9172 case T_DOUBLE: 9173 evpermq(dst, mask, nds, src, merge, vector_len); break; 9174 default: 9175 fatal("Unexpected type argument %s", type2name(type)); break; 9176 } 9177 } 9178 9179 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9180 switch(type) { 9181 case T_BOOLEAN: 9182 case T_BYTE: 9183 evpermb(dst, mask, nds, src, merge, vector_len); break; 9184 case T_CHAR: 9185 case T_SHORT: 9186 evpermw(dst, mask, nds, src, merge, vector_len); break; 9187 case T_INT: 9188 case T_FLOAT: 9189 evpermd(dst, mask, nds, src, merge, vector_len); break; 9190 case T_LONG: 9191 case T_DOUBLE: 9192 evpermq(dst, mask, nds, src, merge, vector_len); break; 9193 default: 9194 fatal("Unexpected type argument %s", type2name(type)); break; 9195 } 9196 } 9197 9198 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9199 switch(type) { 9200 case T_BYTE: 9201 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9202 case T_SHORT: 9203 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9204 case T_INT: 9205 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9206 case T_LONG: 9207 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9208 default: 9209 fatal("Unexpected type argument %s", type2name(type)); break; 9210 } 9211 } 9212 9213 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9214 switch(type) { 9215 case T_BYTE: 9216 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9217 case T_SHORT: 9218 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9219 case T_INT: 9220 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9221 case T_LONG: 9222 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9223 default: 9224 fatal("Unexpected type argument %s", type2name(type)); break; 9225 } 9226 } 9227 9228 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9229 switch(type) { 9230 case T_BYTE: 9231 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9232 case T_SHORT: 9233 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9234 case T_INT: 9235 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9236 case T_LONG: 9237 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9238 default: 9239 fatal("Unexpected type argument %s", type2name(type)); break; 9240 } 9241 } 9242 9243 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9244 switch(type) { 9245 case T_BYTE: 9246 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9247 case T_SHORT: 9248 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9249 case T_INT: 9250 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9251 case T_LONG: 9252 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9253 default: 9254 fatal("Unexpected type argument %s", type2name(type)); break; 9255 } 9256 } 9257 9258 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9259 switch(type) { 9260 case T_INT: 9261 evpxord(dst, mask, nds, src, merge, vector_len); break; 9262 case T_LONG: 9263 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9264 default: 9265 fatal("Unexpected type argument %s", type2name(type)); break; 9266 } 9267 } 9268 9269 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9270 switch(type) { 9271 case T_INT: 9272 evpxord(dst, mask, nds, src, merge, vector_len); break; 9273 case T_LONG: 9274 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9275 default: 9276 fatal("Unexpected type argument %s", type2name(type)); break; 9277 } 9278 } 9279 9280 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9281 switch(type) { 9282 case T_INT: 9283 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9284 case T_LONG: 9285 evporq(dst, mask, nds, src, merge, vector_len); break; 9286 default: 9287 fatal("Unexpected type argument %s", type2name(type)); break; 9288 } 9289 } 9290 9291 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9292 switch(type) { 9293 case T_INT: 9294 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9295 case T_LONG: 9296 evporq(dst, mask, nds, src, merge, vector_len); break; 9297 default: 9298 fatal("Unexpected type argument %s", type2name(type)); break; 9299 } 9300 } 9301 9302 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9303 switch(type) { 9304 case T_INT: 9305 evpandd(dst, mask, nds, src, merge, vector_len); break; 9306 case T_LONG: 9307 evpandq(dst, mask, nds, src, merge, vector_len); break; 9308 default: 9309 fatal("Unexpected type argument %s", type2name(type)); break; 9310 } 9311 } 9312 9313 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9314 switch(type) { 9315 case T_INT: 9316 evpandd(dst, mask, nds, src, merge, vector_len); break; 9317 case T_LONG: 9318 evpandq(dst, mask, nds, src, merge, vector_len); break; 9319 default: 9320 fatal("Unexpected type argument %s", type2name(type)); break; 9321 } 9322 } 9323 9324 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 9325 switch(masklen) { 9326 case 8: 9327 kortestbl(src1, src2); 9328 break; 9329 case 16: 9330 kortestwl(src1, src2); 9331 break; 9332 case 32: 9333 kortestdl(src1, src2); 9334 break; 9335 case 64: 9336 kortestql(src1, src2); 9337 break; 9338 default: 9339 fatal("Unexpected mask length %d", masklen); 9340 break; 9341 } 9342 } 9343 9344 9345 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 9346 switch(masklen) { 9347 case 8: 9348 ktestbl(src1, src2); 9349 break; 9350 case 16: 9351 ktestwl(src1, src2); 9352 break; 9353 case 32: 9354 ktestdl(src1, src2); 9355 break; 9356 case 64: 9357 ktestql(src1, src2); 9358 break; 9359 default: 9360 fatal("Unexpected mask length %d", masklen); 9361 break; 9362 } 9363 } 9364 9365 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9366 switch(type) { 9367 case T_INT: 9368 evprold(dst, mask, src, shift, merge, vlen_enc); break; 9369 case T_LONG: 9370 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 9371 default: 9372 fatal("Unexpected type argument %s", type2name(type)); break; 9373 break; 9374 } 9375 } 9376 9377 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9378 switch(type) { 9379 case T_INT: 9380 evprord(dst, mask, src, shift, merge, vlen_enc); break; 9381 case T_LONG: 9382 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 9383 default: 9384 fatal("Unexpected type argument %s", type2name(type)); break; 9385 } 9386 } 9387 9388 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9389 switch(type) { 9390 case T_INT: 9391 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 9392 case T_LONG: 9393 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 9394 default: 9395 fatal("Unexpected type argument %s", type2name(type)); break; 9396 } 9397 } 9398 9399 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9400 switch(type) { 9401 case T_INT: 9402 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 9403 case T_LONG: 9404 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 9405 default: 9406 fatal("Unexpected type argument %s", type2name(type)); break; 9407 } 9408 } 9409 9410 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9411 assert(rscratch != noreg || always_reachable(src), "missing"); 9412 9413 if (reachable(src)) { 9414 evpandq(dst, nds, as_Address(src), vector_len); 9415 } else { 9416 lea(rscratch, src); 9417 evpandq(dst, nds, Address(rscratch, 0), vector_len); 9418 } 9419 } 9420 9421 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 9422 assert(rscratch != noreg || always_reachable(src), "missing"); 9423 9424 if (reachable(src)) { 9425 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 9426 } else { 9427 lea(rscratch, src); 9428 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 9429 } 9430 } 9431 9432 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9433 assert(rscratch != noreg || always_reachable(src), "missing"); 9434 9435 if (reachable(src)) { 9436 evporq(dst, nds, as_Address(src), vector_len); 9437 } else { 9438 lea(rscratch, src); 9439 evporq(dst, nds, Address(rscratch, 0), vector_len); 9440 } 9441 } 9442 9443 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9444 assert(rscratch != noreg || always_reachable(src), "missing"); 9445 9446 if (reachable(src)) { 9447 vpshufb(dst, nds, as_Address(src), vector_len); 9448 } else { 9449 lea(rscratch, src); 9450 vpshufb(dst, nds, Address(rscratch, 0), vector_len); 9451 } 9452 } 9453 9454 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 9455 assert(rscratch != noreg || always_reachable(src3), "missing"); 9456 9457 if (reachable(src3)) { 9458 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 9459 } else { 9460 lea(rscratch, src3); 9461 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 9462 } 9463 } 9464 9465 #if COMPILER2_OR_JVMCI 9466 9467 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 9468 Register length, Register temp, int vec_enc) { 9469 // Computing mask for predicated vector store. 9470 movptr(temp, -1); 9471 bzhiq(temp, temp, length); 9472 kmov(mask, temp); 9473 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 9474 } 9475 9476 // Set memory operation for length "less than" 64 bytes. 9477 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 9478 XMMRegister xmm, KRegister mask, Register length, 9479 Register temp, bool use64byteVector) { 9480 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9481 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9482 if (!use64byteVector) { 9483 fill32(dst, disp, xmm); 9484 subptr(length, 32 >> shift); 9485 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 9486 } else { 9487 assert(MaxVectorSize == 64, "vector length != 64"); 9488 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 9489 } 9490 } 9491 9492 9493 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 9494 XMMRegister xmm, KRegister mask, Register length, 9495 Register temp) { 9496 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9497 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9498 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 9499 } 9500 9501 9502 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 9503 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9504 vmovdqu(dst, xmm); 9505 } 9506 9507 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 9508 fill32(Address(dst, disp), xmm); 9509 } 9510 9511 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 9512 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9513 if (!use64byteVector) { 9514 fill32(dst, xmm); 9515 fill32(dst.plus_disp(32), xmm); 9516 } else { 9517 evmovdquq(dst, xmm, Assembler::AVX_512bit); 9518 } 9519 } 9520 9521 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 9522 fill64(Address(dst, disp), xmm, use64byteVector); 9523 } 9524 9525 #ifdef _LP64 9526 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 9527 Register count, Register rtmp, XMMRegister xtmp) { 9528 Label L_exit; 9529 Label L_fill_start; 9530 Label L_fill_64_bytes; 9531 Label L_fill_96_bytes; 9532 Label L_fill_128_bytes; 9533 Label L_fill_128_bytes_loop; 9534 Label L_fill_128_loop_header; 9535 Label L_fill_128_bytes_loop_header; 9536 Label L_fill_128_bytes_loop_pre_header; 9537 Label L_fill_zmm_sequence; 9538 9539 int shift = -1; 9540 int avx3threshold = VM_Version::avx3_threshold(); 9541 switch(type) { 9542 case T_BYTE: shift = 0; 9543 break; 9544 case T_SHORT: shift = 1; 9545 break; 9546 case T_INT: shift = 2; 9547 break; 9548 /* Uncomment when LONG fill stubs are supported. 9549 case T_LONG: shift = 3; 9550 break; 9551 */ 9552 default: 9553 fatal("Unhandled type: %s\n", type2name(type)); 9554 } 9555 9556 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 9557 9558 if (MaxVectorSize == 64) { 9559 cmpq(count, avx3threshold >> shift); 9560 jcc(Assembler::greater, L_fill_zmm_sequence); 9561 } 9562 9563 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 9564 9565 bind(L_fill_start); 9566 9567 cmpq(count, 32 >> shift); 9568 jccb(Assembler::greater, L_fill_64_bytes); 9569 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 9570 jmp(L_exit); 9571 9572 bind(L_fill_64_bytes); 9573 cmpq(count, 64 >> shift); 9574 jccb(Assembler::greater, L_fill_96_bytes); 9575 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 9576 jmp(L_exit); 9577 9578 bind(L_fill_96_bytes); 9579 cmpq(count, 96 >> shift); 9580 jccb(Assembler::greater, L_fill_128_bytes); 9581 fill64(to, 0, xtmp); 9582 subq(count, 64 >> shift); 9583 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 9584 jmp(L_exit); 9585 9586 bind(L_fill_128_bytes); 9587 cmpq(count, 128 >> shift); 9588 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 9589 fill64(to, 0, xtmp); 9590 fill32(to, 64, xtmp); 9591 subq(count, 96 >> shift); 9592 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 9593 jmp(L_exit); 9594 9595 bind(L_fill_128_bytes_loop_pre_header); 9596 { 9597 mov(rtmp, to); 9598 andq(rtmp, 31); 9599 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 9600 negq(rtmp); 9601 addq(rtmp, 32); 9602 mov64(r8, -1L); 9603 bzhiq(r8, r8, rtmp); 9604 kmovql(k2, r8); 9605 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 9606 addq(to, rtmp); 9607 shrq(rtmp, shift); 9608 subq(count, rtmp); 9609 } 9610 9611 cmpq(count, 128 >> shift); 9612 jcc(Assembler::less, L_fill_start); 9613 9614 bind(L_fill_128_bytes_loop_header); 9615 subq(count, 128 >> shift); 9616 9617 align32(); 9618 bind(L_fill_128_bytes_loop); 9619 fill64(to, 0, xtmp); 9620 fill64(to, 64, xtmp); 9621 addq(to, 128); 9622 subq(count, 128 >> shift); 9623 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 9624 9625 addq(count, 128 >> shift); 9626 jcc(Assembler::zero, L_exit); 9627 jmp(L_fill_start); 9628 } 9629 9630 if (MaxVectorSize == 64) { 9631 // Sequence using 64 byte ZMM register. 9632 Label L_fill_128_bytes_zmm; 9633 Label L_fill_192_bytes_zmm; 9634 Label L_fill_192_bytes_loop_zmm; 9635 Label L_fill_192_bytes_loop_header_zmm; 9636 Label L_fill_192_bytes_loop_pre_header_zmm; 9637 Label L_fill_start_zmm_sequence; 9638 9639 bind(L_fill_zmm_sequence); 9640 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 9641 9642 bind(L_fill_start_zmm_sequence); 9643 cmpq(count, 64 >> shift); 9644 jccb(Assembler::greater, L_fill_128_bytes_zmm); 9645 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 9646 jmp(L_exit); 9647 9648 bind(L_fill_128_bytes_zmm); 9649 cmpq(count, 128 >> shift); 9650 jccb(Assembler::greater, L_fill_192_bytes_zmm); 9651 fill64(to, 0, xtmp, true); 9652 subq(count, 64 >> shift); 9653 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 9654 jmp(L_exit); 9655 9656 bind(L_fill_192_bytes_zmm); 9657 cmpq(count, 192 >> shift); 9658 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 9659 fill64(to, 0, xtmp, true); 9660 fill64(to, 64, xtmp, true); 9661 subq(count, 128 >> shift); 9662 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 9663 jmp(L_exit); 9664 9665 bind(L_fill_192_bytes_loop_pre_header_zmm); 9666 { 9667 movq(rtmp, to); 9668 andq(rtmp, 63); 9669 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 9670 negq(rtmp); 9671 addq(rtmp, 64); 9672 mov64(r8, -1L); 9673 bzhiq(r8, r8, rtmp); 9674 kmovql(k2, r8); 9675 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 9676 addq(to, rtmp); 9677 shrq(rtmp, shift); 9678 subq(count, rtmp); 9679 } 9680 9681 cmpq(count, 192 >> shift); 9682 jcc(Assembler::less, L_fill_start_zmm_sequence); 9683 9684 bind(L_fill_192_bytes_loop_header_zmm); 9685 subq(count, 192 >> shift); 9686 9687 align32(); 9688 bind(L_fill_192_bytes_loop_zmm); 9689 fill64(to, 0, xtmp, true); 9690 fill64(to, 64, xtmp, true); 9691 fill64(to, 128, xtmp, true); 9692 addq(to, 192); 9693 subq(count, 192 >> shift); 9694 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 9695 9696 addq(count, 192 >> shift); 9697 jcc(Assembler::zero, L_exit); 9698 jmp(L_fill_start_zmm_sequence); 9699 } 9700 bind(L_exit); 9701 } 9702 #endif 9703 #endif //COMPILER2_OR_JVMCI 9704 9705 9706 #ifdef _LP64 9707 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 9708 Label done; 9709 cvttss2sil(dst, src); 9710 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9711 cmpl(dst, 0x80000000); // float_sign_flip 9712 jccb(Assembler::notEqual, done); 9713 subptr(rsp, 8); 9714 movflt(Address(rsp, 0), src); 9715 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 9716 pop(dst); 9717 bind(done); 9718 } 9719 9720 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 9721 Label done; 9722 cvttsd2sil(dst, src); 9723 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9724 cmpl(dst, 0x80000000); // float_sign_flip 9725 jccb(Assembler::notEqual, done); 9726 subptr(rsp, 8); 9727 movdbl(Address(rsp, 0), src); 9728 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 9729 pop(dst); 9730 bind(done); 9731 } 9732 9733 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 9734 Label done; 9735 cvttss2siq(dst, src); 9736 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9737 jccb(Assembler::notEqual, done); 9738 subptr(rsp, 8); 9739 movflt(Address(rsp, 0), src); 9740 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 9741 pop(dst); 9742 bind(done); 9743 } 9744 9745 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9746 // Following code is line by line assembly translation rounding algorithm. 9747 // Please refer to java.lang.Math.round(float) algorithm for details. 9748 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 9749 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 9750 const int32_t FloatConsts_EXP_BIAS = 127; 9751 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 9752 const int32_t MINUS_32 = 0xFFFFFFE0; 9753 Label L_special_case, L_block1, L_exit; 9754 movl(rtmp, FloatConsts_EXP_BIT_MASK); 9755 movdl(dst, src); 9756 andl(dst, rtmp); 9757 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 9758 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 9759 subl(rtmp, dst); 9760 movl(rcx, rtmp); 9761 movl(dst, MINUS_32); 9762 testl(rtmp, dst); 9763 jccb(Assembler::notEqual, L_special_case); 9764 movdl(dst, src); 9765 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 9766 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 9767 movdl(rtmp, src); 9768 testl(rtmp, rtmp); 9769 jccb(Assembler::greaterEqual, L_block1); 9770 negl(dst); 9771 bind(L_block1); 9772 sarl(dst); 9773 addl(dst, 0x1); 9774 sarl(dst, 0x1); 9775 jmp(L_exit); 9776 bind(L_special_case); 9777 convert_f2i(dst, src); 9778 bind(L_exit); 9779 } 9780 9781 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9782 // Following code is line by line assembly translation rounding algorithm. 9783 // Please refer to java.lang.Math.round(double) algorithm for details. 9784 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 9785 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 9786 const int64_t DoubleConsts_EXP_BIAS = 1023; 9787 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 9788 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 9789 Label L_special_case, L_block1, L_exit; 9790 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 9791 movq(dst, src); 9792 andq(dst, rtmp); 9793 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 9794 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 9795 subq(rtmp, dst); 9796 movq(rcx, rtmp); 9797 mov64(dst, MINUS_64); 9798 testq(rtmp, dst); 9799 jccb(Assembler::notEqual, L_special_case); 9800 movq(dst, src); 9801 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 9802 andq(dst, rtmp); 9803 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 9804 orq(dst, rtmp); 9805 movq(rtmp, src); 9806 testq(rtmp, rtmp); 9807 jccb(Assembler::greaterEqual, L_block1); 9808 negq(dst); 9809 bind(L_block1); 9810 sarq(dst); 9811 addq(dst, 0x1); 9812 sarq(dst, 0x1); 9813 jmp(L_exit); 9814 bind(L_special_case); 9815 convert_d2l(dst, src); 9816 bind(L_exit); 9817 } 9818 9819 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 9820 Label done; 9821 cvttsd2siq(dst, src); 9822 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9823 jccb(Assembler::notEqual, done); 9824 subptr(rsp, 8); 9825 movdbl(Address(rsp, 0), src); 9826 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 9827 pop(dst); 9828 bind(done); 9829 } 9830 9831 void MacroAssembler::cache_wb(Address line) 9832 { 9833 // 64 bit cpus always support clflush 9834 assert(VM_Version::supports_clflush(), "clflush should be available"); 9835 bool optimized = VM_Version::supports_clflushopt(); 9836 bool no_evict = VM_Version::supports_clwb(); 9837 9838 // prefer clwb (writeback without evict) otherwise 9839 // prefer clflushopt (potentially parallel writeback with evict) 9840 // otherwise fallback on clflush (serial writeback with evict) 9841 9842 if (optimized) { 9843 if (no_evict) { 9844 clwb(line); 9845 } else { 9846 clflushopt(line); 9847 } 9848 } else { 9849 // no need for fence when using CLFLUSH 9850 clflush(line); 9851 } 9852 } 9853 9854 void MacroAssembler::cache_wbsync(bool is_pre) 9855 { 9856 assert(VM_Version::supports_clflush(), "clflush should be available"); 9857 bool optimized = VM_Version::supports_clflushopt(); 9858 bool no_evict = VM_Version::supports_clwb(); 9859 9860 // pick the correct implementation 9861 9862 if (!is_pre && (optimized || no_evict)) { 9863 // need an sfence for post flush when using clflushopt or clwb 9864 // otherwise no no need for any synchroniaztion 9865 9866 sfence(); 9867 } 9868 } 9869 9870 #endif // _LP64 9871 9872 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 9873 switch (cond) { 9874 // Note some conditions are synonyms for others 9875 case Assembler::zero: return Assembler::notZero; 9876 case Assembler::notZero: return Assembler::zero; 9877 case Assembler::less: return Assembler::greaterEqual; 9878 case Assembler::lessEqual: return Assembler::greater; 9879 case Assembler::greater: return Assembler::lessEqual; 9880 case Assembler::greaterEqual: return Assembler::less; 9881 case Assembler::below: return Assembler::aboveEqual; 9882 case Assembler::belowEqual: return Assembler::above; 9883 case Assembler::above: return Assembler::belowEqual; 9884 case Assembler::aboveEqual: return Assembler::below; 9885 case Assembler::overflow: return Assembler::noOverflow; 9886 case Assembler::noOverflow: return Assembler::overflow; 9887 case Assembler::negative: return Assembler::positive; 9888 case Assembler::positive: return Assembler::negative; 9889 case Assembler::parity: return Assembler::noParity; 9890 case Assembler::noParity: return Assembler::parity; 9891 } 9892 ShouldNotReachHere(); return Assembler::overflow; 9893 } 9894 9895 SkipIfEqual::SkipIfEqual( 9896 MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) { 9897 _masm = masm; 9898 _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch); 9899 _masm->jcc(Assembler::equal, _label); 9900 } 9901 9902 SkipIfEqual::~SkipIfEqual() { 9903 _masm->bind(_label); 9904 } 9905 9906 // 32-bit Windows has its own fast-path implementation 9907 // of get_thread 9908 #if !defined(WIN32) || defined(_LP64) 9909 9910 // This is simply a call to Thread::current() 9911 void MacroAssembler::get_thread(Register thread) { 9912 if (thread != rax) { 9913 push(rax); 9914 } 9915 LP64_ONLY(push(rdi);) 9916 LP64_ONLY(push(rsi);) 9917 push(rdx); 9918 push(rcx); 9919 #ifdef _LP64 9920 push(r8); 9921 push(r9); 9922 push(r10); 9923 push(r11); 9924 #endif 9925 9926 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 9927 9928 #ifdef _LP64 9929 pop(r11); 9930 pop(r10); 9931 pop(r9); 9932 pop(r8); 9933 #endif 9934 pop(rcx); 9935 pop(rdx); 9936 LP64_ONLY(pop(rsi);) 9937 LP64_ONLY(pop(rdi);) 9938 if (thread != rax) { 9939 mov(thread, rax); 9940 pop(rax); 9941 } 9942 } 9943 9944 9945 #endif // !WIN32 || _LP64 9946 9947 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 9948 Label L_stack_ok; 9949 if (bias == 0) { 9950 testptr(sp, 2 * wordSize - 1); 9951 } else { 9952 // lea(tmp, Address(rsp, bias); 9953 mov(tmp, sp); 9954 addptr(tmp, bias); 9955 testptr(tmp, 2 * wordSize - 1); 9956 } 9957 jcc(Assembler::equal, L_stack_ok); 9958 block_comment(msg); 9959 stop(msg); 9960 bind(L_stack_ok); 9961 } 9962 9963 // Implements lightweight-locking. 9964 // Branches to slow upon failure to lock the object, with ZF cleared. 9965 // Falls through upon success with unspecified ZF. 9966 // 9967 // obj: the object to be locked 9968 // hdr: the (pre-loaded) header of the object, must be rax 9969 // thread: the thread which attempts to lock obj 9970 // tmp: a temporary register 9971 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow) { 9972 assert(hdr == rax, "header must be in rax for cmpxchg"); 9973 assert_different_registers(obj, hdr, thread, tmp); 9974 9975 // First we need to check if the lock-stack has room for pushing the object reference. 9976 // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead 9977 // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and 9978 // avoids one branch. 9979 cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1); 9980 jcc(Assembler::greater, slow); 9981 9982 // Now we attempt to take the fast-lock. 9983 // Clear lock_mask bits (locked state). 9984 andptr(hdr, ~(int32_t)markWord::lock_mask_in_place); 9985 movptr(tmp, hdr); 9986 // Set unlocked_value bit. 9987 orptr(hdr, markWord::unlocked_value); 9988 lock(); 9989 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9990 jcc(Assembler::notEqual, slow); 9991 9992 // If successful, push object to lock-stack. 9993 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset())); 9994 movptr(Address(thread, tmp), obj); 9995 incrementl(tmp, oopSize); 9996 movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp); 9997 } 9998 9999 // Implements lightweight-unlocking. 10000 // Branches to slow upon failure, with ZF cleared. 10001 // Falls through upon success, with unspecified ZF. 10002 // 10003 // obj: the object to be unlocked 10004 // hdr: the (pre-loaded) header of the object, must be rax 10005 // tmp: a temporary register 10006 void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) { 10007 assert(hdr == rax, "header must be in rax for cmpxchg"); 10008 assert_different_registers(obj, hdr, tmp); 10009 10010 // Mark-word must be lock_mask now, try to swing it back to unlocked_value. 10011 movptr(tmp, hdr); // The expected old value 10012 orptr(tmp, markWord::unlocked_value); 10013 lock(); 10014 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10015 jcc(Assembler::notEqual, slow); 10016 // Pop the lock object from the lock-stack. 10017 #ifdef _LP64 10018 const Register thread = r15_thread; 10019 #else 10020 const Register thread = rax; 10021 get_thread(thread); 10022 #endif 10023 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10024 #ifdef ASSERT 10025 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset())); 10026 movptr(Address(thread, tmp), 0); 10027 #endif 10028 }