1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "compiler/compiler_globals.hpp" 29 #include "compiler/disassembler.hpp" 30 #include "crc32c.h" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/collectedHeap.inline.hpp" 34 #include "gc/shared/tlab_globals.hpp" 35 #include "interpreter/bytecodeHistogram.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "jvm.h" 38 #include "memory/resourceArea.hpp" 39 #include "memory/universe.hpp" 40 #include "oops/accessDecorators.hpp" 41 #include "oops/compressedOops.inline.hpp" 42 #include "oops/klass.inline.hpp" 43 #include "prims/methodHandles.hpp" 44 #include "runtime/continuation.hpp" 45 #include "runtime/interfaceSupport.inline.hpp" 46 #include "runtime/javaThread.hpp" 47 #include "runtime/jniHandles.hpp" 48 #include "runtime/objectMonitor.hpp" 49 #include "runtime/os.hpp" 50 #include "runtime/safepoint.hpp" 51 #include "runtime/safepointMechanism.hpp" 52 #include "runtime/sharedRuntime.hpp" 53 #include "runtime/stubRoutines.hpp" 54 #include "utilities/macros.hpp" 55 56 #ifdef PRODUCT 57 #define BLOCK_COMMENT(str) /* nothing */ 58 #define STOP(error) stop(error) 59 #else 60 #define BLOCK_COMMENT(str) block_comment(str) 61 #define STOP(error) block_comment(error); stop(error) 62 #endif 63 64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 65 66 #ifdef ASSERT 67 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 68 #endif 69 70 static const Assembler::Condition reverse[] = { 71 Assembler::noOverflow /* overflow = 0x0 */ , 72 Assembler::overflow /* noOverflow = 0x1 */ , 73 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 74 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 75 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 76 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 77 Assembler::above /* belowEqual = 0x6 */ , 78 Assembler::belowEqual /* above = 0x7 */ , 79 Assembler::positive /* negative = 0x8 */ , 80 Assembler::negative /* positive = 0x9 */ , 81 Assembler::noParity /* parity = 0xa */ , 82 Assembler::parity /* noParity = 0xb */ , 83 Assembler::greaterEqual /* less = 0xc */ , 84 Assembler::less /* greaterEqual = 0xd */ , 85 Assembler::greater /* lessEqual = 0xe */ , 86 Assembler::lessEqual /* greater = 0xf, */ 87 88 }; 89 90 91 // Implementation of MacroAssembler 92 93 // First all the versions that have distinct versions depending on 32/64 bit 94 // Unless the difference is trivial (1 line or so). 95 96 #ifndef _LP64 97 98 // 32bit versions 99 100 Address MacroAssembler::as_Address(AddressLiteral adr) { 101 return Address(adr.target(), adr.rspec()); 102 } 103 104 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 105 assert(rscratch == noreg, ""); 106 return Address::make_array(adr); 107 } 108 109 void MacroAssembler::call_VM_leaf_base(address entry_point, 110 int number_of_arguments) { 111 call(RuntimeAddress(entry_point)); 112 increment(rsp, number_of_arguments * wordSize); 113 } 114 115 void MacroAssembler::cmpklass(Address src1, Metadata* obj) { 116 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 117 } 118 119 120 void MacroAssembler::cmpklass(Register src1, Metadata* obj) { 121 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 122 } 123 124 void MacroAssembler::cmpoop(Address src1, jobject obj) { 125 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 126 } 127 128 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) { 129 assert(rscratch == noreg, "redundant"); 130 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 131 } 132 133 void MacroAssembler::extend_sign(Register hi, Register lo) { 134 // According to Intel Doc. AP-526, "Integer Divide", p.18. 135 if (VM_Version::is_P6() && hi == rdx && lo == rax) { 136 cdql(); 137 } else { 138 movl(hi, lo); 139 sarl(hi, 31); 140 } 141 } 142 143 void MacroAssembler::jC2(Register tmp, Label& L) { 144 // set parity bit if FPU flag C2 is set (via rax) 145 save_rax(tmp); 146 fwait(); fnstsw_ax(); 147 sahf(); 148 restore_rax(tmp); 149 // branch 150 jcc(Assembler::parity, L); 151 } 152 153 void MacroAssembler::jnC2(Register tmp, Label& L) { 154 // set parity bit if FPU flag C2 is set (via rax) 155 save_rax(tmp); 156 fwait(); fnstsw_ax(); 157 sahf(); 158 restore_rax(tmp); 159 // branch 160 jcc(Assembler::noParity, L); 161 } 162 163 // 32bit can do a case table jump in one instruction but we no longer allow the base 164 // to be installed in the Address class 165 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 166 assert(rscratch == noreg, "not needed"); 167 jmp(as_Address(entry, noreg)); 168 } 169 170 // Note: y_lo will be destroyed 171 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 172 // Long compare for Java (semantics as described in JVM spec.) 173 Label high, low, done; 174 175 cmpl(x_hi, y_hi); 176 jcc(Assembler::less, low); 177 jcc(Assembler::greater, high); 178 // x_hi is the return register 179 xorl(x_hi, x_hi); 180 cmpl(x_lo, y_lo); 181 jcc(Assembler::below, low); 182 jcc(Assembler::equal, done); 183 184 bind(high); 185 xorl(x_hi, x_hi); 186 increment(x_hi); 187 jmp(done); 188 189 bind(low); 190 xorl(x_hi, x_hi); 191 decrementl(x_hi); 192 193 bind(done); 194 } 195 196 void MacroAssembler::lea(Register dst, AddressLiteral src) { 197 mov_literal32(dst, (int32_t)src.target(), src.rspec()); 198 } 199 200 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 201 assert(rscratch == noreg, "not needed"); 202 203 // leal(dst, as_Address(adr)); 204 // see note in movl as to why we must use a move 205 mov_literal32(dst, (int32_t)adr.target(), adr.rspec()); 206 } 207 208 void MacroAssembler::leave() { 209 mov(rsp, rbp); 210 pop(rbp); 211 } 212 213 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { 214 // Multiplication of two Java long values stored on the stack 215 // as illustrated below. Result is in rdx:rax. 216 // 217 // rsp ---> [ ?? ] \ \ 218 // .... | y_rsp_offset | 219 // [ y_lo ] / (in bytes) | x_rsp_offset 220 // [ y_hi ] | (in bytes) 221 // .... | 222 // [ x_lo ] / 223 // [ x_hi ] 224 // .... 225 // 226 // Basic idea: lo(result) = lo(x_lo * y_lo) 227 // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) 228 Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); 229 Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); 230 Label quick; 231 // load x_hi, y_hi and check if quick 232 // multiplication is possible 233 movl(rbx, x_hi); 234 movl(rcx, y_hi); 235 movl(rax, rbx); 236 orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 237 jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply 238 // do full multiplication 239 // 1st step 240 mull(y_lo); // x_hi * y_lo 241 movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, 242 // 2nd step 243 movl(rax, x_lo); 244 mull(rcx); // x_lo * y_hi 245 addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, 246 // 3rd step 247 bind(quick); // note: rbx, = 0 if quick multiply! 248 movl(rax, x_lo); 249 mull(y_lo); // x_lo * y_lo 250 addl(rdx, rbx); // correct hi(x_lo * y_lo) 251 } 252 253 void MacroAssembler::lneg(Register hi, Register lo) { 254 negl(lo); 255 adcl(hi, 0); 256 negl(hi); 257 } 258 259 void MacroAssembler::lshl(Register hi, Register lo) { 260 // Java shift left long support (semantics as described in JVM spec., p.305) 261 // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) 262 // shift value is in rcx ! 263 assert(hi != rcx, "must not use rcx"); 264 assert(lo != rcx, "must not use rcx"); 265 const Register s = rcx; // shift count 266 const int n = BitsPerWord; 267 Label L; 268 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 269 cmpl(s, n); // if (s < n) 270 jcc(Assembler::less, L); // else (s >= n) 271 movl(hi, lo); // x := x << n 272 xorl(lo, lo); 273 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 274 bind(L); // s (mod n) < n 275 shldl(hi, lo); // x := x << s 276 shll(lo); 277 } 278 279 280 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { 281 // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) 282 // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) 283 assert(hi != rcx, "must not use rcx"); 284 assert(lo != rcx, "must not use rcx"); 285 const Register s = rcx; // shift count 286 const int n = BitsPerWord; 287 Label L; 288 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 289 cmpl(s, n); // if (s < n) 290 jcc(Assembler::less, L); // else (s >= n) 291 movl(lo, hi); // x := x >> n 292 if (sign_extension) sarl(hi, 31); 293 else xorl(hi, hi); 294 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 295 bind(L); // s (mod n) < n 296 shrdl(lo, hi); // x := x >> s 297 if (sign_extension) sarl(hi); 298 else shrl(hi); 299 } 300 301 void MacroAssembler::movoop(Register dst, jobject obj) { 302 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 303 } 304 305 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 306 assert(rscratch == noreg, "redundant"); 307 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 308 } 309 310 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 311 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 312 } 313 314 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 315 assert(rscratch == noreg, "redundant"); 316 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 317 } 318 319 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 320 if (src.is_lval()) { 321 mov_literal32(dst, (intptr_t)src.target(), src.rspec()); 322 } else { 323 movl(dst, as_Address(src)); 324 } 325 } 326 327 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 328 assert(rscratch == noreg, "redundant"); 329 movl(as_Address(dst, noreg), src); 330 } 331 332 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 333 movl(dst, as_Address(src, noreg)); 334 } 335 336 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 337 assert(rscratch == noreg, "redundant"); 338 movl(dst, src); 339 } 340 341 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 342 assert(rscratch == noreg, "redundant"); 343 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); 344 } 345 346 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 347 assert(rscratch == noreg, "redundant"); 348 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); 349 } 350 351 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 352 assert(rscratch == noreg, "redundant"); 353 if (src.is_lval()) { 354 push_literal32((int32_t)src.target(), src.rspec()); 355 } else { 356 pushl(as_Address(src)); 357 } 358 } 359 360 static void pass_arg0(MacroAssembler* masm, Register arg) { 361 masm->push(arg); 362 } 363 364 static void pass_arg1(MacroAssembler* masm, Register arg) { 365 masm->push(arg); 366 } 367 368 static void pass_arg2(MacroAssembler* masm, Register arg) { 369 masm->push(arg); 370 } 371 372 static void pass_arg3(MacroAssembler* masm, Register arg) { 373 masm->push(arg); 374 } 375 376 #ifndef PRODUCT 377 extern "C" void findpc(intptr_t x); 378 #endif 379 380 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { 381 // In order to get locks to work, we need to fake a in_VM state 382 JavaThread* thread = JavaThread::current(); 383 JavaThreadState saved_state = thread->thread_state(); 384 thread->set_thread_state(_thread_in_vm); 385 if (ShowMessageBoxOnError) { 386 JavaThread* thread = JavaThread::current(); 387 JavaThreadState saved_state = thread->thread_state(); 388 thread->set_thread_state(_thread_in_vm); 389 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 390 ttyLocker ttyl; 391 BytecodeCounter::print(); 392 } 393 // To see where a verify_oop failed, get $ebx+40/X for this frame. 394 // This is the value of eip which points to where verify_oop will return. 395 if (os::message_box(msg, "Execution stopped, print registers?")) { 396 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); 397 BREAKPOINT; 398 } 399 } 400 fatal("DEBUG MESSAGE: %s", msg); 401 } 402 403 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { 404 ttyLocker ttyl; 405 DebuggingContext debugging{}; 406 tty->print_cr("eip = 0x%08x", eip); 407 #ifndef PRODUCT 408 if ((WizardMode || Verbose) && PrintMiscellaneous) { 409 tty->cr(); 410 findpc(eip); 411 tty->cr(); 412 } 413 #endif 414 #define PRINT_REG(rax) \ 415 { tty->print("%s = ", #rax); os::print_location(tty, rax); } 416 PRINT_REG(rax); 417 PRINT_REG(rbx); 418 PRINT_REG(rcx); 419 PRINT_REG(rdx); 420 PRINT_REG(rdi); 421 PRINT_REG(rsi); 422 PRINT_REG(rbp); 423 PRINT_REG(rsp); 424 #undef PRINT_REG 425 // Print some words near top of staack. 426 int* dump_sp = (int*) rsp; 427 for (int col1 = 0; col1 < 8; col1++) { 428 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 429 os::print_location(tty, *dump_sp++); 430 } 431 for (int row = 0; row < 16; row++) { 432 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 433 for (int col = 0; col < 8; col++) { 434 tty->print(" 0x%08x", *dump_sp++); 435 } 436 tty->cr(); 437 } 438 // Print some instructions around pc: 439 Disassembler::decode((address)eip-64, (address)eip); 440 tty->print_cr("--------"); 441 Disassembler::decode((address)eip, (address)eip+32); 442 } 443 444 void MacroAssembler::stop(const char* msg) { 445 // push address of message 446 ExternalAddress message((address)msg); 447 pushptr(message.addr(), noreg); 448 { Label L; call(L, relocInfo::none); bind(L); } // push eip 449 pusha(); // push registers 450 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); 451 hlt(); 452 } 453 454 void MacroAssembler::warn(const char* msg) { 455 push_CPU_state(); 456 457 // push address of message 458 ExternalAddress message((address)msg); 459 pushptr(message.addr(), noreg); 460 461 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 462 addl(rsp, wordSize); // discard argument 463 pop_CPU_state(); 464 } 465 466 void MacroAssembler::print_state() { 467 { Label L; call(L, relocInfo::none); bind(L); } // push eip 468 pusha(); // push registers 469 470 push_CPU_state(); 471 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); 472 pop_CPU_state(); 473 474 popa(); 475 addl(rsp, wordSize); 476 } 477 478 #else // _LP64 479 480 // 64 bit versions 481 482 Address MacroAssembler::as_Address(AddressLiteral adr) { 483 // amd64 always does this as a pc-rel 484 // we can be absolute or disp based on the instruction type 485 // jmp/call are displacements others are absolute 486 assert(!adr.is_lval(), "must be rval"); 487 assert(reachable(adr), "must be"); 488 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 489 490 } 491 492 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 493 AddressLiteral base = adr.base(); 494 lea(rscratch, base); 495 Address index = adr.index(); 496 assert(index._disp == 0, "must not have disp"); // maybe it can? 497 Address array(rscratch, index._index, index._scale, index._disp); 498 return array; 499 } 500 501 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 502 Label L, E; 503 504 #ifdef _WIN64 505 // Windows always allocates space for it's register args 506 assert(num_args <= 4, "only register arguments supported"); 507 subq(rsp, frame::arg_reg_save_area_bytes); 508 #endif 509 510 // Align stack if necessary 511 testl(rsp, 15); 512 jcc(Assembler::zero, L); 513 514 subq(rsp, 8); 515 call(RuntimeAddress(entry_point)); 516 addq(rsp, 8); 517 jmp(E); 518 519 bind(L); 520 call(RuntimeAddress(entry_point)); 521 522 bind(E); 523 524 #ifdef _WIN64 525 // restore stack pointer 526 addq(rsp, frame::arg_reg_save_area_bytes); 527 #endif 528 529 } 530 531 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 532 assert(!src2.is_lval(), "should use cmpptr"); 533 assert(rscratch != noreg || always_reachable(src2), "missing"); 534 535 if (reachable(src2)) { 536 cmpq(src1, as_Address(src2)); 537 } else { 538 lea(rscratch, src2); 539 Assembler::cmpq(src1, Address(rscratch, 0)); 540 } 541 } 542 543 int MacroAssembler::corrected_idivq(Register reg) { 544 // Full implementation of Java ldiv and lrem; checks for special 545 // case as described in JVM spec., p.243 & p.271. The function 546 // returns the (pc) offset of the idivl instruction - may be needed 547 // for implicit exceptions. 548 // 549 // normal case special case 550 // 551 // input : rax: dividend min_long 552 // reg: divisor (may not be eax/edx) -1 553 // 554 // output: rax: quotient (= rax idiv reg) min_long 555 // rdx: remainder (= rax irem reg) 0 556 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 557 static const int64_t min_long = 0x8000000000000000; 558 Label normal_case, special_case; 559 560 // check for special case 561 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 562 jcc(Assembler::notEqual, normal_case); 563 xorl(rdx, rdx); // prepare rdx for possible special case (where 564 // remainder = 0) 565 cmpq(reg, -1); 566 jcc(Assembler::equal, special_case); 567 568 // handle normal case 569 bind(normal_case); 570 cdqq(); 571 int idivq_offset = offset(); 572 idivq(reg); 573 574 // normal and special case exit 575 bind(special_case); 576 577 return idivq_offset; 578 } 579 580 void MacroAssembler::decrementq(Register reg, int value) { 581 if (value == min_jint) { subq(reg, value); return; } 582 if (value < 0) { incrementq(reg, -value); return; } 583 if (value == 0) { ; return; } 584 if (value == 1 && UseIncDec) { decq(reg) ; return; } 585 /* else */ { subq(reg, value) ; return; } 586 } 587 588 void MacroAssembler::decrementq(Address dst, int value) { 589 if (value == min_jint) { subq(dst, value); return; } 590 if (value < 0) { incrementq(dst, -value); return; } 591 if (value == 0) { ; return; } 592 if (value == 1 && UseIncDec) { decq(dst) ; return; } 593 /* else */ { subq(dst, value) ; return; } 594 } 595 596 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 597 assert(rscratch != noreg || always_reachable(dst), "missing"); 598 599 if (reachable(dst)) { 600 incrementq(as_Address(dst)); 601 } else { 602 lea(rscratch, dst); 603 incrementq(Address(rscratch, 0)); 604 } 605 } 606 607 void MacroAssembler::incrementq(Register reg, int value) { 608 if (value == min_jint) { addq(reg, value); return; } 609 if (value < 0) { decrementq(reg, -value); return; } 610 if (value == 0) { ; return; } 611 if (value == 1 && UseIncDec) { incq(reg) ; return; } 612 /* else */ { addq(reg, value) ; return; } 613 } 614 615 void MacroAssembler::incrementq(Address dst, int value) { 616 if (value == min_jint) { addq(dst, value); return; } 617 if (value < 0) { decrementq(dst, -value); return; } 618 if (value == 0) { ; return; } 619 if (value == 1 && UseIncDec) { incq(dst) ; return; } 620 /* else */ { addq(dst, value) ; return; } 621 } 622 623 // 32bit can do a case table jump in one instruction but we no longer allow the base 624 // to be installed in the Address class 625 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 626 lea(rscratch, entry.base()); 627 Address dispatch = entry.index(); 628 assert(dispatch._base == noreg, "must be"); 629 dispatch._base = rscratch; 630 jmp(dispatch); 631 } 632 633 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 634 ShouldNotReachHere(); // 64bit doesn't use two regs 635 cmpq(x_lo, y_lo); 636 } 637 638 void MacroAssembler::lea(Register dst, AddressLiteral src) { 639 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 640 } 641 642 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 643 lea(rscratch, adr); 644 movptr(dst, rscratch); 645 } 646 647 void MacroAssembler::leave() { 648 // %%% is this really better? Why not on 32bit too? 649 emit_int8((unsigned char)0xC9); // LEAVE 650 } 651 652 void MacroAssembler::lneg(Register hi, Register lo) { 653 ShouldNotReachHere(); // 64bit doesn't use two regs 654 negq(lo); 655 } 656 657 void MacroAssembler::movoop(Register dst, jobject obj) { 658 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 659 } 660 661 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 662 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 663 movq(dst, rscratch); 664 } 665 666 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 667 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 668 } 669 670 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 671 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 672 movq(dst, rscratch); 673 } 674 675 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 676 if (src.is_lval()) { 677 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 678 } else { 679 if (reachable(src)) { 680 movq(dst, as_Address(src)); 681 } else { 682 lea(dst, src); 683 movq(dst, Address(dst, 0)); 684 } 685 } 686 } 687 688 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 689 movq(as_Address(dst, rscratch), src); 690 } 691 692 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 693 movq(dst, as_Address(src, dst /*rscratch*/)); 694 } 695 696 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 697 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 698 if (is_simm32(src)) { 699 movptr(dst, checked_cast<int32_t>(src)); 700 } else { 701 mov64(rscratch, src); 702 movq(dst, rscratch); 703 } 704 } 705 706 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 707 movoop(rscratch, obj); 708 push(rscratch); 709 } 710 711 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 712 mov_metadata(rscratch, obj); 713 push(rscratch); 714 } 715 716 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 717 lea(rscratch, src); 718 if (src.is_lval()) { 719 push(rscratch); 720 } else { 721 pushq(Address(rscratch, 0)); 722 } 723 } 724 725 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 726 reset_last_Java_frame(r15_thread, clear_fp); 727 } 728 729 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 730 Register last_java_fp, 731 address last_java_pc, 732 Register rscratch) { 733 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch); 734 } 735 736 static void pass_arg0(MacroAssembler* masm, Register arg) { 737 if (c_rarg0 != arg ) { 738 masm->mov(c_rarg0, arg); 739 } 740 } 741 742 static void pass_arg1(MacroAssembler* masm, Register arg) { 743 if (c_rarg1 != arg ) { 744 masm->mov(c_rarg1, arg); 745 } 746 } 747 748 static void pass_arg2(MacroAssembler* masm, Register arg) { 749 if (c_rarg2 != arg ) { 750 masm->mov(c_rarg2, arg); 751 } 752 } 753 754 static void pass_arg3(MacroAssembler* masm, Register arg) { 755 if (c_rarg3 != arg ) { 756 masm->mov(c_rarg3, arg); 757 } 758 } 759 760 void MacroAssembler::stop(const char* msg) { 761 if (ShowMessageBoxOnError) { 762 address rip = pc(); 763 pusha(); // get regs on stack 764 lea(c_rarg1, InternalAddress(rip)); 765 movq(c_rarg2, rsp); // pass pointer to regs array 766 } 767 lea(c_rarg0, ExternalAddress((address) msg)); 768 andq(rsp, -16); // align stack as required by ABI 769 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 770 hlt(); 771 } 772 773 void MacroAssembler::warn(const char* msg) { 774 push(rbp); 775 movq(rbp, rsp); 776 andq(rsp, -16); // align stack as required by push_CPU_state and call 777 push_CPU_state(); // keeps alignment at 16 bytes 778 779 lea(c_rarg0, ExternalAddress((address) msg)); 780 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 781 782 pop_CPU_state(); 783 mov(rsp, rbp); 784 pop(rbp); 785 } 786 787 void MacroAssembler::print_state() { 788 address rip = pc(); 789 pusha(); // get regs on stack 790 push(rbp); 791 movq(rbp, rsp); 792 andq(rsp, -16); // align stack as required by push_CPU_state and call 793 push_CPU_state(); // keeps alignment at 16 bytes 794 795 lea(c_rarg0, InternalAddress(rip)); 796 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 797 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 798 799 pop_CPU_state(); 800 mov(rsp, rbp); 801 pop(rbp); 802 popa(); 803 } 804 805 #ifndef PRODUCT 806 extern "C" void findpc(intptr_t x); 807 #endif 808 809 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 810 // In order to get locks to work, we need to fake a in_VM state 811 if (ShowMessageBoxOnError) { 812 JavaThread* thread = JavaThread::current(); 813 JavaThreadState saved_state = thread->thread_state(); 814 thread->set_thread_state(_thread_in_vm); 815 #ifndef PRODUCT 816 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 817 ttyLocker ttyl; 818 BytecodeCounter::print(); 819 } 820 #endif 821 // To see where a verify_oop failed, get $ebx+40/X for this frame. 822 // XXX correct this offset for amd64 823 // This is the value of eip which points to where verify_oop will return. 824 if (os::message_box(msg, "Execution stopped, print registers?")) { 825 print_state64(pc, regs); 826 BREAKPOINT; 827 } 828 } 829 fatal("DEBUG MESSAGE: %s", msg); 830 } 831 832 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 833 ttyLocker ttyl; 834 DebuggingContext debugging{}; 835 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 836 #ifndef PRODUCT 837 tty->cr(); 838 findpc(pc); 839 tty->cr(); 840 #endif 841 #define PRINT_REG(rax, value) \ 842 { tty->print("%s = ", #rax); os::print_location(tty, value); } 843 PRINT_REG(rax, regs[15]); 844 PRINT_REG(rbx, regs[12]); 845 PRINT_REG(rcx, regs[14]); 846 PRINT_REG(rdx, regs[13]); 847 PRINT_REG(rdi, regs[8]); 848 PRINT_REG(rsi, regs[9]); 849 PRINT_REG(rbp, regs[10]); 850 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 851 PRINT_REG(rsp, (intptr_t)(®s[16])); 852 PRINT_REG(r8 , regs[7]); 853 PRINT_REG(r9 , regs[6]); 854 PRINT_REG(r10, regs[5]); 855 PRINT_REG(r11, regs[4]); 856 PRINT_REG(r12, regs[3]); 857 PRINT_REG(r13, regs[2]); 858 PRINT_REG(r14, regs[1]); 859 PRINT_REG(r15, regs[0]); 860 #undef PRINT_REG 861 // Print some words near the top of the stack. 862 int64_t* rsp = ®s[16]; 863 int64_t* dump_sp = rsp; 864 for (int col1 = 0; col1 < 8; col1++) { 865 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 866 os::print_location(tty, *dump_sp++); 867 } 868 for (int row = 0; row < 25; row++) { 869 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 870 for (int col = 0; col < 4; col++) { 871 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 872 } 873 tty->cr(); 874 } 875 // Print some instructions around pc: 876 Disassembler::decode((address)pc-64, (address)pc); 877 tty->print_cr("--------"); 878 Disassembler::decode((address)pc, (address)pc+32); 879 } 880 881 // The java_calling_convention describes stack locations as ideal slots on 882 // a frame with no abi restrictions. Since we must observe abi restrictions 883 // (like the placement of the register window) the slots must be biased by 884 // the following value. 885 static int reg2offset_in(VMReg r) { 886 // Account for saved rbp and return address 887 // This should really be in_preserve_stack_slots 888 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 889 } 890 891 static int reg2offset_out(VMReg r) { 892 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 893 } 894 895 // A long move 896 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 897 898 // The calling conventions assures us that each VMregpair is either 899 // all really one physical register or adjacent stack slots. 900 901 if (src.is_single_phys_reg() ) { 902 if (dst.is_single_phys_reg()) { 903 if (dst.first() != src.first()) { 904 mov(dst.first()->as_Register(), src.first()->as_Register()); 905 } 906 } else { 907 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 908 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 909 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 910 } 911 } else if (dst.is_single_phys_reg()) { 912 assert(src.is_single_reg(), "not a stack pair"); 913 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 914 } else { 915 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 916 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 917 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 918 } 919 } 920 921 // A double move 922 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 923 924 // The calling conventions assures us that each VMregpair is either 925 // all really one physical register or adjacent stack slots. 926 927 if (src.is_single_phys_reg() ) { 928 if (dst.is_single_phys_reg()) { 929 // In theory these overlap but the ordering is such that this is likely a nop 930 if ( src.first() != dst.first()) { 931 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 932 } 933 } else { 934 assert(dst.is_single_reg(), "not a stack pair"); 935 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 936 } 937 } else if (dst.is_single_phys_reg()) { 938 assert(src.is_single_reg(), "not a stack pair"); 939 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 940 } else { 941 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 942 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 943 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 944 } 945 } 946 947 948 // A float arg may have to do float reg int reg conversion 949 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 950 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 951 952 // The calling conventions assures us that each VMregpair is either 953 // all really one physical register or adjacent stack slots. 954 955 if (src.first()->is_stack()) { 956 if (dst.first()->is_stack()) { 957 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 958 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 959 } else { 960 // stack to reg 961 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 962 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 963 } 964 } else if (dst.first()->is_stack()) { 965 // reg to stack 966 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 967 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 968 } else { 969 // reg to reg 970 // In theory these overlap but the ordering is such that this is likely a nop 971 if ( src.first() != dst.first()) { 972 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 973 } 974 } 975 } 976 977 // On 64 bit we will store integer like items to the stack as 978 // 64 bits items (x86_32/64 abi) even though java would only store 979 // 32bits for a parameter. On 32bit it will simply be 32 bits 980 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 981 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 982 if (src.first()->is_stack()) { 983 if (dst.first()->is_stack()) { 984 // stack to stack 985 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 986 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 987 } else { 988 // stack to reg 989 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 990 } 991 } else if (dst.first()->is_stack()) { 992 // reg to stack 993 // Do we really have to sign extend??? 994 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 995 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 996 } else { 997 // Do we really have to sign extend??? 998 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 999 if (dst.first() != src.first()) { 1000 movq(dst.first()->as_Register(), src.first()->as_Register()); 1001 } 1002 } 1003 } 1004 1005 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 1006 if (src.first()->is_stack()) { 1007 if (dst.first()->is_stack()) { 1008 // stack to stack 1009 movq(rax, Address(rbp, reg2offset_in(src.first()))); 1010 movq(Address(rsp, reg2offset_out(dst.first())), rax); 1011 } else { 1012 // stack to reg 1013 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 1014 } 1015 } else if (dst.first()->is_stack()) { 1016 // reg to stack 1017 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 1018 } else { 1019 if (dst.first() != src.first()) { 1020 movq(dst.first()->as_Register(), src.first()->as_Register()); 1021 } 1022 } 1023 } 1024 1025 // An oop arg. Must pass a handle not the oop itself 1026 void MacroAssembler::object_move(OopMap* map, 1027 int oop_handle_offset, 1028 int framesize_in_slots, 1029 VMRegPair src, 1030 VMRegPair dst, 1031 bool is_receiver, 1032 int* receiver_offset) { 1033 1034 // must pass a handle. First figure out the location we use as a handle 1035 1036 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 1037 1038 // See if oop is null if it is we need no handle 1039 1040 if (src.first()->is_stack()) { 1041 1042 // Oop is already on the stack as an argument 1043 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 1044 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 1045 if (is_receiver) { 1046 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 1047 } 1048 1049 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 1050 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 1051 // conditionally move a null 1052 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 1053 } else { 1054 1055 // Oop is in a register we must store it to the space we reserve 1056 // on the stack for oop_handles and pass a handle if oop is non-null 1057 1058 const Register rOop = src.first()->as_Register(); 1059 int oop_slot; 1060 if (rOop == j_rarg0) 1061 oop_slot = 0; 1062 else if (rOop == j_rarg1) 1063 oop_slot = 1; 1064 else if (rOop == j_rarg2) 1065 oop_slot = 2; 1066 else if (rOop == j_rarg3) 1067 oop_slot = 3; 1068 else if (rOop == j_rarg4) 1069 oop_slot = 4; 1070 else { 1071 assert(rOop == j_rarg5, "wrong register"); 1072 oop_slot = 5; 1073 } 1074 1075 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 1076 int offset = oop_slot*VMRegImpl::stack_slot_size; 1077 1078 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 1079 // Store oop in handle area, may be null 1080 movptr(Address(rsp, offset), rOop); 1081 if (is_receiver) { 1082 *receiver_offset = offset; 1083 } 1084 1085 cmpptr(rOop, NULL_WORD); 1086 lea(rHandle, Address(rsp, offset)); 1087 // conditionally move a null from the handle area where it was just stored 1088 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 1089 } 1090 1091 // If arg is on the stack then place it otherwise it is already in correct reg. 1092 if (dst.first()->is_stack()) { 1093 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 1094 } 1095 } 1096 1097 #endif // _LP64 1098 1099 // Now versions that are common to 32/64 bit 1100 1101 void MacroAssembler::addptr(Register dst, int32_t imm32) { 1102 LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); 1103 } 1104 1105 void MacroAssembler::addptr(Register dst, Register src) { 1106 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1107 } 1108 1109 void MacroAssembler::addptr(Address dst, Register src) { 1110 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1111 } 1112 1113 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1114 assert(rscratch != noreg || always_reachable(src), "missing"); 1115 1116 if (reachable(src)) { 1117 Assembler::addsd(dst, as_Address(src)); 1118 } else { 1119 lea(rscratch, src); 1120 Assembler::addsd(dst, Address(rscratch, 0)); 1121 } 1122 } 1123 1124 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1125 assert(rscratch != noreg || always_reachable(src), "missing"); 1126 1127 if (reachable(src)) { 1128 addss(dst, as_Address(src)); 1129 } else { 1130 lea(rscratch, src); 1131 addss(dst, Address(rscratch, 0)); 1132 } 1133 } 1134 1135 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1136 assert(rscratch != noreg || always_reachable(src), "missing"); 1137 1138 if (reachable(src)) { 1139 Assembler::addpd(dst, as_Address(src)); 1140 } else { 1141 lea(rscratch, src); 1142 Assembler::addpd(dst, Address(rscratch, 0)); 1143 } 1144 } 1145 1146 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 1147 // Stub code is generated once and never copied. 1148 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 1149 void MacroAssembler::align64() { 1150 align(64, (unsigned long long) pc()); 1151 } 1152 1153 void MacroAssembler::align32() { 1154 align(32, (unsigned long long) pc()); 1155 } 1156 1157 void MacroAssembler::align(int modulus) { 1158 // 8273459: Ensure alignment is possible with current segment alignment 1159 assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 1160 align(modulus, offset()); 1161 } 1162 1163 void MacroAssembler::align(int modulus, int target) { 1164 if (target % modulus != 0) { 1165 nop(modulus - (target % modulus)); 1166 } 1167 } 1168 1169 void MacroAssembler::push_f(XMMRegister r) { 1170 subptr(rsp, wordSize); 1171 movflt(Address(rsp, 0), r); 1172 } 1173 1174 void MacroAssembler::pop_f(XMMRegister r) { 1175 movflt(r, Address(rsp, 0)); 1176 addptr(rsp, wordSize); 1177 } 1178 1179 void MacroAssembler::push_d(XMMRegister r) { 1180 subptr(rsp, 2 * wordSize); 1181 movdbl(Address(rsp, 0), r); 1182 } 1183 1184 void MacroAssembler::pop_d(XMMRegister r) { 1185 movdbl(r, Address(rsp, 0)); 1186 addptr(rsp, 2 * Interpreter::stackElementSize); 1187 } 1188 1189 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1190 // Used in sign-masking with aligned address. 1191 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1192 assert(rscratch != noreg || always_reachable(src), "missing"); 1193 1194 if (reachable(src)) { 1195 Assembler::andpd(dst, as_Address(src)); 1196 } else { 1197 lea(rscratch, src); 1198 Assembler::andpd(dst, Address(rscratch, 0)); 1199 } 1200 } 1201 1202 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 1203 // Used in sign-masking with aligned address. 1204 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1205 assert(rscratch != noreg || always_reachable(src), "missing"); 1206 1207 if (reachable(src)) { 1208 Assembler::andps(dst, as_Address(src)); 1209 } else { 1210 lea(rscratch, src); 1211 Assembler::andps(dst, Address(rscratch, 0)); 1212 } 1213 } 1214 1215 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1216 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1217 } 1218 1219 #ifdef _LP64 1220 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 1221 assert(rscratch != noreg || always_reachable(src), "missing"); 1222 1223 if (reachable(src)) { 1224 andq(dst, as_Address(src)); 1225 } else { 1226 lea(rscratch, src); 1227 andq(dst, Address(rscratch, 0)); 1228 } 1229 } 1230 #endif 1231 1232 void MacroAssembler::atomic_incl(Address counter_addr) { 1233 lock(); 1234 incrementl(counter_addr); 1235 } 1236 1237 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 1238 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1239 1240 if (reachable(counter_addr)) { 1241 atomic_incl(as_Address(counter_addr)); 1242 } else { 1243 lea(rscratch, counter_addr); 1244 atomic_incl(Address(rscratch, 0)); 1245 } 1246 } 1247 1248 #ifdef _LP64 1249 void MacroAssembler::atomic_incq(Address counter_addr) { 1250 lock(); 1251 incrementq(counter_addr); 1252 } 1253 1254 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 1255 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1256 1257 if (reachable(counter_addr)) { 1258 atomic_incq(as_Address(counter_addr)); 1259 } else { 1260 lea(rscratch, counter_addr); 1261 atomic_incq(Address(rscratch, 0)); 1262 } 1263 } 1264 #endif 1265 1266 // Writes to stack successive pages until offset reached to check for 1267 // stack overflow + shadow pages. This clobbers tmp. 1268 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 1269 movptr(tmp, rsp); 1270 // Bang stack for total size given plus shadow page size. 1271 // Bang one page at a time because large size can bang beyond yellow and 1272 // red zones. 1273 Label loop; 1274 bind(loop); 1275 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 1276 subptr(tmp, (int)os::vm_page_size()); 1277 subl(size, (int)os::vm_page_size()); 1278 jcc(Assembler::greater, loop); 1279 1280 // Bang down shadow pages too. 1281 // At this point, (tmp-0) is the last address touched, so don't 1282 // touch it again. (It was touched as (tmp-pagesize) but then tmp 1283 // was post-decremented.) Skip this address by starting at i=1, and 1284 // touch a few more pages below. N.B. It is important to touch all 1285 // the way down including all pages in the shadow zone. 1286 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 1287 // this could be any sized move but this is can be a debugging crumb 1288 // so the bigger the better. 1289 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 1290 } 1291 } 1292 1293 void MacroAssembler::reserved_stack_check() { 1294 // testing if reserved zone needs to be enabled 1295 Label no_reserved_zone_enabling; 1296 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 1297 NOT_LP64(get_thread(rsi);) 1298 1299 cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); 1300 jcc(Assembler::below, no_reserved_zone_enabling); 1301 1302 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); 1303 jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); 1304 should_not_reach_here(); 1305 1306 bind(no_reserved_zone_enabling); 1307 } 1308 1309 void MacroAssembler::c2bool(Register x) { 1310 // implements x == 0 ? 0 : 1 1311 // note: must only look at least-significant byte of x 1312 // since C-style booleans are stored in one byte 1313 // only! (was bug) 1314 andl(x, 0xFF); 1315 setb(Assembler::notZero, x); 1316 } 1317 1318 // Wouldn't need if AddressLiteral version had new name 1319 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 1320 Assembler::call(L, rtype); 1321 } 1322 1323 void MacroAssembler::call(Register entry) { 1324 Assembler::call(entry); 1325 } 1326 1327 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 1328 assert(rscratch != noreg || always_reachable(entry), "missing"); 1329 1330 if (reachable(entry)) { 1331 Assembler::call_literal(entry.target(), entry.rspec()); 1332 } else { 1333 lea(rscratch, entry); 1334 Assembler::call(rscratch); 1335 } 1336 } 1337 1338 void MacroAssembler::ic_call(address entry, jint method_index) { 1339 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1340 movptr(rax, (intptr_t)Universe::non_oop_word()); 1341 call(AddressLiteral(entry, rh)); 1342 } 1343 1344 void MacroAssembler::emit_static_call_stub() { 1345 // Static stub relocation also tags the Method* in the code-stream. 1346 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 1347 // This is recognized as unresolved by relocs/nativeinst/ic code. 1348 jump(RuntimeAddress(pc())); 1349 } 1350 1351 // Implementation of call_VM versions 1352 1353 void MacroAssembler::call_VM(Register oop_result, 1354 address entry_point, 1355 bool check_exceptions) { 1356 Label C, E; 1357 call(C, relocInfo::none); 1358 jmp(E); 1359 1360 bind(C); 1361 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1362 ret(0); 1363 1364 bind(E); 1365 } 1366 1367 void MacroAssembler::call_VM(Register oop_result, 1368 address entry_point, 1369 Register arg_1, 1370 bool check_exceptions) { 1371 Label C, E; 1372 call(C, relocInfo::none); 1373 jmp(E); 1374 1375 bind(C); 1376 pass_arg1(this, arg_1); 1377 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1378 ret(0); 1379 1380 bind(E); 1381 } 1382 1383 void MacroAssembler::call_VM(Register oop_result, 1384 address entry_point, 1385 Register arg_1, 1386 Register arg_2, 1387 bool check_exceptions) { 1388 Label C, E; 1389 call(C, relocInfo::none); 1390 jmp(E); 1391 1392 bind(C); 1393 1394 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1395 1396 pass_arg2(this, arg_2); 1397 pass_arg1(this, arg_1); 1398 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1399 ret(0); 1400 1401 bind(E); 1402 } 1403 1404 void MacroAssembler::call_VM(Register oop_result, 1405 address entry_point, 1406 Register arg_1, 1407 Register arg_2, 1408 Register arg_3, 1409 bool check_exceptions) { 1410 Label C, E; 1411 call(C, relocInfo::none); 1412 jmp(E); 1413 1414 bind(C); 1415 1416 LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); 1417 LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); 1418 pass_arg3(this, arg_3); 1419 1420 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1421 pass_arg2(this, arg_2); 1422 1423 pass_arg1(this, arg_1); 1424 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1425 ret(0); 1426 1427 bind(E); 1428 } 1429 1430 void MacroAssembler::call_VM(Register oop_result, 1431 Register last_java_sp, 1432 address entry_point, 1433 int number_of_arguments, 1434 bool check_exceptions) { 1435 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1436 call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1437 } 1438 1439 void MacroAssembler::call_VM(Register oop_result, 1440 Register last_java_sp, 1441 address entry_point, 1442 Register arg_1, 1443 bool check_exceptions) { 1444 pass_arg1(this, arg_1); 1445 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1446 } 1447 1448 void MacroAssembler::call_VM(Register oop_result, 1449 Register last_java_sp, 1450 address entry_point, 1451 Register arg_1, 1452 Register arg_2, 1453 bool check_exceptions) { 1454 1455 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1456 pass_arg2(this, arg_2); 1457 pass_arg1(this, arg_1); 1458 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1459 } 1460 1461 void MacroAssembler::call_VM(Register oop_result, 1462 Register last_java_sp, 1463 address entry_point, 1464 Register arg_1, 1465 Register arg_2, 1466 Register arg_3, 1467 bool check_exceptions) { 1468 LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); 1469 LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); 1470 pass_arg3(this, arg_3); 1471 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1472 pass_arg2(this, arg_2); 1473 pass_arg1(this, arg_1); 1474 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1475 } 1476 1477 void MacroAssembler::super_call_VM(Register oop_result, 1478 Register last_java_sp, 1479 address entry_point, 1480 int number_of_arguments, 1481 bool check_exceptions) { 1482 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1483 MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1484 } 1485 1486 void MacroAssembler::super_call_VM(Register oop_result, 1487 Register last_java_sp, 1488 address entry_point, 1489 Register arg_1, 1490 bool check_exceptions) { 1491 pass_arg1(this, arg_1); 1492 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1493 } 1494 1495 void MacroAssembler::super_call_VM(Register oop_result, 1496 Register last_java_sp, 1497 address entry_point, 1498 Register arg_1, 1499 Register arg_2, 1500 bool check_exceptions) { 1501 1502 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1503 pass_arg2(this, arg_2); 1504 pass_arg1(this, arg_1); 1505 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1506 } 1507 1508 void MacroAssembler::super_call_VM(Register oop_result, 1509 Register last_java_sp, 1510 address entry_point, 1511 Register arg_1, 1512 Register arg_2, 1513 Register arg_3, 1514 bool check_exceptions) { 1515 LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); 1516 LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); 1517 pass_arg3(this, arg_3); 1518 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1519 pass_arg2(this, arg_2); 1520 pass_arg1(this, arg_1); 1521 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1522 } 1523 1524 void MacroAssembler::call_VM_base(Register oop_result, 1525 Register java_thread, 1526 Register last_java_sp, 1527 address entry_point, 1528 int number_of_arguments, 1529 bool check_exceptions) { 1530 // determine java_thread register 1531 if (!java_thread->is_valid()) { 1532 #ifdef _LP64 1533 java_thread = r15_thread; 1534 #else 1535 java_thread = rdi; 1536 get_thread(java_thread); 1537 #endif // LP64 1538 } 1539 // determine last_java_sp register 1540 if (!last_java_sp->is_valid()) { 1541 last_java_sp = rsp; 1542 } 1543 // debugging support 1544 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1545 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); 1546 #ifdef ASSERT 1547 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1548 // r12 is the heapbase. 1549 LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) 1550 #endif // ASSERT 1551 1552 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1553 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1554 1555 // push java thread (becomes first argument of C function) 1556 1557 NOT_LP64(push(java_thread); number_of_arguments++); 1558 LP64_ONLY(mov(c_rarg0, r15_thread)); 1559 1560 // set last Java frame before call 1561 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1562 1563 // Only interpreter should have to set fp 1564 set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1); 1565 1566 // do the call, remove parameters 1567 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1568 1569 // restore the thread (cannot use the pushed argument since arguments 1570 // may be overwritten by C code generated by an optimizing compiler); 1571 // however can use the register value directly if it is callee saved. 1572 if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { 1573 // rdi & rsi (also r15) are callee saved -> nothing to do 1574 #ifdef ASSERT 1575 guarantee(java_thread != rax, "change this code"); 1576 push(rax); 1577 { Label L; 1578 get_thread(rax); 1579 cmpptr(java_thread, rax); 1580 jcc(Assembler::equal, L); 1581 STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); 1582 bind(L); 1583 } 1584 pop(rax); 1585 #endif 1586 } else { 1587 get_thread(java_thread); 1588 } 1589 // reset last Java frame 1590 // Only interpreter should have to clear fp 1591 reset_last_Java_frame(java_thread, true); 1592 1593 // C++ interp handles this in the interpreter 1594 check_and_handle_popframe(java_thread); 1595 check_and_handle_earlyret(java_thread); 1596 1597 if (check_exceptions) { 1598 // check for pending exceptions (java_thread is set upon return) 1599 cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD); 1600 #ifndef _LP64 1601 jump_cc(Assembler::notEqual, 1602 RuntimeAddress(StubRoutines::forward_exception_entry())); 1603 #else 1604 // This used to conditionally jump to forward_exception however it is 1605 // possible if we relocate that the branch will not reach. So we must jump 1606 // around so we can always reach 1607 1608 Label ok; 1609 jcc(Assembler::equal, ok); 1610 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1611 bind(ok); 1612 #endif // LP64 1613 } 1614 1615 // get oop result if there is one and reset the value in the thread 1616 if (oop_result->is_valid()) { 1617 get_vm_result(oop_result, java_thread); 1618 } 1619 } 1620 1621 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1622 1623 // Calculate the value for last_Java_sp 1624 // somewhat subtle. call_VM does an intermediate call 1625 // which places a return address on the stack just under the 1626 // stack pointer as the user finished with it. This allows 1627 // use to retrieve last_Java_pc from last_Java_sp[-1]. 1628 // On 32bit we then have to push additional args on the stack to accomplish 1629 // the actual requested call. On 64bit call_VM only can use register args 1630 // so the only extra space is the return address that call_VM created. 1631 // This hopefully explains the calculations here. 1632 1633 #ifdef _LP64 1634 // We've pushed one address, correct last_Java_sp 1635 lea(rax, Address(rsp, wordSize)); 1636 #else 1637 lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); 1638 #endif // LP64 1639 1640 call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); 1641 1642 } 1643 1644 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1645 void MacroAssembler::call_VM_leaf0(address entry_point) { 1646 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1647 } 1648 1649 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1650 call_VM_leaf_base(entry_point, number_of_arguments); 1651 } 1652 1653 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1654 pass_arg0(this, arg_0); 1655 call_VM_leaf(entry_point, 1); 1656 } 1657 1658 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1659 1660 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1661 pass_arg1(this, arg_1); 1662 pass_arg0(this, arg_0); 1663 call_VM_leaf(entry_point, 2); 1664 } 1665 1666 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1667 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); 1668 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1669 pass_arg2(this, arg_2); 1670 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1671 pass_arg1(this, arg_1); 1672 pass_arg0(this, arg_0); 1673 call_VM_leaf(entry_point, 3); 1674 } 1675 1676 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1677 LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); 1678 LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); 1679 LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); 1680 pass_arg3(this, arg_3); 1681 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); 1682 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1683 pass_arg2(this, arg_2); 1684 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1685 pass_arg1(this, arg_1); 1686 pass_arg0(this, arg_0); 1687 call_VM_leaf(entry_point, 3); 1688 } 1689 1690 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1691 pass_arg0(this, arg_0); 1692 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1693 } 1694 1695 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1696 1697 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1698 pass_arg1(this, arg_1); 1699 pass_arg0(this, arg_0); 1700 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1701 } 1702 1703 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1704 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); 1705 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1706 pass_arg2(this, arg_2); 1707 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1708 pass_arg1(this, arg_1); 1709 pass_arg0(this, arg_0); 1710 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1711 } 1712 1713 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1714 LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); 1715 LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); 1716 LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); 1717 pass_arg3(this, arg_3); 1718 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); 1719 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); 1720 pass_arg2(this, arg_2); 1721 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); 1722 pass_arg1(this, arg_1); 1723 pass_arg0(this, arg_0); 1724 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1725 } 1726 1727 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1728 movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1729 movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); 1730 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1731 } 1732 1733 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1734 movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1735 movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); 1736 } 1737 1738 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { 1739 } 1740 1741 void MacroAssembler::check_and_handle_popframe(Register java_thread) { 1742 } 1743 1744 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1745 assert(rscratch != noreg || always_reachable(src1), "missing"); 1746 1747 if (reachable(src1)) { 1748 cmpl(as_Address(src1), imm); 1749 } else { 1750 lea(rscratch, src1); 1751 cmpl(Address(rscratch, 0), imm); 1752 } 1753 } 1754 1755 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1756 assert(!src2.is_lval(), "use cmpptr"); 1757 assert(rscratch != noreg || always_reachable(src2), "missing"); 1758 1759 if (reachable(src2)) { 1760 cmpl(src1, as_Address(src2)); 1761 } else { 1762 lea(rscratch, src2); 1763 cmpl(src1, Address(rscratch, 0)); 1764 } 1765 } 1766 1767 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1768 Assembler::cmpl(src1, imm); 1769 } 1770 1771 void MacroAssembler::cmp32(Register src1, Address src2) { 1772 Assembler::cmpl(src1, src2); 1773 } 1774 1775 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1776 ucomisd(opr1, opr2); 1777 1778 Label L; 1779 if (unordered_is_less) { 1780 movl(dst, -1); 1781 jcc(Assembler::parity, L); 1782 jcc(Assembler::below , L); 1783 movl(dst, 0); 1784 jcc(Assembler::equal , L); 1785 increment(dst); 1786 } else { // unordered is greater 1787 movl(dst, 1); 1788 jcc(Assembler::parity, L); 1789 jcc(Assembler::above , L); 1790 movl(dst, 0); 1791 jcc(Assembler::equal , L); 1792 decrementl(dst); 1793 } 1794 bind(L); 1795 } 1796 1797 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1798 ucomiss(opr1, opr2); 1799 1800 Label L; 1801 if (unordered_is_less) { 1802 movl(dst, -1); 1803 jcc(Assembler::parity, L); 1804 jcc(Assembler::below , L); 1805 movl(dst, 0); 1806 jcc(Assembler::equal , L); 1807 increment(dst); 1808 } else { // unordered is greater 1809 movl(dst, 1); 1810 jcc(Assembler::parity, L); 1811 jcc(Assembler::above , L); 1812 movl(dst, 0); 1813 jcc(Assembler::equal , L); 1814 decrementl(dst); 1815 } 1816 bind(L); 1817 } 1818 1819 1820 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1821 assert(rscratch != noreg || always_reachable(src1), "missing"); 1822 1823 if (reachable(src1)) { 1824 cmpb(as_Address(src1), imm); 1825 } else { 1826 lea(rscratch, src1); 1827 cmpb(Address(rscratch, 0), imm); 1828 } 1829 } 1830 1831 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1832 #ifdef _LP64 1833 assert(rscratch != noreg || always_reachable(src2), "missing"); 1834 1835 if (src2.is_lval()) { 1836 movptr(rscratch, src2); 1837 Assembler::cmpq(src1, rscratch); 1838 } else if (reachable(src2)) { 1839 cmpq(src1, as_Address(src2)); 1840 } else { 1841 lea(rscratch, src2); 1842 Assembler::cmpq(src1, Address(rscratch, 0)); 1843 } 1844 #else 1845 assert(rscratch == noreg, "not needed"); 1846 if (src2.is_lval()) { 1847 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1848 } else { 1849 cmpl(src1, as_Address(src2)); 1850 } 1851 #endif // _LP64 1852 } 1853 1854 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1855 assert(src2.is_lval(), "not a mem-mem compare"); 1856 #ifdef _LP64 1857 // moves src2's literal address 1858 movptr(rscratch, src2); 1859 Assembler::cmpq(src1, rscratch); 1860 #else 1861 assert(rscratch == noreg, "not needed"); 1862 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1863 #endif // _LP64 1864 } 1865 1866 void MacroAssembler::cmpoop(Register src1, Register src2) { 1867 cmpptr(src1, src2); 1868 } 1869 1870 void MacroAssembler::cmpoop(Register src1, Address src2) { 1871 cmpptr(src1, src2); 1872 } 1873 1874 #ifdef _LP64 1875 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1876 movoop(rscratch, src2); 1877 cmpptr(src1, rscratch); 1878 } 1879 #endif 1880 1881 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1882 assert(rscratch != noreg || always_reachable(adr), "missing"); 1883 1884 if (reachable(adr)) { 1885 lock(); 1886 cmpxchgptr(reg, as_Address(adr)); 1887 } else { 1888 lea(rscratch, adr); 1889 lock(); 1890 cmpxchgptr(reg, Address(rscratch, 0)); 1891 } 1892 } 1893 1894 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1895 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); 1896 } 1897 1898 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1899 assert(rscratch != noreg || always_reachable(src), "missing"); 1900 1901 if (reachable(src)) { 1902 Assembler::comisd(dst, as_Address(src)); 1903 } else { 1904 lea(rscratch, src); 1905 Assembler::comisd(dst, Address(rscratch, 0)); 1906 } 1907 } 1908 1909 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1910 assert(rscratch != noreg || always_reachable(src), "missing"); 1911 1912 if (reachable(src)) { 1913 Assembler::comiss(dst, as_Address(src)); 1914 } else { 1915 lea(rscratch, src); 1916 Assembler::comiss(dst, Address(rscratch, 0)); 1917 } 1918 } 1919 1920 1921 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 1922 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1923 1924 Condition negated_cond = negate_condition(cond); 1925 Label L; 1926 jcc(negated_cond, L); 1927 pushf(); // Preserve flags 1928 atomic_incl(counter_addr, rscratch); 1929 popf(); 1930 bind(L); 1931 } 1932 1933 int MacroAssembler::corrected_idivl(Register reg) { 1934 // Full implementation of Java idiv and irem; checks for 1935 // special case as described in JVM spec., p.243 & p.271. 1936 // The function returns the (pc) offset of the idivl 1937 // instruction - may be needed for implicit exceptions. 1938 // 1939 // normal case special case 1940 // 1941 // input : rax,: dividend min_int 1942 // reg: divisor (may not be rax,/rdx) -1 1943 // 1944 // output: rax,: quotient (= rax, idiv reg) min_int 1945 // rdx: remainder (= rax, irem reg) 0 1946 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 1947 const int min_int = 0x80000000; 1948 Label normal_case, special_case; 1949 1950 // check for special case 1951 cmpl(rax, min_int); 1952 jcc(Assembler::notEqual, normal_case); 1953 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 1954 cmpl(reg, -1); 1955 jcc(Assembler::equal, special_case); 1956 1957 // handle normal case 1958 bind(normal_case); 1959 cdql(); 1960 int idivl_offset = offset(); 1961 idivl(reg); 1962 1963 // normal and special case exit 1964 bind(special_case); 1965 1966 return idivl_offset; 1967 } 1968 1969 1970 1971 void MacroAssembler::decrementl(Register reg, int value) { 1972 if (value == min_jint) {subl(reg, value) ; return; } 1973 if (value < 0) { incrementl(reg, -value); return; } 1974 if (value == 0) { ; return; } 1975 if (value == 1 && UseIncDec) { decl(reg) ; return; } 1976 /* else */ { subl(reg, value) ; return; } 1977 } 1978 1979 void MacroAssembler::decrementl(Address dst, int value) { 1980 if (value == min_jint) {subl(dst, value) ; return; } 1981 if (value < 0) { incrementl(dst, -value); return; } 1982 if (value == 0) { ; return; } 1983 if (value == 1 && UseIncDec) { decl(dst) ; return; } 1984 /* else */ { subl(dst, value) ; return; } 1985 } 1986 1987 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 1988 assert(shift_value > 0, "illegal shift value"); 1989 Label _is_positive; 1990 testl (reg, reg); 1991 jcc (Assembler::positive, _is_positive); 1992 int offset = (1 << shift_value) - 1 ; 1993 1994 if (offset == 1) { 1995 incrementl(reg); 1996 } else { 1997 addl(reg, offset); 1998 } 1999 2000 bind (_is_positive); 2001 sarl(reg, shift_value); 2002 } 2003 2004 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2005 assert(rscratch != noreg || always_reachable(src), "missing"); 2006 2007 if (reachable(src)) { 2008 Assembler::divsd(dst, as_Address(src)); 2009 } else { 2010 lea(rscratch, src); 2011 Assembler::divsd(dst, Address(rscratch, 0)); 2012 } 2013 } 2014 2015 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2016 assert(rscratch != noreg || always_reachable(src), "missing"); 2017 2018 if (reachable(src)) { 2019 Assembler::divss(dst, as_Address(src)); 2020 } else { 2021 lea(rscratch, src); 2022 Assembler::divss(dst, Address(rscratch, 0)); 2023 } 2024 } 2025 2026 void MacroAssembler::enter() { 2027 push(rbp); 2028 mov(rbp, rsp); 2029 } 2030 2031 void MacroAssembler::post_call_nop() { 2032 if (!Continuations::enabled()) { 2033 return; 2034 } 2035 InstructionMark im(this); 2036 relocate(post_call_nop_Relocation::spec()); 2037 InlineSkippedInstructionsCounter skipCounter(this); 2038 emit_int8((int8_t)0x0f); 2039 emit_int8((int8_t)0x1f); 2040 emit_int8((int8_t)0x84); 2041 emit_int8((int8_t)0x00); 2042 emit_int32(0x00); 2043 } 2044 2045 // A 5 byte nop that is safe for patching (see patch_verified_entry) 2046 void MacroAssembler::fat_nop() { 2047 if (UseAddressNop) { 2048 addr_nop_5(); 2049 } else { 2050 emit_int8((int8_t)0x26); // es: 2051 emit_int8((int8_t)0x2e); // cs: 2052 emit_int8((int8_t)0x64); // fs: 2053 emit_int8((int8_t)0x65); // gs: 2054 emit_int8((int8_t)0x90); 2055 } 2056 } 2057 2058 #ifndef _LP64 2059 void MacroAssembler::fcmp(Register tmp) { 2060 fcmp(tmp, 1, true, true); 2061 } 2062 2063 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { 2064 assert(!pop_right || pop_left, "usage error"); 2065 if (VM_Version::supports_cmov()) { 2066 assert(tmp == noreg, "unneeded temp"); 2067 if (pop_left) { 2068 fucomip(index); 2069 } else { 2070 fucomi(index); 2071 } 2072 if (pop_right) { 2073 fpop(); 2074 } 2075 } else { 2076 assert(tmp != noreg, "need temp"); 2077 if (pop_left) { 2078 if (pop_right) { 2079 fcompp(); 2080 } else { 2081 fcomp(index); 2082 } 2083 } else { 2084 fcom(index); 2085 } 2086 // convert FPU condition into eflags condition via rax, 2087 save_rax(tmp); 2088 fwait(); fnstsw_ax(); 2089 sahf(); 2090 restore_rax(tmp); 2091 } 2092 // condition codes set as follows: 2093 // 2094 // CF (corresponds to C0) if x < y 2095 // PF (corresponds to C2) if unordered 2096 // ZF (corresponds to C3) if x = y 2097 } 2098 2099 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { 2100 fcmp2int(dst, unordered_is_less, 1, true, true); 2101 } 2102 2103 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { 2104 fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); 2105 Label L; 2106 if (unordered_is_less) { 2107 movl(dst, -1); 2108 jcc(Assembler::parity, L); 2109 jcc(Assembler::below , L); 2110 movl(dst, 0); 2111 jcc(Assembler::equal , L); 2112 increment(dst); 2113 } else { // unordered is greater 2114 movl(dst, 1); 2115 jcc(Assembler::parity, L); 2116 jcc(Assembler::above , L); 2117 movl(dst, 0); 2118 jcc(Assembler::equal , L); 2119 decrementl(dst); 2120 } 2121 bind(L); 2122 } 2123 2124 void MacroAssembler::fld_d(AddressLiteral src) { 2125 fld_d(as_Address(src)); 2126 } 2127 2128 void MacroAssembler::fld_s(AddressLiteral src) { 2129 fld_s(as_Address(src)); 2130 } 2131 2132 void MacroAssembler::fldcw(AddressLiteral src) { 2133 fldcw(as_Address(src)); 2134 } 2135 2136 void MacroAssembler::fpop() { 2137 ffree(); 2138 fincstp(); 2139 } 2140 2141 void MacroAssembler::fremr(Register tmp) { 2142 save_rax(tmp); 2143 { Label L; 2144 bind(L); 2145 fprem(); 2146 fwait(); fnstsw_ax(); 2147 sahf(); 2148 jcc(Assembler::parity, L); 2149 } 2150 restore_rax(tmp); 2151 // Result is in ST0. 2152 // Note: fxch & fpop to get rid of ST1 2153 // (otherwise FPU stack could overflow eventually) 2154 fxch(1); 2155 fpop(); 2156 } 2157 2158 void MacroAssembler::empty_FPU_stack() { 2159 if (VM_Version::supports_mmx()) { 2160 emms(); 2161 } else { 2162 for (int i = 8; i-- > 0; ) ffree(i); 2163 } 2164 } 2165 #endif // !LP64 2166 2167 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2168 assert(rscratch != noreg || always_reachable(src), "missing"); 2169 if (reachable(src)) { 2170 Assembler::mulpd(dst, as_Address(src)); 2171 } else { 2172 lea(rscratch, src); 2173 Assembler::mulpd(dst, Address(rscratch, 0)); 2174 } 2175 } 2176 2177 void MacroAssembler::load_float(Address src) { 2178 #ifdef _LP64 2179 movflt(xmm0, src); 2180 #else 2181 if (UseSSE >= 1) { 2182 movflt(xmm0, src); 2183 } else { 2184 fld_s(src); 2185 } 2186 #endif // LP64 2187 } 2188 2189 void MacroAssembler::store_float(Address dst) { 2190 #ifdef _LP64 2191 movflt(dst, xmm0); 2192 #else 2193 if (UseSSE >= 1) { 2194 movflt(dst, xmm0); 2195 } else { 2196 fstp_s(dst); 2197 } 2198 #endif // LP64 2199 } 2200 2201 void MacroAssembler::load_double(Address src) { 2202 #ifdef _LP64 2203 movdbl(xmm0, src); 2204 #else 2205 if (UseSSE >= 2) { 2206 movdbl(xmm0, src); 2207 } else { 2208 fld_d(src); 2209 } 2210 #endif // LP64 2211 } 2212 2213 void MacroAssembler::store_double(Address dst) { 2214 #ifdef _LP64 2215 movdbl(dst, xmm0); 2216 #else 2217 if (UseSSE >= 2) { 2218 movdbl(dst, xmm0); 2219 } else { 2220 fstp_d(dst); 2221 } 2222 #endif // LP64 2223 } 2224 2225 // dst = c = a * b + c 2226 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2227 Assembler::vfmadd231sd(c, a, b); 2228 if (dst != c) { 2229 movdbl(dst, c); 2230 } 2231 } 2232 2233 // dst = c = a * b + c 2234 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2235 Assembler::vfmadd231ss(c, a, b); 2236 if (dst != c) { 2237 movflt(dst, c); 2238 } 2239 } 2240 2241 // dst = c = a * b + c 2242 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2243 Assembler::vfmadd231pd(c, a, b, vector_len); 2244 if (dst != c) { 2245 vmovdqu(dst, c); 2246 } 2247 } 2248 2249 // dst = c = a * b + c 2250 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2251 Assembler::vfmadd231ps(c, a, b, vector_len); 2252 if (dst != c) { 2253 vmovdqu(dst, c); 2254 } 2255 } 2256 2257 // dst = c = a * b + c 2258 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2259 Assembler::vfmadd231pd(c, a, b, vector_len); 2260 if (dst != c) { 2261 vmovdqu(dst, c); 2262 } 2263 } 2264 2265 // dst = c = a * b + c 2266 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2267 Assembler::vfmadd231ps(c, a, b, vector_len); 2268 if (dst != c) { 2269 vmovdqu(dst, c); 2270 } 2271 } 2272 2273 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 2274 assert(rscratch != noreg || always_reachable(dst), "missing"); 2275 2276 if (reachable(dst)) { 2277 incrementl(as_Address(dst)); 2278 } else { 2279 lea(rscratch, dst); 2280 incrementl(Address(rscratch, 0)); 2281 } 2282 } 2283 2284 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 2285 incrementl(as_Address(dst, rscratch)); 2286 } 2287 2288 void MacroAssembler::incrementl(Register reg, int value) { 2289 if (value == min_jint) {addl(reg, value) ; return; } 2290 if (value < 0) { decrementl(reg, -value); return; } 2291 if (value == 0) { ; return; } 2292 if (value == 1 && UseIncDec) { incl(reg) ; return; } 2293 /* else */ { addl(reg, value) ; return; } 2294 } 2295 2296 void MacroAssembler::incrementl(Address dst, int value) { 2297 if (value == min_jint) {addl(dst, value) ; return; } 2298 if (value < 0) { decrementl(dst, -value); return; } 2299 if (value == 0) { ; return; } 2300 if (value == 1 && UseIncDec) { incl(dst) ; return; } 2301 /* else */ { addl(dst, value) ; return; } 2302 } 2303 2304 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 2305 assert(rscratch != noreg || always_reachable(dst), "missing"); 2306 2307 if (reachable(dst)) { 2308 jmp_literal(dst.target(), dst.rspec()); 2309 } else { 2310 lea(rscratch, dst); 2311 jmp(rscratch); 2312 } 2313 } 2314 2315 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 2316 assert(rscratch != noreg || always_reachable(dst), "missing"); 2317 2318 if (reachable(dst)) { 2319 InstructionMark im(this); 2320 relocate(dst.reloc()); 2321 const int short_size = 2; 2322 const int long_size = 6; 2323 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 2324 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 2325 // 0111 tttn #8-bit disp 2326 emit_int8(0x70 | cc); 2327 emit_int8((offs - short_size) & 0xFF); 2328 } else { 2329 // 0000 1111 1000 tttn #32-bit disp 2330 emit_int8(0x0F); 2331 emit_int8((unsigned char)(0x80 | cc)); 2332 emit_int32(offs - long_size); 2333 } 2334 } else { 2335 #ifdef ASSERT 2336 warning("reversing conditional branch"); 2337 #endif /* ASSERT */ 2338 Label skip; 2339 jccb(reverse[cc], skip); 2340 lea(rscratch, dst); 2341 Assembler::jmp(rscratch); 2342 bind(skip); 2343 } 2344 } 2345 2346 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 2347 assert(rscratch != noreg || always_reachable(src), "missing"); 2348 2349 if (reachable(src)) { 2350 Assembler::ldmxcsr(as_Address(src)); 2351 } else { 2352 lea(rscratch, src); 2353 Assembler::ldmxcsr(Address(rscratch, 0)); 2354 } 2355 } 2356 2357 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2358 int off; 2359 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2360 off = offset(); 2361 movsbl(dst, src); // movsxb 2362 } else { 2363 off = load_unsigned_byte(dst, src); 2364 shll(dst, 24); 2365 sarl(dst, 24); 2366 } 2367 return off; 2368 } 2369 2370 // Note: load_signed_short used to be called load_signed_word. 2371 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 2372 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 2373 // The term "word" in HotSpot means a 32- or 64-bit machine word. 2374 int MacroAssembler::load_signed_short(Register dst, Address src) { 2375 int off; 2376 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2377 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 2378 // version but this is what 64bit has always done. This seems to imply 2379 // that users are only using 32bits worth. 2380 off = offset(); 2381 movswl(dst, src); // movsxw 2382 } else { 2383 off = load_unsigned_short(dst, src); 2384 shll(dst, 16); 2385 sarl(dst, 16); 2386 } 2387 return off; 2388 } 2389 2390 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2391 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2392 // and "3.9 Partial Register Penalties", p. 22). 2393 int off; 2394 if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { 2395 off = offset(); 2396 movzbl(dst, src); // movzxb 2397 } else { 2398 xorl(dst, dst); 2399 off = offset(); 2400 movb(dst, src); 2401 } 2402 return off; 2403 } 2404 2405 // Note: load_unsigned_short used to be called load_unsigned_word. 2406 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2407 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2408 // and "3.9 Partial Register Penalties", p. 22). 2409 int off; 2410 if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { 2411 off = offset(); 2412 movzwl(dst, src); // movzxw 2413 } else { 2414 xorl(dst, dst); 2415 off = offset(); 2416 movw(dst, src); 2417 } 2418 return off; 2419 } 2420 2421 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 2422 switch (size_in_bytes) { 2423 #ifndef _LP64 2424 case 8: 2425 assert(dst2 != noreg, "second dest register required"); 2426 movl(dst, src); 2427 movl(dst2, src.plus_disp(BytesPerInt)); 2428 break; 2429 #else 2430 case 8: movq(dst, src); break; 2431 #endif 2432 case 4: movl(dst, src); break; 2433 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2434 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2435 default: ShouldNotReachHere(); 2436 } 2437 } 2438 2439 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 2440 switch (size_in_bytes) { 2441 #ifndef _LP64 2442 case 8: 2443 assert(src2 != noreg, "second source register required"); 2444 movl(dst, src); 2445 movl(dst.plus_disp(BytesPerInt), src2); 2446 break; 2447 #else 2448 case 8: movq(dst, src); break; 2449 #endif 2450 case 4: movl(dst, src); break; 2451 case 2: movw(dst, src); break; 2452 case 1: movb(dst, src); break; 2453 default: ShouldNotReachHere(); 2454 } 2455 } 2456 2457 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 2458 assert(rscratch != noreg || always_reachable(dst), "missing"); 2459 2460 if (reachable(dst)) { 2461 movl(as_Address(dst), src); 2462 } else { 2463 lea(rscratch, dst); 2464 movl(Address(rscratch, 0), src); 2465 } 2466 } 2467 2468 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 2469 if (reachable(src)) { 2470 movl(dst, as_Address(src)); 2471 } else { 2472 lea(dst, src); 2473 movl(dst, Address(dst, 0)); 2474 } 2475 } 2476 2477 // C++ bool manipulation 2478 2479 void MacroAssembler::movbool(Register dst, Address src) { 2480 if(sizeof(bool) == 1) 2481 movb(dst, src); 2482 else if(sizeof(bool) == 2) 2483 movw(dst, src); 2484 else if(sizeof(bool) == 4) 2485 movl(dst, src); 2486 else 2487 // unsupported 2488 ShouldNotReachHere(); 2489 } 2490 2491 void MacroAssembler::movbool(Address dst, bool boolconst) { 2492 if(sizeof(bool) == 1) 2493 movb(dst, (int) boolconst); 2494 else if(sizeof(bool) == 2) 2495 movw(dst, (int) boolconst); 2496 else if(sizeof(bool) == 4) 2497 movl(dst, (int) boolconst); 2498 else 2499 // unsupported 2500 ShouldNotReachHere(); 2501 } 2502 2503 void MacroAssembler::movbool(Address dst, Register src) { 2504 if(sizeof(bool) == 1) 2505 movb(dst, src); 2506 else if(sizeof(bool) == 2) 2507 movw(dst, src); 2508 else if(sizeof(bool) == 4) 2509 movl(dst, src); 2510 else 2511 // unsupported 2512 ShouldNotReachHere(); 2513 } 2514 2515 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2516 assert(rscratch != noreg || always_reachable(src), "missing"); 2517 2518 if (reachable(src)) { 2519 movdl(dst, as_Address(src)); 2520 } else { 2521 lea(rscratch, src); 2522 movdl(dst, Address(rscratch, 0)); 2523 } 2524 } 2525 2526 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 2527 assert(rscratch != noreg || always_reachable(src), "missing"); 2528 2529 if (reachable(src)) { 2530 movq(dst, as_Address(src)); 2531 } else { 2532 lea(rscratch, src); 2533 movq(dst, Address(rscratch, 0)); 2534 } 2535 } 2536 2537 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2538 assert(rscratch != noreg || always_reachable(src), "missing"); 2539 2540 if (reachable(src)) { 2541 if (UseXmmLoadAndClearUpper) { 2542 movsd (dst, as_Address(src)); 2543 } else { 2544 movlpd(dst, as_Address(src)); 2545 } 2546 } else { 2547 lea(rscratch, src); 2548 if (UseXmmLoadAndClearUpper) { 2549 movsd (dst, Address(rscratch, 0)); 2550 } else { 2551 movlpd(dst, Address(rscratch, 0)); 2552 } 2553 } 2554 } 2555 2556 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 2557 assert(rscratch != noreg || always_reachable(src), "missing"); 2558 2559 if (reachable(src)) { 2560 movss(dst, as_Address(src)); 2561 } else { 2562 lea(rscratch, src); 2563 movss(dst, Address(rscratch, 0)); 2564 } 2565 } 2566 2567 void MacroAssembler::movptr(Register dst, Register src) { 2568 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2569 } 2570 2571 void MacroAssembler::movptr(Register dst, Address src) { 2572 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2573 } 2574 2575 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 2576 void MacroAssembler::movptr(Register dst, intptr_t src) { 2577 LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src)); 2578 } 2579 2580 void MacroAssembler::movptr(Address dst, Register src) { 2581 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2582 } 2583 2584 void MacroAssembler::movptr(Address dst, int32_t src) { 2585 LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); 2586 } 2587 2588 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 2589 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2590 Assembler::movdqu(dst, src); 2591 } 2592 2593 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 2594 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2595 Assembler::movdqu(dst, src); 2596 } 2597 2598 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 2599 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2600 Assembler::movdqu(dst, src); 2601 } 2602 2603 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2604 assert(rscratch != noreg || always_reachable(src), "missing"); 2605 2606 if (reachable(src)) { 2607 movdqu(dst, as_Address(src)); 2608 } else { 2609 lea(rscratch, src); 2610 movdqu(dst, Address(rscratch, 0)); 2611 } 2612 } 2613 2614 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2615 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2616 Assembler::vmovdqu(dst, src); 2617 } 2618 2619 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2620 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2621 Assembler::vmovdqu(dst, src); 2622 } 2623 2624 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2625 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2626 Assembler::vmovdqu(dst, src); 2627 } 2628 2629 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2630 assert(rscratch != noreg || always_reachable(src), "missing"); 2631 2632 if (reachable(src)) { 2633 vmovdqu(dst, as_Address(src)); 2634 } 2635 else { 2636 lea(rscratch, src); 2637 vmovdqu(dst, Address(rscratch, 0)); 2638 } 2639 } 2640 2641 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2642 assert(rscratch != noreg || always_reachable(src), "missing"); 2643 2644 if (vector_len == AVX_512bit) { 2645 evmovdquq(dst, src, AVX_512bit, rscratch); 2646 } else if (vector_len == AVX_256bit) { 2647 vmovdqu(dst, src, rscratch); 2648 } else { 2649 movdqu(dst, src, rscratch); 2650 } 2651 } 2652 2653 void MacroAssembler::kmov(KRegister dst, Address src) { 2654 if (VM_Version::supports_avx512bw()) { 2655 kmovql(dst, src); 2656 } else { 2657 assert(VM_Version::supports_evex(), ""); 2658 kmovwl(dst, src); 2659 } 2660 } 2661 2662 void MacroAssembler::kmov(Address dst, KRegister src) { 2663 if (VM_Version::supports_avx512bw()) { 2664 kmovql(dst, src); 2665 } else { 2666 assert(VM_Version::supports_evex(), ""); 2667 kmovwl(dst, src); 2668 } 2669 } 2670 2671 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2672 if (VM_Version::supports_avx512bw()) { 2673 kmovql(dst, src); 2674 } else { 2675 assert(VM_Version::supports_evex(), ""); 2676 kmovwl(dst, src); 2677 } 2678 } 2679 2680 void MacroAssembler::kmov(Register dst, KRegister src) { 2681 if (VM_Version::supports_avx512bw()) { 2682 kmovql(dst, src); 2683 } else { 2684 assert(VM_Version::supports_evex(), ""); 2685 kmovwl(dst, src); 2686 } 2687 } 2688 2689 void MacroAssembler::kmov(KRegister dst, Register src) { 2690 if (VM_Version::supports_avx512bw()) { 2691 kmovql(dst, src); 2692 } else { 2693 assert(VM_Version::supports_evex(), ""); 2694 kmovwl(dst, src); 2695 } 2696 } 2697 2698 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2699 assert(rscratch != noreg || always_reachable(src), "missing"); 2700 2701 if (reachable(src)) { 2702 kmovql(dst, as_Address(src)); 2703 } else { 2704 lea(rscratch, src); 2705 kmovql(dst, Address(rscratch, 0)); 2706 } 2707 } 2708 2709 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2710 assert(rscratch != noreg || always_reachable(src), "missing"); 2711 2712 if (reachable(src)) { 2713 kmovwl(dst, as_Address(src)); 2714 } else { 2715 lea(rscratch, src); 2716 kmovwl(dst, Address(rscratch, 0)); 2717 } 2718 } 2719 2720 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2721 int vector_len, Register rscratch) { 2722 assert(rscratch != noreg || always_reachable(src), "missing"); 2723 2724 if (reachable(src)) { 2725 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2726 } else { 2727 lea(rscratch, src); 2728 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2729 } 2730 } 2731 2732 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2733 int vector_len, Register rscratch) { 2734 assert(rscratch != noreg || always_reachable(src), "missing"); 2735 2736 if (reachable(src)) { 2737 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2738 } else { 2739 lea(rscratch, src); 2740 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2741 } 2742 } 2743 2744 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2745 assert(rscratch != noreg || always_reachable(src), "missing"); 2746 2747 if (reachable(src)) { 2748 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2749 } else { 2750 lea(rscratch, src); 2751 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2752 } 2753 } 2754 2755 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2756 assert(rscratch != noreg || always_reachable(src), "missing"); 2757 2758 if (reachable(src)) { 2759 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2760 } else { 2761 lea(rscratch, src); 2762 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2763 } 2764 } 2765 2766 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2767 assert(rscratch != noreg || always_reachable(src), "missing"); 2768 2769 if (reachable(src)) { 2770 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2771 } else { 2772 lea(rscratch, src); 2773 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2774 } 2775 } 2776 2777 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2778 assert(rscratch != noreg || always_reachable(src), "missing"); 2779 2780 if (reachable(src)) { 2781 Assembler::movdqa(dst, as_Address(src)); 2782 } else { 2783 lea(rscratch, src); 2784 Assembler::movdqa(dst, Address(rscratch, 0)); 2785 } 2786 } 2787 2788 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2789 assert(rscratch != noreg || always_reachable(src), "missing"); 2790 2791 if (reachable(src)) { 2792 Assembler::movsd(dst, as_Address(src)); 2793 } else { 2794 lea(rscratch, src); 2795 Assembler::movsd(dst, Address(rscratch, 0)); 2796 } 2797 } 2798 2799 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2800 assert(rscratch != noreg || always_reachable(src), "missing"); 2801 2802 if (reachable(src)) { 2803 Assembler::movss(dst, as_Address(src)); 2804 } else { 2805 lea(rscratch, src); 2806 Assembler::movss(dst, Address(rscratch, 0)); 2807 } 2808 } 2809 2810 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2811 assert(rscratch != noreg || always_reachable(src), "missing"); 2812 2813 if (reachable(src)) { 2814 Assembler::movddup(dst, as_Address(src)); 2815 } else { 2816 lea(rscratch, src); 2817 Assembler::movddup(dst, Address(rscratch, 0)); 2818 } 2819 } 2820 2821 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2822 assert(rscratch != noreg || always_reachable(src), "missing"); 2823 2824 if (reachable(src)) { 2825 Assembler::vmovddup(dst, as_Address(src), vector_len); 2826 } else { 2827 lea(rscratch, src); 2828 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2829 } 2830 } 2831 2832 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2833 assert(rscratch != noreg || always_reachable(src), "missing"); 2834 2835 if (reachable(src)) { 2836 Assembler::mulsd(dst, as_Address(src)); 2837 } else { 2838 lea(rscratch, src); 2839 Assembler::mulsd(dst, Address(rscratch, 0)); 2840 } 2841 } 2842 2843 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2844 assert(rscratch != noreg || always_reachable(src), "missing"); 2845 2846 if (reachable(src)) { 2847 Assembler::mulss(dst, as_Address(src)); 2848 } else { 2849 lea(rscratch, src); 2850 Assembler::mulss(dst, Address(rscratch, 0)); 2851 } 2852 } 2853 2854 void MacroAssembler::null_check(Register reg, int offset) { 2855 if (needs_explicit_null_check(offset)) { 2856 // provoke OS null exception if reg is null by 2857 // accessing M[reg] w/o changing any (non-CC) registers 2858 // NOTE: cmpl is plenty here to provoke a segv 2859 cmpptr(rax, Address(reg, 0)); 2860 // Note: should probably use testl(rax, Address(reg, 0)); 2861 // may be shorter code (however, this version of 2862 // testl needs to be implemented first) 2863 } else { 2864 // nothing to do, (later) access of M[reg + offset] 2865 // will provoke OS null exception if reg is null 2866 } 2867 } 2868 2869 void MacroAssembler::os_breakpoint() { 2870 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2871 // (e.g., MSVC can't call ps() otherwise) 2872 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2873 } 2874 2875 void MacroAssembler::unimplemented(const char* what) { 2876 const char* buf = nullptr; 2877 { 2878 ResourceMark rm; 2879 stringStream ss; 2880 ss.print("unimplemented: %s", what); 2881 buf = code_string(ss.as_string()); 2882 } 2883 stop(buf); 2884 } 2885 2886 #ifdef _LP64 2887 #define XSTATE_BV 0x200 2888 #endif 2889 2890 void MacroAssembler::pop_CPU_state() { 2891 pop_FPU_state(); 2892 pop_IU_state(); 2893 } 2894 2895 void MacroAssembler::pop_FPU_state() { 2896 #ifndef _LP64 2897 frstor(Address(rsp, 0)); 2898 #else 2899 fxrstor(Address(rsp, 0)); 2900 #endif 2901 addptr(rsp, FPUStateSizeInWords * wordSize); 2902 } 2903 2904 void MacroAssembler::pop_IU_state() { 2905 popa(); 2906 LP64_ONLY(addq(rsp, 8)); 2907 popf(); 2908 } 2909 2910 // Save Integer and Float state 2911 // Warning: Stack must be 16 byte aligned (64bit) 2912 void MacroAssembler::push_CPU_state() { 2913 push_IU_state(); 2914 push_FPU_state(); 2915 } 2916 2917 void MacroAssembler::push_FPU_state() { 2918 subptr(rsp, FPUStateSizeInWords * wordSize); 2919 #ifndef _LP64 2920 fnsave(Address(rsp, 0)); 2921 fwait(); 2922 #else 2923 fxsave(Address(rsp, 0)); 2924 #endif // LP64 2925 } 2926 2927 void MacroAssembler::push_IU_state() { 2928 // Push flags first because pusha kills them 2929 pushf(); 2930 // Make sure rsp stays 16-byte aligned 2931 LP64_ONLY(subq(rsp, 8)); 2932 pusha(); 2933 } 2934 2935 void MacroAssembler::push_cont_fastpath() { 2936 if (!Continuations::enabled()) return; 2937 2938 #ifndef _LP64 2939 Register rthread = rax; 2940 Register rrealsp = rbx; 2941 push(rthread); 2942 push(rrealsp); 2943 2944 get_thread(rthread); 2945 2946 // The code below wants the original RSP. 2947 // Move it back after the pushes above. 2948 movptr(rrealsp, rsp); 2949 addptr(rrealsp, 2*wordSize); 2950 #else 2951 Register rthread = r15_thread; 2952 Register rrealsp = rsp; 2953 #endif 2954 2955 Label done; 2956 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 2957 jccb(Assembler::belowEqual, done); 2958 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp); 2959 bind(done); 2960 2961 #ifndef _LP64 2962 pop(rrealsp); 2963 pop(rthread); 2964 #endif 2965 } 2966 2967 void MacroAssembler::pop_cont_fastpath() { 2968 if (!Continuations::enabled()) return; 2969 2970 #ifndef _LP64 2971 Register rthread = rax; 2972 Register rrealsp = rbx; 2973 push(rthread); 2974 push(rrealsp); 2975 2976 get_thread(rthread); 2977 2978 // The code below wants the original RSP. 2979 // Move it back after the pushes above. 2980 movptr(rrealsp, rsp); 2981 addptr(rrealsp, 2*wordSize); 2982 #else 2983 Register rthread = r15_thread; 2984 Register rrealsp = rsp; 2985 #endif 2986 2987 Label done; 2988 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 2989 jccb(Assembler::below, done); 2990 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0); 2991 bind(done); 2992 2993 #ifndef _LP64 2994 pop(rrealsp); 2995 pop(rthread); 2996 #endif 2997 } 2998 2999 void MacroAssembler::inc_held_monitor_count() { 3000 #ifndef _LP64 3001 Register thread = rax; 3002 push(thread); 3003 get_thread(thread); 3004 incrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3005 pop(thread); 3006 #else // LP64 3007 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3008 #endif 3009 } 3010 3011 void MacroAssembler::dec_held_monitor_count() { 3012 #ifndef _LP64 3013 Register thread = rax; 3014 push(thread); 3015 get_thread(thread); 3016 decrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3017 pop(thread); 3018 #else // LP64 3019 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3020 #endif 3021 } 3022 3023 #ifdef ASSERT 3024 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 3025 #ifdef _LP64 3026 Label no_cont; 3027 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 3028 testl(cont, cont); 3029 jcc(Assembler::zero, no_cont); 3030 stop(name); 3031 bind(no_cont); 3032 #else 3033 Unimplemented(); 3034 #endif 3035 } 3036 #endif 3037 3038 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register 3039 if (!java_thread->is_valid()) { 3040 java_thread = rdi; 3041 get_thread(java_thread); 3042 } 3043 // we must set sp to zero to clear frame 3044 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 3045 // must clear fp, so that compiled frames are not confused; it is 3046 // possible that we need it only for debugging 3047 if (clear_fp) { 3048 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 3049 } 3050 // Always clear the pc because it could have been set by make_walkable() 3051 movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 3052 vzeroupper(); 3053 } 3054 3055 void MacroAssembler::restore_rax(Register tmp) { 3056 if (tmp == noreg) pop(rax); 3057 else if (tmp != rax) mov(rax, tmp); 3058 } 3059 3060 void MacroAssembler::round_to(Register reg, int modulus) { 3061 addptr(reg, modulus - 1); 3062 andptr(reg, -modulus); 3063 } 3064 3065 void MacroAssembler::save_rax(Register tmp) { 3066 if (tmp == noreg) push(rax); 3067 else if (tmp != rax) mov(tmp, rax); 3068 } 3069 3070 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) { 3071 if (at_return) { 3072 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 3073 // we may safely use rsp instead to perform the stack watermark check. 3074 cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset())); 3075 jcc(Assembler::above, slow_path); 3076 return; 3077 } 3078 testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 3079 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 3080 } 3081 3082 // Calls to C land 3083 // 3084 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 3085 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 3086 // has to be reset to 0. This is required to allow proper stack traversal. 3087 void MacroAssembler::set_last_Java_frame(Register java_thread, 3088 Register last_java_sp, 3089 Register last_java_fp, 3090 address last_java_pc, 3091 Register rscratch) { 3092 vzeroupper(); 3093 // determine java_thread register 3094 if (!java_thread->is_valid()) { 3095 java_thread = rdi; 3096 get_thread(java_thread); 3097 } 3098 // determine last_java_sp register 3099 if (!last_java_sp->is_valid()) { 3100 last_java_sp = rsp; 3101 } 3102 // last_java_fp is optional 3103 if (last_java_fp->is_valid()) { 3104 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 3105 } 3106 // last_java_pc is optional 3107 if (last_java_pc != nullptr) { 3108 Address java_pc(java_thread, 3109 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 3110 lea(java_pc, InternalAddress(last_java_pc), rscratch); 3111 } 3112 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 3113 } 3114 3115 void MacroAssembler::shlptr(Register dst, int imm8) { 3116 LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); 3117 } 3118 3119 void MacroAssembler::shrptr(Register dst, int imm8) { 3120 LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); 3121 } 3122 3123 void MacroAssembler::sign_extend_byte(Register reg) { 3124 if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { 3125 movsbl(reg, reg); // movsxb 3126 } else { 3127 shll(reg, 24); 3128 sarl(reg, 24); 3129 } 3130 } 3131 3132 void MacroAssembler::sign_extend_short(Register reg) { 3133 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 3134 movswl(reg, reg); // movsxw 3135 } else { 3136 shll(reg, 16); 3137 sarl(reg, 16); 3138 } 3139 } 3140 3141 void MacroAssembler::testl(Address dst, int32_t imm32) { 3142 if (imm32 >= 0 && is8bit(imm32)) { 3143 testb(dst, imm32); 3144 } else { 3145 Assembler::testl(dst, imm32); 3146 } 3147 } 3148 3149 void MacroAssembler::testl(Register dst, int32_t imm32) { 3150 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 3151 testb(dst, imm32); 3152 } else { 3153 Assembler::testl(dst, imm32); 3154 } 3155 } 3156 3157 void MacroAssembler::testl(Register dst, AddressLiteral src) { 3158 assert(always_reachable(src), "Address should be reachable"); 3159 testl(dst, as_Address(src)); 3160 } 3161 3162 #ifdef _LP64 3163 3164 void MacroAssembler::testq(Address dst, int32_t imm32) { 3165 if (imm32 >= 0) { 3166 testl(dst, imm32); 3167 } else { 3168 Assembler::testq(dst, imm32); 3169 } 3170 } 3171 3172 void MacroAssembler::testq(Register dst, int32_t imm32) { 3173 if (imm32 >= 0) { 3174 testl(dst, imm32); 3175 } else { 3176 Assembler::testq(dst, imm32); 3177 } 3178 } 3179 3180 #endif 3181 3182 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 3183 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3184 Assembler::pcmpeqb(dst, src); 3185 } 3186 3187 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 3188 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3189 Assembler::pcmpeqw(dst, src); 3190 } 3191 3192 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 3193 assert((dst->encoding() < 16),"XMM register should be 0-15"); 3194 Assembler::pcmpestri(dst, src, imm8); 3195 } 3196 3197 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 3198 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3199 Assembler::pcmpestri(dst, src, imm8); 3200 } 3201 3202 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 3203 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3204 Assembler::pmovzxbw(dst, src); 3205 } 3206 3207 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 3208 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3209 Assembler::pmovzxbw(dst, src); 3210 } 3211 3212 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 3213 assert((src->encoding() < 16),"XMM register should be 0-15"); 3214 Assembler::pmovmskb(dst, src); 3215 } 3216 3217 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 3218 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3219 Assembler::ptest(dst, src); 3220 } 3221 3222 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3223 assert(rscratch != noreg || always_reachable(src), "missing"); 3224 3225 if (reachable(src)) { 3226 Assembler::sqrtss(dst, as_Address(src)); 3227 } else { 3228 lea(rscratch, src); 3229 Assembler::sqrtss(dst, Address(rscratch, 0)); 3230 } 3231 } 3232 3233 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3234 assert(rscratch != noreg || always_reachable(src), "missing"); 3235 3236 if (reachable(src)) { 3237 Assembler::subsd(dst, as_Address(src)); 3238 } else { 3239 lea(rscratch, src); 3240 Assembler::subsd(dst, Address(rscratch, 0)); 3241 } 3242 } 3243 3244 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 3245 assert(rscratch != noreg || always_reachable(src), "missing"); 3246 3247 if (reachable(src)) { 3248 Assembler::roundsd(dst, as_Address(src), rmode); 3249 } else { 3250 lea(rscratch, src); 3251 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 3252 } 3253 } 3254 3255 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3256 assert(rscratch != noreg || always_reachable(src), "missing"); 3257 3258 if (reachable(src)) { 3259 Assembler::subss(dst, as_Address(src)); 3260 } else { 3261 lea(rscratch, src); 3262 Assembler::subss(dst, Address(rscratch, 0)); 3263 } 3264 } 3265 3266 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3267 assert(rscratch != noreg || always_reachable(src), "missing"); 3268 3269 if (reachable(src)) { 3270 Assembler::ucomisd(dst, as_Address(src)); 3271 } else { 3272 lea(rscratch, src); 3273 Assembler::ucomisd(dst, Address(rscratch, 0)); 3274 } 3275 } 3276 3277 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3278 assert(rscratch != noreg || always_reachable(src), "missing"); 3279 3280 if (reachable(src)) { 3281 Assembler::ucomiss(dst, as_Address(src)); 3282 } else { 3283 lea(rscratch, src); 3284 Assembler::ucomiss(dst, Address(rscratch, 0)); 3285 } 3286 } 3287 3288 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3289 assert(rscratch != noreg || always_reachable(src), "missing"); 3290 3291 // Used in sign-bit flipping with aligned address. 3292 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3293 if (reachable(src)) { 3294 Assembler::xorpd(dst, as_Address(src)); 3295 } else { 3296 lea(rscratch, src); 3297 Assembler::xorpd(dst, Address(rscratch, 0)); 3298 } 3299 } 3300 3301 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3302 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3303 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3304 } 3305 else { 3306 Assembler::xorpd(dst, src); 3307 } 3308 } 3309 3310 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3311 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3312 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3313 } else { 3314 Assembler::xorps(dst, src); 3315 } 3316 } 3317 3318 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 3319 assert(rscratch != noreg || always_reachable(src), "missing"); 3320 3321 // Used in sign-bit flipping with aligned address. 3322 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3323 if (reachable(src)) { 3324 Assembler::xorps(dst, as_Address(src)); 3325 } else { 3326 lea(rscratch, src); 3327 Assembler::xorps(dst, Address(rscratch, 0)); 3328 } 3329 } 3330 3331 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 3332 assert(rscratch != noreg || always_reachable(src), "missing"); 3333 3334 // Used in sign-bit flipping with aligned address. 3335 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3336 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3337 if (reachable(src)) { 3338 Assembler::pshufb(dst, as_Address(src)); 3339 } else { 3340 lea(rscratch, src); 3341 Assembler::pshufb(dst, Address(rscratch, 0)); 3342 } 3343 } 3344 3345 // AVX 3-operands instructions 3346 3347 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3348 assert(rscratch != noreg || always_reachable(src), "missing"); 3349 3350 if (reachable(src)) { 3351 vaddsd(dst, nds, as_Address(src)); 3352 } else { 3353 lea(rscratch, src); 3354 vaddsd(dst, nds, Address(rscratch, 0)); 3355 } 3356 } 3357 3358 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3359 assert(rscratch != noreg || always_reachable(src), "missing"); 3360 3361 if (reachable(src)) { 3362 vaddss(dst, nds, as_Address(src)); 3363 } else { 3364 lea(rscratch, src); 3365 vaddss(dst, nds, Address(rscratch, 0)); 3366 } 3367 } 3368 3369 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3370 assert(UseAVX > 0, "requires some form of AVX"); 3371 assert(rscratch != noreg || always_reachable(src), "missing"); 3372 3373 if (reachable(src)) { 3374 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 3375 } else { 3376 lea(rscratch, src); 3377 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 3378 } 3379 } 3380 3381 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3382 assert(UseAVX > 0, "requires some form of AVX"); 3383 assert(rscratch != noreg || always_reachable(src), "missing"); 3384 3385 if (reachable(src)) { 3386 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 3387 } else { 3388 lea(rscratch, src); 3389 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 3390 } 3391 } 3392 3393 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3394 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3395 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3396 3397 vandps(dst, nds, negate_field, vector_len, rscratch); 3398 } 3399 3400 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3401 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3402 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3403 3404 vandpd(dst, nds, negate_field, vector_len, rscratch); 3405 } 3406 3407 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3408 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3409 Assembler::vpaddb(dst, nds, src, vector_len); 3410 } 3411 3412 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3413 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3414 Assembler::vpaddb(dst, nds, src, vector_len); 3415 } 3416 3417 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3418 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3419 Assembler::vpaddw(dst, nds, src, vector_len); 3420 } 3421 3422 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3423 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3424 Assembler::vpaddw(dst, nds, src, vector_len); 3425 } 3426 3427 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3428 assert(rscratch != noreg || always_reachable(src), "missing"); 3429 3430 if (reachable(src)) { 3431 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3432 } else { 3433 lea(rscratch, src); 3434 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 3435 } 3436 } 3437 3438 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3439 assert(rscratch != noreg || always_reachable(src), "missing"); 3440 3441 if (reachable(src)) { 3442 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 3443 } else { 3444 lea(rscratch, src); 3445 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 3446 } 3447 } 3448 3449 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3450 assert(rscratch != noreg || always_reachable(src), "missing"); 3451 3452 if (reachable(src)) { 3453 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 3454 } else { 3455 lea(rscratch, src); 3456 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 3457 } 3458 } 3459 3460 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3461 assert(rscratch != noreg || always_reachable(src), "missing"); 3462 3463 if (reachable(src)) { 3464 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 3465 } else { 3466 lea(rscratch, src); 3467 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 3468 } 3469 } 3470 3471 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3472 assert(rscratch != noreg || always_reachable(src), "missing"); 3473 3474 if (reachable(src)) { 3475 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 3476 } else { 3477 lea(rscratch, src); 3478 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 3479 } 3480 } 3481 3482 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3483 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3484 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3485 } 3486 3487 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3488 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3489 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3490 } 3491 3492 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3493 assert(rscratch != noreg || always_reachable(src), "missing"); 3494 3495 if (reachable(src)) { 3496 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 3497 } else { 3498 lea(rscratch, src); 3499 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 3500 } 3501 } 3502 3503 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3504 int comparison, bool is_signed, int vector_len, Register rscratch) { 3505 assert(rscratch != noreg || always_reachable(src), "missing"); 3506 3507 if (reachable(src)) { 3508 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3509 } else { 3510 lea(rscratch, src); 3511 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3512 } 3513 } 3514 3515 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3516 int comparison, bool is_signed, int vector_len, Register rscratch) { 3517 assert(rscratch != noreg || always_reachable(src), "missing"); 3518 3519 if (reachable(src)) { 3520 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3521 } else { 3522 lea(rscratch, src); 3523 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3524 } 3525 } 3526 3527 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3528 int comparison, bool is_signed, int vector_len, Register rscratch) { 3529 assert(rscratch != noreg || always_reachable(src), "missing"); 3530 3531 if (reachable(src)) { 3532 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3533 } else { 3534 lea(rscratch, src); 3535 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3536 } 3537 } 3538 3539 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3540 int comparison, bool is_signed, int vector_len, Register rscratch) { 3541 assert(rscratch != noreg || always_reachable(src), "missing"); 3542 3543 if (reachable(src)) { 3544 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3545 } else { 3546 lea(rscratch, src); 3547 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3548 } 3549 } 3550 3551 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3552 if (width == Assembler::Q) { 3553 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3554 } else { 3555 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3556 } 3557 } 3558 3559 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3560 int eq_cond_enc = 0x29; 3561 int gt_cond_enc = 0x37; 3562 if (width != Assembler::Q) { 3563 eq_cond_enc = 0x74 + width; 3564 gt_cond_enc = 0x64 + width; 3565 } 3566 switch (cond) { 3567 case eq: 3568 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3569 break; 3570 case neq: 3571 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3572 vallones(xtmp, vector_len); 3573 vpxor(dst, xtmp, dst, vector_len); 3574 break; 3575 case le: 3576 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3577 vallones(xtmp, vector_len); 3578 vpxor(dst, xtmp, dst, vector_len); 3579 break; 3580 case nlt: 3581 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3582 vallones(xtmp, vector_len); 3583 vpxor(dst, xtmp, dst, vector_len); 3584 break; 3585 case lt: 3586 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3587 break; 3588 case nle: 3589 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3590 break; 3591 default: 3592 assert(false, "Should not reach here"); 3593 } 3594 } 3595 3596 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3597 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3598 Assembler::vpmovzxbw(dst, src, vector_len); 3599 } 3600 3601 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3602 assert((src->encoding() < 16),"XMM register should be 0-15"); 3603 Assembler::vpmovmskb(dst, src, vector_len); 3604 } 3605 3606 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3607 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3608 Assembler::vpmullw(dst, nds, src, vector_len); 3609 } 3610 3611 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3612 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3613 Assembler::vpmullw(dst, nds, src, vector_len); 3614 } 3615 3616 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3617 assert((UseAVX > 0), "AVX support is needed"); 3618 assert(rscratch != noreg || always_reachable(src), "missing"); 3619 3620 if (reachable(src)) { 3621 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3622 } else { 3623 lea(rscratch, src); 3624 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3625 } 3626 } 3627 3628 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3629 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3630 Assembler::vpsubb(dst, nds, src, vector_len); 3631 } 3632 3633 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3634 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3635 Assembler::vpsubb(dst, nds, src, vector_len); 3636 } 3637 3638 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3639 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3640 Assembler::vpsubw(dst, nds, src, vector_len); 3641 } 3642 3643 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3644 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3645 Assembler::vpsubw(dst, nds, src, vector_len); 3646 } 3647 3648 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3649 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3650 Assembler::vpsraw(dst, nds, shift, vector_len); 3651 } 3652 3653 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3654 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3655 Assembler::vpsraw(dst, nds, shift, vector_len); 3656 } 3657 3658 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3659 assert(UseAVX > 2,""); 3660 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3661 vector_len = 2; 3662 } 3663 Assembler::evpsraq(dst, nds, shift, vector_len); 3664 } 3665 3666 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3667 assert(UseAVX > 2,""); 3668 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3669 vector_len = 2; 3670 } 3671 Assembler::evpsraq(dst, nds, shift, vector_len); 3672 } 3673 3674 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3675 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3676 Assembler::vpsrlw(dst, nds, shift, vector_len); 3677 } 3678 3679 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3680 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3681 Assembler::vpsrlw(dst, nds, shift, vector_len); 3682 } 3683 3684 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3685 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3686 Assembler::vpsllw(dst, nds, shift, vector_len); 3687 } 3688 3689 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3690 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3691 Assembler::vpsllw(dst, nds, shift, vector_len); 3692 } 3693 3694 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3695 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3696 Assembler::vptest(dst, src); 3697 } 3698 3699 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3700 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3701 Assembler::punpcklbw(dst, src); 3702 } 3703 3704 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3705 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3706 Assembler::pshufd(dst, src, mode); 3707 } 3708 3709 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3710 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3711 Assembler::pshuflw(dst, src, mode); 3712 } 3713 3714 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3715 assert(rscratch != noreg || always_reachable(src), "missing"); 3716 3717 if (reachable(src)) { 3718 vandpd(dst, nds, as_Address(src), vector_len); 3719 } else { 3720 lea(rscratch, src); 3721 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3722 } 3723 } 3724 3725 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3726 assert(rscratch != noreg || always_reachable(src), "missing"); 3727 3728 if (reachable(src)) { 3729 vandps(dst, nds, as_Address(src), vector_len); 3730 } else { 3731 lea(rscratch, src); 3732 vandps(dst, nds, Address(rscratch, 0), vector_len); 3733 } 3734 } 3735 3736 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3737 bool merge, int vector_len, Register rscratch) { 3738 assert(rscratch != noreg || always_reachable(src), "missing"); 3739 3740 if (reachable(src)) { 3741 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3742 } else { 3743 lea(rscratch, src); 3744 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3745 } 3746 } 3747 3748 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3749 assert(rscratch != noreg || always_reachable(src), "missing"); 3750 3751 if (reachable(src)) { 3752 vdivsd(dst, nds, as_Address(src)); 3753 } else { 3754 lea(rscratch, src); 3755 vdivsd(dst, nds, Address(rscratch, 0)); 3756 } 3757 } 3758 3759 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3760 assert(rscratch != noreg || always_reachable(src), "missing"); 3761 3762 if (reachable(src)) { 3763 vdivss(dst, nds, as_Address(src)); 3764 } else { 3765 lea(rscratch, src); 3766 vdivss(dst, nds, Address(rscratch, 0)); 3767 } 3768 } 3769 3770 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3771 assert(rscratch != noreg || always_reachable(src), "missing"); 3772 3773 if (reachable(src)) { 3774 vmulsd(dst, nds, as_Address(src)); 3775 } else { 3776 lea(rscratch, src); 3777 vmulsd(dst, nds, Address(rscratch, 0)); 3778 } 3779 } 3780 3781 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3782 assert(rscratch != noreg || always_reachable(src), "missing"); 3783 3784 if (reachable(src)) { 3785 vmulss(dst, nds, as_Address(src)); 3786 } else { 3787 lea(rscratch, src); 3788 vmulss(dst, nds, Address(rscratch, 0)); 3789 } 3790 } 3791 3792 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3793 assert(rscratch != noreg || always_reachable(src), "missing"); 3794 3795 if (reachable(src)) { 3796 vsubsd(dst, nds, as_Address(src)); 3797 } else { 3798 lea(rscratch, src); 3799 vsubsd(dst, nds, Address(rscratch, 0)); 3800 } 3801 } 3802 3803 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3804 assert(rscratch != noreg || always_reachable(src), "missing"); 3805 3806 if (reachable(src)) { 3807 vsubss(dst, nds, as_Address(src)); 3808 } else { 3809 lea(rscratch, src); 3810 vsubss(dst, nds, Address(rscratch, 0)); 3811 } 3812 } 3813 3814 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3815 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3816 assert(rscratch != noreg || always_reachable(src), "missing"); 3817 3818 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3819 } 3820 3821 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3822 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3823 assert(rscratch != noreg || always_reachable(src), "missing"); 3824 3825 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3826 } 3827 3828 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3829 assert(rscratch != noreg || always_reachable(src), "missing"); 3830 3831 if (reachable(src)) { 3832 vxorpd(dst, nds, as_Address(src), vector_len); 3833 } else { 3834 lea(rscratch, src); 3835 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3836 } 3837 } 3838 3839 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3840 assert(rscratch != noreg || always_reachable(src), "missing"); 3841 3842 if (reachable(src)) { 3843 vxorps(dst, nds, as_Address(src), vector_len); 3844 } else { 3845 lea(rscratch, src); 3846 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3847 } 3848 } 3849 3850 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3851 assert(rscratch != noreg || always_reachable(src), "missing"); 3852 3853 if (UseAVX > 1 || (vector_len < 1)) { 3854 if (reachable(src)) { 3855 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3856 } else { 3857 lea(rscratch, src); 3858 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3859 } 3860 } else { 3861 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3862 } 3863 } 3864 3865 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3866 assert(rscratch != noreg || always_reachable(src), "missing"); 3867 3868 if (reachable(src)) { 3869 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 3870 } else { 3871 lea(rscratch, src); 3872 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 3873 } 3874 } 3875 3876 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 3877 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 3878 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 3879 // The inverted mask is sign-extended 3880 andptr(possibly_non_local, inverted_mask); 3881 } 3882 3883 void MacroAssembler::resolve_jobject(Register value, 3884 Register thread, 3885 Register tmp) { 3886 assert_different_registers(value, thread, tmp); 3887 Label done, tagged, weak_tagged; 3888 testptr(value, value); 3889 jcc(Assembler::zero, done); // Use null as-is. 3890 testptr(value, JNIHandles::tag_mask); // Test for tag. 3891 jcc(Assembler::notZero, tagged); 3892 3893 // Resolve local handle 3894 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread); 3895 verify_oop(value); 3896 jmp(done); 3897 3898 bind(tagged); 3899 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 3900 jcc(Assembler::notZero, weak_tagged); 3901 3902 // Resolve global handle 3903 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 3904 verify_oop(value); 3905 jmp(done); 3906 3907 bind(weak_tagged); 3908 // Resolve jweak. 3909 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 3910 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread); 3911 verify_oop(value); 3912 3913 bind(done); 3914 } 3915 3916 void MacroAssembler::resolve_global_jobject(Register value, 3917 Register thread, 3918 Register tmp) { 3919 assert_different_registers(value, thread, tmp); 3920 Label done; 3921 3922 testptr(value, value); 3923 jcc(Assembler::zero, done); // Use null as-is. 3924 3925 #ifdef ASSERT 3926 { 3927 Label valid_global_tag; 3928 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 3929 jcc(Assembler::notZero, valid_global_tag); 3930 stop("non global jobject using resolve_global_jobject"); 3931 bind(valid_global_tag); 3932 } 3933 #endif 3934 3935 // Resolve global handle 3936 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 3937 verify_oop(value); 3938 3939 bind(done); 3940 } 3941 3942 void MacroAssembler::subptr(Register dst, int32_t imm32) { 3943 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 3944 } 3945 3946 // Force generation of a 4 byte immediate value even if it fits into 8bit 3947 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 3948 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 3949 } 3950 3951 void MacroAssembler::subptr(Register dst, Register src) { 3952 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 3953 } 3954 3955 // C++ bool manipulation 3956 void MacroAssembler::testbool(Register dst) { 3957 if(sizeof(bool) == 1) 3958 testb(dst, 0xff); 3959 else if(sizeof(bool) == 2) { 3960 // testw implementation needed for two byte bools 3961 ShouldNotReachHere(); 3962 } else if(sizeof(bool) == 4) 3963 testl(dst, dst); 3964 else 3965 // unsupported 3966 ShouldNotReachHere(); 3967 } 3968 3969 void MacroAssembler::testptr(Register dst, Register src) { 3970 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); 3971 } 3972 3973 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 3974 void MacroAssembler::tlab_allocate(Register thread, Register obj, 3975 Register var_size_in_bytes, 3976 int con_size_in_bytes, 3977 Register t1, 3978 Register t2, 3979 Label& slow_case) { 3980 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 3981 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 3982 } 3983 3984 RegSet MacroAssembler::call_clobbered_gp_registers() { 3985 RegSet regs; 3986 #ifdef _LP64 3987 regs += RegSet::of(rax, rcx, rdx); 3988 #ifndef WINDOWS 3989 regs += RegSet::of(rsi, rdi); 3990 #endif 3991 regs += RegSet::range(r8, r11); 3992 #else 3993 regs += RegSet::of(rax, rcx, rdx); 3994 #endif 3995 return regs; 3996 } 3997 3998 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 3999 int num_xmm_registers = XMMRegister::available_xmm_registers(); 4000 #if defined(WINDOWS) && defined(_LP64) 4001 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 4002 if (num_xmm_registers > 16) { 4003 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 4004 } 4005 return result; 4006 #else 4007 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 4008 #endif 4009 } 4010 4011 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor 4012 4013 #ifndef _LP64 4014 static bool use_x87_registers() { return UseSSE < 2; } 4015 #endif 4016 static bool use_xmm_registers() { return UseSSE >= 1; } 4017 4018 // C1 only ever uses the first double/float of the XMM register. 4019 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); } 4020 4021 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4022 if (UseSSE == 1) { 4023 masm->movflt(Address(rsp, offset), reg); 4024 } else { 4025 masm->movdbl(Address(rsp, offset), reg); 4026 } 4027 } 4028 4029 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4030 if (UseSSE == 1) { 4031 masm->movflt(reg, Address(rsp, offset)); 4032 } else { 4033 masm->movdbl(reg, Address(rsp, offset)); 4034 } 4035 } 4036 4037 int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu, 4038 int& gp_area_size, int& fp_area_size, int& xmm_area_size) { 4039 4040 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 4041 StackAlignmentInBytes); 4042 #ifdef _LP64 4043 fp_area_size = 0; 4044 #else 4045 fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0; 4046 #endif 4047 xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0; 4048 4049 return gp_area_size + fp_area_size + xmm_area_size; 4050 } 4051 4052 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 4053 block_comment("push_call_clobbered_registers start"); 4054 // Regular registers 4055 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 4056 4057 int gp_area_size; 4058 int fp_area_size; 4059 int xmm_area_size; 4060 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 4061 gp_area_size, fp_area_size, xmm_area_size); 4062 subptr(rsp, total_save_size); 4063 4064 push_set(gp_registers_to_push, 0); 4065 4066 #ifndef _LP64 4067 if (save_fpu && use_x87_registers()) { 4068 fnsave(Address(rsp, gp_area_size)); 4069 fwait(); 4070 } 4071 #endif 4072 if (save_fpu && use_xmm_registers()) { 4073 push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4074 } 4075 4076 block_comment("push_call_clobbered_registers end"); 4077 } 4078 4079 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 4080 block_comment("pop_call_clobbered_registers start"); 4081 4082 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 4083 4084 int gp_area_size; 4085 int fp_area_size; 4086 int xmm_area_size; 4087 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 4088 gp_area_size, fp_area_size, xmm_area_size); 4089 4090 if (restore_fpu && use_xmm_registers()) { 4091 pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4092 } 4093 #ifndef _LP64 4094 if (restore_fpu && use_x87_registers()) { 4095 frstor(Address(rsp, gp_area_size)); 4096 } 4097 #endif 4098 4099 pop_set(gp_registers_to_pop, 0); 4100 4101 addptr(rsp, total_save_size); 4102 4103 vzeroupper(); 4104 4105 block_comment("pop_call_clobbered_registers end"); 4106 } 4107 4108 void MacroAssembler::push_set(XMMRegSet set, int offset) { 4109 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 4110 int spill_offset = offset; 4111 4112 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 4113 save_xmm_register(this, spill_offset, *it); 4114 spill_offset += xmm_save_size(); 4115 } 4116 } 4117 4118 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 4119 int restore_size = set.size() * xmm_save_size(); 4120 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 4121 4122 int restore_offset = offset + restore_size - xmm_save_size(); 4123 4124 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 4125 restore_xmm_register(this, restore_offset, *it); 4126 restore_offset -= xmm_save_size(); 4127 } 4128 } 4129 4130 void MacroAssembler::push_set(RegSet set, int offset) { 4131 int spill_offset; 4132 if (offset == -1) { 4133 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4134 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4135 subptr(rsp, aligned_size); 4136 spill_offset = 0; 4137 } else { 4138 spill_offset = offset; 4139 } 4140 4141 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 4142 movptr(Address(rsp, spill_offset), *it); 4143 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4144 } 4145 } 4146 4147 void MacroAssembler::pop_set(RegSet set, int offset) { 4148 4149 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4150 int restore_size = set.size() * gp_reg_size; 4151 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 4152 4153 int restore_offset; 4154 if (offset == -1) { 4155 restore_offset = restore_size - gp_reg_size; 4156 } else { 4157 restore_offset = offset + restore_size - gp_reg_size; 4158 } 4159 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 4160 movptr(*it, Address(rsp, restore_offset)); 4161 restore_offset -= gp_reg_size; 4162 } 4163 4164 if (offset == -1) { 4165 addptr(rsp, aligned_size); 4166 } 4167 } 4168 4169 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 4170 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 4171 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 4172 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 4173 Label done; 4174 4175 testptr(length_in_bytes, length_in_bytes); 4176 jcc(Assembler::zero, done); 4177 4178 // initialize topmost word, divide index by 2, check if odd and test if zero 4179 // note: for the remaining code to work, index must be a multiple of BytesPerWord 4180 #ifdef ASSERT 4181 { 4182 Label L; 4183 testptr(length_in_bytes, BytesPerWord - 1); 4184 jcc(Assembler::zero, L); 4185 stop("length must be a multiple of BytesPerWord"); 4186 bind(L); 4187 } 4188 #endif 4189 Register index = length_in_bytes; 4190 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 4191 if (UseIncDec) { 4192 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 4193 } else { 4194 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 4195 shrptr(index, 1); 4196 } 4197 #ifndef _LP64 4198 // index could have not been a multiple of 8 (i.e., bit 2 was set) 4199 { 4200 Label even; 4201 // note: if index was a multiple of 8, then it cannot 4202 // be 0 now otherwise it must have been 0 before 4203 // => if it is even, we don't need to check for 0 again 4204 jcc(Assembler::carryClear, even); 4205 // clear topmost word (no jump would be needed if conditional assignment worked here) 4206 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); 4207 // index could be 0 now, must check again 4208 jcc(Assembler::zero, done); 4209 bind(even); 4210 } 4211 #endif // !_LP64 4212 // initialize remaining object fields: index is a multiple of 2 now 4213 { 4214 Label loop; 4215 bind(loop); 4216 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 4217 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) 4218 decrement(index); 4219 jcc(Assembler::notZero, loop); 4220 } 4221 4222 bind(done); 4223 } 4224 4225 // Look up the method for a megamorphic invokeinterface call. 4226 // The target method is determined by <intf_klass, itable_index>. 4227 // The receiver klass is in recv_klass. 4228 // On success, the result will be in method_result, and execution falls through. 4229 // On failure, execution transfers to the given label. 4230 void MacroAssembler::lookup_interface_method(Register recv_klass, 4231 Register intf_klass, 4232 RegisterOrConstant itable_index, 4233 Register method_result, 4234 Register scan_temp, 4235 Label& L_no_such_interface, 4236 bool return_method) { 4237 assert_different_registers(recv_klass, intf_klass, scan_temp); 4238 assert_different_registers(method_result, intf_klass, scan_temp); 4239 assert(recv_klass != method_result || !return_method, 4240 "recv_klass can be destroyed when method isn't needed"); 4241 4242 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 4243 "caller must use same register for non-constant itable index as for method"); 4244 4245 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 4246 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4247 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4248 int scan_step = itableOffsetEntry::size() * wordSize; 4249 int vte_size = vtableEntry::size_in_bytes(); 4250 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4251 assert(vte_size == wordSize, "else adjust times_vte_scale"); 4252 4253 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4254 4255 // %%% Could store the aligned, prescaled offset in the klassoop. 4256 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 4257 4258 if (return_method) { 4259 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 4260 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4261 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 4262 } 4263 4264 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 4265 // if (scan->interface() == intf) { 4266 // result = (klass + scan->offset() + itable_index); 4267 // } 4268 // } 4269 Label search, found_method; 4270 4271 for (int peel = 1; peel >= 0; peel--) { 4272 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 4273 cmpptr(intf_klass, method_result); 4274 4275 if (peel) { 4276 jccb(Assembler::equal, found_method); 4277 } else { 4278 jccb(Assembler::notEqual, search); 4279 // (invert the test to fall through to found_method...) 4280 } 4281 4282 if (!peel) break; 4283 4284 bind(search); 4285 4286 // Check that the previous entry is non-null. A null entry means that 4287 // the receiver class doesn't implement the interface, and wasn't the 4288 // same as when the caller was compiled. 4289 testptr(method_result, method_result); 4290 jcc(Assembler::zero, L_no_such_interface); 4291 addptr(scan_temp, scan_step); 4292 } 4293 4294 bind(found_method); 4295 4296 if (return_method) { 4297 // Got a hit. 4298 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 4299 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 4300 } 4301 } 4302 4303 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 4304 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder 4305 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 4306 // The target method is determined by <holder_klass, itable_index>. 4307 // The receiver klass is in recv_klass. 4308 // On success, the result will be in method_result, and execution falls through. 4309 // On failure, execution transfers to the given label. 4310 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 4311 Register holder_klass, 4312 Register resolved_klass, 4313 Register method_result, 4314 Register scan_temp, 4315 Register temp_reg2, 4316 Register receiver, 4317 int itable_index, 4318 Label& L_no_such_interface) { 4319 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 4320 Register temp_itbl_klass = method_result; 4321 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 4322 4323 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4324 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4325 int scan_step = itableOffsetEntry::size() * wordSize; 4326 int vte_size = vtableEntry::size_in_bytes(); 4327 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 4328 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 4329 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4330 assert(vte_size == wordSize, "adjust times_vte_scale"); 4331 4332 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 4333 4334 // temp_itbl_klass = recv_klass.itable[0] 4335 // scan_temp = &recv_klass.itable[0] + step 4336 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4337 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 4338 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 4339 xorptr(temp_reg, temp_reg); 4340 4341 // Initial checks: 4342 // - if (holder_klass != resolved_klass), go to "scan for resolved" 4343 // - if (itable[0] == 0), no such interface 4344 // - if (itable[0] == holder_klass), shortcut to "holder found" 4345 cmpptr(holder_klass, resolved_klass); 4346 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 4347 testptr(temp_itbl_klass, temp_itbl_klass); 4348 jccb(Assembler::zero, L_no_such_interface); 4349 cmpptr(holder_klass, temp_itbl_klass); 4350 jccb(Assembler::equal, L_holder_found); 4351 4352 // Loop: Look for holder_klass record in itable 4353 // do { 4354 // tmp = itable[index]; 4355 // index += step; 4356 // if (tmp == holder_klass) { 4357 // goto L_holder_found; // Found! 4358 // } 4359 // } while (tmp != 0); 4360 // goto L_no_such_interface // Not found. 4361 Label L_scan_holder; 4362 bind(L_scan_holder); 4363 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4364 addptr(scan_temp, scan_step); 4365 cmpptr(holder_klass, temp_itbl_klass); 4366 jccb(Assembler::equal, L_holder_found); 4367 testptr(temp_itbl_klass, temp_itbl_klass); 4368 jccb(Assembler::notZero, L_scan_holder); 4369 4370 jmpb(L_no_such_interface); 4371 4372 // Loop: Look for resolved_class record in itable 4373 // do { 4374 // tmp = itable[index]; 4375 // index += step; 4376 // if (tmp == holder_klass) { 4377 // // Also check if we have met a holder klass 4378 // holder_tmp = itable[index-step-ioffset]; 4379 // } 4380 // if (tmp == resolved_klass) { 4381 // goto L_resolved_found; // Found! 4382 // } 4383 // } while (tmp != 0); 4384 // goto L_no_such_interface // Not found. 4385 // 4386 Label L_loop_scan_resolved; 4387 bind(L_loop_scan_resolved); 4388 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4389 addptr(scan_temp, scan_step); 4390 bind(L_loop_scan_resolved_entry); 4391 cmpptr(holder_klass, temp_itbl_klass); 4392 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4393 cmpptr(resolved_klass, temp_itbl_klass); 4394 jccb(Assembler::equal, L_resolved_found); 4395 testptr(temp_itbl_klass, temp_itbl_klass); 4396 jccb(Assembler::notZero, L_loop_scan_resolved); 4397 4398 jmpb(L_no_such_interface); 4399 4400 Label L_ready; 4401 4402 // See if we already have a holder klass. If not, go and scan for it. 4403 bind(L_resolved_found); 4404 testptr(temp_reg, temp_reg); 4405 jccb(Assembler::zero, L_scan_holder); 4406 jmpb(L_ready); 4407 4408 bind(L_holder_found); 4409 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4410 4411 // Finally, temp_reg contains holder_klass vtable offset 4412 bind(L_ready); 4413 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4414 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 4415 load_klass(scan_temp, receiver, noreg); 4416 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4417 } else { 4418 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4419 } 4420 } 4421 4422 4423 // virtual method calling 4424 void MacroAssembler::lookup_virtual_method(Register recv_klass, 4425 RegisterOrConstant vtable_index, 4426 Register method_result) { 4427 const ByteSize base = Klass::vtable_start_offset(); 4428 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 4429 Address vtable_entry_addr(recv_klass, 4430 vtable_index, Address::times_ptr, 4431 base + vtableEntry::method_offset()); 4432 movptr(method_result, vtable_entry_addr); 4433 } 4434 4435 4436 void MacroAssembler::check_klass_subtype(Register sub_klass, 4437 Register super_klass, 4438 Register temp_reg, 4439 Label& L_success) { 4440 Label L_failure; 4441 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 4442 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 4443 bind(L_failure); 4444 } 4445 4446 4447 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 4448 Register super_klass, 4449 Register temp_reg, 4450 Label* L_success, 4451 Label* L_failure, 4452 Label* L_slow_path, 4453 RegisterOrConstant super_check_offset) { 4454 assert_different_registers(sub_klass, super_klass, temp_reg); 4455 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 4456 if (super_check_offset.is_register()) { 4457 assert_different_registers(sub_klass, super_klass, 4458 super_check_offset.as_register()); 4459 } else if (must_load_sco) { 4460 assert(temp_reg != noreg, "supply either a temp or a register offset"); 4461 } 4462 4463 Label L_fallthrough; 4464 int label_nulls = 0; 4465 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4466 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4467 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 4468 assert(label_nulls <= 1, "at most one null in the batch"); 4469 4470 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4471 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 4472 Address super_check_offset_addr(super_klass, sco_offset); 4473 4474 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 4475 // range of a jccb. If this routine grows larger, reconsider at 4476 // least some of these. 4477 #define local_jcc(assembler_cond, label) \ 4478 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 4479 else jcc( assembler_cond, label) /*omit semi*/ 4480 4481 // Hacked jmp, which may only be used just before L_fallthrough. 4482 #define final_jmp(label) \ 4483 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 4484 else jmp(label) /*omit semi*/ 4485 4486 // If the pointers are equal, we are done (e.g., String[] elements). 4487 // This self-check enables sharing of secondary supertype arrays among 4488 // non-primary types such as array-of-interface. Otherwise, each such 4489 // type would need its own customized SSA. 4490 // We move this check to the front of the fast path because many 4491 // type checks are in fact trivially successful in this manner, 4492 // so we get a nicely predicted branch right at the start of the check. 4493 cmpptr(sub_klass, super_klass); 4494 local_jcc(Assembler::equal, *L_success); 4495 4496 // Check the supertype display: 4497 if (must_load_sco) { 4498 // Positive movl does right thing on LP64. 4499 movl(temp_reg, super_check_offset_addr); 4500 super_check_offset = RegisterOrConstant(temp_reg); 4501 } 4502 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 4503 cmpptr(super_klass, super_check_addr); // load displayed supertype 4504 4505 // This check has worked decisively for primary supers. 4506 // Secondary supers are sought in the super_cache ('super_cache_addr'). 4507 // (Secondary supers are interfaces and very deeply nested subtypes.) 4508 // This works in the same check above because of a tricky aliasing 4509 // between the super_cache and the primary super display elements. 4510 // (The 'super_check_addr' can address either, as the case requires.) 4511 // Note that the cache is updated below if it does not help us find 4512 // what we need immediately. 4513 // So if it was a primary super, we can just fail immediately. 4514 // Otherwise, it's the slow path for us (no success at this point). 4515 4516 if (super_check_offset.is_register()) { 4517 local_jcc(Assembler::equal, *L_success); 4518 cmpl(super_check_offset.as_register(), sc_offset); 4519 if (L_failure == &L_fallthrough) { 4520 local_jcc(Assembler::equal, *L_slow_path); 4521 } else { 4522 local_jcc(Assembler::notEqual, *L_failure); 4523 final_jmp(*L_slow_path); 4524 } 4525 } else if (super_check_offset.as_constant() == sc_offset) { 4526 // Need a slow path; fast failure is impossible. 4527 if (L_slow_path == &L_fallthrough) { 4528 local_jcc(Assembler::equal, *L_success); 4529 } else { 4530 local_jcc(Assembler::notEqual, *L_slow_path); 4531 final_jmp(*L_success); 4532 } 4533 } else { 4534 // No slow path; it's a fast decision. 4535 if (L_failure == &L_fallthrough) { 4536 local_jcc(Assembler::equal, *L_success); 4537 } else { 4538 local_jcc(Assembler::notEqual, *L_failure); 4539 final_jmp(*L_success); 4540 } 4541 } 4542 4543 bind(L_fallthrough); 4544 4545 #undef local_jcc 4546 #undef final_jmp 4547 } 4548 4549 4550 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4551 Register super_klass, 4552 Register temp_reg, 4553 Register temp2_reg, 4554 Label* L_success, 4555 Label* L_failure, 4556 bool set_cond_codes) { 4557 assert_different_registers(sub_klass, super_klass, temp_reg); 4558 if (temp2_reg != noreg) 4559 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 4560 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 4561 4562 Label L_fallthrough; 4563 int label_nulls = 0; 4564 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4565 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4566 assert(label_nulls <= 1, "at most one null in the batch"); 4567 4568 // a couple of useful fields in sub_klass: 4569 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 4570 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4571 Address secondary_supers_addr(sub_klass, ss_offset); 4572 Address super_cache_addr( sub_klass, sc_offset); 4573 4574 // Do a linear scan of the secondary super-klass chain. 4575 // This code is rarely used, so simplicity is a virtue here. 4576 // The repne_scan instruction uses fixed registers, which we must spill. 4577 // Don't worry too much about pre-existing connections with the input regs. 4578 4579 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 4580 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 4581 4582 // Get super_klass value into rax (even if it was in rdi or rcx). 4583 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 4584 if (super_klass != rax) { 4585 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 4586 mov(rax, super_klass); 4587 } 4588 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 4589 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 4590 4591 #ifndef PRODUCT 4592 int* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4593 ExternalAddress pst_counter_addr((address) pst_counter); 4594 NOT_LP64( incrementl(pst_counter_addr) ); 4595 LP64_ONLY( lea(rcx, pst_counter_addr) ); 4596 LP64_ONLY( incrementl(Address(rcx, 0)) ); 4597 #endif //PRODUCT 4598 4599 // We will consult the secondary-super array. 4600 movptr(rdi, secondary_supers_addr); 4601 // Load the array length. (Positive movl does right thing on LP64.) 4602 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4603 // Skip to start of data. 4604 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4605 4606 // Scan RCX words at [RDI] for an occurrence of RAX. 4607 // Set NZ/Z based on last compare. 4608 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4609 // not change flags (only scas instruction which is repeated sets flags). 4610 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4611 4612 testptr(rax,rax); // Set Z = 0 4613 repne_scan(); 4614 4615 // Unspill the temp. registers: 4616 if (pushed_rdi) pop(rdi); 4617 if (pushed_rcx) pop(rcx); 4618 if (pushed_rax) pop(rax); 4619 4620 if (set_cond_codes) { 4621 // Special hack for the AD files: rdi is guaranteed non-zero. 4622 assert(!pushed_rdi, "rdi must be left non-null"); 4623 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4624 } 4625 4626 if (L_failure == &L_fallthrough) 4627 jccb(Assembler::notEqual, *L_failure); 4628 else jcc(Assembler::notEqual, *L_failure); 4629 4630 // Success. Cache the super we found and proceed in triumph. 4631 movptr(super_cache_addr, super_klass); 4632 4633 if (L_success != &L_fallthrough) { 4634 jmp(*L_success); 4635 } 4636 4637 #undef IS_A_TEMP 4638 4639 bind(L_fallthrough); 4640 } 4641 4642 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { 4643 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 4644 4645 Label L_fallthrough; 4646 if (L_fast_path == nullptr) { 4647 L_fast_path = &L_fallthrough; 4648 } else if (L_slow_path == nullptr) { 4649 L_slow_path = &L_fallthrough; 4650 } 4651 4652 // Fast path check: class is fully initialized 4653 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 4654 jcc(Assembler::equal, *L_fast_path); 4655 4656 // Fast path check: current thread is initializer thread 4657 cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); 4658 if (L_slow_path == &L_fallthrough) { 4659 jcc(Assembler::equal, *L_fast_path); 4660 bind(*L_slow_path); 4661 } else if (L_fast_path == &L_fallthrough) { 4662 jcc(Assembler::notEqual, *L_slow_path); 4663 bind(*L_fast_path); 4664 } else { 4665 Unimplemented(); 4666 } 4667 } 4668 4669 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 4670 if (VM_Version::supports_cmov()) { 4671 cmovl(cc, dst, src); 4672 } else { 4673 Label L; 4674 jccb(negate_condition(cc), L); 4675 movl(dst, src); 4676 bind(L); 4677 } 4678 } 4679 4680 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 4681 if (VM_Version::supports_cmov()) { 4682 cmovl(cc, dst, src); 4683 } else { 4684 Label L; 4685 jccb(negate_condition(cc), L); 4686 movl(dst, src); 4687 bind(L); 4688 } 4689 } 4690 4691 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 4692 if (!VerifyOops) return; 4693 4694 BLOCK_COMMENT("verify_oop {"); 4695 #ifdef _LP64 4696 push(rscratch1); 4697 #endif 4698 push(rax); // save rax 4699 push(reg); // pass register argument 4700 4701 // Pass register number to verify_oop_subroutine 4702 const char* b = nullptr; 4703 { 4704 ResourceMark rm; 4705 stringStream ss; 4706 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 4707 b = code_string(ss.as_string()); 4708 } 4709 ExternalAddress buffer((address) b); 4710 pushptr(buffer.addr(), rscratch1); 4711 4712 // call indirectly to solve generation ordering problem 4713 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4714 call(rax); 4715 // Caller pops the arguments (oop, message) and restores rax, r10 4716 BLOCK_COMMENT("} verify_oop"); 4717 } 4718 4719 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 4720 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 4721 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 4722 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 4723 vpternlogd(dst, 0xFF, dst, dst, vector_len); 4724 } else if (VM_Version::supports_avx()) { 4725 vpcmpeqd(dst, dst, dst, vector_len); 4726 } else { 4727 assert(VM_Version::supports_sse2(), ""); 4728 pcmpeqd(dst, dst); 4729 } 4730 } 4731 4732 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 4733 int extra_slot_offset) { 4734 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 4735 int stackElementSize = Interpreter::stackElementSize; 4736 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 4737 #ifdef ASSERT 4738 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 4739 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 4740 #endif 4741 Register scale_reg = noreg; 4742 Address::ScaleFactor scale_factor = Address::no_scale; 4743 if (arg_slot.is_constant()) { 4744 offset += arg_slot.as_constant() * stackElementSize; 4745 } else { 4746 scale_reg = arg_slot.as_register(); 4747 scale_factor = Address::times(stackElementSize); 4748 } 4749 offset += wordSize; // return PC is on stack 4750 return Address(rsp, scale_reg, scale_factor, offset); 4751 } 4752 4753 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 4754 if (!VerifyOops) return; 4755 4756 #ifdef _LP64 4757 push(rscratch1); 4758 #endif 4759 push(rax); // save rax, 4760 // addr may contain rsp so we will have to adjust it based on the push 4761 // we just did (and on 64 bit we do two pushes) 4762 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 4763 // stores rax into addr which is backwards of what was intended. 4764 if (addr.uses(rsp)) { 4765 lea(rax, addr); 4766 pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); 4767 } else { 4768 pushptr(addr); 4769 } 4770 4771 // Pass register number to verify_oop_subroutine 4772 const char* b = nullptr; 4773 { 4774 ResourceMark rm; 4775 stringStream ss; 4776 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 4777 b = code_string(ss.as_string()); 4778 } 4779 ExternalAddress buffer((address) b); 4780 pushptr(buffer.addr(), rscratch1); 4781 4782 // call indirectly to solve generation ordering problem 4783 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4784 call(rax); 4785 // Caller pops the arguments (addr, message) and restores rax, r10. 4786 } 4787 4788 void MacroAssembler::verify_tlab() { 4789 #ifdef ASSERT 4790 if (UseTLAB && VerifyOops) { 4791 Label next, ok; 4792 Register t1 = rsi; 4793 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); 4794 4795 push(t1); 4796 NOT_LP64(push(thread_reg)); 4797 NOT_LP64(get_thread(thread_reg)); 4798 4799 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4800 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); 4801 jcc(Assembler::aboveEqual, next); 4802 STOP("assert(top >= start)"); 4803 should_not_reach_here(); 4804 4805 bind(next); 4806 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); 4807 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4808 jcc(Assembler::aboveEqual, ok); 4809 STOP("assert(top <= end)"); 4810 should_not_reach_here(); 4811 4812 bind(ok); 4813 NOT_LP64(pop(thread_reg)); 4814 pop(t1); 4815 } 4816 #endif 4817 } 4818 4819 class ControlWord { 4820 public: 4821 int32_t _value; 4822 4823 int rounding_control() const { return (_value >> 10) & 3 ; } 4824 int precision_control() const { return (_value >> 8) & 3 ; } 4825 bool precision() const { return ((_value >> 5) & 1) != 0; } 4826 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4827 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4828 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4829 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4830 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4831 4832 void print() const { 4833 // rounding control 4834 const char* rc; 4835 switch (rounding_control()) { 4836 case 0: rc = "round near"; break; 4837 case 1: rc = "round down"; break; 4838 case 2: rc = "round up "; break; 4839 case 3: rc = "chop "; break; 4840 default: 4841 rc = nullptr; // silence compiler warnings 4842 fatal("Unknown rounding control: %d", rounding_control()); 4843 }; 4844 // precision control 4845 const char* pc; 4846 switch (precision_control()) { 4847 case 0: pc = "24 bits "; break; 4848 case 1: pc = "reserved"; break; 4849 case 2: pc = "53 bits "; break; 4850 case 3: pc = "64 bits "; break; 4851 default: 4852 pc = nullptr; // silence compiler warnings 4853 fatal("Unknown precision control: %d", precision_control()); 4854 }; 4855 // flags 4856 char f[9]; 4857 f[0] = ' '; 4858 f[1] = ' '; 4859 f[2] = (precision ()) ? 'P' : 'p'; 4860 f[3] = (underflow ()) ? 'U' : 'u'; 4861 f[4] = (overflow ()) ? 'O' : 'o'; 4862 f[5] = (zero_divide ()) ? 'Z' : 'z'; 4863 f[6] = (denormalized()) ? 'D' : 'd'; 4864 f[7] = (invalid ()) ? 'I' : 'i'; 4865 f[8] = '\x0'; 4866 // output 4867 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 4868 } 4869 4870 }; 4871 4872 class StatusWord { 4873 public: 4874 int32_t _value; 4875 4876 bool busy() const { return ((_value >> 15) & 1) != 0; } 4877 bool C3() const { return ((_value >> 14) & 1) != 0; } 4878 bool C2() const { return ((_value >> 10) & 1) != 0; } 4879 bool C1() const { return ((_value >> 9) & 1) != 0; } 4880 bool C0() const { return ((_value >> 8) & 1) != 0; } 4881 int top() const { return (_value >> 11) & 7 ; } 4882 bool error_status() const { return ((_value >> 7) & 1) != 0; } 4883 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 4884 bool precision() const { return ((_value >> 5) & 1) != 0; } 4885 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4886 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4887 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4888 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4889 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4890 4891 void print() const { 4892 // condition codes 4893 char c[5]; 4894 c[0] = (C3()) ? '3' : '-'; 4895 c[1] = (C2()) ? '2' : '-'; 4896 c[2] = (C1()) ? '1' : '-'; 4897 c[3] = (C0()) ? '0' : '-'; 4898 c[4] = '\x0'; 4899 // flags 4900 char f[9]; 4901 f[0] = (error_status()) ? 'E' : '-'; 4902 f[1] = (stack_fault ()) ? 'S' : '-'; 4903 f[2] = (precision ()) ? 'P' : '-'; 4904 f[3] = (underflow ()) ? 'U' : '-'; 4905 f[4] = (overflow ()) ? 'O' : '-'; 4906 f[5] = (zero_divide ()) ? 'Z' : '-'; 4907 f[6] = (denormalized()) ? 'D' : '-'; 4908 f[7] = (invalid ()) ? 'I' : '-'; 4909 f[8] = '\x0'; 4910 // output 4911 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 4912 } 4913 4914 }; 4915 4916 class TagWord { 4917 public: 4918 int32_t _value; 4919 4920 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 4921 4922 void print() const { 4923 printf("%04x", _value & 0xFFFF); 4924 } 4925 4926 }; 4927 4928 class FPU_Register { 4929 public: 4930 int32_t _m0; 4931 int32_t _m1; 4932 int16_t _ex; 4933 4934 bool is_indefinite() const { 4935 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 4936 } 4937 4938 void print() const { 4939 char sign = (_ex < 0) ? '-' : '+'; 4940 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 4941 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 4942 }; 4943 4944 }; 4945 4946 class FPU_State { 4947 public: 4948 enum { 4949 register_size = 10, 4950 number_of_registers = 8, 4951 register_mask = 7 4952 }; 4953 4954 ControlWord _control_word; 4955 StatusWord _status_word; 4956 TagWord _tag_word; 4957 int32_t _error_offset; 4958 int32_t _error_selector; 4959 int32_t _data_offset; 4960 int32_t _data_selector; 4961 int8_t _register[register_size * number_of_registers]; 4962 4963 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 4964 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 4965 4966 const char* tag_as_string(int tag) const { 4967 switch (tag) { 4968 case 0: return "valid"; 4969 case 1: return "zero"; 4970 case 2: return "special"; 4971 case 3: return "empty"; 4972 } 4973 ShouldNotReachHere(); 4974 return nullptr; 4975 } 4976 4977 void print() const { 4978 // print computation registers 4979 { int t = _status_word.top(); 4980 for (int i = 0; i < number_of_registers; i++) { 4981 int j = (i - t) & register_mask; 4982 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 4983 st(j)->print(); 4984 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 4985 } 4986 } 4987 printf("\n"); 4988 // print control registers 4989 printf("ctrl = "); _control_word.print(); printf("\n"); 4990 printf("stat = "); _status_word .print(); printf("\n"); 4991 printf("tags = "); _tag_word .print(); printf("\n"); 4992 } 4993 4994 }; 4995 4996 class Flag_Register { 4997 public: 4998 int32_t _value; 4999 5000 bool overflow() const { return ((_value >> 11) & 1) != 0; } 5001 bool direction() const { return ((_value >> 10) & 1) != 0; } 5002 bool sign() const { return ((_value >> 7) & 1) != 0; } 5003 bool zero() const { return ((_value >> 6) & 1) != 0; } 5004 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 5005 bool parity() const { return ((_value >> 2) & 1) != 0; } 5006 bool carry() const { return ((_value >> 0) & 1) != 0; } 5007 5008 void print() const { 5009 // flags 5010 char f[8]; 5011 f[0] = (overflow ()) ? 'O' : '-'; 5012 f[1] = (direction ()) ? 'D' : '-'; 5013 f[2] = (sign ()) ? 'S' : '-'; 5014 f[3] = (zero ()) ? 'Z' : '-'; 5015 f[4] = (auxiliary_carry()) ? 'A' : '-'; 5016 f[5] = (parity ()) ? 'P' : '-'; 5017 f[6] = (carry ()) ? 'C' : '-'; 5018 f[7] = '\x0'; 5019 // output 5020 printf("%08x flags = %s", _value, f); 5021 } 5022 5023 }; 5024 5025 class IU_Register { 5026 public: 5027 int32_t _value; 5028 5029 void print() const { 5030 printf("%08x %11d", _value, _value); 5031 } 5032 5033 }; 5034 5035 class IU_State { 5036 public: 5037 Flag_Register _eflags; 5038 IU_Register _rdi; 5039 IU_Register _rsi; 5040 IU_Register _rbp; 5041 IU_Register _rsp; 5042 IU_Register _rbx; 5043 IU_Register _rdx; 5044 IU_Register _rcx; 5045 IU_Register _rax; 5046 5047 void print() const { 5048 // computation registers 5049 printf("rax, = "); _rax.print(); printf("\n"); 5050 printf("rbx, = "); _rbx.print(); printf("\n"); 5051 printf("rcx = "); _rcx.print(); printf("\n"); 5052 printf("rdx = "); _rdx.print(); printf("\n"); 5053 printf("rdi = "); _rdi.print(); printf("\n"); 5054 printf("rsi = "); _rsi.print(); printf("\n"); 5055 printf("rbp, = "); _rbp.print(); printf("\n"); 5056 printf("rsp = "); _rsp.print(); printf("\n"); 5057 printf("\n"); 5058 // control registers 5059 printf("flgs = "); _eflags.print(); printf("\n"); 5060 } 5061 }; 5062 5063 5064 class CPU_State { 5065 public: 5066 FPU_State _fpu_state; 5067 IU_State _iu_state; 5068 5069 void print() const { 5070 printf("--------------------------------------------------\n"); 5071 _iu_state .print(); 5072 printf("\n"); 5073 _fpu_state.print(); 5074 printf("--------------------------------------------------\n"); 5075 } 5076 5077 }; 5078 5079 5080 static void _print_CPU_state(CPU_State* state) { 5081 state->print(); 5082 }; 5083 5084 5085 void MacroAssembler::print_CPU_state() { 5086 push_CPU_state(); 5087 push(rsp); // pass CPU state 5088 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5089 addptr(rsp, wordSize); // discard argument 5090 pop_CPU_state(); 5091 } 5092 5093 5094 #ifndef _LP64 5095 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { 5096 static int counter = 0; 5097 FPU_State* fs = &state->_fpu_state; 5098 counter++; 5099 // For leaf calls, only verify that the top few elements remain empty. 5100 // We only need 1 empty at the top for C2 code. 5101 if( stack_depth < 0 ) { 5102 if( fs->tag_for_st(7) != 3 ) { 5103 printf("FPR7 not empty\n"); 5104 state->print(); 5105 assert(false, "error"); 5106 return false; 5107 } 5108 return true; // All other stack states do not matter 5109 } 5110 5111 assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(), 5112 "bad FPU control word"); 5113 5114 // compute stack depth 5115 int i = 0; 5116 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; 5117 int d = i; 5118 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; 5119 // verify findings 5120 if (i != FPU_State::number_of_registers) { 5121 // stack not contiguous 5122 printf("%s: stack not contiguous at ST%d\n", s, i); 5123 state->print(); 5124 assert(false, "error"); 5125 return false; 5126 } 5127 // check if computed stack depth corresponds to expected stack depth 5128 if (stack_depth < 0) { 5129 // expected stack depth is -stack_depth or less 5130 if (d > -stack_depth) { 5131 // too many elements on the stack 5132 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); 5133 state->print(); 5134 assert(false, "error"); 5135 return false; 5136 } 5137 } else { 5138 // expected stack depth is stack_depth 5139 if (d != stack_depth) { 5140 // wrong stack depth 5141 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); 5142 state->print(); 5143 assert(false, "error"); 5144 return false; 5145 } 5146 } 5147 // everything is cool 5148 return true; 5149 } 5150 5151 void MacroAssembler::verify_FPU(int stack_depth, const char* s) { 5152 if (!VerifyFPU) return; 5153 push_CPU_state(); 5154 push(rsp); // pass CPU state 5155 ExternalAddress msg((address) s); 5156 // pass message string s 5157 pushptr(msg.addr(), noreg); 5158 push(stack_depth); // pass stack depth 5159 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); 5160 addptr(rsp, 3 * wordSize); // discard arguments 5161 // check for error 5162 { Label L; 5163 testl(rax, rax); 5164 jcc(Assembler::notZero, L); 5165 int3(); // break if error condition 5166 bind(L); 5167 } 5168 pop_CPU_state(); 5169 } 5170 #endif // _LP64 5171 5172 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5173 // Either restore the MXCSR register after returning from the JNI Call 5174 // or verify that it wasn't changed (with -Xcheck:jni flag). 5175 if (VM_Version::supports_sse()) { 5176 if (RestoreMXCSROnJNICalls) { 5177 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5178 } else if (CheckJNICalls) { 5179 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5180 } 5181 } 5182 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5183 vzeroupper(); 5184 5185 #ifndef _LP64 5186 // Either restore the x87 floating pointer control word after returning 5187 // from the JNI call or verify that it wasn't changed. 5188 if (CheckJNICalls) { 5189 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); 5190 } 5191 #endif // _LP64 5192 } 5193 5194 // ((OopHandle)result).resolve(); 5195 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5196 assert_different_registers(result, tmp); 5197 5198 // Only 64 bit platforms support GCs that require a tmp register 5199 // Only IN_HEAP loads require a thread_tmp register 5200 // OopHandle::resolve is an indirection like jobject. 5201 access_load_at(T_OBJECT, IN_NATIVE, 5202 result, Address(result, 0), tmp, /*tmp_thread*/noreg); 5203 } 5204 5205 // ((WeakHandle)result).resolve(); 5206 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5207 assert_different_registers(rresult, rtmp); 5208 Label resolved; 5209 5210 // A null weak handle resolves to null. 5211 cmpptr(rresult, 0); 5212 jcc(Assembler::equal, resolved); 5213 5214 // Only 64 bit platforms support GCs that require a tmp register 5215 // Only IN_HEAP loads require a thread_tmp register 5216 // WeakHandle::resolve is an indirection like jweak. 5217 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5218 rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); 5219 bind(resolved); 5220 } 5221 5222 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5223 // get mirror 5224 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5225 load_method_holder(mirror, method); 5226 movptr(mirror, Address(mirror, mirror_offset)); 5227 resolve_oop_handle(mirror, tmp); 5228 } 5229 5230 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5231 load_method_holder(rresult, rmethod); 5232 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5233 } 5234 5235 void MacroAssembler::load_method_holder(Register holder, Register method) { 5236 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5237 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5238 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5239 } 5240 5241 #ifdef _LP64 5242 void MacroAssembler::load_nklass_compact(Register dst, Register src) { 5243 assert(UseCompactObjectHeaders, "expect compact object headers"); 5244 5245 Label fast; 5246 movq(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5247 testb(dst, markWord::monitor_value); 5248 jccb(Assembler::zero, fast); 5249 5250 // Fetch displaced header 5251 movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); 5252 5253 bind(fast); 5254 shrq(dst, markWord::klass_shift); 5255 } 5256 #endif 5257 5258 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5259 assert_different_registers(src, tmp); 5260 assert_different_registers(dst, tmp); 5261 #ifdef _LP64 5262 if (UseCompactObjectHeaders) { 5263 load_nklass_compact(dst, src); 5264 decode_klass_not_null(dst, tmp); 5265 } else if (UseCompressedClassPointers) { 5266 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5267 decode_klass_not_null(dst, tmp); 5268 } else 5269 #endif 5270 { 5271 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5272 } 5273 } 5274 5275 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 5276 assert(!UseCompactObjectHeaders, "not with compact headers"); 5277 assert_different_registers(src, tmp); 5278 assert_different_registers(dst, tmp); 5279 #ifdef _LP64 5280 if (UseCompressedClassPointers) { 5281 encode_klass_not_null(src, tmp); 5282 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5283 } else 5284 #endif 5285 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5286 } 5287 5288 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { 5289 #ifdef _LP64 5290 if (UseCompactObjectHeaders) { 5291 load_nklass_compact(tmp, obj); 5292 cmpl(klass, tmp); 5293 } else if (UseCompressedClassPointers) { 5294 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5295 } else 5296 #endif 5297 { 5298 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5299 } 5300 } 5301 5302 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) { 5303 #ifdef _LP64 5304 if (UseCompactObjectHeaders) { 5305 assert(tmp2 != noreg, "need tmp2"); 5306 assert_different_registers(src, dst, tmp1, tmp2); 5307 load_nklass_compact(tmp1, src); 5308 load_nklass_compact(tmp2, dst); 5309 cmpl(tmp1, tmp2); 5310 } else if (UseCompressedClassPointers) { 5311 movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes())); 5312 cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes())); 5313 } else 5314 #endif 5315 { 5316 movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes())); 5317 cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes())); 5318 } 5319 } 5320 5321 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 5322 Register tmp1, Register thread_tmp) { 5323 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5324 decorators = AccessInternal::decorator_fixup(decorators, type); 5325 bool as_raw = (decorators & AS_RAW) != 0; 5326 if (as_raw) { 5327 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5328 } else { 5329 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5330 } 5331 } 5332 5333 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 5334 Register tmp1, Register tmp2, Register tmp3) { 5335 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5336 decorators = AccessInternal::decorator_fixup(decorators, type); 5337 bool as_raw = (decorators & AS_RAW) != 0; 5338 if (as_raw) { 5339 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5340 } else { 5341 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5342 } 5343 } 5344 5345 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5346 Register thread_tmp, DecoratorSet decorators) { 5347 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); 5348 } 5349 5350 // Doesn't do verification, generates fixed size code 5351 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5352 Register thread_tmp, DecoratorSet decorators) { 5353 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); 5354 } 5355 5356 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5357 Register tmp2, Register tmp3, DecoratorSet decorators) { 5358 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5359 } 5360 5361 // Used for storing nulls. 5362 void MacroAssembler::store_heap_oop_null(Address dst) { 5363 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5364 } 5365 5366 #ifdef _LP64 5367 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5368 assert(!UseCompactObjectHeaders, "Don't use with compact headers"); 5369 if (UseCompressedClassPointers) { 5370 // Store to klass gap in destination 5371 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 5372 } 5373 } 5374 5375 #ifdef ASSERT 5376 void MacroAssembler::verify_heapbase(const char* msg) { 5377 assert (UseCompressedOops, "should be compressed"); 5378 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5379 if (CheckCompressedOops) { 5380 Label ok; 5381 ExternalAddress src2(CompressedOops::ptrs_base_addr()); 5382 const bool is_src2_reachable = reachable(src2); 5383 if (!is_src2_reachable) { 5384 push(rscratch1); // cmpptr trashes rscratch1 5385 } 5386 cmpptr(r12_heapbase, src2, rscratch1); 5387 jcc(Assembler::equal, ok); 5388 STOP(msg); 5389 bind(ok); 5390 if (!is_src2_reachable) { 5391 pop(rscratch1); 5392 } 5393 } 5394 } 5395 #endif 5396 5397 // Algorithm must match oop.inline.hpp encode_heap_oop. 5398 void MacroAssembler::encode_heap_oop(Register r) { 5399 #ifdef ASSERT 5400 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5401 #endif 5402 verify_oop_msg(r, "broken oop in encode_heap_oop"); 5403 if (CompressedOops::base() == nullptr) { 5404 if (CompressedOops::shift() != 0) { 5405 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5406 shrq(r, LogMinObjAlignmentInBytes); 5407 } 5408 return; 5409 } 5410 testq(r, r); 5411 cmovq(Assembler::equal, r, r12_heapbase); 5412 subq(r, r12_heapbase); 5413 shrq(r, LogMinObjAlignmentInBytes); 5414 } 5415 5416 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5417 #ifdef ASSERT 5418 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5419 if (CheckCompressedOops) { 5420 Label ok; 5421 testq(r, r); 5422 jcc(Assembler::notEqual, ok); 5423 STOP("null oop passed to encode_heap_oop_not_null"); 5424 bind(ok); 5425 } 5426 #endif 5427 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5428 if (CompressedOops::base() != nullptr) { 5429 subq(r, r12_heapbase); 5430 } 5431 if (CompressedOops::shift() != 0) { 5432 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5433 shrq(r, LogMinObjAlignmentInBytes); 5434 } 5435 } 5436 5437 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5438 #ifdef ASSERT 5439 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5440 if (CheckCompressedOops) { 5441 Label ok; 5442 testq(src, src); 5443 jcc(Assembler::notEqual, ok); 5444 STOP("null oop passed to encode_heap_oop_not_null2"); 5445 bind(ok); 5446 } 5447 #endif 5448 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5449 if (dst != src) { 5450 movq(dst, src); 5451 } 5452 if (CompressedOops::base() != nullptr) { 5453 subq(dst, r12_heapbase); 5454 } 5455 if (CompressedOops::shift() != 0) { 5456 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5457 shrq(dst, LogMinObjAlignmentInBytes); 5458 } 5459 } 5460 5461 void MacroAssembler::decode_heap_oop(Register r) { 5462 #ifdef ASSERT 5463 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5464 #endif 5465 if (CompressedOops::base() == nullptr) { 5466 if (CompressedOops::shift() != 0) { 5467 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5468 shlq(r, LogMinObjAlignmentInBytes); 5469 } 5470 } else { 5471 Label done; 5472 shlq(r, LogMinObjAlignmentInBytes); 5473 jccb(Assembler::equal, done); 5474 addq(r, r12_heapbase); 5475 bind(done); 5476 } 5477 verify_oop_msg(r, "broken oop in decode_heap_oop"); 5478 } 5479 5480 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5481 // Note: it will change flags 5482 assert (UseCompressedOops, "should only be used for compressed headers"); 5483 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5484 // Cannot assert, unverified entry point counts instructions (see .ad file) 5485 // vtableStubs also counts instructions in pd_code_size_limit. 5486 // Also do not verify_oop as this is called by verify_oop. 5487 if (CompressedOops::shift() != 0) { 5488 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5489 shlq(r, LogMinObjAlignmentInBytes); 5490 if (CompressedOops::base() != nullptr) { 5491 addq(r, r12_heapbase); 5492 } 5493 } else { 5494 assert (CompressedOops::base() == nullptr, "sanity"); 5495 } 5496 } 5497 5498 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5499 // Note: it will change flags 5500 assert (UseCompressedOops, "should only be used for compressed headers"); 5501 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5502 // Cannot assert, unverified entry point counts instructions (see .ad file) 5503 // vtableStubs also counts instructions in pd_code_size_limit. 5504 // Also do not verify_oop as this is called by verify_oop. 5505 if (CompressedOops::shift() != 0) { 5506 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5507 if (LogMinObjAlignmentInBytes == Address::times_8) { 5508 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 5509 } else { 5510 if (dst != src) { 5511 movq(dst, src); 5512 } 5513 shlq(dst, LogMinObjAlignmentInBytes); 5514 if (CompressedOops::base() != nullptr) { 5515 addq(dst, r12_heapbase); 5516 } 5517 } 5518 } else { 5519 assert (CompressedOops::base() == nullptr, "sanity"); 5520 if (dst != src) { 5521 movq(dst, src); 5522 } 5523 } 5524 } 5525 5526 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 5527 assert_different_registers(r, tmp); 5528 if (CompressedKlassPointers::base() != nullptr) { 5529 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5530 subq(r, tmp); 5531 } 5532 if (CompressedKlassPointers::shift() != 0) { 5533 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5534 shrq(r, LogKlassAlignmentInBytes); 5535 } 5536 } 5537 5538 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 5539 assert_different_registers(src, dst); 5540 if (CompressedKlassPointers::base() != nullptr) { 5541 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 5542 addq(dst, src); 5543 } else { 5544 movptr(dst, src); 5545 } 5546 if (CompressedKlassPointers::shift() != 0) { 5547 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5548 shrq(dst, LogKlassAlignmentInBytes); 5549 } 5550 } 5551 5552 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 5553 assert_different_registers(r, tmp); 5554 // Note: it will change flags 5555 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 5556 // Cannot assert, unverified entry point counts instructions (see .ad file) 5557 // vtableStubs also counts instructions in pd_code_size_limit. 5558 // Also do not verify_oop as this is called by verify_oop. 5559 if (CompressedKlassPointers::shift() != 0) { 5560 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5561 shlq(r, LogKlassAlignmentInBytes); 5562 } 5563 if (CompressedKlassPointers::base() != nullptr) { 5564 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5565 addq(r, tmp); 5566 } 5567 } 5568 5569 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 5570 assert_different_registers(src, dst); 5571 // Note: it will change flags 5572 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5573 // Cannot assert, unverified entry point counts instructions (see .ad file) 5574 // vtableStubs also counts instructions in pd_code_size_limit. 5575 // Also do not verify_oop as this is called by verify_oop. 5576 5577 if (CompressedKlassPointers::base() == nullptr && 5578 CompressedKlassPointers::shift() == 0) { 5579 // The best case scenario is that there is no base or shift. Then it is already 5580 // a pointer that needs nothing but a register rename. 5581 movl(dst, src); 5582 } else { 5583 if (CompressedKlassPointers::base() != nullptr) { 5584 mov64(dst, (int64_t)CompressedKlassPointers::base()); 5585 } else { 5586 xorq(dst, dst); 5587 } 5588 if (CompressedKlassPointers::shift() != 0) { 5589 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 5590 assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); 5591 leaq(dst, Address(dst, src, Address::times_8, 0)); 5592 } else { 5593 addq(dst, src); 5594 } 5595 } 5596 } 5597 5598 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5599 assert (UseCompressedOops, "should only be used for compressed headers"); 5600 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5601 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5602 int oop_index = oop_recorder()->find_index(obj); 5603 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5604 mov_narrow_oop(dst, oop_index, rspec); 5605 } 5606 5607 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 5608 assert (UseCompressedOops, "should only be used for compressed headers"); 5609 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5610 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5611 int oop_index = oop_recorder()->find_index(obj); 5612 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5613 mov_narrow_oop(dst, oop_index, rspec); 5614 } 5615 5616 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5617 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5618 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5619 int klass_index = oop_recorder()->find_index(k); 5620 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5621 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5622 } 5623 5624 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 5625 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5626 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5627 int klass_index = oop_recorder()->find_index(k); 5628 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5629 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5630 } 5631 5632 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 5633 assert (UseCompressedOops, "should only be used for compressed headers"); 5634 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5635 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5636 int oop_index = oop_recorder()->find_index(obj); 5637 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5638 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5639 } 5640 5641 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 5642 assert (UseCompressedOops, "should only be used for compressed headers"); 5643 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5644 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5645 int oop_index = oop_recorder()->find_index(obj); 5646 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5647 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5648 } 5649 5650 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 5651 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5652 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5653 int klass_index = oop_recorder()->find_index(k); 5654 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5655 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5656 } 5657 5658 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 5659 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5660 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5661 int klass_index = oop_recorder()->find_index(k); 5662 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5663 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5664 } 5665 5666 void MacroAssembler::reinit_heapbase() { 5667 if (UseCompressedOops) { 5668 if (Universe::heap() != nullptr) { 5669 if (CompressedOops::base() == nullptr) { 5670 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 5671 } else { 5672 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base()); 5673 } 5674 } else { 5675 movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr())); 5676 } 5677 } 5678 } 5679 5680 #endif // _LP64 5681 5682 #if COMPILER2_OR_JVMCI 5683 5684 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 5685 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5686 // cnt - number of qwords (8-byte words). 5687 // base - start address, qword aligned. 5688 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 5689 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 5690 if (use64byteVector) { 5691 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 5692 } else if (MaxVectorSize >= 32) { 5693 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 5694 } else { 5695 pxor(xtmp, xtmp); 5696 } 5697 jmp(L_zero_64_bytes); 5698 5699 BIND(L_loop); 5700 if (MaxVectorSize >= 32) { 5701 fill64(base, 0, xtmp, use64byteVector); 5702 } else { 5703 movdqu(Address(base, 0), xtmp); 5704 movdqu(Address(base, 16), xtmp); 5705 movdqu(Address(base, 32), xtmp); 5706 movdqu(Address(base, 48), xtmp); 5707 } 5708 addptr(base, 64); 5709 5710 BIND(L_zero_64_bytes); 5711 subptr(cnt, 8); 5712 jccb(Assembler::greaterEqual, L_loop); 5713 5714 // Copy trailing 64 bytes 5715 if (use64byteVector) { 5716 addptr(cnt, 8); 5717 jccb(Assembler::equal, L_end); 5718 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 5719 jmp(L_end); 5720 } else { 5721 addptr(cnt, 4); 5722 jccb(Assembler::less, L_tail); 5723 if (MaxVectorSize >= 32) { 5724 vmovdqu(Address(base, 0), xtmp); 5725 } else { 5726 movdqu(Address(base, 0), xtmp); 5727 movdqu(Address(base, 16), xtmp); 5728 } 5729 } 5730 addptr(base, 32); 5731 subptr(cnt, 4); 5732 5733 BIND(L_tail); 5734 addptr(cnt, 4); 5735 jccb(Assembler::lessEqual, L_end); 5736 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 5737 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 5738 } else { 5739 decrement(cnt); 5740 5741 BIND(L_sloop); 5742 movq(Address(base, 0), xtmp); 5743 addptr(base, 8); 5744 decrement(cnt); 5745 jccb(Assembler::greaterEqual, L_sloop); 5746 } 5747 BIND(L_end); 5748 } 5749 5750 // Clearing constant sized memory using YMM/ZMM registers. 5751 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5752 assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), ""); 5753 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 5754 5755 int vector64_count = (cnt & (~0x7)) >> 3; 5756 cnt = cnt & 0x7; 5757 const int fill64_per_loop = 4; 5758 const int max_unrolled_fill64 = 8; 5759 5760 // 64 byte initialization loop. 5761 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 5762 int start64 = 0; 5763 if (vector64_count > max_unrolled_fill64) { 5764 Label LOOP; 5765 Register index = rtmp; 5766 5767 start64 = vector64_count - (vector64_count % fill64_per_loop); 5768 5769 movl(index, 0); 5770 BIND(LOOP); 5771 for (int i = 0; i < fill64_per_loop; i++) { 5772 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 5773 } 5774 addl(index, fill64_per_loop * 64); 5775 cmpl(index, start64 * 64); 5776 jccb(Assembler::less, LOOP); 5777 } 5778 for (int i = start64; i < vector64_count; i++) { 5779 fill64(base, i * 64, xtmp, use64byteVector); 5780 } 5781 5782 // Clear remaining 64 byte tail. 5783 int disp = vector64_count * 64; 5784 if (cnt) { 5785 switch (cnt) { 5786 case 1: 5787 movq(Address(base, disp), xtmp); 5788 break; 5789 case 2: 5790 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 5791 break; 5792 case 3: 5793 movl(rtmp, 0x7); 5794 kmovwl(mask, rtmp); 5795 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 5796 break; 5797 case 4: 5798 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5799 break; 5800 case 5: 5801 if (use64byteVector) { 5802 movl(rtmp, 0x1F); 5803 kmovwl(mask, rtmp); 5804 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5805 } else { 5806 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5807 movq(Address(base, disp + 32), xtmp); 5808 } 5809 break; 5810 case 6: 5811 if (use64byteVector) { 5812 movl(rtmp, 0x3F); 5813 kmovwl(mask, rtmp); 5814 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5815 } else { 5816 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5817 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 5818 } 5819 break; 5820 case 7: 5821 if (use64byteVector) { 5822 movl(rtmp, 0x7F); 5823 kmovwl(mask, rtmp); 5824 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5825 } else { 5826 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5827 movl(rtmp, 0x7); 5828 kmovwl(mask, rtmp); 5829 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 5830 } 5831 break; 5832 default: 5833 fatal("Unexpected length : %d\n",cnt); 5834 break; 5835 } 5836 } 5837 } 5838 5839 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 5840 bool is_large, KRegister mask) { 5841 // cnt - number of qwords (8-byte words). 5842 // base - start address, qword aligned. 5843 // is_large - if optimizers know cnt is larger than InitArrayShortSize 5844 assert(base==rdi, "base register must be edi for rep stos"); 5845 assert(tmp==rax, "tmp register must be eax for rep stos"); 5846 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 5847 assert(InitArrayShortSize % BytesPerLong == 0, 5848 "InitArrayShortSize should be the multiple of BytesPerLong"); 5849 5850 Label DONE; 5851 if (!is_large || !UseXMMForObjInit) { 5852 xorptr(tmp, tmp); 5853 } 5854 5855 if (!is_large) { 5856 Label LOOP, LONG; 5857 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 5858 jccb(Assembler::greater, LONG); 5859 5860 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5861 5862 decrement(cnt); 5863 jccb(Assembler::negative, DONE); // Zero length 5864 5865 // Use individual pointer-sized stores for small counts: 5866 BIND(LOOP); 5867 movptr(Address(base, cnt, Address::times_ptr), tmp); 5868 decrement(cnt); 5869 jccb(Assembler::greaterEqual, LOOP); 5870 jmpb(DONE); 5871 5872 BIND(LONG); 5873 } 5874 5875 // Use longer rep-prefixed ops for non-small counts: 5876 if (UseFastStosb) { 5877 shlptr(cnt, 3); // convert to number of bytes 5878 rep_stosb(); 5879 } else if (UseXMMForObjInit) { 5880 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 5881 } else { 5882 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5883 rep_stos(); 5884 } 5885 5886 BIND(DONE); 5887 } 5888 5889 #endif //COMPILER2_OR_JVMCI 5890 5891 5892 void MacroAssembler::generate_fill(BasicType t, bool aligned, 5893 Register to, Register value, Register count, 5894 Register rtmp, XMMRegister xtmp) { 5895 ShortBranchVerifier sbv(this); 5896 assert_different_registers(to, value, count, rtmp); 5897 Label L_exit; 5898 Label L_fill_2_bytes, L_fill_4_bytes; 5899 5900 #if defined(COMPILER2) && defined(_LP64) 5901 if(MaxVectorSize >=32 && 5902 VM_Version::supports_avx512vlbw() && 5903 VM_Version::supports_bmi2()) { 5904 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 5905 return; 5906 } 5907 #endif 5908 5909 int shift = -1; 5910 switch (t) { 5911 case T_BYTE: 5912 shift = 2; 5913 break; 5914 case T_SHORT: 5915 shift = 1; 5916 break; 5917 case T_INT: 5918 shift = 0; 5919 break; 5920 default: ShouldNotReachHere(); 5921 } 5922 5923 if (t == T_BYTE) { 5924 andl(value, 0xff); 5925 movl(rtmp, value); 5926 shll(rtmp, 8); 5927 orl(value, rtmp); 5928 } 5929 if (t == T_SHORT) { 5930 andl(value, 0xffff); 5931 } 5932 if (t == T_BYTE || t == T_SHORT) { 5933 movl(rtmp, value); 5934 shll(rtmp, 16); 5935 orl(value, rtmp); 5936 } 5937 5938 cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 5939 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 5940 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 5941 Label L_skip_align2; 5942 // align source address at 4 bytes address boundary 5943 if (t == T_BYTE) { 5944 Label L_skip_align1; 5945 // One byte misalignment happens only for byte arrays 5946 testptr(to, 1); 5947 jccb(Assembler::zero, L_skip_align1); 5948 movb(Address(to, 0), value); 5949 increment(to); 5950 decrement(count); 5951 BIND(L_skip_align1); 5952 } 5953 // Two bytes misalignment happens only for byte and short (char) arrays 5954 testptr(to, 2); 5955 jccb(Assembler::zero, L_skip_align2); 5956 movw(Address(to, 0), value); 5957 addptr(to, 2); 5958 subl(count, 1<<(shift-1)); 5959 BIND(L_skip_align2); 5960 } 5961 if (UseSSE < 2) { 5962 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 5963 // Fill 32-byte chunks 5964 subl(count, 8 << shift); 5965 jcc(Assembler::less, L_check_fill_8_bytes); 5966 align(16); 5967 5968 BIND(L_fill_32_bytes_loop); 5969 5970 for (int i = 0; i < 32; i += 4) { 5971 movl(Address(to, i), value); 5972 } 5973 5974 addptr(to, 32); 5975 subl(count, 8 << shift); 5976 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 5977 BIND(L_check_fill_8_bytes); 5978 addl(count, 8 << shift); 5979 jccb(Assembler::zero, L_exit); 5980 jmpb(L_fill_8_bytes); 5981 5982 // 5983 // length is too short, just fill qwords 5984 // 5985 BIND(L_fill_8_bytes_loop); 5986 movl(Address(to, 0), value); 5987 movl(Address(to, 4), value); 5988 addptr(to, 8); 5989 BIND(L_fill_8_bytes); 5990 subl(count, 1 << (shift + 1)); 5991 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 5992 // fall through to fill 4 bytes 5993 } else { 5994 Label L_fill_32_bytes; 5995 if (!UseUnalignedLoadStores) { 5996 // align to 8 bytes, we know we are 4 byte aligned to start 5997 testptr(to, 4); 5998 jccb(Assembler::zero, L_fill_32_bytes); 5999 movl(Address(to, 0), value); 6000 addptr(to, 4); 6001 subl(count, 1<<shift); 6002 } 6003 BIND(L_fill_32_bytes); 6004 { 6005 assert( UseSSE >= 2, "supported cpu only" ); 6006 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6007 movdl(xtmp, value); 6008 if (UseAVX >= 2 && UseUnalignedLoadStores) { 6009 Label L_check_fill_32_bytes; 6010 if (UseAVX > 2) { 6011 // Fill 64-byte chunks 6012 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 6013 6014 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 6015 cmpl(count, VM_Version::avx3_threshold()); 6016 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 6017 6018 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 6019 6020 subl(count, 16 << shift); 6021 jccb(Assembler::less, L_check_fill_32_bytes); 6022 align(16); 6023 6024 BIND(L_fill_64_bytes_loop_avx3); 6025 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 6026 addptr(to, 64); 6027 subl(count, 16 << shift); 6028 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 6029 jmpb(L_check_fill_32_bytes); 6030 6031 BIND(L_check_fill_64_bytes_avx2); 6032 } 6033 // Fill 64-byte chunks 6034 Label L_fill_64_bytes_loop; 6035 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 6036 6037 subl(count, 16 << shift); 6038 jcc(Assembler::less, L_check_fill_32_bytes); 6039 align(16); 6040 6041 BIND(L_fill_64_bytes_loop); 6042 vmovdqu(Address(to, 0), xtmp); 6043 vmovdqu(Address(to, 32), xtmp); 6044 addptr(to, 64); 6045 subl(count, 16 << shift); 6046 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 6047 6048 BIND(L_check_fill_32_bytes); 6049 addl(count, 8 << shift); 6050 jccb(Assembler::less, L_check_fill_8_bytes); 6051 vmovdqu(Address(to, 0), xtmp); 6052 addptr(to, 32); 6053 subl(count, 8 << shift); 6054 6055 BIND(L_check_fill_8_bytes); 6056 // clean upper bits of YMM registers 6057 movdl(xtmp, value); 6058 pshufd(xtmp, xtmp, 0); 6059 } else { 6060 // Fill 32-byte chunks 6061 pshufd(xtmp, xtmp, 0); 6062 6063 subl(count, 8 << shift); 6064 jcc(Assembler::less, L_check_fill_8_bytes); 6065 align(16); 6066 6067 BIND(L_fill_32_bytes_loop); 6068 6069 if (UseUnalignedLoadStores) { 6070 movdqu(Address(to, 0), xtmp); 6071 movdqu(Address(to, 16), xtmp); 6072 } else { 6073 movq(Address(to, 0), xtmp); 6074 movq(Address(to, 8), xtmp); 6075 movq(Address(to, 16), xtmp); 6076 movq(Address(to, 24), xtmp); 6077 } 6078 6079 addptr(to, 32); 6080 subl(count, 8 << shift); 6081 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6082 6083 BIND(L_check_fill_8_bytes); 6084 } 6085 addl(count, 8 << shift); 6086 jccb(Assembler::zero, L_exit); 6087 jmpb(L_fill_8_bytes); 6088 6089 // 6090 // length is too short, just fill qwords 6091 // 6092 BIND(L_fill_8_bytes_loop); 6093 movq(Address(to, 0), xtmp); 6094 addptr(to, 8); 6095 BIND(L_fill_8_bytes); 6096 subl(count, 1 << (shift + 1)); 6097 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6098 } 6099 } 6100 // fill trailing 4 bytes 6101 BIND(L_fill_4_bytes); 6102 testl(count, 1<<shift); 6103 jccb(Assembler::zero, L_fill_2_bytes); 6104 movl(Address(to, 0), value); 6105 if (t == T_BYTE || t == T_SHORT) { 6106 Label L_fill_byte; 6107 addptr(to, 4); 6108 BIND(L_fill_2_bytes); 6109 // fill trailing 2 bytes 6110 testl(count, 1<<(shift-1)); 6111 jccb(Assembler::zero, L_fill_byte); 6112 movw(Address(to, 0), value); 6113 if (t == T_BYTE) { 6114 addptr(to, 2); 6115 BIND(L_fill_byte); 6116 // fill trailing byte 6117 testl(count, 1); 6118 jccb(Assembler::zero, L_exit); 6119 movb(Address(to, 0), value); 6120 } else { 6121 BIND(L_fill_byte); 6122 } 6123 } else { 6124 BIND(L_fill_2_bytes); 6125 } 6126 BIND(L_exit); 6127 } 6128 6129 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 6130 switch(type) { 6131 case T_BYTE: 6132 case T_BOOLEAN: 6133 evpbroadcastb(dst, src, vector_len); 6134 break; 6135 case T_SHORT: 6136 case T_CHAR: 6137 evpbroadcastw(dst, src, vector_len); 6138 break; 6139 case T_INT: 6140 case T_FLOAT: 6141 evpbroadcastd(dst, src, vector_len); 6142 break; 6143 case T_LONG: 6144 case T_DOUBLE: 6145 evpbroadcastq(dst, src, vector_len); 6146 break; 6147 default: 6148 fatal("Unhandled type : %s", type2name(type)); 6149 break; 6150 } 6151 } 6152 6153 // encode char[] to byte[] in ISO_8859_1 or ASCII 6154 //@IntrinsicCandidate 6155 //private static int implEncodeISOArray(byte[] sa, int sp, 6156 //byte[] da, int dp, int len) { 6157 // int i = 0; 6158 // for (; i < len; i++) { 6159 // char c = StringUTF16.getChar(sa, sp++); 6160 // if (c > '\u00FF') 6161 // break; 6162 // da[dp++] = (byte)c; 6163 // } 6164 // return i; 6165 //} 6166 // 6167 //@IntrinsicCandidate 6168 //private static int implEncodeAsciiArray(char[] sa, int sp, 6169 // byte[] da, int dp, int len) { 6170 // int i = 0; 6171 // for (; i < len; i++) { 6172 // char c = sa[sp++]; 6173 // if (c >= '\u0080') 6174 // break; 6175 // da[dp++] = (byte)c; 6176 // } 6177 // return i; 6178 //} 6179 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6180 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6181 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6182 Register tmp5, Register result, bool ascii) { 6183 6184 // rsi: src 6185 // rdi: dst 6186 // rdx: len 6187 // rcx: tmp5 6188 // rax: result 6189 ShortBranchVerifier sbv(this); 6190 assert_different_registers(src, dst, len, tmp5, result); 6191 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6192 6193 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6194 int short_mask = ascii ? 0xff80 : 0xff00; 6195 6196 // set result 6197 xorl(result, result); 6198 // check for zero length 6199 testl(len, len); 6200 jcc(Assembler::zero, L_done); 6201 6202 movl(result, len); 6203 6204 // Setup pointers 6205 lea(src, Address(src, len, Address::times_2)); // char[] 6206 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6207 negptr(len); 6208 6209 if (UseSSE42Intrinsics || UseAVX >= 2) { 6210 Label L_copy_8_chars, L_copy_8_chars_exit; 6211 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6212 6213 if (UseAVX >= 2) { 6214 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6215 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6216 movdl(tmp1Reg, tmp5); 6217 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6218 jmp(L_chars_32_check); 6219 6220 bind(L_copy_32_chars); 6221 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6222 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6223 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6224 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6225 jccb(Assembler::notZero, L_copy_32_chars_exit); 6226 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6227 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6228 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6229 6230 bind(L_chars_32_check); 6231 addptr(len, 32); 6232 jcc(Assembler::lessEqual, L_copy_32_chars); 6233 6234 bind(L_copy_32_chars_exit); 6235 subptr(len, 16); 6236 jccb(Assembler::greater, L_copy_16_chars_exit); 6237 6238 } else if (UseSSE42Intrinsics) { 6239 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6240 movdl(tmp1Reg, tmp5); 6241 pshufd(tmp1Reg, tmp1Reg, 0); 6242 jmpb(L_chars_16_check); 6243 } 6244 6245 bind(L_copy_16_chars); 6246 if (UseAVX >= 2) { 6247 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6248 vptest(tmp2Reg, tmp1Reg); 6249 jcc(Assembler::notZero, L_copy_16_chars_exit); 6250 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6251 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6252 } else { 6253 if (UseAVX > 0) { 6254 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6255 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6256 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6257 } else { 6258 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6259 por(tmp2Reg, tmp3Reg); 6260 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6261 por(tmp2Reg, tmp4Reg); 6262 } 6263 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6264 jccb(Assembler::notZero, L_copy_16_chars_exit); 6265 packuswb(tmp3Reg, tmp4Reg); 6266 } 6267 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 6268 6269 bind(L_chars_16_check); 6270 addptr(len, 16); 6271 jcc(Assembler::lessEqual, L_copy_16_chars); 6272 6273 bind(L_copy_16_chars_exit); 6274 if (UseAVX >= 2) { 6275 // clean upper bits of YMM registers 6276 vpxor(tmp2Reg, tmp2Reg); 6277 vpxor(tmp3Reg, tmp3Reg); 6278 vpxor(tmp4Reg, tmp4Reg); 6279 movdl(tmp1Reg, tmp5); 6280 pshufd(tmp1Reg, tmp1Reg, 0); 6281 } 6282 subptr(len, 8); 6283 jccb(Assembler::greater, L_copy_8_chars_exit); 6284 6285 bind(L_copy_8_chars); 6286 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 6287 ptest(tmp3Reg, tmp1Reg); 6288 jccb(Assembler::notZero, L_copy_8_chars_exit); 6289 packuswb(tmp3Reg, tmp1Reg); 6290 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 6291 addptr(len, 8); 6292 jccb(Assembler::lessEqual, L_copy_8_chars); 6293 6294 bind(L_copy_8_chars_exit); 6295 subptr(len, 8); 6296 jccb(Assembler::zero, L_done); 6297 } 6298 6299 bind(L_copy_1_char); 6300 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 6301 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 6302 jccb(Assembler::notZero, L_copy_1_char_exit); 6303 movb(Address(dst, len, Address::times_1, 0), tmp5); 6304 addptr(len, 1); 6305 jccb(Assembler::less, L_copy_1_char); 6306 6307 bind(L_copy_1_char_exit); 6308 addptr(result, len); // len is negative count of not processed elements 6309 6310 bind(L_done); 6311 } 6312 6313 #ifdef _LP64 6314 /** 6315 * Helper for multiply_to_len(). 6316 */ 6317 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 6318 addq(dest_lo, src1); 6319 adcq(dest_hi, 0); 6320 addq(dest_lo, src2); 6321 adcq(dest_hi, 0); 6322 } 6323 6324 /** 6325 * Multiply 64 bit by 64 bit first loop. 6326 */ 6327 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 6328 Register y, Register y_idx, Register z, 6329 Register carry, Register product, 6330 Register idx, Register kdx) { 6331 // 6332 // jlong carry, x[], y[], z[]; 6333 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6334 // huge_128 product = y[idx] * x[xstart] + carry; 6335 // z[kdx] = (jlong)product; 6336 // carry = (jlong)(product >>> 64); 6337 // } 6338 // z[xstart] = carry; 6339 // 6340 6341 Label L_first_loop, L_first_loop_exit; 6342 Label L_one_x, L_one_y, L_multiply; 6343 6344 decrementl(xstart); 6345 jcc(Assembler::negative, L_one_x); 6346 6347 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6348 rorq(x_xstart, 32); // convert big-endian to little-endian 6349 6350 bind(L_first_loop); 6351 decrementl(idx); 6352 jcc(Assembler::negative, L_first_loop_exit); 6353 decrementl(idx); 6354 jcc(Assembler::negative, L_one_y); 6355 movq(y_idx, Address(y, idx, Address::times_4, 0)); 6356 rorq(y_idx, 32); // convert big-endian to little-endian 6357 bind(L_multiply); 6358 movq(product, x_xstart); 6359 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 6360 addq(product, carry); 6361 adcq(rdx, 0); 6362 subl(kdx, 2); 6363 movl(Address(z, kdx, Address::times_4, 4), product); 6364 shrq(product, 32); 6365 movl(Address(z, kdx, Address::times_4, 0), product); 6366 movq(carry, rdx); 6367 jmp(L_first_loop); 6368 6369 bind(L_one_y); 6370 movl(y_idx, Address(y, 0)); 6371 jmp(L_multiply); 6372 6373 bind(L_one_x); 6374 movl(x_xstart, Address(x, 0)); 6375 jmp(L_first_loop); 6376 6377 bind(L_first_loop_exit); 6378 } 6379 6380 /** 6381 * Multiply 64 bit by 64 bit and add 128 bit. 6382 */ 6383 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 6384 Register yz_idx, Register idx, 6385 Register carry, Register product, int offset) { 6386 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 6387 // z[kdx] = (jlong)product; 6388 6389 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 6390 rorq(yz_idx, 32); // convert big-endian to little-endian 6391 movq(product, x_xstart); 6392 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6393 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 6394 rorq(yz_idx, 32); // convert big-endian to little-endian 6395 6396 add2_with_carry(rdx, product, carry, yz_idx); 6397 6398 movl(Address(z, idx, Address::times_4, offset+4), product); 6399 shrq(product, 32); 6400 movl(Address(z, idx, Address::times_4, offset), product); 6401 6402 } 6403 6404 /** 6405 * Multiply 128 bit by 128 bit. Unrolled inner loop. 6406 */ 6407 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 6408 Register yz_idx, Register idx, Register jdx, 6409 Register carry, Register product, 6410 Register carry2) { 6411 // jlong carry, x[], y[], z[]; 6412 // int kdx = ystart+1; 6413 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6414 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 6415 // z[kdx+idx+1] = (jlong)product; 6416 // jlong carry2 = (jlong)(product >>> 64); 6417 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 6418 // z[kdx+idx] = (jlong)product; 6419 // carry = (jlong)(product >>> 64); 6420 // } 6421 // idx += 2; 6422 // if (idx > 0) { 6423 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 6424 // z[kdx+idx] = (jlong)product; 6425 // carry = (jlong)(product >>> 64); 6426 // } 6427 // 6428 6429 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6430 6431 movl(jdx, idx); 6432 andl(jdx, 0xFFFFFFFC); 6433 shrl(jdx, 2); 6434 6435 bind(L_third_loop); 6436 subl(jdx, 1); 6437 jcc(Assembler::negative, L_third_loop_exit); 6438 subl(idx, 4); 6439 6440 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 6441 movq(carry2, rdx); 6442 6443 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 6444 movq(carry, rdx); 6445 jmp(L_third_loop); 6446 6447 bind (L_third_loop_exit); 6448 6449 andl (idx, 0x3); 6450 jcc(Assembler::zero, L_post_third_loop_done); 6451 6452 Label L_check_1; 6453 subl(idx, 2); 6454 jcc(Assembler::negative, L_check_1); 6455 6456 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 6457 movq(carry, rdx); 6458 6459 bind (L_check_1); 6460 addl (idx, 0x2); 6461 andl (idx, 0x1); 6462 subl(idx, 1); 6463 jcc(Assembler::negative, L_post_third_loop_done); 6464 6465 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 6466 movq(product, x_xstart); 6467 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6468 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 6469 6470 add2_with_carry(rdx, product, yz_idx, carry); 6471 6472 movl(Address(z, idx, Address::times_4, 0), product); 6473 shrq(product, 32); 6474 6475 shlq(rdx, 32); 6476 orq(product, rdx); 6477 movq(carry, product); 6478 6479 bind(L_post_third_loop_done); 6480 } 6481 6482 /** 6483 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 6484 * 6485 */ 6486 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 6487 Register carry, Register carry2, 6488 Register idx, Register jdx, 6489 Register yz_idx1, Register yz_idx2, 6490 Register tmp, Register tmp3, Register tmp4) { 6491 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 6492 6493 // jlong carry, x[], y[], z[]; 6494 // int kdx = ystart+1; 6495 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6496 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 6497 // jlong carry2 = (jlong)(tmp3 >>> 64); 6498 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 6499 // carry = (jlong)(tmp4 >>> 64); 6500 // z[kdx+idx+1] = (jlong)tmp3; 6501 // z[kdx+idx] = (jlong)tmp4; 6502 // } 6503 // idx += 2; 6504 // if (idx > 0) { 6505 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 6506 // z[kdx+idx] = (jlong)yz_idx1; 6507 // carry = (jlong)(yz_idx1 >>> 64); 6508 // } 6509 // 6510 6511 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6512 6513 movl(jdx, idx); 6514 andl(jdx, 0xFFFFFFFC); 6515 shrl(jdx, 2); 6516 6517 bind(L_third_loop); 6518 subl(jdx, 1); 6519 jcc(Assembler::negative, L_third_loop_exit); 6520 subl(idx, 4); 6521 6522 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 6523 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 6524 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 6525 rorxq(yz_idx2, yz_idx2, 32); 6526 6527 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6528 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 6529 6530 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 6531 rorxq(yz_idx1, yz_idx1, 32); 6532 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6533 rorxq(yz_idx2, yz_idx2, 32); 6534 6535 if (VM_Version::supports_adx()) { 6536 adcxq(tmp3, carry); 6537 adoxq(tmp3, yz_idx1); 6538 6539 adcxq(tmp4, tmp); 6540 adoxq(tmp4, yz_idx2); 6541 6542 movl(carry, 0); // does not affect flags 6543 adcxq(carry2, carry); 6544 adoxq(carry2, carry); 6545 } else { 6546 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 6547 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 6548 } 6549 movq(carry, carry2); 6550 6551 movl(Address(z, idx, Address::times_4, 12), tmp3); 6552 shrq(tmp3, 32); 6553 movl(Address(z, idx, Address::times_4, 8), tmp3); 6554 6555 movl(Address(z, idx, Address::times_4, 4), tmp4); 6556 shrq(tmp4, 32); 6557 movl(Address(z, idx, Address::times_4, 0), tmp4); 6558 6559 jmp(L_third_loop); 6560 6561 bind (L_third_loop_exit); 6562 6563 andl (idx, 0x3); 6564 jcc(Assembler::zero, L_post_third_loop_done); 6565 6566 Label L_check_1; 6567 subl(idx, 2); 6568 jcc(Assembler::negative, L_check_1); 6569 6570 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 6571 rorxq(yz_idx1, yz_idx1, 32); 6572 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6573 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6574 rorxq(yz_idx2, yz_idx2, 32); 6575 6576 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 6577 6578 movl(Address(z, idx, Address::times_4, 4), tmp3); 6579 shrq(tmp3, 32); 6580 movl(Address(z, idx, Address::times_4, 0), tmp3); 6581 movq(carry, tmp4); 6582 6583 bind (L_check_1); 6584 addl (idx, 0x2); 6585 andl (idx, 0x1); 6586 subl(idx, 1); 6587 jcc(Assembler::negative, L_post_third_loop_done); 6588 movl(tmp4, Address(y, idx, Address::times_4, 0)); 6589 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 6590 movl(tmp4, Address(z, idx, Address::times_4, 0)); 6591 6592 add2_with_carry(carry2, tmp3, tmp4, carry); 6593 6594 movl(Address(z, idx, Address::times_4, 0), tmp3); 6595 shrq(tmp3, 32); 6596 6597 shlq(carry2, 32); 6598 orq(tmp3, carry2); 6599 movq(carry, tmp3); 6600 6601 bind(L_post_third_loop_done); 6602 } 6603 6604 /** 6605 * Code for BigInteger::multiplyToLen() intrinsic. 6606 * 6607 * rdi: x 6608 * rax: xlen 6609 * rsi: y 6610 * rcx: ylen 6611 * r8: z 6612 * r11: zlen 6613 * r12: tmp1 6614 * r13: tmp2 6615 * r14: tmp3 6616 * r15: tmp4 6617 * rbx: tmp5 6618 * 6619 */ 6620 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, 6621 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 6622 ShortBranchVerifier sbv(this); 6623 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 6624 6625 push(tmp1); 6626 push(tmp2); 6627 push(tmp3); 6628 push(tmp4); 6629 push(tmp5); 6630 6631 push(xlen); 6632 push(zlen); 6633 6634 const Register idx = tmp1; 6635 const Register kdx = tmp2; 6636 const Register xstart = tmp3; 6637 6638 const Register y_idx = tmp4; 6639 const Register carry = tmp5; 6640 const Register product = xlen; 6641 const Register x_xstart = zlen; // reuse register 6642 6643 // First Loop. 6644 // 6645 // final static long LONG_MASK = 0xffffffffL; 6646 // int xstart = xlen - 1; 6647 // int ystart = ylen - 1; 6648 // long carry = 0; 6649 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6650 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 6651 // z[kdx] = (int)product; 6652 // carry = product >>> 32; 6653 // } 6654 // z[xstart] = (int)carry; 6655 // 6656 6657 movl(idx, ylen); // idx = ylen; 6658 movl(kdx, zlen); // kdx = xlen+ylen; 6659 xorq(carry, carry); // carry = 0; 6660 6661 Label L_done; 6662 6663 movl(xstart, xlen); 6664 decrementl(xstart); 6665 jcc(Assembler::negative, L_done); 6666 6667 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 6668 6669 Label L_second_loop; 6670 testl(kdx, kdx); 6671 jcc(Assembler::zero, L_second_loop); 6672 6673 Label L_carry; 6674 subl(kdx, 1); 6675 jcc(Assembler::zero, L_carry); 6676 6677 movl(Address(z, kdx, Address::times_4, 0), carry); 6678 shrq(carry, 32); 6679 subl(kdx, 1); 6680 6681 bind(L_carry); 6682 movl(Address(z, kdx, Address::times_4, 0), carry); 6683 6684 // Second and third (nested) loops. 6685 // 6686 // for (int i = xstart-1; i >= 0; i--) { // Second loop 6687 // carry = 0; 6688 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 6689 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 6690 // (z[k] & LONG_MASK) + carry; 6691 // z[k] = (int)product; 6692 // carry = product >>> 32; 6693 // } 6694 // z[i] = (int)carry; 6695 // } 6696 // 6697 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 6698 6699 const Register jdx = tmp1; 6700 6701 bind(L_second_loop); 6702 xorl(carry, carry); // carry = 0; 6703 movl(jdx, ylen); // j = ystart+1 6704 6705 subl(xstart, 1); // i = xstart-1; 6706 jcc(Assembler::negative, L_done); 6707 6708 push (z); 6709 6710 Label L_last_x; 6711 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 6712 subl(xstart, 1); // i = xstart-1; 6713 jcc(Assembler::negative, L_last_x); 6714 6715 if (UseBMI2Instructions) { 6716 movq(rdx, Address(x, xstart, Address::times_4, 0)); 6717 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 6718 } else { 6719 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6720 rorq(x_xstart, 32); // convert big-endian to little-endian 6721 } 6722 6723 Label L_third_loop_prologue; 6724 bind(L_third_loop_prologue); 6725 6726 push (x); 6727 push (xstart); 6728 push (ylen); 6729 6730 6731 if (UseBMI2Instructions) { 6732 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 6733 } else { // !UseBMI2Instructions 6734 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 6735 } 6736 6737 pop(ylen); 6738 pop(xlen); 6739 pop(x); 6740 pop(z); 6741 6742 movl(tmp3, xlen); 6743 addl(tmp3, 1); 6744 movl(Address(z, tmp3, Address::times_4, 0), carry); 6745 subl(tmp3, 1); 6746 jccb(Assembler::negative, L_done); 6747 6748 shrq(carry, 32); 6749 movl(Address(z, tmp3, Address::times_4, 0), carry); 6750 jmp(L_second_loop); 6751 6752 // Next infrequent code is moved outside loops. 6753 bind(L_last_x); 6754 if (UseBMI2Instructions) { 6755 movl(rdx, Address(x, 0)); 6756 } else { 6757 movl(x_xstart, Address(x, 0)); 6758 } 6759 jmp(L_third_loop_prologue); 6760 6761 bind(L_done); 6762 6763 pop(zlen); 6764 pop(xlen); 6765 6766 pop(tmp5); 6767 pop(tmp4); 6768 pop(tmp3); 6769 pop(tmp2); 6770 pop(tmp1); 6771 } 6772 6773 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 6774 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 6775 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 6776 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 6777 Label VECTOR8_TAIL, VECTOR4_TAIL; 6778 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 6779 Label SAME_TILL_END, DONE; 6780 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 6781 6782 //scale is in rcx in both Win64 and Unix 6783 ShortBranchVerifier sbv(this); 6784 6785 shlq(length); 6786 xorq(result, result); 6787 6788 if ((AVX3Threshold == 0) && (UseAVX > 2) && 6789 VM_Version::supports_avx512vlbw()) { 6790 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 6791 6792 cmpq(length, 64); 6793 jcc(Assembler::less, VECTOR32_TAIL); 6794 6795 movq(tmp1, length); 6796 andq(tmp1, 0x3F); // tail count 6797 andq(length, ~(0x3F)); //vector count 6798 6799 bind(VECTOR64_LOOP); 6800 // AVX512 code to compare 64 byte vectors. 6801 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 6802 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 6803 kortestql(k7, k7); 6804 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 6805 addq(result, 64); 6806 subq(length, 64); 6807 jccb(Assembler::notZero, VECTOR64_LOOP); 6808 6809 //bind(VECTOR64_TAIL); 6810 testq(tmp1, tmp1); 6811 jcc(Assembler::zero, SAME_TILL_END); 6812 6813 //bind(VECTOR64_TAIL); 6814 // AVX512 code to compare up to 63 byte vectors. 6815 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 6816 shlxq(tmp2, tmp2, tmp1); 6817 notq(tmp2); 6818 kmovql(k3, tmp2); 6819 6820 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 6821 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 6822 6823 ktestql(k7, k3); 6824 jcc(Assembler::below, SAME_TILL_END); // not mismatch 6825 6826 bind(VECTOR64_NOT_EQUAL); 6827 kmovql(tmp1, k7); 6828 notq(tmp1); 6829 tzcntq(tmp1, tmp1); 6830 addq(result, tmp1); 6831 shrq(result); 6832 jmp(DONE); 6833 bind(VECTOR32_TAIL); 6834 } 6835 6836 cmpq(length, 8); 6837 jcc(Assembler::equal, VECTOR8_LOOP); 6838 jcc(Assembler::less, VECTOR4_TAIL); 6839 6840 if (UseAVX >= 2) { 6841 Label VECTOR16_TAIL, VECTOR32_LOOP; 6842 6843 cmpq(length, 16); 6844 jcc(Assembler::equal, VECTOR16_LOOP); 6845 jcc(Assembler::less, VECTOR8_LOOP); 6846 6847 cmpq(length, 32); 6848 jccb(Assembler::less, VECTOR16_TAIL); 6849 6850 subq(length, 32); 6851 bind(VECTOR32_LOOP); 6852 vmovdqu(rymm0, Address(obja, result)); 6853 vmovdqu(rymm1, Address(objb, result)); 6854 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 6855 vptest(rymm2, rymm2); 6856 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 6857 addq(result, 32); 6858 subq(length, 32); 6859 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 6860 addq(length, 32); 6861 jcc(Assembler::equal, SAME_TILL_END); 6862 //falling through if less than 32 bytes left //close the branch here. 6863 6864 bind(VECTOR16_TAIL); 6865 cmpq(length, 16); 6866 jccb(Assembler::less, VECTOR8_TAIL); 6867 bind(VECTOR16_LOOP); 6868 movdqu(rymm0, Address(obja, result)); 6869 movdqu(rymm1, Address(objb, result)); 6870 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 6871 ptest(rymm2, rymm2); 6872 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6873 addq(result, 16); 6874 subq(length, 16); 6875 jcc(Assembler::equal, SAME_TILL_END); 6876 //falling through if less than 16 bytes left 6877 } else {//regular intrinsics 6878 6879 cmpq(length, 16); 6880 jccb(Assembler::less, VECTOR8_TAIL); 6881 6882 subq(length, 16); 6883 bind(VECTOR16_LOOP); 6884 movdqu(rymm0, Address(obja, result)); 6885 movdqu(rymm1, Address(objb, result)); 6886 pxor(rymm0, rymm1); 6887 ptest(rymm0, rymm0); 6888 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6889 addq(result, 16); 6890 subq(length, 16); 6891 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 6892 addq(length, 16); 6893 jcc(Assembler::equal, SAME_TILL_END); 6894 //falling through if less than 16 bytes left 6895 } 6896 6897 bind(VECTOR8_TAIL); 6898 cmpq(length, 8); 6899 jccb(Assembler::less, VECTOR4_TAIL); 6900 bind(VECTOR8_LOOP); 6901 movq(tmp1, Address(obja, result)); 6902 movq(tmp2, Address(objb, result)); 6903 xorq(tmp1, tmp2); 6904 testq(tmp1, tmp1); 6905 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 6906 addq(result, 8); 6907 subq(length, 8); 6908 jcc(Assembler::equal, SAME_TILL_END); 6909 //falling through if less than 8 bytes left 6910 6911 bind(VECTOR4_TAIL); 6912 cmpq(length, 4); 6913 jccb(Assembler::less, BYTES_TAIL); 6914 bind(VECTOR4_LOOP); 6915 movl(tmp1, Address(obja, result)); 6916 xorl(tmp1, Address(objb, result)); 6917 testl(tmp1, tmp1); 6918 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 6919 addq(result, 4); 6920 subq(length, 4); 6921 jcc(Assembler::equal, SAME_TILL_END); 6922 //falling through if less than 4 bytes left 6923 6924 bind(BYTES_TAIL); 6925 bind(BYTES_LOOP); 6926 load_unsigned_byte(tmp1, Address(obja, result)); 6927 load_unsigned_byte(tmp2, Address(objb, result)); 6928 xorl(tmp1, tmp2); 6929 testl(tmp1, tmp1); 6930 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6931 decq(length); 6932 jcc(Assembler::zero, SAME_TILL_END); 6933 incq(result); 6934 load_unsigned_byte(tmp1, Address(obja, result)); 6935 load_unsigned_byte(tmp2, Address(objb, result)); 6936 xorl(tmp1, tmp2); 6937 testl(tmp1, tmp1); 6938 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6939 decq(length); 6940 jcc(Assembler::zero, SAME_TILL_END); 6941 incq(result); 6942 load_unsigned_byte(tmp1, Address(obja, result)); 6943 load_unsigned_byte(tmp2, Address(objb, result)); 6944 xorl(tmp1, tmp2); 6945 testl(tmp1, tmp1); 6946 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6947 jmp(SAME_TILL_END); 6948 6949 if (UseAVX >= 2) { 6950 bind(VECTOR32_NOT_EQUAL); 6951 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 6952 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 6953 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 6954 vpmovmskb(tmp1, rymm0); 6955 bsfq(tmp1, tmp1); 6956 addq(result, tmp1); 6957 shrq(result); 6958 jmp(DONE); 6959 } 6960 6961 bind(VECTOR16_NOT_EQUAL); 6962 if (UseAVX >= 2) { 6963 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 6964 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 6965 pxor(rymm0, rymm2); 6966 } else { 6967 pcmpeqb(rymm2, rymm2); 6968 pxor(rymm0, rymm1); 6969 pcmpeqb(rymm0, rymm1); 6970 pxor(rymm0, rymm2); 6971 } 6972 pmovmskb(tmp1, rymm0); 6973 bsfq(tmp1, tmp1); 6974 addq(result, tmp1); 6975 shrq(result); 6976 jmpb(DONE); 6977 6978 bind(VECTOR8_NOT_EQUAL); 6979 bind(VECTOR4_NOT_EQUAL); 6980 bsfq(tmp1, tmp1); 6981 shrq(tmp1, 3); 6982 addq(result, tmp1); 6983 bind(BYTES_NOT_EQUAL); 6984 shrq(result); 6985 jmpb(DONE); 6986 6987 bind(SAME_TILL_END); 6988 mov64(result, -1); 6989 6990 bind(DONE); 6991 } 6992 6993 //Helper functions for square_to_len() 6994 6995 /** 6996 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 6997 * Preserves x and z and modifies rest of the registers. 6998 */ 6999 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7000 // Perform square and right shift by 1 7001 // Handle odd xlen case first, then for even xlen do the following 7002 // jlong carry = 0; 7003 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 7004 // huge_128 product = x[j:j+1] * x[j:j+1]; 7005 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 7006 // z[i+2:i+3] = (jlong)(product >>> 1); 7007 // carry = (jlong)product; 7008 // } 7009 7010 xorq(tmp5, tmp5); // carry 7011 xorq(rdxReg, rdxReg); 7012 xorl(tmp1, tmp1); // index for x 7013 xorl(tmp4, tmp4); // index for z 7014 7015 Label L_first_loop, L_first_loop_exit; 7016 7017 testl(xlen, 1); 7018 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 7019 7020 // Square and right shift by 1 the odd element using 32 bit multiply 7021 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 7022 imulq(raxReg, raxReg); 7023 shrq(raxReg, 1); 7024 adcq(tmp5, 0); 7025 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 7026 incrementl(tmp1); 7027 addl(tmp4, 2); 7028 7029 // Square and right shift by 1 the rest using 64 bit multiply 7030 bind(L_first_loop); 7031 cmpptr(tmp1, xlen); 7032 jccb(Assembler::equal, L_first_loop_exit); 7033 7034 // Square 7035 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 7036 rorq(raxReg, 32); // convert big-endian to little-endian 7037 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 7038 7039 // Right shift by 1 and save carry 7040 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 7041 rcrq(rdxReg, 1); 7042 rcrq(raxReg, 1); 7043 adcq(tmp5, 0); 7044 7045 // Store result in z 7046 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 7047 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 7048 7049 // Update indices for x and z 7050 addl(tmp1, 2); 7051 addl(tmp4, 4); 7052 jmp(L_first_loop); 7053 7054 bind(L_first_loop_exit); 7055 } 7056 7057 7058 /** 7059 * Perform the following multiply add operation using BMI2 instructions 7060 * carry:sum = sum + op1*op2 + carry 7061 * op2 should be in rdx 7062 * op2 is preserved, all other registers are modified 7063 */ 7064 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 7065 // assert op2 is rdx 7066 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 7067 addq(sum, carry); 7068 adcq(tmp2, 0); 7069 addq(sum, op1); 7070 adcq(tmp2, 0); 7071 movq(carry, tmp2); 7072 } 7073 7074 /** 7075 * Perform the following multiply add operation: 7076 * carry:sum = sum + op1*op2 + carry 7077 * Preserves op1, op2 and modifies rest of registers 7078 */ 7079 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 7080 // rdx:rax = op1 * op2 7081 movq(raxReg, op2); 7082 mulq(op1); 7083 7084 // rdx:rax = sum + carry + rdx:rax 7085 addq(sum, carry); 7086 adcq(rdxReg, 0); 7087 addq(sum, raxReg); 7088 adcq(rdxReg, 0); 7089 7090 // carry:sum = rdx:sum 7091 movq(carry, rdxReg); 7092 } 7093 7094 /** 7095 * Add 64 bit long carry into z[] with carry propagation. 7096 * Preserves z and carry register values and modifies rest of registers. 7097 * 7098 */ 7099 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 7100 Label L_fourth_loop, L_fourth_loop_exit; 7101 7102 movl(tmp1, 1); 7103 subl(zlen, 2); 7104 addq(Address(z, zlen, Address::times_4, 0), carry); 7105 7106 bind(L_fourth_loop); 7107 jccb(Assembler::carryClear, L_fourth_loop_exit); 7108 subl(zlen, 2); 7109 jccb(Assembler::negative, L_fourth_loop_exit); 7110 addq(Address(z, zlen, Address::times_4, 0), tmp1); 7111 jmp(L_fourth_loop); 7112 bind(L_fourth_loop_exit); 7113 } 7114 7115 /** 7116 * Shift z[] left by 1 bit. 7117 * Preserves x, len, z and zlen registers and modifies rest of the registers. 7118 * 7119 */ 7120 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 7121 7122 Label L_fifth_loop, L_fifth_loop_exit; 7123 7124 // Fifth loop 7125 // Perform primitiveLeftShift(z, zlen, 1) 7126 7127 const Register prev_carry = tmp1; 7128 const Register new_carry = tmp4; 7129 const Register value = tmp2; 7130 const Register zidx = tmp3; 7131 7132 // int zidx, carry; 7133 // long value; 7134 // carry = 0; 7135 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 7136 // (carry:value) = (z[i] << 1) | carry ; 7137 // z[i] = value; 7138 // } 7139 7140 movl(zidx, zlen); 7141 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 7142 7143 bind(L_fifth_loop); 7144 decl(zidx); // Use decl to preserve carry flag 7145 decl(zidx); 7146 jccb(Assembler::negative, L_fifth_loop_exit); 7147 7148 if (UseBMI2Instructions) { 7149 movq(value, Address(z, zidx, Address::times_4, 0)); 7150 rclq(value, 1); 7151 rorxq(value, value, 32); 7152 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7153 } 7154 else { 7155 // clear new_carry 7156 xorl(new_carry, new_carry); 7157 7158 // Shift z[i] by 1, or in previous carry and save new carry 7159 movq(value, Address(z, zidx, Address::times_4, 0)); 7160 shlq(value, 1); 7161 adcl(new_carry, 0); 7162 7163 orq(value, prev_carry); 7164 rorq(value, 0x20); 7165 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7166 7167 // Set previous carry = new carry 7168 movl(prev_carry, new_carry); 7169 } 7170 jmp(L_fifth_loop); 7171 7172 bind(L_fifth_loop_exit); 7173 } 7174 7175 7176 /** 7177 * Code for BigInteger::squareToLen() intrinsic 7178 * 7179 * rdi: x 7180 * rsi: len 7181 * r8: z 7182 * rcx: zlen 7183 * r12: tmp1 7184 * r13: tmp2 7185 * r14: tmp3 7186 * r15: tmp4 7187 * rbx: tmp5 7188 * 7189 */ 7190 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7191 7192 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7193 push(tmp1); 7194 push(tmp2); 7195 push(tmp3); 7196 push(tmp4); 7197 push(tmp5); 7198 7199 // First loop 7200 // Store the squares, right shifted one bit (i.e., divided by 2). 7201 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7202 7203 // Add in off-diagonal sums. 7204 // 7205 // Second, third (nested) and fourth loops. 7206 // zlen +=2; 7207 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7208 // carry = 0; 7209 // long op2 = x[xidx:xidx+1]; 7210 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7211 // k -= 2; 7212 // long op1 = x[j:j+1]; 7213 // long sum = z[k:k+1]; 7214 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7215 // z[k:k+1] = sum; 7216 // } 7217 // add_one_64(z, k, carry, tmp_regs); 7218 // } 7219 7220 const Register carry = tmp5; 7221 const Register sum = tmp3; 7222 const Register op1 = tmp4; 7223 Register op2 = tmp2; 7224 7225 push(zlen); 7226 push(len); 7227 addl(zlen,2); 7228 bind(L_second_loop); 7229 xorq(carry, carry); 7230 subl(zlen, 4); 7231 subl(len, 2); 7232 push(zlen); 7233 push(len); 7234 cmpl(len, 0); 7235 jccb(Assembler::lessEqual, L_second_loop_exit); 7236 7237 // Multiply an array by one 64 bit long. 7238 if (UseBMI2Instructions) { 7239 op2 = rdxReg; 7240 movq(op2, Address(x, len, Address::times_4, 0)); 7241 rorxq(op2, op2, 32); 7242 } 7243 else { 7244 movq(op2, Address(x, len, Address::times_4, 0)); 7245 rorq(op2, 32); 7246 } 7247 7248 bind(L_third_loop); 7249 decrementl(len); 7250 jccb(Assembler::negative, L_third_loop_exit); 7251 decrementl(len); 7252 jccb(Assembler::negative, L_last_x); 7253 7254 movq(op1, Address(x, len, Address::times_4, 0)); 7255 rorq(op1, 32); 7256 7257 bind(L_multiply); 7258 subl(zlen, 2); 7259 movq(sum, Address(z, zlen, Address::times_4, 0)); 7260 7261 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7262 if (UseBMI2Instructions) { 7263 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 7264 } 7265 else { 7266 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7267 } 7268 7269 movq(Address(z, zlen, Address::times_4, 0), sum); 7270 7271 jmp(L_third_loop); 7272 bind(L_third_loop_exit); 7273 7274 // Fourth loop 7275 // Add 64 bit long carry into z with carry propagation. 7276 // Uses offsetted zlen. 7277 add_one_64(z, zlen, carry, tmp1); 7278 7279 pop(len); 7280 pop(zlen); 7281 jmp(L_second_loop); 7282 7283 // Next infrequent code is moved outside loops. 7284 bind(L_last_x); 7285 movl(op1, Address(x, 0)); 7286 jmp(L_multiply); 7287 7288 bind(L_second_loop_exit); 7289 pop(len); 7290 pop(zlen); 7291 pop(len); 7292 pop(zlen); 7293 7294 // Fifth loop 7295 // Shift z left 1 bit. 7296 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 7297 7298 // z[zlen-1] |= x[len-1] & 1; 7299 movl(tmp3, Address(x, len, Address::times_4, -4)); 7300 andl(tmp3, 1); 7301 orl(Address(z, zlen, Address::times_4, -4), tmp3); 7302 7303 pop(tmp5); 7304 pop(tmp4); 7305 pop(tmp3); 7306 pop(tmp2); 7307 pop(tmp1); 7308 } 7309 7310 /** 7311 * Helper function for mul_add() 7312 * Multiply the in[] by int k and add to out[] starting at offset offs using 7313 * 128 bit by 32 bit multiply and return the carry in tmp5. 7314 * Only quad int aligned length of in[] is operated on in this function. 7315 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 7316 * This function preserves out, in and k registers. 7317 * len and offset point to the appropriate index in "in" & "out" correspondingly 7318 * tmp5 has the carry. 7319 * other registers are temporary and are modified. 7320 * 7321 */ 7322 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 7323 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 7324 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7325 7326 Label L_first_loop, L_first_loop_exit; 7327 7328 movl(tmp1, len); 7329 shrl(tmp1, 2); 7330 7331 bind(L_first_loop); 7332 subl(tmp1, 1); 7333 jccb(Assembler::negative, L_first_loop_exit); 7334 7335 subl(len, 4); 7336 subl(offset, 4); 7337 7338 Register op2 = tmp2; 7339 const Register sum = tmp3; 7340 const Register op1 = tmp4; 7341 const Register carry = tmp5; 7342 7343 if (UseBMI2Instructions) { 7344 op2 = rdxReg; 7345 } 7346 7347 movq(op1, Address(in, len, Address::times_4, 8)); 7348 rorq(op1, 32); 7349 movq(sum, Address(out, offset, Address::times_4, 8)); 7350 rorq(sum, 32); 7351 if (UseBMI2Instructions) { 7352 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7353 } 7354 else { 7355 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7356 } 7357 // Store back in big endian from little endian 7358 rorq(sum, 0x20); 7359 movq(Address(out, offset, Address::times_4, 8), sum); 7360 7361 movq(op1, Address(in, len, Address::times_4, 0)); 7362 rorq(op1, 32); 7363 movq(sum, Address(out, offset, Address::times_4, 0)); 7364 rorq(sum, 32); 7365 if (UseBMI2Instructions) { 7366 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7367 } 7368 else { 7369 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7370 } 7371 // Store back in big endian from little endian 7372 rorq(sum, 0x20); 7373 movq(Address(out, offset, Address::times_4, 0), sum); 7374 7375 jmp(L_first_loop); 7376 bind(L_first_loop_exit); 7377 } 7378 7379 /** 7380 * Code for BigInteger::mulAdd() intrinsic 7381 * 7382 * rdi: out 7383 * rsi: in 7384 * r11: offs (out.length - offset) 7385 * rcx: len 7386 * r8: k 7387 * r12: tmp1 7388 * r13: tmp2 7389 * r14: tmp3 7390 * r15: tmp4 7391 * rbx: tmp5 7392 * Multiply the in[] by word k and add to out[], return the carry in rax 7393 */ 7394 void MacroAssembler::mul_add(Register out, Register in, Register offs, 7395 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 7396 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7397 7398 Label L_carry, L_last_in, L_done; 7399 7400 // carry = 0; 7401 // for (int j=len-1; j >= 0; j--) { 7402 // long product = (in[j] & LONG_MASK) * kLong + 7403 // (out[offs] & LONG_MASK) + carry; 7404 // out[offs--] = (int)product; 7405 // carry = product >>> 32; 7406 // } 7407 // 7408 push(tmp1); 7409 push(tmp2); 7410 push(tmp3); 7411 push(tmp4); 7412 push(tmp5); 7413 7414 Register op2 = tmp2; 7415 const Register sum = tmp3; 7416 const Register op1 = tmp4; 7417 const Register carry = tmp5; 7418 7419 if (UseBMI2Instructions) { 7420 op2 = rdxReg; 7421 movl(op2, k); 7422 } 7423 else { 7424 movl(op2, k); 7425 } 7426 7427 xorq(carry, carry); 7428 7429 //First loop 7430 7431 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 7432 //The carry is in tmp5 7433 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 7434 7435 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 7436 decrementl(len); 7437 jccb(Assembler::negative, L_carry); 7438 decrementl(len); 7439 jccb(Assembler::negative, L_last_in); 7440 7441 movq(op1, Address(in, len, Address::times_4, 0)); 7442 rorq(op1, 32); 7443 7444 subl(offs, 2); 7445 movq(sum, Address(out, offs, Address::times_4, 0)); 7446 rorq(sum, 32); 7447 7448 if (UseBMI2Instructions) { 7449 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7450 } 7451 else { 7452 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7453 } 7454 7455 // Store back in big endian from little endian 7456 rorq(sum, 0x20); 7457 movq(Address(out, offs, Address::times_4, 0), sum); 7458 7459 testl(len, len); 7460 jccb(Assembler::zero, L_carry); 7461 7462 //Multiply the last in[] entry, if any 7463 bind(L_last_in); 7464 movl(op1, Address(in, 0)); 7465 movl(sum, Address(out, offs, Address::times_4, -4)); 7466 7467 movl(raxReg, k); 7468 mull(op1); //tmp4 * eax -> edx:eax 7469 addl(sum, carry); 7470 adcl(rdxReg, 0); 7471 addl(sum, raxReg); 7472 adcl(rdxReg, 0); 7473 movl(carry, rdxReg); 7474 7475 movl(Address(out, offs, Address::times_4, -4), sum); 7476 7477 bind(L_carry); 7478 //return tmp5/carry as carry in rax 7479 movl(rax, carry); 7480 7481 bind(L_done); 7482 pop(tmp5); 7483 pop(tmp4); 7484 pop(tmp3); 7485 pop(tmp2); 7486 pop(tmp1); 7487 } 7488 #endif 7489 7490 /** 7491 * Emits code to update CRC-32 with a byte value according to constants in table 7492 * 7493 * @param [in,out]crc Register containing the crc. 7494 * @param [in]val Register containing the byte to fold into the CRC. 7495 * @param [in]table Register containing the table of crc constants. 7496 * 7497 * uint32_t crc; 7498 * val = crc_table[(val ^ crc) & 0xFF]; 7499 * crc = val ^ (crc >> 8); 7500 * 7501 */ 7502 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 7503 xorl(val, crc); 7504 andl(val, 0xFF); 7505 shrl(crc, 8); // unsigned shift 7506 xorl(crc, Address(table, val, Address::times_4, 0)); 7507 } 7508 7509 /** 7510 * Fold 128-bit data chunk 7511 */ 7512 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 7513 if (UseAVX > 0) { 7514 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 7515 vpclmulldq(xcrc, xK, xcrc); // [63:0] 7516 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 7517 pxor(xcrc, xtmp); 7518 } else { 7519 movdqa(xtmp, xcrc); 7520 pclmulhdq(xtmp, xK); // [123:64] 7521 pclmulldq(xcrc, xK); // [63:0] 7522 pxor(xcrc, xtmp); 7523 movdqu(xtmp, Address(buf, offset)); 7524 pxor(xcrc, xtmp); 7525 } 7526 } 7527 7528 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 7529 if (UseAVX > 0) { 7530 vpclmulhdq(xtmp, xK, xcrc); 7531 vpclmulldq(xcrc, xK, xcrc); 7532 pxor(xcrc, xbuf); 7533 pxor(xcrc, xtmp); 7534 } else { 7535 movdqa(xtmp, xcrc); 7536 pclmulhdq(xtmp, xK); 7537 pclmulldq(xcrc, xK); 7538 pxor(xcrc, xbuf); 7539 pxor(xcrc, xtmp); 7540 } 7541 } 7542 7543 /** 7544 * 8-bit folds to compute 32-bit CRC 7545 * 7546 * uint64_t xcrc; 7547 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 7548 */ 7549 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 7550 movdl(tmp, xcrc); 7551 andl(tmp, 0xFF); 7552 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 7553 psrldq(xcrc, 1); // unsigned shift one byte 7554 pxor(xcrc, xtmp); 7555 } 7556 7557 /** 7558 * uint32_t crc; 7559 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 7560 */ 7561 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 7562 movl(tmp, crc); 7563 andl(tmp, 0xFF); 7564 shrl(crc, 8); 7565 xorl(crc, Address(table, tmp, Address::times_4, 0)); 7566 } 7567 7568 /** 7569 * @param crc register containing existing CRC (32-bit) 7570 * @param buf register pointing to input byte buffer (byte*) 7571 * @param len register containing number of bytes 7572 * @param table register that will contain address of CRC table 7573 * @param tmp scratch register 7574 */ 7575 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 7576 assert_different_registers(crc, buf, len, table, tmp, rax); 7577 7578 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7579 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7580 7581 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7582 // context for the registers used, where all instructions below are using 128-bit mode 7583 // On EVEX without VL and BW, these instructions will all be AVX. 7584 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 7585 notl(crc); // ~crc 7586 cmpl(len, 16); 7587 jcc(Assembler::less, L_tail); 7588 7589 // Align buffer to 16 bytes 7590 movl(tmp, buf); 7591 andl(tmp, 0xF); 7592 jccb(Assembler::zero, L_aligned); 7593 subl(tmp, 16); 7594 addl(len, tmp); 7595 7596 align(4); 7597 BIND(L_align_loop); 7598 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7599 update_byte_crc32(crc, rax, table); 7600 increment(buf); 7601 incrementl(tmp); 7602 jccb(Assembler::less, L_align_loop); 7603 7604 BIND(L_aligned); 7605 movl(tmp, len); // save 7606 shrl(len, 4); 7607 jcc(Assembler::zero, L_tail_restore); 7608 7609 // Fold crc into first bytes of vector 7610 movdqa(xmm1, Address(buf, 0)); 7611 movdl(rax, xmm1); 7612 xorl(crc, rax); 7613 if (VM_Version::supports_sse4_1()) { 7614 pinsrd(xmm1, crc, 0); 7615 } else { 7616 pinsrw(xmm1, crc, 0); 7617 shrl(crc, 16); 7618 pinsrw(xmm1, crc, 1); 7619 } 7620 addptr(buf, 16); 7621 subl(len, 4); // len > 0 7622 jcc(Assembler::less, L_fold_tail); 7623 7624 movdqa(xmm2, Address(buf, 0)); 7625 movdqa(xmm3, Address(buf, 16)); 7626 movdqa(xmm4, Address(buf, 32)); 7627 addptr(buf, 48); 7628 subl(len, 3); 7629 jcc(Assembler::lessEqual, L_fold_512b); 7630 7631 // Fold total 512 bits of polynomial on each iteration, 7632 // 128 bits per each of 4 parallel streams. 7633 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 7634 7635 align32(); 7636 BIND(L_fold_512b_loop); 7637 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7638 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 7639 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 7640 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 7641 addptr(buf, 64); 7642 subl(len, 4); 7643 jcc(Assembler::greater, L_fold_512b_loop); 7644 7645 // Fold 512 bits to 128 bits. 7646 BIND(L_fold_512b); 7647 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7648 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 7649 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 7650 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 7651 7652 // Fold the rest of 128 bits data chunks 7653 BIND(L_fold_tail); 7654 addl(len, 3); 7655 jccb(Assembler::lessEqual, L_fold_128b); 7656 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7657 7658 BIND(L_fold_tail_loop); 7659 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7660 addptr(buf, 16); 7661 decrementl(len); 7662 jccb(Assembler::greater, L_fold_tail_loop); 7663 7664 // Fold 128 bits in xmm1 down into 32 bits in crc register. 7665 BIND(L_fold_128b); 7666 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 7667 if (UseAVX > 0) { 7668 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 7669 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 7670 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 7671 } else { 7672 movdqa(xmm2, xmm0); 7673 pclmulqdq(xmm2, xmm1, 0x1); 7674 movdqa(xmm3, xmm0); 7675 pand(xmm3, xmm2); 7676 pclmulqdq(xmm0, xmm3, 0x1); 7677 } 7678 psrldq(xmm1, 8); 7679 psrldq(xmm2, 4); 7680 pxor(xmm0, xmm1); 7681 pxor(xmm0, xmm2); 7682 7683 // 8 8-bit folds to compute 32-bit CRC. 7684 for (int j = 0; j < 4; j++) { 7685 fold_8bit_crc32(xmm0, table, xmm1, rax); 7686 } 7687 movdl(crc, xmm0); // mov 32 bits to general register 7688 for (int j = 0; j < 4; j++) { 7689 fold_8bit_crc32(crc, table, rax); 7690 } 7691 7692 BIND(L_tail_restore); 7693 movl(len, tmp); // restore 7694 BIND(L_tail); 7695 andl(len, 0xf); 7696 jccb(Assembler::zero, L_exit); 7697 7698 // Fold the rest of bytes 7699 align(4); 7700 BIND(L_tail_loop); 7701 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7702 update_byte_crc32(crc, rax, table); 7703 increment(buf); 7704 decrementl(len); 7705 jccb(Assembler::greater, L_tail_loop); 7706 7707 BIND(L_exit); 7708 notl(crc); // ~c 7709 } 7710 7711 #ifdef _LP64 7712 // Helper function for AVX 512 CRC32 7713 // Fold 512-bit data chunks 7714 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 7715 Register pos, int offset) { 7716 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 7717 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 7718 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 7719 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 7720 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 7721 } 7722 7723 // Helper function for AVX 512 CRC32 7724 // Compute CRC32 for < 256B buffers 7725 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 7726 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 7727 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 7728 7729 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 7730 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 7731 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 7732 7733 // check if there is enough buffer to be able to fold 16B at a time 7734 cmpl(len, 32); 7735 jcc(Assembler::less, L_less_than_32); 7736 7737 // if there is, load the constants 7738 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 7739 movdl(xmm0, crc); // get the initial crc value 7740 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7741 pxor(xmm7, xmm0); 7742 7743 // update the buffer pointer 7744 addl(pos, 16); 7745 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 7746 subl(len, 32); 7747 jmp(L_16B_reduction_loop); 7748 7749 bind(L_less_than_32); 7750 //mov initial crc to the return value. this is necessary for zero - length buffers. 7751 movl(rax, crc); 7752 testl(len, len); 7753 jcc(Assembler::equal, L_cleanup); 7754 7755 movdl(xmm0, crc); //get the initial crc value 7756 7757 cmpl(len, 16); 7758 jcc(Assembler::equal, L_exact_16_left); 7759 jcc(Assembler::less, L_less_than_16_left); 7760 7761 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7762 pxor(xmm7, xmm0); //xor the initial crc value 7763 addl(pos, 16); 7764 subl(len, 16); 7765 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 7766 jmp(L_get_last_two_xmms); 7767 7768 bind(L_less_than_16_left); 7769 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 7770 pxor(xmm1, xmm1); 7771 movptr(tmp1, rsp); 7772 movdqu(Address(tmp1, 0 * 16), xmm1); 7773 7774 cmpl(len, 4); 7775 jcc(Assembler::less, L_only_less_than_4); 7776 7777 //backup the counter value 7778 movl(tmp2, len); 7779 cmpl(len, 8); 7780 jcc(Assembler::less, L_less_than_8_left); 7781 7782 //load 8 Bytes 7783 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 7784 movq(Address(tmp1, 0 * 16), rax); 7785 addptr(tmp1, 8); 7786 subl(len, 8); 7787 addl(pos, 8); 7788 7789 bind(L_less_than_8_left); 7790 cmpl(len, 4); 7791 jcc(Assembler::less, L_less_than_4_left); 7792 7793 //load 4 Bytes 7794 movl(rax, Address(buf, pos, Address::times_1, 0)); 7795 movl(Address(tmp1, 0 * 16), rax); 7796 addptr(tmp1, 4); 7797 subl(len, 4); 7798 addl(pos, 4); 7799 7800 bind(L_less_than_4_left); 7801 cmpl(len, 2); 7802 jcc(Assembler::less, L_less_than_2_left); 7803 7804 // load 2 Bytes 7805 movw(rax, Address(buf, pos, Address::times_1, 0)); 7806 movl(Address(tmp1, 0 * 16), rax); 7807 addptr(tmp1, 2); 7808 subl(len, 2); 7809 addl(pos, 2); 7810 7811 bind(L_less_than_2_left); 7812 cmpl(len, 1); 7813 jcc(Assembler::less, L_zero_left); 7814 7815 // load 1 Byte 7816 movb(rax, Address(buf, pos, Address::times_1, 0)); 7817 movb(Address(tmp1, 0 * 16), rax); 7818 7819 bind(L_zero_left); 7820 movdqu(xmm7, Address(rsp, 0)); 7821 pxor(xmm7, xmm0); //xor the initial crc value 7822 7823 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7824 movdqu(xmm0, Address(rax, tmp2)); 7825 pshufb(xmm7, xmm0); 7826 jmp(L_128_done); 7827 7828 bind(L_exact_16_left); 7829 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 7830 pxor(xmm7, xmm0); //xor the initial crc value 7831 jmp(L_128_done); 7832 7833 bind(L_only_less_than_4); 7834 cmpl(len, 3); 7835 jcc(Assembler::less, L_only_less_than_3); 7836 7837 // load 3 Bytes 7838 movb(rax, Address(buf, pos, Address::times_1, 0)); 7839 movb(Address(tmp1, 0), rax); 7840 7841 movb(rax, Address(buf, pos, Address::times_1, 1)); 7842 movb(Address(tmp1, 1), rax); 7843 7844 movb(rax, Address(buf, pos, Address::times_1, 2)); 7845 movb(Address(tmp1, 2), rax); 7846 7847 movdqu(xmm7, Address(rsp, 0)); 7848 pxor(xmm7, xmm0); //xor the initial crc value 7849 7850 pslldq(xmm7, 0x5); 7851 jmp(L_barrett); 7852 bind(L_only_less_than_3); 7853 cmpl(len, 2); 7854 jcc(Assembler::less, L_only_less_than_2); 7855 7856 // load 2 Bytes 7857 movb(rax, Address(buf, pos, Address::times_1, 0)); 7858 movb(Address(tmp1, 0), rax); 7859 7860 movb(rax, Address(buf, pos, Address::times_1, 1)); 7861 movb(Address(tmp1, 1), rax); 7862 7863 movdqu(xmm7, Address(rsp, 0)); 7864 pxor(xmm7, xmm0); //xor the initial crc value 7865 7866 pslldq(xmm7, 0x6); 7867 jmp(L_barrett); 7868 7869 bind(L_only_less_than_2); 7870 //load 1 Byte 7871 movb(rax, Address(buf, pos, Address::times_1, 0)); 7872 movb(Address(tmp1, 0), rax); 7873 7874 movdqu(xmm7, Address(rsp, 0)); 7875 pxor(xmm7, xmm0); //xor the initial crc value 7876 7877 pslldq(xmm7, 0x7); 7878 } 7879 7880 /** 7881 * Compute CRC32 using AVX512 instructions 7882 * param crc register containing existing CRC (32-bit) 7883 * param buf register pointing to input byte buffer (byte*) 7884 * param len register containing number of bytes 7885 * param table address of crc or crc32c table 7886 * param tmp1 scratch register 7887 * param tmp2 scratch register 7888 * return rax result register 7889 * 7890 * This routine is identical for crc32c with the exception of the precomputed constant 7891 * table which will be passed as the table argument. The calculation steps are 7892 * the same for both variants. 7893 */ 7894 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 7895 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 7896 7897 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7898 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7899 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 7900 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 7901 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 7902 7903 const Register pos = r12; 7904 push(r12); 7905 subptr(rsp, 16 * 2 + 8); 7906 7907 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7908 // context for the registers used, where all instructions below are using 128-bit mode 7909 // On EVEX without VL and BW, these instructions will all be AVX. 7910 movl(pos, 0); 7911 7912 // check if smaller than 256B 7913 cmpl(len, 256); 7914 jcc(Assembler::less, L_less_than_256); 7915 7916 // load the initial crc value 7917 movdl(xmm10, crc); 7918 7919 // receive the initial 64B data, xor the initial crc value 7920 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 7921 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 7922 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 7923 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 7924 7925 subl(len, 256); 7926 cmpl(len, 256); 7927 jcc(Assembler::less, L_fold_128_B_loop); 7928 7929 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 7930 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 7931 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 7932 subl(len, 256); 7933 7934 bind(L_fold_256_B_loop); 7935 addl(pos, 256); 7936 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 7937 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 7938 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 7939 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 7940 7941 subl(len, 256); 7942 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 7943 7944 // Fold 256 into 128 7945 addl(pos, 256); 7946 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 7947 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 7948 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 7949 7950 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 7951 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 7952 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 7953 7954 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 7955 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 7956 7957 addl(len, 128); 7958 jmp(L_fold_128_B_register); 7959 7960 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 7961 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 7962 7963 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 7964 bind(L_fold_128_B_loop); 7965 addl(pos, 128); 7966 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 7967 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 7968 7969 subl(len, 128); 7970 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 7971 7972 addl(pos, 128); 7973 7974 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 7975 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 7976 bind(L_fold_128_B_register); 7977 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 7978 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 7979 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 7980 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 7981 // save last that has no multiplicand 7982 vextracti64x2(xmm7, xmm4, 3); 7983 7984 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 7985 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 7986 // Needed later in reduction loop 7987 movdqu(xmm10, Address(table, 1 * 16)); 7988 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 7989 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 7990 7991 // Swap 1,0,3,2 - 01 00 11 10 7992 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 7993 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 7994 vextracti128(xmm5, xmm8, 1); 7995 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 7996 7997 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 7998 // instead of a cmp instruction, we use the negative flag with the jl instruction 7999 addl(len, 128 - 16); 8000 jcc(Assembler::less, L_final_reduction_for_128); 8001 8002 bind(L_16B_reduction_loop); 8003 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8004 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8005 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8006 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 8007 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8008 addl(pos, 16); 8009 subl(len, 16); 8010 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 8011 8012 bind(L_final_reduction_for_128); 8013 addl(len, 16); 8014 jcc(Assembler::equal, L_128_done); 8015 8016 bind(L_get_last_two_xmms); 8017 movdqu(xmm2, xmm7); 8018 addl(pos, len); 8019 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 8020 subl(pos, len); 8021 8022 // get rid of the extra data that was loaded before 8023 // load the shift constant 8024 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8025 movdqu(xmm0, Address(rax, len)); 8026 addl(rax, len); 8027 8028 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8029 //Change mask to 512 8030 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 8031 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 8032 8033 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 8034 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8035 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8036 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8037 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 8038 8039 bind(L_128_done); 8040 // compute crc of a 128-bit value 8041 movdqu(xmm10, Address(table, 3 * 16)); 8042 movdqu(xmm0, xmm7); 8043 8044 // 64b fold 8045 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 8046 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 8047 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8048 8049 // 32b fold 8050 movdqu(xmm0, xmm7); 8051 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 8052 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8053 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8054 jmp(L_barrett); 8055 8056 bind(L_less_than_256); 8057 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 8058 8059 //barrett reduction 8060 bind(L_barrett); 8061 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 8062 movdqu(xmm1, xmm7); 8063 movdqu(xmm2, xmm7); 8064 movdqu(xmm10, Address(table, 4 * 16)); 8065 8066 pclmulqdq(xmm7, xmm10, 0x0); 8067 pxor(xmm7, xmm2); 8068 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 8069 movdqu(xmm2, xmm7); 8070 pclmulqdq(xmm7, xmm10, 0x10); 8071 pxor(xmm7, xmm2); 8072 pxor(xmm7, xmm1); 8073 pextrd(crc, xmm7, 2); 8074 8075 bind(L_cleanup); 8076 addptr(rsp, 16 * 2 + 8); 8077 pop(r12); 8078 } 8079 8080 // S. Gueron / Information Processing Letters 112 (2012) 184 8081 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 8082 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 8083 // Output: the 64-bit carry-less product of B * CONST 8084 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 8085 Register tmp1, Register tmp2, Register tmp3) { 8086 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8087 if (n > 0) { 8088 addq(tmp3, n * 256 * 8); 8089 } 8090 // Q1 = TABLEExt[n][B & 0xFF]; 8091 movl(tmp1, in); 8092 andl(tmp1, 0x000000FF); 8093 shll(tmp1, 3); 8094 addq(tmp1, tmp3); 8095 movq(tmp1, Address(tmp1, 0)); 8096 8097 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8098 movl(tmp2, in); 8099 shrl(tmp2, 8); 8100 andl(tmp2, 0x000000FF); 8101 shll(tmp2, 3); 8102 addq(tmp2, tmp3); 8103 movq(tmp2, Address(tmp2, 0)); 8104 8105 shlq(tmp2, 8); 8106 xorq(tmp1, tmp2); 8107 8108 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8109 movl(tmp2, in); 8110 shrl(tmp2, 16); 8111 andl(tmp2, 0x000000FF); 8112 shll(tmp2, 3); 8113 addq(tmp2, tmp3); 8114 movq(tmp2, Address(tmp2, 0)); 8115 8116 shlq(tmp2, 16); 8117 xorq(tmp1, tmp2); 8118 8119 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8120 shrl(in, 24); 8121 andl(in, 0x000000FF); 8122 shll(in, 3); 8123 addq(in, tmp3); 8124 movq(in, Address(in, 0)); 8125 8126 shlq(in, 24); 8127 xorq(in, tmp1); 8128 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8129 } 8130 8131 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8132 Register in_out, 8133 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8134 XMMRegister w_xtmp2, 8135 Register tmp1, 8136 Register n_tmp2, Register n_tmp3) { 8137 if (is_pclmulqdq_supported) { 8138 movdl(w_xtmp1, in_out); // modified blindly 8139 8140 movl(tmp1, const_or_pre_comp_const_index); 8141 movdl(w_xtmp2, tmp1); 8142 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8143 8144 movdq(in_out, w_xtmp1); 8145 } else { 8146 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 8147 } 8148 } 8149 8150 // Recombination Alternative 2: No bit-reflections 8151 // T1 = (CRC_A * U1) << 1 8152 // T2 = (CRC_B * U2) << 1 8153 // C1 = T1 >> 32 8154 // C2 = T2 >> 32 8155 // T1 = T1 & 0xFFFFFFFF 8156 // T2 = T2 & 0xFFFFFFFF 8157 // T1 = CRC32(0, T1) 8158 // T2 = CRC32(0, T2) 8159 // C1 = C1 ^ T1 8160 // C2 = C2 ^ T2 8161 // CRC = C1 ^ C2 ^ CRC_C 8162 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8163 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8164 Register tmp1, Register tmp2, 8165 Register n_tmp3) { 8166 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8167 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8168 shlq(in_out, 1); 8169 movl(tmp1, in_out); 8170 shrq(in_out, 32); 8171 xorl(tmp2, tmp2); 8172 crc32(tmp2, tmp1, 4); 8173 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8174 shlq(in1, 1); 8175 movl(tmp1, in1); 8176 shrq(in1, 32); 8177 xorl(tmp2, tmp2); 8178 crc32(tmp2, tmp1, 4); 8179 xorl(in1, tmp2); 8180 xorl(in_out, in1); 8181 xorl(in_out, in2); 8182 } 8183 8184 // Set N to predefined value 8185 // Subtract from a length of a buffer 8186 // execute in a loop: 8187 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8188 // for i = 1 to N do 8189 // CRC_A = CRC32(CRC_A, A[i]) 8190 // CRC_B = CRC32(CRC_B, B[i]) 8191 // CRC_C = CRC32(CRC_C, C[i]) 8192 // end for 8193 // Recombine 8194 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8195 Register in_out1, Register in_out2, Register in_out3, 8196 Register tmp1, Register tmp2, Register tmp3, 8197 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8198 Register tmp4, Register tmp5, 8199 Register n_tmp6) { 8200 Label L_processPartitions; 8201 Label L_processPartition; 8202 Label L_exit; 8203 8204 bind(L_processPartitions); 8205 cmpl(in_out1, 3 * size); 8206 jcc(Assembler::less, L_exit); 8207 xorl(tmp1, tmp1); 8208 xorl(tmp2, tmp2); 8209 movq(tmp3, in_out2); 8210 addq(tmp3, size); 8211 8212 bind(L_processPartition); 8213 crc32(in_out3, Address(in_out2, 0), 8); 8214 crc32(tmp1, Address(in_out2, size), 8); 8215 crc32(tmp2, Address(in_out2, size * 2), 8); 8216 addq(in_out2, 8); 8217 cmpq(in_out2, tmp3); 8218 jcc(Assembler::less, L_processPartition); 8219 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8220 w_xtmp1, w_xtmp2, w_xtmp3, 8221 tmp4, tmp5, 8222 n_tmp6); 8223 addq(in_out2, 2 * size); 8224 subl(in_out1, 3 * size); 8225 jmp(L_processPartitions); 8226 8227 bind(L_exit); 8228 } 8229 #else 8230 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, 8231 Register tmp1, Register tmp2, Register tmp3, 8232 XMMRegister xtmp1, XMMRegister xtmp2) { 8233 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8234 if (n > 0) { 8235 addl(tmp3, n * 256 * 8); 8236 } 8237 // Q1 = TABLEExt[n][B & 0xFF]; 8238 movl(tmp1, in_out); 8239 andl(tmp1, 0x000000FF); 8240 shll(tmp1, 3); 8241 addl(tmp1, tmp3); 8242 movq(xtmp1, Address(tmp1, 0)); 8243 8244 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8245 movl(tmp2, in_out); 8246 shrl(tmp2, 8); 8247 andl(tmp2, 0x000000FF); 8248 shll(tmp2, 3); 8249 addl(tmp2, tmp3); 8250 movq(xtmp2, Address(tmp2, 0)); 8251 8252 psllq(xtmp2, 8); 8253 pxor(xtmp1, xtmp2); 8254 8255 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8256 movl(tmp2, in_out); 8257 shrl(tmp2, 16); 8258 andl(tmp2, 0x000000FF); 8259 shll(tmp2, 3); 8260 addl(tmp2, tmp3); 8261 movq(xtmp2, Address(tmp2, 0)); 8262 8263 psllq(xtmp2, 16); 8264 pxor(xtmp1, xtmp2); 8265 8266 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8267 shrl(in_out, 24); 8268 andl(in_out, 0x000000FF); 8269 shll(in_out, 3); 8270 addl(in_out, tmp3); 8271 movq(xtmp2, Address(in_out, 0)); 8272 8273 psllq(xtmp2, 24); 8274 pxor(xtmp1, xtmp2); // Result in CXMM 8275 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8276 } 8277 8278 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8279 Register in_out, 8280 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8281 XMMRegister w_xtmp2, 8282 Register tmp1, 8283 Register n_tmp2, Register n_tmp3) { 8284 if (is_pclmulqdq_supported) { 8285 movdl(w_xtmp1, in_out); 8286 8287 movl(tmp1, const_or_pre_comp_const_index); 8288 movdl(w_xtmp2, tmp1); 8289 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8290 // Keep result in XMM since GPR is 32 bit in length 8291 } else { 8292 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); 8293 } 8294 } 8295 8296 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8297 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8298 Register tmp1, Register tmp2, 8299 Register n_tmp3) { 8300 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8301 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8302 8303 psllq(w_xtmp1, 1); 8304 movdl(tmp1, w_xtmp1); 8305 psrlq(w_xtmp1, 32); 8306 movdl(in_out, w_xtmp1); 8307 8308 xorl(tmp2, tmp2); 8309 crc32(tmp2, tmp1, 4); 8310 xorl(in_out, tmp2); 8311 8312 psllq(w_xtmp2, 1); 8313 movdl(tmp1, w_xtmp2); 8314 psrlq(w_xtmp2, 32); 8315 movdl(in1, w_xtmp2); 8316 8317 xorl(tmp2, tmp2); 8318 crc32(tmp2, tmp1, 4); 8319 xorl(in1, tmp2); 8320 xorl(in_out, in1); 8321 xorl(in_out, in2); 8322 } 8323 8324 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8325 Register in_out1, Register in_out2, Register in_out3, 8326 Register tmp1, Register tmp2, Register tmp3, 8327 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8328 Register tmp4, Register tmp5, 8329 Register n_tmp6) { 8330 Label L_processPartitions; 8331 Label L_processPartition; 8332 Label L_exit; 8333 8334 bind(L_processPartitions); 8335 cmpl(in_out1, 3 * size); 8336 jcc(Assembler::less, L_exit); 8337 xorl(tmp1, tmp1); 8338 xorl(tmp2, tmp2); 8339 movl(tmp3, in_out2); 8340 addl(tmp3, size); 8341 8342 bind(L_processPartition); 8343 crc32(in_out3, Address(in_out2, 0), 4); 8344 crc32(tmp1, Address(in_out2, size), 4); 8345 crc32(tmp2, Address(in_out2, size*2), 4); 8346 crc32(in_out3, Address(in_out2, 0+4), 4); 8347 crc32(tmp1, Address(in_out2, size+4), 4); 8348 crc32(tmp2, Address(in_out2, size*2+4), 4); 8349 addl(in_out2, 8); 8350 cmpl(in_out2, tmp3); 8351 jcc(Assembler::less, L_processPartition); 8352 8353 push(tmp3); 8354 push(in_out1); 8355 push(in_out2); 8356 tmp4 = tmp3; 8357 tmp5 = in_out1; 8358 n_tmp6 = in_out2; 8359 8360 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8361 w_xtmp1, w_xtmp2, w_xtmp3, 8362 tmp4, tmp5, 8363 n_tmp6); 8364 8365 pop(in_out2); 8366 pop(in_out1); 8367 pop(tmp3); 8368 8369 addl(in_out2, 2 * size); 8370 subl(in_out1, 3 * size); 8371 jmp(L_processPartitions); 8372 8373 bind(L_exit); 8374 } 8375 #endif //LP64 8376 8377 #ifdef _LP64 8378 // Algorithm 2: Pipelined usage of the CRC32 instruction. 8379 // Input: A buffer I of L bytes. 8380 // Output: the CRC32C value of the buffer. 8381 // Notations: 8382 // Write L = 24N + r, with N = floor (L/24). 8383 // r = L mod 24 (0 <= r < 24). 8384 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 8385 // N quadwords, and R consists of r bytes. 8386 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 8387 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 8388 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 8389 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 8390 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8391 Register tmp1, Register tmp2, Register tmp3, 8392 Register tmp4, Register tmp5, Register tmp6, 8393 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8394 bool is_pclmulqdq_supported) { 8395 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8396 Label L_wordByWord; 8397 Label L_byteByByteProlog; 8398 Label L_byteByByte; 8399 Label L_exit; 8400 8401 if (is_pclmulqdq_supported ) { 8402 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8403 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); 8404 8405 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8406 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8407 8408 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8409 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8410 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 8411 } else { 8412 const_or_pre_comp_const_index[0] = 1; 8413 const_or_pre_comp_const_index[1] = 0; 8414 8415 const_or_pre_comp_const_index[2] = 3; 8416 const_or_pre_comp_const_index[3] = 2; 8417 8418 const_or_pre_comp_const_index[4] = 5; 8419 const_or_pre_comp_const_index[5] = 4; 8420 } 8421 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8422 in2, in1, in_out, 8423 tmp1, tmp2, tmp3, 8424 w_xtmp1, w_xtmp2, w_xtmp3, 8425 tmp4, tmp5, 8426 tmp6); 8427 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8428 in2, in1, in_out, 8429 tmp1, tmp2, tmp3, 8430 w_xtmp1, w_xtmp2, w_xtmp3, 8431 tmp4, tmp5, 8432 tmp6); 8433 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8434 in2, in1, in_out, 8435 tmp1, tmp2, tmp3, 8436 w_xtmp1, w_xtmp2, w_xtmp3, 8437 tmp4, tmp5, 8438 tmp6); 8439 movl(tmp1, in2); 8440 andl(tmp1, 0x00000007); 8441 negl(tmp1); 8442 addl(tmp1, in2); 8443 addq(tmp1, in1); 8444 8445 cmpq(in1, tmp1); 8446 jccb(Assembler::greaterEqual, L_byteByByteProlog); 8447 align(16); 8448 BIND(L_wordByWord); 8449 crc32(in_out, Address(in1, 0), 8); 8450 addq(in1, 8); 8451 cmpq(in1, tmp1); 8452 jcc(Assembler::less, L_wordByWord); 8453 8454 BIND(L_byteByByteProlog); 8455 andl(in2, 0x00000007); 8456 movl(tmp2, 1); 8457 8458 cmpl(tmp2, in2); 8459 jccb(Assembler::greater, L_exit); 8460 BIND(L_byteByByte); 8461 crc32(in_out, Address(in1, 0), 1); 8462 incq(in1); 8463 incl(tmp2); 8464 cmpl(tmp2, in2); 8465 jcc(Assembler::lessEqual, L_byteByByte); 8466 8467 BIND(L_exit); 8468 } 8469 #else 8470 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8471 Register tmp1, Register tmp2, Register tmp3, 8472 Register tmp4, Register tmp5, Register tmp6, 8473 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8474 bool is_pclmulqdq_supported) { 8475 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8476 Label L_wordByWord; 8477 Label L_byteByByteProlog; 8478 Label L_byteByByte; 8479 Label L_exit; 8480 8481 if (is_pclmulqdq_supported) { 8482 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8483 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); 8484 8485 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8486 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8487 8488 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8489 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8490 } else { 8491 const_or_pre_comp_const_index[0] = 1; 8492 const_or_pre_comp_const_index[1] = 0; 8493 8494 const_or_pre_comp_const_index[2] = 3; 8495 const_or_pre_comp_const_index[3] = 2; 8496 8497 const_or_pre_comp_const_index[4] = 5; 8498 const_or_pre_comp_const_index[5] = 4; 8499 } 8500 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8501 in2, in1, in_out, 8502 tmp1, tmp2, tmp3, 8503 w_xtmp1, w_xtmp2, w_xtmp3, 8504 tmp4, tmp5, 8505 tmp6); 8506 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8507 in2, in1, in_out, 8508 tmp1, tmp2, tmp3, 8509 w_xtmp1, w_xtmp2, w_xtmp3, 8510 tmp4, tmp5, 8511 tmp6); 8512 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8513 in2, in1, in_out, 8514 tmp1, tmp2, tmp3, 8515 w_xtmp1, w_xtmp2, w_xtmp3, 8516 tmp4, tmp5, 8517 tmp6); 8518 movl(tmp1, in2); 8519 andl(tmp1, 0x00000007); 8520 negl(tmp1); 8521 addl(tmp1, in2); 8522 addl(tmp1, in1); 8523 8524 BIND(L_wordByWord); 8525 cmpl(in1, tmp1); 8526 jcc(Assembler::greaterEqual, L_byteByByteProlog); 8527 crc32(in_out, Address(in1,0), 4); 8528 addl(in1, 4); 8529 jmp(L_wordByWord); 8530 8531 BIND(L_byteByByteProlog); 8532 andl(in2, 0x00000007); 8533 movl(tmp2, 1); 8534 8535 BIND(L_byteByByte); 8536 cmpl(tmp2, in2); 8537 jccb(Assembler::greater, L_exit); 8538 movb(tmp1, Address(in1, 0)); 8539 crc32(in_out, tmp1, 1); 8540 incl(in1); 8541 incl(tmp2); 8542 jmp(L_byteByByte); 8543 8544 BIND(L_exit); 8545 } 8546 #endif // LP64 8547 #undef BIND 8548 #undef BLOCK_COMMENT 8549 8550 // Compress char[] array to byte[]. 8551 // ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java 8552 // @IntrinsicCandidate 8553 // private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 8554 // for (int i = 0; i < len; i++) { 8555 // int c = src[srcOff++]; 8556 // if (c >>> 8 != 0) { 8557 // return 0; 8558 // } 8559 // dst[dstOff++] = (byte)c; 8560 // } 8561 // return len; 8562 // } 8563 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 8564 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 8565 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 8566 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 8567 Label copy_chars_loop, return_length, return_zero, done; 8568 8569 // rsi: src 8570 // rdi: dst 8571 // rdx: len 8572 // rcx: tmp5 8573 // rax: result 8574 8575 // rsi holds start addr of source char[] to be compressed 8576 // rdi holds start addr of destination byte[] 8577 // rdx holds length 8578 8579 assert(len != result, ""); 8580 8581 // save length for return 8582 push(len); 8583 8584 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 8585 VM_Version::supports_avx512vlbw() && 8586 VM_Version::supports_bmi2()) { 8587 8588 Label copy_32_loop, copy_loop_tail, below_threshold; 8589 8590 // alignment 8591 Label post_alignment; 8592 8593 // if length of the string is less than 16, handle it in an old fashioned way 8594 testl(len, -32); 8595 jcc(Assembler::zero, below_threshold); 8596 8597 // First check whether a character is compressible ( <= 0xFF). 8598 // Create mask to test for Unicode chars inside zmm vector 8599 movl(result, 0x00FF); 8600 evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit); 8601 8602 testl(len, -64); 8603 jcc(Assembler::zero, post_alignment); 8604 8605 movl(tmp5, dst); 8606 andl(tmp5, (32 - 1)); 8607 negl(tmp5); 8608 andl(tmp5, (32 - 1)); 8609 8610 // bail out when there is nothing to be done 8611 testl(tmp5, 0xFFFFFFFF); 8612 jcc(Assembler::zero, post_alignment); 8613 8614 // ~(~0 << len), where len is the # of remaining elements to process 8615 movl(result, 0xFFFFFFFF); 8616 shlxl(result, result, tmp5); 8617 notl(result); 8618 kmovdl(mask2, result); 8619 8620 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8621 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8622 ktestd(mask1, mask2); 8623 jcc(Assembler::carryClear, return_zero); 8624 8625 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8626 8627 addptr(src, tmp5); 8628 addptr(src, tmp5); 8629 addptr(dst, tmp5); 8630 subl(len, tmp5); 8631 8632 bind(post_alignment); 8633 // end of alignment 8634 8635 movl(tmp5, len); 8636 andl(tmp5, (32 - 1)); // tail count (in chars) 8637 andl(len, ~(32 - 1)); // vector count (in chars) 8638 jcc(Assembler::zero, copy_loop_tail); 8639 8640 lea(src, Address(src, len, Address::times_2)); 8641 lea(dst, Address(dst, len, Address::times_1)); 8642 negptr(len); 8643 8644 bind(copy_32_loop); 8645 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 8646 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 8647 kortestdl(mask1, mask1); 8648 jcc(Assembler::carryClear, return_zero); 8649 8650 // All elements in current processed chunk are valid candidates for 8651 // compression. Write a truncated byte elements to the memory. 8652 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 8653 addptr(len, 32); 8654 jcc(Assembler::notZero, copy_32_loop); 8655 8656 bind(copy_loop_tail); 8657 // bail out when there is nothing to be done 8658 testl(tmp5, 0xFFFFFFFF); 8659 jcc(Assembler::zero, return_length); 8660 8661 movl(len, tmp5); 8662 8663 // ~(~0 << len), where len is the # of remaining elements to process 8664 movl(result, 0xFFFFFFFF); 8665 shlxl(result, result, len); 8666 notl(result); 8667 8668 kmovdl(mask2, result); 8669 8670 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8671 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8672 ktestd(mask1, mask2); 8673 jcc(Assembler::carryClear, return_zero); 8674 8675 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8676 jmp(return_length); 8677 8678 bind(below_threshold); 8679 } 8680 8681 if (UseSSE42Intrinsics) { 8682 Label copy_32_loop, copy_16, copy_tail; 8683 8684 movl(result, len); 8685 8686 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 8687 8688 // vectored compression 8689 andl(len, 0xfffffff0); // vector count (in chars) 8690 andl(result, 0x0000000f); // tail count (in chars) 8691 testl(len, len); 8692 jcc(Assembler::zero, copy_16); 8693 8694 // compress 16 chars per iter 8695 movdl(tmp1Reg, tmp5); 8696 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8697 pxor(tmp4Reg, tmp4Reg); 8698 8699 lea(src, Address(src, len, Address::times_2)); 8700 lea(dst, Address(dst, len, Address::times_1)); 8701 negptr(len); 8702 8703 bind(copy_32_loop); 8704 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 8705 por(tmp4Reg, tmp2Reg); 8706 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 8707 por(tmp4Reg, tmp3Reg); 8708 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 8709 jcc(Assembler::notZero, return_zero); 8710 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 8711 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 8712 addptr(len, 16); 8713 jcc(Assembler::notZero, copy_32_loop); 8714 8715 // compress next vector of 8 chars (if any) 8716 bind(copy_16); 8717 movl(len, result); 8718 andl(len, 0xfffffff8); // vector count (in chars) 8719 andl(result, 0x00000007); // tail count (in chars) 8720 testl(len, len); 8721 jccb(Assembler::zero, copy_tail); 8722 8723 movdl(tmp1Reg, tmp5); 8724 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8725 pxor(tmp3Reg, tmp3Reg); 8726 8727 movdqu(tmp2Reg, Address(src, 0)); 8728 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 8729 jccb(Assembler::notZero, return_zero); 8730 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 8731 movq(Address(dst, 0), tmp2Reg); 8732 addptr(src, 16); 8733 addptr(dst, 8); 8734 8735 bind(copy_tail); 8736 movl(len, result); 8737 } 8738 // compress 1 char per iter 8739 testl(len, len); 8740 jccb(Assembler::zero, return_length); 8741 lea(src, Address(src, len, Address::times_2)); 8742 lea(dst, Address(dst, len, Address::times_1)); 8743 negptr(len); 8744 8745 bind(copy_chars_loop); 8746 load_unsigned_short(result, Address(src, len, Address::times_2)); 8747 testl(result, 0xff00); // check if Unicode char 8748 jccb(Assembler::notZero, return_zero); 8749 movb(Address(dst, len, Address::times_1), result); // ASCII char; compress to 1 byte 8750 increment(len); 8751 jcc(Assembler::notZero, copy_chars_loop); 8752 8753 // if compression succeeded, return length 8754 bind(return_length); 8755 pop(result); 8756 jmpb(done); 8757 8758 // if compression failed, return 0 8759 bind(return_zero); 8760 xorl(result, result); 8761 addptr(rsp, wordSize); 8762 8763 bind(done); 8764 } 8765 8766 // Inflate byte[] array to char[]. 8767 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 8768 // @IntrinsicCandidate 8769 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 8770 // for (int i = 0; i < len; i++) { 8771 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 8772 // } 8773 // } 8774 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 8775 XMMRegister tmp1, Register tmp2, KRegister mask) { 8776 Label copy_chars_loop, done, below_threshold, avx3_threshold; 8777 // rsi: src 8778 // rdi: dst 8779 // rdx: len 8780 // rcx: tmp2 8781 8782 // rsi holds start addr of source byte[] to be inflated 8783 // rdi holds start addr of destination char[] 8784 // rdx holds length 8785 assert_different_registers(src, dst, len, tmp2); 8786 movl(tmp2, len); 8787 if ((UseAVX > 2) && // AVX512 8788 VM_Version::supports_avx512vlbw() && 8789 VM_Version::supports_bmi2()) { 8790 8791 Label copy_32_loop, copy_tail; 8792 Register tmp3_aliased = len; 8793 8794 // if length of the string is less than 16, handle it in an old fashioned way 8795 testl(len, -16); 8796 jcc(Assembler::zero, below_threshold); 8797 8798 testl(len, -1 * AVX3Threshold); 8799 jcc(Assembler::zero, avx3_threshold); 8800 8801 // In order to use only one arithmetic operation for the main loop we use 8802 // this pre-calculation 8803 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 8804 andl(len, -32); // vector count 8805 jccb(Assembler::zero, copy_tail); 8806 8807 lea(src, Address(src, len, Address::times_1)); 8808 lea(dst, Address(dst, len, Address::times_2)); 8809 negptr(len); 8810 8811 8812 // inflate 32 chars per iter 8813 bind(copy_32_loop); 8814 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 8815 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 8816 addptr(len, 32); 8817 jcc(Assembler::notZero, copy_32_loop); 8818 8819 bind(copy_tail); 8820 // bail out when there is nothing to be done 8821 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 8822 jcc(Assembler::zero, done); 8823 8824 // ~(~0 << length), where length is the # of remaining elements to process 8825 movl(tmp3_aliased, -1); 8826 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 8827 notl(tmp3_aliased); 8828 kmovdl(mask, tmp3_aliased); 8829 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 8830 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 8831 8832 jmp(done); 8833 bind(avx3_threshold); 8834 } 8835 if (UseSSE42Intrinsics) { 8836 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 8837 8838 if (UseAVX > 1) { 8839 andl(tmp2, (16 - 1)); 8840 andl(len, -16); 8841 jccb(Assembler::zero, copy_new_tail); 8842 } else { 8843 andl(tmp2, 0x00000007); // tail count (in chars) 8844 andl(len, 0xfffffff8); // vector count (in chars) 8845 jccb(Assembler::zero, copy_tail); 8846 } 8847 8848 // vectored inflation 8849 lea(src, Address(src, len, Address::times_1)); 8850 lea(dst, Address(dst, len, Address::times_2)); 8851 negptr(len); 8852 8853 if (UseAVX > 1) { 8854 bind(copy_16_loop); 8855 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 8856 vmovdqu(Address(dst, len, Address::times_2), tmp1); 8857 addptr(len, 16); 8858 jcc(Assembler::notZero, copy_16_loop); 8859 8860 bind(below_threshold); 8861 bind(copy_new_tail); 8862 movl(len, tmp2); 8863 andl(tmp2, 0x00000007); 8864 andl(len, 0xFFFFFFF8); 8865 jccb(Assembler::zero, copy_tail); 8866 8867 pmovzxbw(tmp1, Address(src, 0)); 8868 movdqu(Address(dst, 0), tmp1); 8869 addptr(src, 8); 8870 addptr(dst, 2 * 8); 8871 8872 jmp(copy_tail, true); 8873 } 8874 8875 // inflate 8 chars per iter 8876 bind(copy_8_loop); 8877 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 8878 movdqu(Address(dst, len, Address::times_2), tmp1); 8879 addptr(len, 8); 8880 jcc(Assembler::notZero, copy_8_loop); 8881 8882 bind(copy_tail); 8883 movl(len, tmp2); 8884 8885 cmpl(len, 4); 8886 jccb(Assembler::less, copy_bytes); 8887 8888 movdl(tmp1, Address(src, 0)); // load 4 byte chars 8889 pmovzxbw(tmp1, tmp1); 8890 movq(Address(dst, 0), tmp1); 8891 subptr(len, 4); 8892 addptr(src, 4); 8893 addptr(dst, 8); 8894 8895 bind(copy_bytes); 8896 } else { 8897 bind(below_threshold); 8898 } 8899 8900 testl(len, len); 8901 jccb(Assembler::zero, done); 8902 lea(src, Address(src, len, Address::times_1)); 8903 lea(dst, Address(dst, len, Address::times_2)); 8904 negptr(len); 8905 8906 // inflate 1 char per iter 8907 bind(copy_chars_loop); 8908 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 8909 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 8910 increment(len); 8911 jcc(Assembler::notZero, copy_chars_loop); 8912 8913 bind(done); 8914 } 8915 8916 8917 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 8918 switch(type) { 8919 case T_BYTE: 8920 case T_BOOLEAN: 8921 evmovdqub(dst, kmask, src, merge, vector_len); 8922 break; 8923 case T_CHAR: 8924 case T_SHORT: 8925 evmovdquw(dst, kmask, src, merge, vector_len); 8926 break; 8927 case T_INT: 8928 case T_FLOAT: 8929 evmovdqul(dst, kmask, src, merge, vector_len); 8930 break; 8931 case T_LONG: 8932 case T_DOUBLE: 8933 evmovdquq(dst, kmask, src, merge, vector_len); 8934 break; 8935 default: 8936 fatal("Unexpected type argument %s", type2name(type)); 8937 break; 8938 } 8939 } 8940 8941 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 8942 switch(type) { 8943 case T_BYTE: 8944 case T_BOOLEAN: 8945 evmovdqub(dst, kmask, src, merge, vector_len); 8946 break; 8947 case T_CHAR: 8948 case T_SHORT: 8949 evmovdquw(dst, kmask, src, merge, vector_len); 8950 break; 8951 case T_INT: 8952 case T_FLOAT: 8953 evmovdqul(dst, kmask, src, merge, vector_len); 8954 break; 8955 case T_LONG: 8956 case T_DOUBLE: 8957 evmovdquq(dst, kmask, src, merge, vector_len); 8958 break; 8959 default: 8960 fatal("Unexpected type argument %s", type2name(type)); 8961 break; 8962 } 8963 } 8964 8965 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 8966 switch(masklen) { 8967 case 2: 8968 knotbl(dst, src); 8969 movl(rtmp, 3); 8970 kmovbl(ktmp, rtmp); 8971 kandbl(dst, ktmp, dst); 8972 break; 8973 case 4: 8974 knotbl(dst, src); 8975 movl(rtmp, 15); 8976 kmovbl(ktmp, rtmp); 8977 kandbl(dst, ktmp, dst); 8978 break; 8979 case 8: 8980 knotbl(dst, src); 8981 break; 8982 case 16: 8983 knotwl(dst, src); 8984 break; 8985 case 32: 8986 knotdl(dst, src); 8987 break; 8988 case 64: 8989 knotql(dst, src); 8990 break; 8991 default: 8992 fatal("Unexpected vector length %d", masklen); 8993 break; 8994 } 8995 } 8996 8997 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 8998 switch(type) { 8999 case T_BOOLEAN: 9000 case T_BYTE: 9001 kandbl(dst, src1, src2); 9002 break; 9003 case T_CHAR: 9004 case T_SHORT: 9005 kandwl(dst, src1, src2); 9006 break; 9007 case T_INT: 9008 case T_FLOAT: 9009 kanddl(dst, src1, src2); 9010 break; 9011 case T_LONG: 9012 case T_DOUBLE: 9013 kandql(dst, src1, src2); 9014 break; 9015 default: 9016 fatal("Unexpected type argument %s", type2name(type)); 9017 break; 9018 } 9019 } 9020 9021 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9022 switch(type) { 9023 case T_BOOLEAN: 9024 case T_BYTE: 9025 korbl(dst, src1, src2); 9026 break; 9027 case T_CHAR: 9028 case T_SHORT: 9029 korwl(dst, src1, src2); 9030 break; 9031 case T_INT: 9032 case T_FLOAT: 9033 kordl(dst, src1, src2); 9034 break; 9035 case T_LONG: 9036 case T_DOUBLE: 9037 korql(dst, src1, src2); 9038 break; 9039 default: 9040 fatal("Unexpected type argument %s", type2name(type)); 9041 break; 9042 } 9043 } 9044 9045 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9046 switch(type) { 9047 case T_BOOLEAN: 9048 case T_BYTE: 9049 kxorbl(dst, src1, src2); 9050 break; 9051 case T_CHAR: 9052 case T_SHORT: 9053 kxorwl(dst, src1, src2); 9054 break; 9055 case T_INT: 9056 case T_FLOAT: 9057 kxordl(dst, src1, src2); 9058 break; 9059 case T_LONG: 9060 case T_DOUBLE: 9061 kxorql(dst, src1, src2); 9062 break; 9063 default: 9064 fatal("Unexpected type argument %s", type2name(type)); 9065 break; 9066 } 9067 } 9068 9069 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9070 switch(type) { 9071 case T_BOOLEAN: 9072 case T_BYTE: 9073 evpermb(dst, mask, nds, src, merge, vector_len); break; 9074 case T_CHAR: 9075 case T_SHORT: 9076 evpermw(dst, mask, nds, src, merge, vector_len); break; 9077 case T_INT: 9078 case T_FLOAT: 9079 evpermd(dst, mask, nds, src, merge, vector_len); break; 9080 case T_LONG: 9081 case T_DOUBLE: 9082 evpermq(dst, mask, nds, src, merge, vector_len); break; 9083 default: 9084 fatal("Unexpected type argument %s", type2name(type)); break; 9085 } 9086 } 9087 9088 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9089 switch(type) { 9090 case T_BOOLEAN: 9091 case T_BYTE: 9092 evpermb(dst, mask, nds, src, merge, vector_len); break; 9093 case T_CHAR: 9094 case T_SHORT: 9095 evpermw(dst, mask, nds, src, merge, vector_len); break; 9096 case T_INT: 9097 case T_FLOAT: 9098 evpermd(dst, mask, nds, src, merge, vector_len); break; 9099 case T_LONG: 9100 case T_DOUBLE: 9101 evpermq(dst, mask, nds, src, merge, vector_len); break; 9102 default: 9103 fatal("Unexpected type argument %s", type2name(type)); break; 9104 } 9105 } 9106 9107 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9108 switch(type) { 9109 case T_BYTE: 9110 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9111 case T_SHORT: 9112 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9113 case T_INT: 9114 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9115 case T_LONG: 9116 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9117 default: 9118 fatal("Unexpected type argument %s", type2name(type)); break; 9119 } 9120 } 9121 9122 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9123 switch(type) { 9124 case T_BYTE: 9125 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9126 case T_SHORT: 9127 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9128 case T_INT: 9129 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9130 case T_LONG: 9131 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9132 default: 9133 fatal("Unexpected type argument %s", type2name(type)); break; 9134 } 9135 } 9136 9137 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9138 switch(type) { 9139 case T_BYTE: 9140 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9141 case T_SHORT: 9142 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9143 case T_INT: 9144 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9145 case T_LONG: 9146 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9147 default: 9148 fatal("Unexpected type argument %s", type2name(type)); break; 9149 } 9150 } 9151 9152 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9153 switch(type) { 9154 case T_BYTE: 9155 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9156 case T_SHORT: 9157 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9158 case T_INT: 9159 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9160 case T_LONG: 9161 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9162 default: 9163 fatal("Unexpected type argument %s", type2name(type)); break; 9164 } 9165 } 9166 9167 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9168 switch(type) { 9169 case T_INT: 9170 evpxord(dst, mask, nds, src, merge, vector_len); break; 9171 case T_LONG: 9172 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9173 default: 9174 fatal("Unexpected type argument %s", type2name(type)); break; 9175 } 9176 } 9177 9178 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9179 switch(type) { 9180 case T_INT: 9181 evpxord(dst, mask, nds, src, merge, vector_len); break; 9182 case T_LONG: 9183 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9184 default: 9185 fatal("Unexpected type argument %s", type2name(type)); break; 9186 } 9187 } 9188 9189 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9190 switch(type) { 9191 case T_INT: 9192 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9193 case T_LONG: 9194 evporq(dst, mask, nds, src, merge, vector_len); break; 9195 default: 9196 fatal("Unexpected type argument %s", type2name(type)); break; 9197 } 9198 } 9199 9200 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9201 switch(type) { 9202 case T_INT: 9203 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9204 case T_LONG: 9205 evporq(dst, mask, nds, src, merge, vector_len); break; 9206 default: 9207 fatal("Unexpected type argument %s", type2name(type)); break; 9208 } 9209 } 9210 9211 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9212 switch(type) { 9213 case T_INT: 9214 evpandd(dst, mask, nds, src, merge, vector_len); break; 9215 case T_LONG: 9216 evpandq(dst, mask, nds, src, merge, vector_len); break; 9217 default: 9218 fatal("Unexpected type argument %s", type2name(type)); break; 9219 } 9220 } 9221 9222 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9223 switch(type) { 9224 case T_INT: 9225 evpandd(dst, mask, nds, src, merge, vector_len); break; 9226 case T_LONG: 9227 evpandq(dst, mask, nds, src, merge, vector_len); break; 9228 default: 9229 fatal("Unexpected type argument %s", type2name(type)); break; 9230 } 9231 } 9232 9233 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 9234 switch(masklen) { 9235 case 8: 9236 kortestbl(src1, src2); 9237 break; 9238 case 16: 9239 kortestwl(src1, src2); 9240 break; 9241 case 32: 9242 kortestdl(src1, src2); 9243 break; 9244 case 64: 9245 kortestql(src1, src2); 9246 break; 9247 default: 9248 fatal("Unexpected mask length %d", masklen); 9249 break; 9250 } 9251 } 9252 9253 9254 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 9255 switch(masklen) { 9256 case 8: 9257 ktestbl(src1, src2); 9258 break; 9259 case 16: 9260 ktestwl(src1, src2); 9261 break; 9262 case 32: 9263 ktestdl(src1, src2); 9264 break; 9265 case 64: 9266 ktestql(src1, src2); 9267 break; 9268 default: 9269 fatal("Unexpected mask length %d", masklen); 9270 break; 9271 } 9272 } 9273 9274 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9275 switch(type) { 9276 case T_INT: 9277 evprold(dst, mask, src, shift, merge, vlen_enc); break; 9278 case T_LONG: 9279 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 9280 default: 9281 fatal("Unexpected type argument %s", type2name(type)); break; 9282 break; 9283 } 9284 } 9285 9286 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9287 switch(type) { 9288 case T_INT: 9289 evprord(dst, mask, src, shift, merge, vlen_enc); break; 9290 case T_LONG: 9291 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 9292 default: 9293 fatal("Unexpected type argument %s", type2name(type)); break; 9294 } 9295 } 9296 9297 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9298 switch(type) { 9299 case T_INT: 9300 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 9301 case T_LONG: 9302 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 9303 default: 9304 fatal("Unexpected type argument %s", type2name(type)); break; 9305 } 9306 } 9307 9308 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9309 switch(type) { 9310 case T_INT: 9311 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 9312 case T_LONG: 9313 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 9314 default: 9315 fatal("Unexpected type argument %s", type2name(type)); break; 9316 } 9317 } 9318 9319 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9320 assert(rscratch != noreg || always_reachable(src), "missing"); 9321 9322 if (reachable(src)) { 9323 evpandq(dst, nds, as_Address(src), vector_len); 9324 } else { 9325 lea(rscratch, src); 9326 evpandq(dst, nds, Address(rscratch, 0), vector_len); 9327 } 9328 } 9329 9330 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 9331 assert(rscratch != noreg || always_reachable(src), "missing"); 9332 9333 if (reachable(src)) { 9334 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 9335 } else { 9336 lea(rscratch, src); 9337 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 9338 } 9339 } 9340 9341 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9342 assert(rscratch != noreg || always_reachable(src), "missing"); 9343 9344 if (reachable(src)) { 9345 evporq(dst, nds, as_Address(src), vector_len); 9346 } else { 9347 lea(rscratch, src); 9348 evporq(dst, nds, Address(rscratch, 0), vector_len); 9349 } 9350 } 9351 9352 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 9353 assert(rscratch != noreg || always_reachable(src3), "missing"); 9354 9355 if (reachable(src3)) { 9356 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 9357 } else { 9358 lea(rscratch, src3); 9359 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 9360 } 9361 } 9362 9363 #if COMPILER2_OR_JVMCI 9364 9365 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 9366 Register length, Register temp, int vec_enc) { 9367 // Computing mask for predicated vector store. 9368 movptr(temp, -1); 9369 bzhiq(temp, temp, length); 9370 kmov(mask, temp); 9371 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 9372 } 9373 9374 // Set memory operation for length "less than" 64 bytes. 9375 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 9376 XMMRegister xmm, KRegister mask, Register length, 9377 Register temp, bool use64byteVector) { 9378 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9379 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9380 if (!use64byteVector) { 9381 fill32(dst, disp, xmm); 9382 subptr(length, 32 >> shift); 9383 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 9384 } else { 9385 assert(MaxVectorSize == 64, "vector length != 64"); 9386 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 9387 } 9388 } 9389 9390 9391 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 9392 XMMRegister xmm, KRegister mask, Register length, 9393 Register temp) { 9394 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9395 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9396 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 9397 } 9398 9399 9400 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 9401 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9402 vmovdqu(dst, xmm); 9403 } 9404 9405 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 9406 fill32(Address(dst, disp), xmm); 9407 } 9408 9409 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 9410 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9411 if (!use64byteVector) { 9412 fill32(dst, xmm); 9413 fill32(dst.plus_disp(32), xmm); 9414 } else { 9415 evmovdquq(dst, xmm, Assembler::AVX_512bit); 9416 } 9417 } 9418 9419 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 9420 fill64(Address(dst, disp), xmm, use64byteVector); 9421 } 9422 9423 #ifdef _LP64 9424 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 9425 Register count, Register rtmp, XMMRegister xtmp) { 9426 Label L_exit; 9427 Label L_fill_start; 9428 Label L_fill_64_bytes; 9429 Label L_fill_96_bytes; 9430 Label L_fill_128_bytes; 9431 Label L_fill_128_bytes_loop; 9432 Label L_fill_128_loop_header; 9433 Label L_fill_128_bytes_loop_header; 9434 Label L_fill_128_bytes_loop_pre_header; 9435 Label L_fill_zmm_sequence; 9436 9437 int shift = -1; 9438 int avx3threshold = VM_Version::avx3_threshold(); 9439 switch(type) { 9440 case T_BYTE: shift = 0; 9441 break; 9442 case T_SHORT: shift = 1; 9443 break; 9444 case T_INT: shift = 2; 9445 break; 9446 /* Uncomment when LONG fill stubs are supported. 9447 case T_LONG: shift = 3; 9448 break; 9449 */ 9450 default: 9451 fatal("Unhandled type: %s\n", type2name(type)); 9452 } 9453 9454 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 9455 9456 if (MaxVectorSize == 64) { 9457 cmpq(count, avx3threshold >> shift); 9458 jcc(Assembler::greater, L_fill_zmm_sequence); 9459 } 9460 9461 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 9462 9463 bind(L_fill_start); 9464 9465 cmpq(count, 32 >> shift); 9466 jccb(Assembler::greater, L_fill_64_bytes); 9467 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 9468 jmp(L_exit); 9469 9470 bind(L_fill_64_bytes); 9471 cmpq(count, 64 >> shift); 9472 jccb(Assembler::greater, L_fill_96_bytes); 9473 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 9474 jmp(L_exit); 9475 9476 bind(L_fill_96_bytes); 9477 cmpq(count, 96 >> shift); 9478 jccb(Assembler::greater, L_fill_128_bytes); 9479 fill64(to, 0, xtmp); 9480 subq(count, 64 >> shift); 9481 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 9482 jmp(L_exit); 9483 9484 bind(L_fill_128_bytes); 9485 cmpq(count, 128 >> shift); 9486 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 9487 fill64(to, 0, xtmp); 9488 fill32(to, 64, xtmp); 9489 subq(count, 96 >> shift); 9490 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 9491 jmp(L_exit); 9492 9493 bind(L_fill_128_bytes_loop_pre_header); 9494 { 9495 mov(rtmp, to); 9496 andq(rtmp, 31); 9497 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 9498 negq(rtmp); 9499 addq(rtmp, 32); 9500 mov64(r8, -1L); 9501 bzhiq(r8, r8, rtmp); 9502 kmovql(k2, r8); 9503 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 9504 addq(to, rtmp); 9505 shrq(rtmp, shift); 9506 subq(count, rtmp); 9507 } 9508 9509 cmpq(count, 128 >> shift); 9510 jcc(Assembler::less, L_fill_start); 9511 9512 bind(L_fill_128_bytes_loop_header); 9513 subq(count, 128 >> shift); 9514 9515 align32(); 9516 bind(L_fill_128_bytes_loop); 9517 fill64(to, 0, xtmp); 9518 fill64(to, 64, xtmp); 9519 addq(to, 128); 9520 subq(count, 128 >> shift); 9521 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 9522 9523 addq(count, 128 >> shift); 9524 jcc(Assembler::zero, L_exit); 9525 jmp(L_fill_start); 9526 } 9527 9528 if (MaxVectorSize == 64) { 9529 // Sequence using 64 byte ZMM register. 9530 Label L_fill_128_bytes_zmm; 9531 Label L_fill_192_bytes_zmm; 9532 Label L_fill_192_bytes_loop_zmm; 9533 Label L_fill_192_bytes_loop_header_zmm; 9534 Label L_fill_192_bytes_loop_pre_header_zmm; 9535 Label L_fill_start_zmm_sequence; 9536 9537 bind(L_fill_zmm_sequence); 9538 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 9539 9540 bind(L_fill_start_zmm_sequence); 9541 cmpq(count, 64 >> shift); 9542 jccb(Assembler::greater, L_fill_128_bytes_zmm); 9543 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 9544 jmp(L_exit); 9545 9546 bind(L_fill_128_bytes_zmm); 9547 cmpq(count, 128 >> shift); 9548 jccb(Assembler::greater, L_fill_192_bytes_zmm); 9549 fill64(to, 0, xtmp, true); 9550 subq(count, 64 >> shift); 9551 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 9552 jmp(L_exit); 9553 9554 bind(L_fill_192_bytes_zmm); 9555 cmpq(count, 192 >> shift); 9556 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 9557 fill64(to, 0, xtmp, true); 9558 fill64(to, 64, xtmp, true); 9559 subq(count, 128 >> shift); 9560 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 9561 jmp(L_exit); 9562 9563 bind(L_fill_192_bytes_loop_pre_header_zmm); 9564 { 9565 movq(rtmp, to); 9566 andq(rtmp, 63); 9567 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 9568 negq(rtmp); 9569 addq(rtmp, 64); 9570 mov64(r8, -1L); 9571 bzhiq(r8, r8, rtmp); 9572 kmovql(k2, r8); 9573 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 9574 addq(to, rtmp); 9575 shrq(rtmp, shift); 9576 subq(count, rtmp); 9577 } 9578 9579 cmpq(count, 192 >> shift); 9580 jcc(Assembler::less, L_fill_start_zmm_sequence); 9581 9582 bind(L_fill_192_bytes_loop_header_zmm); 9583 subq(count, 192 >> shift); 9584 9585 align32(); 9586 bind(L_fill_192_bytes_loop_zmm); 9587 fill64(to, 0, xtmp, true); 9588 fill64(to, 64, xtmp, true); 9589 fill64(to, 128, xtmp, true); 9590 addq(to, 192); 9591 subq(count, 192 >> shift); 9592 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 9593 9594 addq(count, 192 >> shift); 9595 jcc(Assembler::zero, L_exit); 9596 jmp(L_fill_start_zmm_sequence); 9597 } 9598 bind(L_exit); 9599 } 9600 #endif 9601 #endif //COMPILER2_OR_JVMCI 9602 9603 9604 #ifdef _LP64 9605 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 9606 Label done; 9607 cvttss2sil(dst, src); 9608 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9609 cmpl(dst, 0x80000000); // float_sign_flip 9610 jccb(Assembler::notEqual, done); 9611 subptr(rsp, 8); 9612 movflt(Address(rsp, 0), src); 9613 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 9614 pop(dst); 9615 bind(done); 9616 } 9617 9618 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 9619 Label done; 9620 cvttsd2sil(dst, src); 9621 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9622 cmpl(dst, 0x80000000); // float_sign_flip 9623 jccb(Assembler::notEqual, done); 9624 subptr(rsp, 8); 9625 movdbl(Address(rsp, 0), src); 9626 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 9627 pop(dst); 9628 bind(done); 9629 } 9630 9631 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 9632 Label done; 9633 cvttss2siq(dst, src); 9634 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9635 jccb(Assembler::notEqual, done); 9636 subptr(rsp, 8); 9637 movflt(Address(rsp, 0), src); 9638 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 9639 pop(dst); 9640 bind(done); 9641 } 9642 9643 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9644 // Following code is line by line assembly translation rounding algorithm. 9645 // Please refer to java.lang.Math.round(float) algorithm for details. 9646 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 9647 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 9648 const int32_t FloatConsts_EXP_BIAS = 127; 9649 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 9650 const int32_t MINUS_32 = 0xFFFFFFE0; 9651 Label L_special_case, L_block1, L_exit; 9652 movl(rtmp, FloatConsts_EXP_BIT_MASK); 9653 movdl(dst, src); 9654 andl(dst, rtmp); 9655 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 9656 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 9657 subl(rtmp, dst); 9658 movl(rcx, rtmp); 9659 movl(dst, MINUS_32); 9660 testl(rtmp, dst); 9661 jccb(Assembler::notEqual, L_special_case); 9662 movdl(dst, src); 9663 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 9664 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 9665 movdl(rtmp, src); 9666 testl(rtmp, rtmp); 9667 jccb(Assembler::greaterEqual, L_block1); 9668 negl(dst); 9669 bind(L_block1); 9670 sarl(dst); 9671 addl(dst, 0x1); 9672 sarl(dst, 0x1); 9673 jmp(L_exit); 9674 bind(L_special_case); 9675 convert_f2i(dst, src); 9676 bind(L_exit); 9677 } 9678 9679 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9680 // Following code is line by line assembly translation rounding algorithm. 9681 // Please refer to java.lang.Math.round(double) algorithm for details. 9682 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 9683 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 9684 const int64_t DoubleConsts_EXP_BIAS = 1023; 9685 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 9686 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 9687 Label L_special_case, L_block1, L_exit; 9688 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 9689 movq(dst, src); 9690 andq(dst, rtmp); 9691 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 9692 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 9693 subq(rtmp, dst); 9694 movq(rcx, rtmp); 9695 mov64(dst, MINUS_64); 9696 testq(rtmp, dst); 9697 jccb(Assembler::notEqual, L_special_case); 9698 movq(dst, src); 9699 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 9700 andq(dst, rtmp); 9701 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 9702 orq(dst, rtmp); 9703 movq(rtmp, src); 9704 testq(rtmp, rtmp); 9705 jccb(Assembler::greaterEqual, L_block1); 9706 negq(dst); 9707 bind(L_block1); 9708 sarq(dst); 9709 addq(dst, 0x1); 9710 sarq(dst, 0x1); 9711 jmp(L_exit); 9712 bind(L_special_case); 9713 convert_d2l(dst, src); 9714 bind(L_exit); 9715 } 9716 9717 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 9718 Label done; 9719 cvttsd2siq(dst, src); 9720 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9721 jccb(Assembler::notEqual, done); 9722 subptr(rsp, 8); 9723 movdbl(Address(rsp, 0), src); 9724 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 9725 pop(dst); 9726 bind(done); 9727 } 9728 9729 void MacroAssembler::cache_wb(Address line) 9730 { 9731 // 64 bit cpus always support clflush 9732 assert(VM_Version::supports_clflush(), "clflush should be available"); 9733 bool optimized = VM_Version::supports_clflushopt(); 9734 bool no_evict = VM_Version::supports_clwb(); 9735 9736 // prefer clwb (writeback without evict) otherwise 9737 // prefer clflushopt (potentially parallel writeback with evict) 9738 // otherwise fallback on clflush (serial writeback with evict) 9739 9740 if (optimized) { 9741 if (no_evict) { 9742 clwb(line); 9743 } else { 9744 clflushopt(line); 9745 } 9746 } else { 9747 // no need for fence when using CLFLUSH 9748 clflush(line); 9749 } 9750 } 9751 9752 void MacroAssembler::cache_wbsync(bool is_pre) 9753 { 9754 assert(VM_Version::supports_clflush(), "clflush should be available"); 9755 bool optimized = VM_Version::supports_clflushopt(); 9756 bool no_evict = VM_Version::supports_clwb(); 9757 9758 // pick the correct implementation 9759 9760 if (!is_pre && (optimized || no_evict)) { 9761 // need an sfence for post flush when using clflushopt or clwb 9762 // otherwise no no need for any synchroniaztion 9763 9764 sfence(); 9765 } 9766 } 9767 9768 #endif // _LP64 9769 9770 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 9771 switch (cond) { 9772 // Note some conditions are synonyms for others 9773 case Assembler::zero: return Assembler::notZero; 9774 case Assembler::notZero: return Assembler::zero; 9775 case Assembler::less: return Assembler::greaterEqual; 9776 case Assembler::lessEqual: return Assembler::greater; 9777 case Assembler::greater: return Assembler::lessEqual; 9778 case Assembler::greaterEqual: return Assembler::less; 9779 case Assembler::below: return Assembler::aboveEqual; 9780 case Assembler::belowEqual: return Assembler::above; 9781 case Assembler::above: return Assembler::belowEqual; 9782 case Assembler::aboveEqual: return Assembler::below; 9783 case Assembler::overflow: return Assembler::noOverflow; 9784 case Assembler::noOverflow: return Assembler::overflow; 9785 case Assembler::negative: return Assembler::positive; 9786 case Assembler::positive: return Assembler::negative; 9787 case Assembler::parity: return Assembler::noParity; 9788 case Assembler::noParity: return Assembler::parity; 9789 } 9790 ShouldNotReachHere(); return Assembler::overflow; 9791 } 9792 9793 SkipIfEqual::SkipIfEqual( 9794 MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) { 9795 _masm = masm; 9796 _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch); 9797 _masm->jcc(Assembler::equal, _label); 9798 } 9799 9800 SkipIfEqual::~SkipIfEqual() { 9801 _masm->bind(_label); 9802 } 9803 9804 // 32-bit Windows has its own fast-path implementation 9805 // of get_thread 9806 #if !defined(WIN32) || defined(_LP64) 9807 9808 // This is simply a call to Thread::current() 9809 void MacroAssembler::get_thread(Register thread) { 9810 if (thread != rax) { 9811 push(rax); 9812 } 9813 LP64_ONLY(push(rdi);) 9814 LP64_ONLY(push(rsi);) 9815 push(rdx); 9816 push(rcx); 9817 #ifdef _LP64 9818 push(r8); 9819 push(r9); 9820 push(r10); 9821 push(r11); 9822 #endif 9823 9824 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 9825 9826 #ifdef _LP64 9827 pop(r11); 9828 pop(r10); 9829 pop(r9); 9830 pop(r8); 9831 #endif 9832 pop(rcx); 9833 pop(rdx); 9834 LP64_ONLY(pop(rsi);) 9835 LP64_ONLY(pop(rdi);) 9836 if (thread != rax) { 9837 mov(thread, rax); 9838 pop(rax); 9839 } 9840 } 9841 9842 9843 #endif // !WIN32 || _LP64 9844 9845 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 9846 Label L_stack_ok; 9847 if (bias == 0) { 9848 testptr(sp, 2 * wordSize - 1); 9849 } else { 9850 // lea(tmp, Address(rsp, bias); 9851 mov(tmp, sp); 9852 addptr(tmp, bias); 9853 testptr(tmp, 2 * wordSize - 1); 9854 } 9855 jcc(Assembler::equal, L_stack_ok); 9856 block_comment(msg); 9857 stop(msg); 9858 bind(L_stack_ok); 9859 } 9860 9861 // Implements fast-locking. 9862 // Branches to slow upon failure to lock the object, with ZF cleared. 9863 // Falls through upon success with unspecified ZF. 9864 // 9865 // obj: the object to be locked 9866 // hdr: the (pre-loaded) header of the object, must be rax 9867 // thread: the thread which attempts to lock obj 9868 // tmp: a temporary register 9869 void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow) { 9870 assert(hdr == rax, "header must be in rax for cmpxchg"); 9871 assert_different_registers(obj, hdr, thread, tmp); 9872 9873 // First we need to check if the lock-stack has room for pushing the object reference. 9874 // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead 9875 // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and 9876 // avoids one branch. 9877 cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1); 9878 jcc(Assembler::greater, slow); 9879 9880 // Now we attempt to take the fast-lock. 9881 // Clear lock_mask bits (locked state). 9882 andptr(hdr, ~(int32_t)markWord::lock_mask_in_place); 9883 movptr(tmp, hdr); 9884 // Set unlocked_value bit. 9885 orptr(hdr, markWord::unlocked_value); 9886 lock(); 9887 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9888 jcc(Assembler::notEqual, slow); 9889 9890 // If successful, push object to lock-stack. 9891 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset())); 9892 movptr(Address(thread, tmp), obj); 9893 incrementl(tmp, oopSize); 9894 movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp); 9895 } 9896 9897 // Implements fast-unlocking. 9898 // Branches to slow upon failure, with ZF cleared. 9899 // Falls through upon success, with unspecified ZF. 9900 // 9901 // obj: the object to be unlocked 9902 // hdr: the (pre-loaded) header of the object, must be rax 9903 // tmp: a temporary register 9904 void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) { 9905 assert(hdr == rax, "header must be in rax for cmpxchg"); 9906 assert_different_registers(obj, hdr, tmp); 9907 9908 // Mark-word must be lock_mask now, try to swing it back to unlocked_value. 9909 movptr(tmp, hdr); // The expected old value 9910 orptr(tmp, markWord::unlocked_value); 9911 lock(); 9912 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9913 jcc(Assembler::notEqual, slow); 9914 // Pop the lock object from the lock-stack. 9915 #ifdef _LP64 9916 const Register thread = r15_thread; 9917 #else 9918 const Register thread = rax; 9919 get_thread(thread); 9920 #endif 9921 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 9922 #ifdef ASSERT 9923 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset())); 9924 movptr(Address(thread, tmp), 0); 9925 #endif 9926 }