1 /* 2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "code/compiledIC.hpp" 29 #include "compiler/compiler_globals.hpp" 30 #include "compiler/disassembler.hpp" 31 #include "crc32c.h" 32 #include "gc/shared/barrierSet.hpp" 33 #include "gc/shared/barrierSetAssembler.hpp" 34 #include "gc/shared/collectedHeap.inline.hpp" 35 #include "gc/shared/tlab_globals.hpp" 36 #include "interpreter/bytecodeHistogram.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "jvm.h" 39 #include "memory/resourceArea.hpp" 40 #include "memory/universe.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedKlass.inline.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/klass.inline.hpp" 45 #include "prims/methodHandles.hpp" 46 #include "runtime/continuation.hpp" 47 #include "runtime/interfaceSupport.inline.hpp" 48 #include "runtime/javaThread.hpp" 49 #include "runtime/jniHandles.hpp" 50 #include "runtime/objectMonitor.hpp" 51 #include "runtime/os.hpp" 52 #include "runtime/safepoint.hpp" 53 #include "runtime/safepointMechanism.hpp" 54 #include "runtime/sharedRuntime.hpp" 55 #include "runtime/stubRoutines.hpp" 56 #include "utilities/checkedCast.hpp" 57 #include "utilities/macros.hpp" 58 59 #ifdef PRODUCT 60 #define BLOCK_COMMENT(str) /* nothing */ 61 #define STOP(error) stop(error) 62 #else 63 #define BLOCK_COMMENT(str) block_comment(str) 64 #define STOP(error) block_comment(error); stop(error) 65 #endif 66 67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 68 69 #ifdef ASSERT 70 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 71 #endif 72 73 static const Assembler::Condition reverse[] = { 74 Assembler::noOverflow /* overflow = 0x0 */ , 75 Assembler::overflow /* noOverflow = 0x1 */ , 76 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 77 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 78 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 79 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 80 Assembler::above /* belowEqual = 0x6 */ , 81 Assembler::belowEqual /* above = 0x7 */ , 82 Assembler::positive /* negative = 0x8 */ , 83 Assembler::negative /* positive = 0x9 */ , 84 Assembler::noParity /* parity = 0xa */ , 85 Assembler::parity /* noParity = 0xb */ , 86 Assembler::greaterEqual /* less = 0xc */ , 87 Assembler::less /* greaterEqual = 0xd */ , 88 Assembler::greater /* lessEqual = 0xe */ , 89 Assembler::lessEqual /* greater = 0xf, */ 90 91 }; 92 93 94 // Implementation of MacroAssembler 95 96 // First all the versions that have distinct versions depending on 32/64 bit 97 // Unless the difference is trivial (1 line or so). 98 99 #ifndef _LP64 100 101 // 32bit versions 102 103 Address MacroAssembler::as_Address(AddressLiteral adr) { 104 return Address(adr.target(), adr.rspec()); 105 } 106 107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 108 assert(rscratch == noreg, ""); 109 return Address::make_array(adr); 110 } 111 112 void MacroAssembler::call_VM_leaf_base(address entry_point, 113 int number_of_arguments) { 114 call(RuntimeAddress(entry_point)); 115 increment(rsp, number_of_arguments * wordSize); 116 } 117 118 void MacroAssembler::cmpklass(Address src1, Metadata* obj) { 119 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 120 } 121 122 123 void MacroAssembler::cmpklass(Register src1, Metadata* obj) { 124 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 125 } 126 127 void MacroAssembler::cmpoop(Address src1, jobject obj) { 128 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 129 } 130 131 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) { 132 assert(rscratch == noreg, "redundant"); 133 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 134 } 135 136 void MacroAssembler::extend_sign(Register hi, Register lo) { 137 // According to Intel Doc. AP-526, "Integer Divide", p.18. 138 if (VM_Version::is_P6() && hi == rdx && lo == rax) { 139 cdql(); 140 } else { 141 movl(hi, lo); 142 sarl(hi, 31); 143 } 144 } 145 146 void MacroAssembler::jC2(Register tmp, Label& L) { 147 // set parity bit if FPU flag C2 is set (via rax) 148 save_rax(tmp); 149 fwait(); fnstsw_ax(); 150 sahf(); 151 restore_rax(tmp); 152 // branch 153 jcc(Assembler::parity, L); 154 } 155 156 void MacroAssembler::jnC2(Register tmp, Label& L) { 157 // set parity bit if FPU flag C2 is set (via rax) 158 save_rax(tmp); 159 fwait(); fnstsw_ax(); 160 sahf(); 161 restore_rax(tmp); 162 // branch 163 jcc(Assembler::noParity, L); 164 } 165 166 // 32bit can do a case table jump in one instruction but we no longer allow the base 167 // to be installed in the Address class 168 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 169 assert(rscratch == noreg, "not needed"); 170 jmp(as_Address(entry, noreg)); 171 } 172 173 // Note: y_lo will be destroyed 174 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 175 // Long compare for Java (semantics as described in JVM spec.) 176 Label high, low, done; 177 178 cmpl(x_hi, y_hi); 179 jcc(Assembler::less, low); 180 jcc(Assembler::greater, high); 181 // x_hi is the return register 182 xorl(x_hi, x_hi); 183 cmpl(x_lo, y_lo); 184 jcc(Assembler::below, low); 185 jcc(Assembler::equal, done); 186 187 bind(high); 188 xorl(x_hi, x_hi); 189 increment(x_hi); 190 jmp(done); 191 192 bind(low); 193 xorl(x_hi, x_hi); 194 decrementl(x_hi); 195 196 bind(done); 197 } 198 199 void MacroAssembler::lea(Register dst, AddressLiteral src) { 200 mov_literal32(dst, (int32_t)src.target(), src.rspec()); 201 } 202 203 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 204 assert(rscratch == noreg, "not needed"); 205 206 // leal(dst, as_Address(adr)); 207 // see note in movl as to why we must use a move 208 mov_literal32(dst, (int32_t)adr.target(), adr.rspec()); 209 } 210 211 void MacroAssembler::leave() { 212 mov(rsp, rbp); 213 pop(rbp); 214 } 215 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { 217 // Multiplication of two Java long values stored on the stack 218 // as illustrated below. Result is in rdx:rax. 219 // 220 // rsp ---> [ ?? ] \ \ 221 // .... | y_rsp_offset | 222 // [ y_lo ] / (in bytes) | x_rsp_offset 223 // [ y_hi ] | (in bytes) 224 // .... | 225 // [ x_lo ] / 226 // [ x_hi ] 227 // .... 228 // 229 // Basic idea: lo(result) = lo(x_lo * y_lo) 230 // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) 231 Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); 232 Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); 233 Label quick; 234 // load x_hi, y_hi and check if quick 235 // multiplication is possible 236 movl(rbx, x_hi); 237 movl(rcx, y_hi); 238 movl(rax, rbx); 239 orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 240 jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply 241 // do full multiplication 242 // 1st step 243 mull(y_lo); // x_hi * y_lo 244 movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, 245 // 2nd step 246 movl(rax, x_lo); 247 mull(rcx); // x_lo * y_hi 248 addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, 249 // 3rd step 250 bind(quick); // note: rbx, = 0 if quick multiply! 251 movl(rax, x_lo); 252 mull(y_lo); // x_lo * y_lo 253 addl(rdx, rbx); // correct hi(x_lo * y_lo) 254 } 255 256 void MacroAssembler::lneg(Register hi, Register lo) { 257 negl(lo); 258 adcl(hi, 0); 259 negl(hi); 260 } 261 262 void MacroAssembler::lshl(Register hi, Register lo) { 263 // Java shift left long support (semantics as described in JVM spec., p.305) 264 // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) 265 // shift value is in rcx ! 266 assert(hi != rcx, "must not use rcx"); 267 assert(lo != rcx, "must not use rcx"); 268 const Register s = rcx; // shift count 269 const int n = BitsPerWord; 270 Label L; 271 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 272 cmpl(s, n); // if (s < n) 273 jcc(Assembler::less, L); // else (s >= n) 274 movl(hi, lo); // x := x << n 275 xorl(lo, lo); 276 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 277 bind(L); // s (mod n) < n 278 shldl(hi, lo); // x := x << s 279 shll(lo); 280 } 281 282 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { 284 // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) 285 // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) 286 assert(hi != rcx, "must not use rcx"); 287 assert(lo != rcx, "must not use rcx"); 288 const Register s = rcx; // shift count 289 const int n = BitsPerWord; 290 Label L; 291 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 292 cmpl(s, n); // if (s < n) 293 jcc(Assembler::less, L); // else (s >= n) 294 movl(lo, hi); // x := x >> n 295 if (sign_extension) sarl(hi, 31); 296 else xorl(hi, hi); 297 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 298 bind(L); // s (mod n) < n 299 shrdl(lo, hi); // x := x >> s 300 if (sign_extension) sarl(hi); 301 else shrl(hi); 302 } 303 304 void MacroAssembler::movoop(Register dst, jobject obj) { 305 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 306 } 307 308 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 309 assert(rscratch == noreg, "redundant"); 310 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 311 } 312 313 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 314 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 315 } 316 317 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 318 assert(rscratch == noreg, "redundant"); 319 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 320 } 321 322 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 323 if (src.is_lval()) { 324 mov_literal32(dst, (intptr_t)src.target(), src.rspec()); 325 } else { 326 movl(dst, as_Address(src)); 327 } 328 } 329 330 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 331 assert(rscratch == noreg, "redundant"); 332 movl(as_Address(dst, noreg), src); 333 } 334 335 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 336 movl(dst, as_Address(src, noreg)); 337 } 338 339 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 340 assert(rscratch == noreg, "redundant"); 341 movl(dst, src); 342 } 343 344 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 345 assert(rscratch == noreg, "redundant"); 346 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); 347 } 348 349 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 350 assert(rscratch == noreg, "redundant"); 351 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); 352 } 353 354 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 355 assert(rscratch == noreg, "redundant"); 356 if (src.is_lval()) { 357 push_literal32((int32_t)src.target(), src.rspec()); 358 } else { 359 pushl(as_Address(src)); 360 } 361 } 362 363 static void pass_arg0(MacroAssembler* masm, Register arg) { 364 masm->push(arg); 365 } 366 367 static void pass_arg1(MacroAssembler* masm, Register arg) { 368 masm->push(arg); 369 } 370 371 static void pass_arg2(MacroAssembler* masm, Register arg) { 372 masm->push(arg); 373 } 374 375 static void pass_arg3(MacroAssembler* masm, Register arg) { 376 masm->push(arg); 377 } 378 379 #ifndef PRODUCT 380 extern "C" void findpc(intptr_t x); 381 #endif 382 383 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { 384 // In order to get locks to work, we need to fake a in_VM state 385 JavaThread* thread = JavaThread::current(); 386 JavaThreadState saved_state = thread->thread_state(); 387 thread->set_thread_state(_thread_in_vm); 388 if (ShowMessageBoxOnError) { 389 JavaThread* thread = JavaThread::current(); 390 JavaThreadState saved_state = thread->thread_state(); 391 thread->set_thread_state(_thread_in_vm); 392 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 393 ttyLocker ttyl; 394 BytecodeCounter::print(); 395 } 396 // To see where a verify_oop failed, get $ebx+40/X for this frame. 397 // This is the value of eip which points to where verify_oop will return. 398 if (os::message_box(msg, "Execution stopped, print registers?")) { 399 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); 400 BREAKPOINT; 401 } 402 } 403 fatal("DEBUG MESSAGE: %s", msg); 404 } 405 406 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { 407 ttyLocker ttyl; 408 DebuggingContext debugging{}; 409 tty->print_cr("eip = 0x%08x", eip); 410 #ifndef PRODUCT 411 if ((WizardMode || Verbose) && PrintMiscellaneous) { 412 tty->cr(); 413 findpc(eip); 414 tty->cr(); 415 } 416 #endif 417 #define PRINT_REG(rax) \ 418 { tty->print("%s = ", #rax); os::print_location(tty, rax); } 419 PRINT_REG(rax); 420 PRINT_REG(rbx); 421 PRINT_REG(rcx); 422 PRINT_REG(rdx); 423 PRINT_REG(rdi); 424 PRINT_REG(rsi); 425 PRINT_REG(rbp); 426 PRINT_REG(rsp); 427 #undef PRINT_REG 428 // Print some words near top of staack. 429 int* dump_sp = (int*) rsp; 430 for (int col1 = 0; col1 < 8; col1++) { 431 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 432 os::print_location(tty, *dump_sp++); 433 } 434 for (int row = 0; row < 16; row++) { 435 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 436 for (int col = 0; col < 8; col++) { 437 tty->print(" 0x%08x", *dump_sp++); 438 } 439 tty->cr(); 440 } 441 // Print some instructions around pc: 442 Disassembler::decode((address)eip-64, (address)eip); 443 tty->print_cr("--------"); 444 Disassembler::decode((address)eip, (address)eip+32); 445 } 446 447 void MacroAssembler::stop(const char* msg) { 448 // push address of message 449 ExternalAddress message((address)msg); 450 pushptr(message.addr(), noreg); 451 { Label L; call(L, relocInfo::none); bind(L); } // push eip 452 pusha(); // push registers 453 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); 454 hlt(); 455 } 456 457 void MacroAssembler::warn(const char* msg) { 458 push_CPU_state(); 459 460 // push address of message 461 ExternalAddress message((address)msg); 462 pushptr(message.addr(), noreg); 463 464 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 465 addl(rsp, wordSize); // discard argument 466 pop_CPU_state(); 467 } 468 469 void MacroAssembler::print_state() { 470 { Label L; call(L, relocInfo::none); bind(L); } // push eip 471 pusha(); // push registers 472 473 push_CPU_state(); 474 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); 475 pop_CPU_state(); 476 477 popa(); 478 addl(rsp, wordSize); 479 } 480 481 #else // _LP64 482 483 // 64 bit versions 484 485 Address MacroAssembler::as_Address(AddressLiteral adr) { 486 // amd64 always does this as a pc-rel 487 // we can be absolute or disp based on the instruction type 488 // jmp/call are displacements others are absolute 489 assert(!adr.is_lval(), "must be rval"); 490 assert(reachable(adr), "must be"); 491 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 492 493 } 494 495 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 496 AddressLiteral base = adr.base(); 497 lea(rscratch, base); 498 Address index = adr.index(); 499 assert(index._disp == 0, "must not have disp"); // maybe it can? 500 Address array(rscratch, index._index, index._scale, index._disp); 501 return array; 502 } 503 504 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 505 Label L, E; 506 507 #ifdef _WIN64 508 // Windows always allocates space for it's register args 509 assert(num_args <= 4, "only register arguments supported"); 510 subq(rsp, frame::arg_reg_save_area_bytes); 511 #endif 512 513 // Align stack if necessary 514 testl(rsp, 15); 515 jcc(Assembler::zero, L); 516 517 subq(rsp, 8); 518 call(RuntimeAddress(entry_point)); 519 addq(rsp, 8); 520 jmp(E); 521 522 bind(L); 523 call(RuntimeAddress(entry_point)); 524 525 bind(E); 526 527 #ifdef _WIN64 528 // restore stack pointer 529 addq(rsp, frame::arg_reg_save_area_bytes); 530 #endif 531 532 } 533 534 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 535 assert(!src2.is_lval(), "should use cmpptr"); 536 assert(rscratch != noreg || always_reachable(src2), "missing"); 537 538 if (reachable(src2)) { 539 cmpq(src1, as_Address(src2)); 540 } else { 541 lea(rscratch, src2); 542 Assembler::cmpq(src1, Address(rscratch, 0)); 543 } 544 } 545 546 int MacroAssembler::corrected_idivq(Register reg) { 547 // Full implementation of Java ldiv and lrem; checks for special 548 // case as described in JVM spec., p.243 & p.271. The function 549 // returns the (pc) offset of the idivl instruction - may be needed 550 // for implicit exceptions. 551 // 552 // normal case special case 553 // 554 // input : rax: dividend min_long 555 // reg: divisor (may not be eax/edx) -1 556 // 557 // output: rax: quotient (= rax idiv reg) min_long 558 // rdx: remainder (= rax irem reg) 0 559 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 560 static const int64_t min_long = 0x8000000000000000; 561 Label normal_case, special_case; 562 563 // check for special case 564 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 565 jcc(Assembler::notEqual, normal_case); 566 xorl(rdx, rdx); // prepare rdx for possible special case (where 567 // remainder = 0) 568 cmpq(reg, -1); 569 jcc(Assembler::equal, special_case); 570 571 // handle normal case 572 bind(normal_case); 573 cdqq(); 574 int idivq_offset = offset(); 575 idivq(reg); 576 577 // normal and special case exit 578 bind(special_case); 579 580 return idivq_offset; 581 } 582 583 void MacroAssembler::decrementq(Register reg, int value) { 584 if (value == min_jint) { subq(reg, value); return; } 585 if (value < 0) { incrementq(reg, -value); return; } 586 if (value == 0) { ; return; } 587 if (value == 1 && UseIncDec) { decq(reg) ; return; } 588 /* else */ { subq(reg, value) ; return; } 589 } 590 591 void MacroAssembler::decrementq(Address dst, int value) { 592 if (value == min_jint) { subq(dst, value); return; } 593 if (value < 0) { incrementq(dst, -value); return; } 594 if (value == 0) { ; return; } 595 if (value == 1 && UseIncDec) { decq(dst) ; return; } 596 /* else */ { subq(dst, value) ; return; } 597 } 598 599 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 600 assert(rscratch != noreg || always_reachable(dst), "missing"); 601 602 if (reachable(dst)) { 603 incrementq(as_Address(dst)); 604 } else { 605 lea(rscratch, dst); 606 incrementq(Address(rscratch, 0)); 607 } 608 } 609 610 void MacroAssembler::incrementq(Register reg, int value) { 611 if (value == min_jint) { addq(reg, value); return; } 612 if (value < 0) { decrementq(reg, -value); return; } 613 if (value == 0) { ; return; } 614 if (value == 1 && UseIncDec) { incq(reg) ; return; } 615 /* else */ { addq(reg, value) ; return; } 616 } 617 618 void MacroAssembler::incrementq(Address dst, int value) { 619 if (value == min_jint) { addq(dst, value); return; } 620 if (value < 0) { decrementq(dst, -value); return; } 621 if (value == 0) { ; return; } 622 if (value == 1 && UseIncDec) { incq(dst) ; return; } 623 /* else */ { addq(dst, value) ; return; } 624 } 625 626 // 32bit can do a case table jump in one instruction but we no longer allow the base 627 // to be installed in the Address class 628 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 629 lea(rscratch, entry.base()); 630 Address dispatch = entry.index(); 631 assert(dispatch._base == noreg, "must be"); 632 dispatch._base = rscratch; 633 jmp(dispatch); 634 } 635 636 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 637 ShouldNotReachHere(); // 64bit doesn't use two regs 638 cmpq(x_lo, y_lo); 639 } 640 641 void MacroAssembler::lea(Register dst, AddressLiteral src) { 642 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 643 } 644 645 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 646 lea(rscratch, adr); 647 movptr(dst, rscratch); 648 } 649 650 void MacroAssembler::leave() { 651 // %%% is this really better? Why not on 32bit too? 652 emit_int8((unsigned char)0xC9); // LEAVE 653 } 654 655 void MacroAssembler::lneg(Register hi, Register lo) { 656 ShouldNotReachHere(); // 64bit doesn't use two regs 657 negq(lo); 658 } 659 660 void MacroAssembler::movoop(Register dst, jobject obj) { 661 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 662 } 663 664 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 665 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 666 movq(dst, rscratch); 667 } 668 669 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 670 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 671 } 672 673 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 674 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 675 movq(dst, rscratch); 676 } 677 678 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 679 if (src.is_lval()) { 680 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 681 } else { 682 if (reachable(src)) { 683 movq(dst, as_Address(src)); 684 } else { 685 lea(dst, src); 686 movq(dst, Address(dst, 0)); 687 } 688 } 689 } 690 691 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 692 movq(as_Address(dst, rscratch), src); 693 } 694 695 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 696 movq(dst, as_Address(src, dst /*rscratch*/)); 697 } 698 699 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 700 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 701 if (is_simm32(src)) { 702 movptr(dst, checked_cast<int32_t>(src)); 703 } else { 704 mov64(rscratch, src); 705 movq(dst, rscratch); 706 } 707 } 708 709 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 710 movoop(rscratch, obj); 711 push(rscratch); 712 } 713 714 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 715 mov_metadata(rscratch, obj); 716 push(rscratch); 717 } 718 719 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 720 lea(rscratch, src); 721 if (src.is_lval()) { 722 push(rscratch); 723 } else { 724 pushq(Address(rscratch, 0)); 725 } 726 } 727 728 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 729 reset_last_Java_frame(r15_thread, clear_fp); 730 } 731 732 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 733 Register last_java_fp, 734 address last_java_pc, 735 Register rscratch) { 736 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch); 737 } 738 739 static void pass_arg0(MacroAssembler* masm, Register arg) { 740 if (c_rarg0 != arg ) { 741 masm->mov(c_rarg0, arg); 742 } 743 } 744 745 static void pass_arg1(MacroAssembler* masm, Register arg) { 746 if (c_rarg1 != arg ) { 747 masm->mov(c_rarg1, arg); 748 } 749 } 750 751 static void pass_arg2(MacroAssembler* masm, Register arg) { 752 if (c_rarg2 != arg ) { 753 masm->mov(c_rarg2, arg); 754 } 755 } 756 757 static void pass_arg3(MacroAssembler* masm, Register arg) { 758 if (c_rarg3 != arg ) { 759 masm->mov(c_rarg3, arg); 760 } 761 } 762 763 void MacroAssembler::stop(const char* msg) { 764 if (ShowMessageBoxOnError) { 765 address rip = pc(); 766 pusha(); // get regs on stack 767 lea(c_rarg1, InternalAddress(rip)); 768 movq(c_rarg2, rsp); // pass pointer to regs array 769 } 770 lea(c_rarg0, ExternalAddress((address) msg)); 771 andq(rsp, -16); // align stack as required by ABI 772 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 773 hlt(); 774 } 775 776 void MacroAssembler::warn(const char* msg) { 777 push(rbp); 778 movq(rbp, rsp); 779 andq(rsp, -16); // align stack as required by push_CPU_state and call 780 push_CPU_state(); // keeps alignment at 16 bytes 781 782 lea(c_rarg0, ExternalAddress((address) msg)); 783 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 784 785 pop_CPU_state(); 786 mov(rsp, rbp); 787 pop(rbp); 788 } 789 790 void MacroAssembler::print_state() { 791 address rip = pc(); 792 pusha(); // get regs on stack 793 push(rbp); 794 movq(rbp, rsp); 795 andq(rsp, -16); // align stack as required by push_CPU_state and call 796 push_CPU_state(); // keeps alignment at 16 bytes 797 798 lea(c_rarg0, InternalAddress(rip)); 799 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 800 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 801 802 pop_CPU_state(); 803 mov(rsp, rbp); 804 pop(rbp); 805 popa(); 806 } 807 808 #ifndef PRODUCT 809 extern "C" void findpc(intptr_t x); 810 #endif 811 812 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 813 // In order to get locks to work, we need to fake a in_VM state 814 if (ShowMessageBoxOnError) { 815 JavaThread* thread = JavaThread::current(); 816 JavaThreadState saved_state = thread->thread_state(); 817 thread->set_thread_state(_thread_in_vm); 818 #ifndef PRODUCT 819 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 820 ttyLocker ttyl; 821 BytecodeCounter::print(); 822 } 823 #endif 824 // To see where a verify_oop failed, get $ebx+40/X for this frame. 825 // XXX correct this offset for amd64 826 // This is the value of eip which points to where verify_oop will return. 827 if (os::message_box(msg, "Execution stopped, print registers?")) { 828 print_state64(pc, regs); 829 BREAKPOINT; 830 } 831 } 832 fatal("DEBUG MESSAGE: %s", msg); 833 } 834 835 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 836 ttyLocker ttyl; 837 DebuggingContext debugging{}; 838 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 839 #ifndef PRODUCT 840 tty->cr(); 841 findpc(pc); 842 tty->cr(); 843 #endif 844 #define PRINT_REG(rax, value) \ 845 { tty->print("%s = ", #rax); os::print_location(tty, value); } 846 PRINT_REG(rax, regs[15]); 847 PRINT_REG(rbx, regs[12]); 848 PRINT_REG(rcx, regs[14]); 849 PRINT_REG(rdx, regs[13]); 850 PRINT_REG(rdi, regs[8]); 851 PRINT_REG(rsi, regs[9]); 852 PRINT_REG(rbp, regs[10]); 853 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 854 PRINT_REG(rsp, (intptr_t)(®s[16])); 855 PRINT_REG(r8 , regs[7]); 856 PRINT_REG(r9 , regs[6]); 857 PRINT_REG(r10, regs[5]); 858 PRINT_REG(r11, regs[4]); 859 PRINT_REG(r12, regs[3]); 860 PRINT_REG(r13, regs[2]); 861 PRINT_REG(r14, regs[1]); 862 PRINT_REG(r15, regs[0]); 863 #undef PRINT_REG 864 // Print some words near the top of the stack. 865 int64_t* rsp = ®s[16]; 866 int64_t* dump_sp = rsp; 867 for (int col1 = 0; col1 < 8; col1++) { 868 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 869 os::print_location(tty, *dump_sp++); 870 } 871 for (int row = 0; row < 25; row++) { 872 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 873 for (int col = 0; col < 4; col++) { 874 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 875 } 876 tty->cr(); 877 } 878 // Print some instructions around pc: 879 Disassembler::decode((address)pc-64, (address)pc); 880 tty->print_cr("--------"); 881 Disassembler::decode((address)pc, (address)pc+32); 882 } 883 884 // The java_calling_convention describes stack locations as ideal slots on 885 // a frame with no abi restrictions. Since we must observe abi restrictions 886 // (like the placement of the register window) the slots must be biased by 887 // the following value. 888 static int reg2offset_in(VMReg r) { 889 // Account for saved rbp and return address 890 // This should really be in_preserve_stack_slots 891 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 892 } 893 894 static int reg2offset_out(VMReg r) { 895 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 896 } 897 898 // A long move 899 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 900 901 // The calling conventions assures us that each VMregpair is either 902 // all really one physical register or adjacent stack slots. 903 904 if (src.is_single_phys_reg() ) { 905 if (dst.is_single_phys_reg()) { 906 if (dst.first() != src.first()) { 907 mov(dst.first()->as_Register(), src.first()->as_Register()); 908 } 909 } else { 910 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 911 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 912 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 913 } 914 } else if (dst.is_single_phys_reg()) { 915 assert(src.is_single_reg(), "not a stack pair"); 916 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 917 } else { 918 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 919 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 920 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 921 } 922 } 923 924 // A double move 925 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 926 927 // The calling conventions assures us that each VMregpair is either 928 // all really one physical register or adjacent stack slots. 929 930 if (src.is_single_phys_reg() ) { 931 if (dst.is_single_phys_reg()) { 932 // In theory these overlap but the ordering is such that this is likely a nop 933 if ( src.first() != dst.first()) { 934 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 935 } 936 } else { 937 assert(dst.is_single_reg(), "not a stack pair"); 938 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 939 } 940 } else if (dst.is_single_phys_reg()) { 941 assert(src.is_single_reg(), "not a stack pair"); 942 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 943 } else { 944 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 945 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 946 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 947 } 948 } 949 950 951 // A float arg may have to do float reg int reg conversion 952 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 953 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 954 955 // The calling conventions assures us that each VMregpair is either 956 // all really one physical register or adjacent stack slots. 957 958 if (src.first()->is_stack()) { 959 if (dst.first()->is_stack()) { 960 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 961 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 962 } else { 963 // stack to reg 964 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 965 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 966 } 967 } else if (dst.first()->is_stack()) { 968 // reg to stack 969 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 970 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 971 } else { 972 // reg to reg 973 // In theory these overlap but the ordering is such that this is likely a nop 974 if ( src.first() != dst.first()) { 975 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 976 } 977 } 978 } 979 980 // On 64 bit we will store integer like items to the stack as 981 // 64 bits items (x86_32/64 abi) even though java would only store 982 // 32bits for a parameter. On 32bit it will simply be 32 bits 983 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 984 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 985 if (src.first()->is_stack()) { 986 if (dst.first()->is_stack()) { 987 // stack to stack 988 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 989 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 990 } else { 991 // stack to reg 992 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 993 } 994 } else if (dst.first()->is_stack()) { 995 // reg to stack 996 // Do we really have to sign extend??? 997 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 998 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 999 } else { 1000 // Do we really have to sign extend??? 1001 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 1002 if (dst.first() != src.first()) { 1003 movq(dst.first()->as_Register(), src.first()->as_Register()); 1004 } 1005 } 1006 } 1007 1008 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 1009 if (src.first()->is_stack()) { 1010 if (dst.first()->is_stack()) { 1011 // stack to stack 1012 movq(rax, Address(rbp, reg2offset_in(src.first()))); 1013 movq(Address(rsp, reg2offset_out(dst.first())), rax); 1014 } else { 1015 // stack to reg 1016 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 1017 } 1018 } else if (dst.first()->is_stack()) { 1019 // reg to stack 1020 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 1021 } else { 1022 if (dst.first() != src.first()) { 1023 movq(dst.first()->as_Register(), src.first()->as_Register()); 1024 } 1025 } 1026 } 1027 1028 // An oop arg. Must pass a handle not the oop itself 1029 void MacroAssembler::object_move(OopMap* map, 1030 int oop_handle_offset, 1031 int framesize_in_slots, 1032 VMRegPair src, 1033 VMRegPair dst, 1034 bool is_receiver, 1035 int* receiver_offset) { 1036 1037 // must pass a handle. First figure out the location we use as a handle 1038 1039 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 1040 1041 // See if oop is null if it is we need no handle 1042 1043 if (src.first()->is_stack()) { 1044 1045 // Oop is already on the stack as an argument 1046 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 1047 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 1048 if (is_receiver) { 1049 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 1050 } 1051 1052 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 1053 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 1054 // conditionally move a null 1055 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 1056 } else { 1057 1058 // Oop is in a register we must store it to the space we reserve 1059 // on the stack for oop_handles and pass a handle if oop is non-null 1060 1061 const Register rOop = src.first()->as_Register(); 1062 int oop_slot; 1063 if (rOop == j_rarg0) 1064 oop_slot = 0; 1065 else if (rOop == j_rarg1) 1066 oop_slot = 1; 1067 else if (rOop == j_rarg2) 1068 oop_slot = 2; 1069 else if (rOop == j_rarg3) 1070 oop_slot = 3; 1071 else if (rOop == j_rarg4) 1072 oop_slot = 4; 1073 else { 1074 assert(rOop == j_rarg5, "wrong register"); 1075 oop_slot = 5; 1076 } 1077 1078 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 1079 int offset = oop_slot*VMRegImpl::stack_slot_size; 1080 1081 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 1082 // Store oop in handle area, may be null 1083 movptr(Address(rsp, offset), rOop); 1084 if (is_receiver) { 1085 *receiver_offset = offset; 1086 } 1087 1088 cmpptr(rOop, NULL_WORD); 1089 lea(rHandle, Address(rsp, offset)); 1090 // conditionally move a null from the handle area where it was just stored 1091 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 1092 } 1093 1094 // If arg is on the stack then place it otherwise it is already in correct reg. 1095 if (dst.first()->is_stack()) { 1096 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 1097 } 1098 } 1099 1100 #endif // _LP64 1101 1102 // Now versions that are common to 32/64 bit 1103 1104 void MacroAssembler::addptr(Register dst, int32_t imm32) { 1105 LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); 1106 } 1107 1108 void MacroAssembler::addptr(Register dst, Register src) { 1109 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1110 } 1111 1112 void MacroAssembler::addptr(Address dst, Register src) { 1113 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1114 } 1115 1116 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1117 assert(rscratch != noreg || always_reachable(src), "missing"); 1118 1119 if (reachable(src)) { 1120 Assembler::addsd(dst, as_Address(src)); 1121 } else { 1122 lea(rscratch, src); 1123 Assembler::addsd(dst, Address(rscratch, 0)); 1124 } 1125 } 1126 1127 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1128 assert(rscratch != noreg || always_reachable(src), "missing"); 1129 1130 if (reachable(src)) { 1131 addss(dst, as_Address(src)); 1132 } else { 1133 lea(rscratch, src); 1134 addss(dst, Address(rscratch, 0)); 1135 } 1136 } 1137 1138 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1139 assert(rscratch != noreg || always_reachable(src), "missing"); 1140 1141 if (reachable(src)) { 1142 Assembler::addpd(dst, as_Address(src)); 1143 } else { 1144 lea(rscratch, src); 1145 Assembler::addpd(dst, Address(rscratch, 0)); 1146 } 1147 } 1148 1149 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 1150 // Stub code is generated once and never copied. 1151 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 1152 void MacroAssembler::align64() { 1153 align(64, (unsigned long long) pc()); 1154 } 1155 1156 void MacroAssembler::align32() { 1157 align(32, (unsigned long long) pc()); 1158 } 1159 1160 void MacroAssembler::align(int modulus) { 1161 // 8273459: Ensure alignment is possible with current segment alignment 1162 assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 1163 align(modulus, offset()); 1164 } 1165 1166 void MacroAssembler::align(int modulus, int target) { 1167 if (target % modulus != 0) { 1168 nop(modulus - (target % modulus)); 1169 } 1170 } 1171 1172 void MacroAssembler::push_f(XMMRegister r) { 1173 subptr(rsp, wordSize); 1174 movflt(Address(rsp, 0), r); 1175 } 1176 1177 void MacroAssembler::pop_f(XMMRegister r) { 1178 movflt(r, Address(rsp, 0)); 1179 addptr(rsp, wordSize); 1180 } 1181 1182 void MacroAssembler::push_d(XMMRegister r) { 1183 subptr(rsp, 2 * wordSize); 1184 movdbl(Address(rsp, 0), r); 1185 } 1186 1187 void MacroAssembler::pop_d(XMMRegister r) { 1188 movdbl(r, Address(rsp, 0)); 1189 addptr(rsp, 2 * Interpreter::stackElementSize); 1190 } 1191 1192 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1193 // Used in sign-masking with aligned address. 1194 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1195 assert(rscratch != noreg || always_reachable(src), "missing"); 1196 1197 if (reachable(src)) { 1198 Assembler::andpd(dst, as_Address(src)); 1199 } else { 1200 lea(rscratch, src); 1201 Assembler::andpd(dst, Address(rscratch, 0)); 1202 } 1203 } 1204 1205 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 1206 // Used in sign-masking with aligned address. 1207 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1208 assert(rscratch != noreg || always_reachable(src), "missing"); 1209 1210 if (reachable(src)) { 1211 Assembler::andps(dst, as_Address(src)); 1212 } else { 1213 lea(rscratch, src); 1214 Assembler::andps(dst, Address(rscratch, 0)); 1215 } 1216 } 1217 1218 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1219 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1220 } 1221 1222 #ifdef _LP64 1223 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 1224 assert(rscratch != noreg || always_reachable(src), "missing"); 1225 1226 if (reachable(src)) { 1227 andq(dst, as_Address(src)); 1228 } else { 1229 lea(rscratch, src); 1230 andq(dst, Address(rscratch, 0)); 1231 } 1232 } 1233 #endif 1234 1235 void MacroAssembler::atomic_incl(Address counter_addr) { 1236 lock(); 1237 incrementl(counter_addr); 1238 } 1239 1240 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 1241 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1242 1243 if (reachable(counter_addr)) { 1244 atomic_incl(as_Address(counter_addr)); 1245 } else { 1246 lea(rscratch, counter_addr); 1247 atomic_incl(Address(rscratch, 0)); 1248 } 1249 } 1250 1251 #ifdef _LP64 1252 void MacroAssembler::atomic_incq(Address counter_addr) { 1253 lock(); 1254 incrementq(counter_addr); 1255 } 1256 1257 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 1258 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1259 1260 if (reachable(counter_addr)) { 1261 atomic_incq(as_Address(counter_addr)); 1262 } else { 1263 lea(rscratch, counter_addr); 1264 atomic_incq(Address(rscratch, 0)); 1265 } 1266 } 1267 #endif 1268 1269 // Writes to stack successive pages until offset reached to check for 1270 // stack overflow + shadow pages. This clobbers tmp. 1271 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 1272 movptr(tmp, rsp); 1273 // Bang stack for total size given plus shadow page size. 1274 // Bang one page at a time because large size can bang beyond yellow and 1275 // red zones. 1276 Label loop; 1277 bind(loop); 1278 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 1279 subptr(tmp, (int)os::vm_page_size()); 1280 subl(size, (int)os::vm_page_size()); 1281 jcc(Assembler::greater, loop); 1282 1283 // Bang down shadow pages too. 1284 // At this point, (tmp-0) is the last address touched, so don't 1285 // touch it again. (It was touched as (tmp-pagesize) but then tmp 1286 // was post-decremented.) Skip this address by starting at i=1, and 1287 // touch a few more pages below. N.B. It is important to touch all 1288 // the way down including all pages in the shadow zone. 1289 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 1290 // this could be any sized move but this is can be a debugging crumb 1291 // so the bigger the better. 1292 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 1293 } 1294 } 1295 1296 void MacroAssembler::reserved_stack_check() { 1297 // testing if reserved zone needs to be enabled 1298 Label no_reserved_zone_enabling; 1299 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 1300 NOT_LP64(get_thread(rsi);) 1301 1302 cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); 1303 jcc(Assembler::below, no_reserved_zone_enabling); 1304 1305 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); 1306 jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); 1307 should_not_reach_here(); 1308 1309 bind(no_reserved_zone_enabling); 1310 } 1311 1312 void MacroAssembler::c2bool(Register x) { 1313 // implements x == 0 ? 0 : 1 1314 // note: must only look at least-significant byte of x 1315 // since C-style booleans are stored in one byte 1316 // only! (was bug) 1317 andl(x, 0xFF); 1318 setb(Assembler::notZero, x); 1319 } 1320 1321 // Wouldn't need if AddressLiteral version had new name 1322 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 1323 Assembler::call(L, rtype); 1324 } 1325 1326 void MacroAssembler::call(Register entry) { 1327 Assembler::call(entry); 1328 } 1329 1330 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 1331 assert(rscratch != noreg || always_reachable(entry), "missing"); 1332 1333 if (reachable(entry)) { 1334 Assembler::call_literal(entry.target(), entry.rspec()); 1335 } else { 1336 lea(rscratch, entry); 1337 Assembler::call(rscratch); 1338 } 1339 } 1340 1341 void MacroAssembler::ic_call(address entry, jint method_index) { 1342 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1343 #ifdef _LP64 1344 // Needs full 64-bit immediate for later patching. 1345 mov64(rax, (int64_t)Universe::non_oop_word()); 1346 #else 1347 movptr(rax, (intptr_t)Universe::non_oop_word()); 1348 #endif 1349 call(AddressLiteral(entry, rh)); 1350 } 1351 1352 int MacroAssembler::ic_check_size() { 1353 return 1354 LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12); 1355 } 1356 1357 int MacroAssembler::ic_check(int end_alignment) { 1358 Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx); 1359 Register data = rax; 1360 Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx); 1361 1362 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 1363 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 1364 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 1365 // before the inline cache check here, and not after 1366 align(end_alignment, offset() + ic_check_size()); 1367 1368 int uep_offset = offset(); 1369 1370 #ifdef _LP64 1371 if (UseCompactObjectHeaders) { 1372 load_nklass_compact(temp, receiver); 1373 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 1374 } else 1375 #endif 1376 if (UseCompressedClassPointers) { 1377 movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1378 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 1379 } else { 1380 movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1381 cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset())); 1382 } 1383 1384 // if inline cache check fails, then jump to runtime routine 1385 jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1386 assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment); 1387 1388 return uep_offset; 1389 } 1390 1391 void MacroAssembler::emit_static_call_stub() { 1392 // Static stub relocation also tags the Method* in the code-stream. 1393 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 1394 // This is recognized as unresolved by relocs/nativeinst/ic code. 1395 jump(RuntimeAddress(pc())); 1396 } 1397 1398 // Implementation of call_VM versions 1399 1400 void MacroAssembler::call_VM(Register oop_result, 1401 address entry_point, 1402 bool check_exceptions) { 1403 Label C, E; 1404 call(C, relocInfo::none); 1405 jmp(E); 1406 1407 bind(C); 1408 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1409 ret(0); 1410 1411 bind(E); 1412 } 1413 1414 void MacroAssembler::call_VM(Register oop_result, 1415 address entry_point, 1416 Register arg_1, 1417 bool check_exceptions) { 1418 Label C, E; 1419 call(C, relocInfo::none); 1420 jmp(E); 1421 1422 bind(C); 1423 pass_arg1(this, arg_1); 1424 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1425 ret(0); 1426 1427 bind(E); 1428 } 1429 1430 void MacroAssembler::call_VM(Register oop_result, 1431 address entry_point, 1432 Register arg_1, 1433 Register arg_2, 1434 bool check_exceptions) { 1435 Label C, E; 1436 call(C, relocInfo::none); 1437 jmp(E); 1438 1439 bind(C); 1440 1441 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1442 1443 pass_arg2(this, arg_2); 1444 pass_arg1(this, arg_1); 1445 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1446 ret(0); 1447 1448 bind(E); 1449 } 1450 1451 void MacroAssembler::call_VM(Register oop_result, 1452 address entry_point, 1453 Register arg_1, 1454 Register arg_2, 1455 Register arg_3, 1456 bool check_exceptions) { 1457 Label C, E; 1458 call(C, relocInfo::none); 1459 jmp(E); 1460 1461 bind(C); 1462 1463 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1464 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1465 pass_arg3(this, arg_3); 1466 pass_arg2(this, arg_2); 1467 pass_arg1(this, arg_1); 1468 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1469 ret(0); 1470 1471 bind(E); 1472 } 1473 1474 void MacroAssembler::call_VM(Register oop_result, 1475 Register last_java_sp, 1476 address entry_point, 1477 int number_of_arguments, 1478 bool check_exceptions) { 1479 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1480 call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1481 } 1482 1483 void MacroAssembler::call_VM(Register oop_result, 1484 Register last_java_sp, 1485 address entry_point, 1486 Register arg_1, 1487 bool check_exceptions) { 1488 pass_arg1(this, arg_1); 1489 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1490 } 1491 1492 void MacroAssembler::call_VM(Register oop_result, 1493 Register last_java_sp, 1494 address entry_point, 1495 Register arg_1, 1496 Register arg_2, 1497 bool check_exceptions) { 1498 1499 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1500 pass_arg2(this, arg_2); 1501 pass_arg1(this, arg_1); 1502 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1503 } 1504 1505 void MacroAssembler::call_VM(Register oop_result, 1506 Register last_java_sp, 1507 address entry_point, 1508 Register arg_1, 1509 Register arg_2, 1510 Register arg_3, 1511 bool check_exceptions) { 1512 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1513 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1514 pass_arg3(this, arg_3); 1515 pass_arg2(this, arg_2); 1516 pass_arg1(this, arg_1); 1517 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1518 } 1519 1520 void MacroAssembler::super_call_VM(Register oop_result, 1521 Register last_java_sp, 1522 address entry_point, 1523 int number_of_arguments, 1524 bool check_exceptions) { 1525 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1526 MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1527 } 1528 1529 void MacroAssembler::super_call_VM(Register oop_result, 1530 Register last_java_sp, 1531 address entry_point, 1532 Register arg_1, 1533 bool check_exceptions) { 1534 pass_arg1(this, arg_1); 1535 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1536 } 1537 1538 void MacroAssembler::super_call_VM(Register oop_result, 1539 Register last_java_sp, 1540 address entry_point, 1541 Register arg_1, 1542 Register arg_2, 1543 bool check_exceptions) { 1544 1545 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1546 pass_arg2(this, arg_2); 1547 pass_arg1(this, arg_1); 1548 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1549 } 1550 1551 void MacroAssembler::super_call_VM(Register oop_result, 1552 Register last_java_sp, 1553 address entry_point, 1554 Register arg_1, 1555 Register arg_2, 1556 Register arg_3, 1557 bool check_exceptions) { 1558 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1559 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1560 pass_arg3(this, arg_3); 1561 pass_arg2(this, arg_2); 1562 pass_arg1(this, arg_1); 1563 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1564 } 1565 1566 void MacroAssembler::call_VM_base(Register oop_result, 1567 Register java_thread, 1568 Register last_java_sp, 1569 address entry_point, 1570 int number_of_arguments, 1571 bool check_exceptions) { 1572 // determine java_thread register 1573 if (!java_thread->is_valid()) { 1574 #ifdef _LP64 1575 java_thread = r15_thread; 1576 #else 1577 java_thread = rdi; 1578 get_thread(java_thread); 1579 #endif // LP64 1580 } 1581 // determine last_java_sp register 1582 if (!last_java_sp->is_valid()) { 1583 last_java_sp = rsp; 1584 } 1585 // debugging support 1586 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1587 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); 1588 #ifdef ASSERT 1589 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1590 // r12 is the heapbase. 1591 LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) 1592 #endif // ASSERT 1593 1594 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1595 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1596 1597 // push java thread (becomes first argument of C function) 1598 1599 NOT_LP64(push(java_thread); number_of_arguments++); 1600 LP64_ONLY(mov(c_rarg0, r15_thread)); 1601 1602 // set last Java frame before call 1603 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1604 1605 // Only interpreter should have to set fp 1606 set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1); 1607 1608 // do the call, remove parameters 1609 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1610 1611 // restore the thread (cannot use the pushed argument since arguments 1612 // may be overwritten by C code generated by an optimizing compiler); 1613 // however can use the register value directly if it is callee saved. 1614 if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { 1615 // rdi & rsi (also r15) are callee saved -> nothing to do 1616 #ifdef ASSERT 1617 guarantee(java_thread != rax, "change this code"); 1618 push(rax); 1619 { Label L; 1620 get_thread(rax); 1621 cmpptr(java_thread, rax); 1622 jcc(Assembler::equal, L); 1623 STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); 1624 bind(L); 1625 } 1626 pop(rax); 1627 #endif 1628 } else { 1629 get_thread(java_thread); 1630 } 1631 // reset last Java frame 1632 // Only interpreter should have to clear fp 1633 reset_last_Java_frame(java_thread, true); 1634 1635 // C++ interp handles this in the interpreter 1636 check_and_handle_popframe(java_thread); 1637 check_and_handle_earlyret(java_thread); 1638 1639 if (check_exceptions) { 1640 // check for pending exceptions (java_thread is set upon return) 1641 cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD); 1642 #ifndef _LP64 1643 jump_cc(Assembler::notEqual, 1644 RuntimeAddress(StubRoutines::forward_exception_entry())); 1645 #else 1646 // This used to conditionally jump to forward_exception however it is 1647 // possible if we relocate that the branch will not reach. So we must jump 1648 // around so we can always reach 1649 1650 Label ok; 1651 jcc(Assembler::equal, ok); 1652 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1653 bind(ok); 1654 #endif // LP64 1655 } 1656 1657 // get oop result if there is one and reset the value in the thread 1658 if (oop_result->is_valid()) { 1659 get_vm_result(oop_result, java_thread); 1660 } 1661 } 1662 1663 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1664 1665 // Calculate the value for last_Java_sp 1666 // somewhat subtle. call_VM does an intermediate call 1667 // which places a return address on the stack just under the 1668 // stack pointer as the user finished with it. This allows 1669 // use to retrieve last_Java_pc from last_Java_sp[-1]. 1670 // On 32bit we then have to push additional args on the stack to accomplish 1671 // the actual requested call. On 64bit call_VM only can use register args 1672 // so the only extra space is the return address that call_VM created. 1673 // This hopefully explains the calculations here. 1674 1675 #ifdef _LP64 1676 // We've pushed one address, correct last_Java_sp 1677 lea(rax, Address(rsp, wordSize)); 1678 #else 1679 lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); 1680 #endif // LP64 1681 1682 call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); 1683 1684 } 1685 1686 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1687 void MacroAssembler::call_VM_leaf0(address entry_point) { 1688 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1689 } 1690 1691 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1692 call_VM_leaf_base(entry_point, number_of_arguments); 1693 } 1694 1695 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1696 pass_arg0(this, arg_0); 1697 call_VM_leaf(entry_point, 1); 1698 } 1699 1700 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1701 1702 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1703 pass_arg1(this, arg_1); 1704 pass_arg0(this, arg_0); 1705 call_VM_leaf(entry_point, 2); 1706 } 1707 1708 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1709 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1710 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1711 pass_arg2(this, arg_2); 1712 pass_arg1(this, arg_1); 1713 pass_arg0(this, arg_0); 1714 call_VM_leaf(entry_point, 3); 1715 } 1716 1717 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1718 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1719 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1720 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1721 pass_arg3(this, arg_3); 1722 pass_arg2(this, arg_2); 1723 pass_arg1(this, arg_1); 1724 pass_arg0(this, arg_0); 1725 call_VM_leaf(entry_point, 3); 1726 } 1727 1728 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1729 pass_arg0(this, arg_0); 1730 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1731 } 1732 1733 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1734 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1735 pass_arg1(this, arg_1); 1736 pass_arg0(this, arg_0); 1737 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1738 } 1739 1740 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1741 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1742 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1743 pass_arg2(this, arg_2); 1744 pass_arg1(this, arg_1); 1745 pass_arg0(this, arg_0); 1746 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1747 } 1748 1749 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1750 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1751 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1752 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1753 pass_arg3(this, arg_3); 1754 pass_arg2(this, arg_2); 1755 pass_arg1(this, arg_1); 1756 pass_arg0(this, arg_0); 1757 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1758 } 1759 1760 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1761 movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1762 movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); 1763 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1764 } 1765 1766 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1767 movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1768 movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); 1769 } 1770 1771 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { 1772 } 1773 1774 void MacroAssembler::check_and_handle_popframe(Register java_thread) { 1775 } 1776 1777 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1778 assert(rscratch != noreg || always_reachable(src1), "missing"); 1779 1780 if (reachable(src1)) { 1781 cmpl(as_Address(src1), imm); 1782 } else { 1783 lea(rscratch, src1); 1784 cmpl(Address(rscratch, 0), imm); 1785 } 1786 } 1787 1788 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1789 assert(!src2.is_lval(), "use cmpptr"); 1790 assert(rscratch != noreg || always_reachable(src2), "missing"); 1791 1792 if (reachable(src2)) { 1793 cmpl(src1, as_Address(src2)); 1794 } else { 1795 lea(rscratch, src2); 1796 cmpl(src1, Address(rscratch, 0)); 1797 } 1798 } 1799 1800 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1801 Assembler::cmpl(src1, imm); 1802 } 1803 1804 void MacroAssembler::cmp32(Register src1, Address src2) { 1805 Assembler::cmpl(src1, src2); 1806 } 1807 1808 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1809 ucomisd(opr1, opr2); 1810 1811 Label L; 1812 if (unordered_is_less) { 1813 movl(dst, -1); 1814 jcc(Assembler::parity, L); 1815 jcc(Assembler::below , L); 1816 movl(dst, 0); 1817 jcc(Assembler::equal , L); 1818 increment(dst); 1819 } else { // unordered is greater 1820 movl(dst, 1); 1821 jcc(Assembler::parity, L); 1822 jcc(Assembler::above , L); 1823 movl(dst, 0); 1824 jcc(Assembler::equal , L); 1825 decrementl(dst); 1826 } 1827 bind(L); 1828 } 1829 1830 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1831 ucomiss(opr1, opr2); 1832 1833 Label L; 1834 if (unordered_is_less) { 1835 movl(dst, -1); 1836 jcc(Assembler::parity, L); 1837 jcc(Assembler::below , L); 1838 movl(dst, 0); 1839 jcc(Assembler::equal , L); 1840 increment(dst); 1841 } else { // unordered is greater 1842 movl(dst, 1); 1843 jcc(Assembler::parity, L); 1844 jcc(Assembler::above , L); 1845 movl(dst, 0); 1846 jcc(Assembler::equal , L); 1847 decrementl(dst); 1848 } 1849 bind(L); 1850 } 1851 1852 1853 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1854 assert(rscratch != noreg || always_reachable(src1), "missing"); 1855 1856 if (reachable(src1)) { 1857 cmpb(as_Address(src1), imm); 1858 } else { 1859 lea(rscratch, src1); 1860 cmpb(Address(rscratch, 0), imm); 1861 } 1862 } 1863 1864 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1865 #ifdef _LP64 1866 assert(rscratch != noreg || always_reachable(src2), "missing"); 1867 1868 if (src2.is_lval()) { 1869 movptr(rscratch, src2); 1870 Assembler::cmpq(src1, rscratch); 1871 } else if (reachable(src2)) { 1872 cmpq(src1, as_Address(src2)); 1873 } else { 1874 lea(rscratch, src2); 1875 Assembler::cmpq(src1, Address(rscratch, 0)); 1876 } 1877 #else 1878 assert(rscratch == noreg, "not needed"); 1879 if (src2.is_lval()) { 1880 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1881 } else { 1882 cmpl(src1, as_Address(src2)); 1883 } 1884 #endif // _LP64 1885 } 1886 1887 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1888 assert(src2.is_lval(), "not a mem-mem compare"); 1889 #ifdef _LP64 1890 // moves src2's literal address 1891 movptr(rscratch, src2); 1892 Assembler::cmpq(src1, rscratch); 1893 #else 1894 assert(rscratch == noreg, "not needed"); 1895 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1896 #endif // _LP64 1897 } 1898 1899 void MacroAssembler::cmpoop(Register src1, Register src2) { 1900 cmpptr(src1, src2); 1901 } 1902 1903 void MacroAssembler::cmpoop(Register src1, Address src2) { 1904 cmpptr(src1, src2); 1905 } 1906 1907 #ifdef _LP64 1908 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1909 movoop(rscratch, src2); 1910 cmpptr(src1, rscratch); 1911 } 1912 #endif 1913 1914 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1915 assert(rscratch != noreg || always_reachable(adr), "missing"); 1916 1917 if (reachable(adr)) { 1918 lock(); 1919 cmpxchgptr(reg, as_Address(adr)); 1920 } else { 1921 lea(rscratch, adr); 1922 lock(); 1923 cmpxchgptr(reg, Address(rscratch, 0)); 1924 } 1925 } 1926 1927 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1928 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); 1929 } 1930 1931 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1932 assert(rscratch != noreg || always_reachable(src), "missing"); 1933 1934 if (reachable(src)) { 1935 Assembler::comisd(dst, as_Address(src)); 1936 } else { 1937 lea(rscratch, src); 1938 Assembler::comisd(dst, Address(rscratch, 0)); 1939 } 1940 } 1941 1942 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1943 assert(rscratch != noreg || always_reachable(src), "missing"); 1944 1945 if (reachable(src)) { 1946 Assembler::comiss(dst, as_Address(src)); 1947 } else { 1948 lea(rscratch, src); 1949 Assembler::comiss(dst, Address(rscratch, 0)); 1950 } 1951 } 1952 1953 1954 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 1955 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1956 1957 Condition negated_cond = negate_condition(cond); 1958 Label L; 1959 jcc(negated_cond, L); 1960 pushf(); // Preserve flags 1961 atomic_incl(counter_addr, rscratch); 1962 popf(); 1963 bind(L); 1964 } 1965 1966 int MacroAssembler::corrected_idivl(Register reg) { 1967 // Full implementation of Java idiv and irem; checks for 1968 // special case as described in JVM spec., p.243 & p.271. 1969 // The function returns the (pc) offset of the idivl 1970 // instruction - may be needed for implicit exceptions. 1971 // 1972 // normal case special case 1973 // 1974 // input : rax,: dividend min_int 1975 // reg: divisor (may not be rax,/rdx) -1 1976 // 1977 // output: rax,: quotient (= rax, idiv reg) min_int 1978 // rdx: remainder (= rax, irem reg) 0 1979 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 1980 const int min_int = 0x80000000; 1981 Label normal_case, special_case; 1982 1983 // check for special case 1984 cmpl(rax, min_int); 1985 jcc(Assembler::notEqual, normal_case); 1986 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 1987 cmpl(reg, -1); 1988 jcc(Assembler::equal, special_case); 1989 1990 // handle normal case 1991 bind(normal_case); 1992 cdql(); 1993 int idivl_offset = offset(); 1994 idivl(reg); 1995 1996 // normal and special case exit 1997 bind(special_case); 1998 1999 return idivl_offset; 2000 } 2001 2002 2003 2004 void MacroAssembler::decrementl(Register reg, int value) { 2005 if (value == min_jint) {subl(reg, value) ; return; } 2006 if (value < 0) { incrementl(reg, -value); return; } 2007 if (value == 0) { ; return; } 2008 if (value == 1 && UseIncDec) { decl(reg) ; return; } 2009 /* else */ { subl(reg, value) ; return; } 2010 } 2011 2012 void MacroAssembler::decrementl(Address dst, int value) { 2013 if (value == min_jint) {subl(dst, value) ; return; } 2014 if (value < 0) { incrementl(dst, -value); return; } 2015 if (value == 0) { ; return; } 2016 if (value == 1 && UseIncDec) { decl(dst) ; return; } 2017 /* else */ { subl(dst, value) ; return; } 2018 } 2019 2020 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 2021 assert(shift_value > 0, "illegal shift value"); 2022 Label _is_positive; 2023 testl (reg, reg); 2024 jcc (Assembler::positive, _is_positive); 2025 int offset = (1 << shift_value) - 1 ; 2026 2027 if (offset == 1) { 2028 incrementl(reg); 2029 } else { 2030 addl(reg, offset); 2031 } 2032 2033 bind (_is_positive); 2034 sarl(reg, shift_value); 2035 } 2036 2037 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2038 assert(rscratch != noreg || always_reachable(src), "missing"); 2039 2040 if (reachable(src)) { 2041 Assembler::divsd(dst, as_Address(src)); 2042 } else { 2043 lea(rscratch, src); 2044 Assembler::divsd(dst, Address(rscratch, 0)); 2045 } 2046 } 2047 2048 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2049 assert(rscratch != noreg || always_reachable(src), "missing"); 2050 2051 if (reachable(src)) { 2052 Assembler::divss(dst, as_Address(src)); 2053 } else { 2054 lea(rscratch, src); 2055 Assembler::divss(dst, Address(rscratch, 0)); 2056 } 2057 } 2058 2059 void MacroAssembler::enter() { 2060 push(rbp); 2061 mov(rbp, rsp); 2062 } 2063 2064 void MacroAssembler::post_call_nop() { 2065 if (!Continuations::enabled()) { 2066 return; 2067 } 2068 InstructionMark im(this); 2069 relocate(post_call_nop_Relocation::spec()); 2070 InlineSkippedInstructionsCounter skipCounter(this); 2071 emit_int8((uint8_t)0x0f); 2072 emit_int8((uint8_t)0x1f); 2073 emit_int8((uint8_t)0x84); 2074 emit_int8((uint8_t)0x00); 2075 emit_int32(0x00); 2076 } 2077 2078 // A 5 byte nop that is safe for patching (see patch_verified_entry) 2079 void MacroAssembler::fat_nop() { 2080 if (UseAddressNop) { 2081 addr_nop_5(); 2082 } else { 2083 emit_int8((uint8_t)0x26); // es: 2084 emit_int8((uint8_t)0x2e); // cs: 2085 emit_int8((uint8_t)0x64); // fs: 2086 emit_int8((uint8_t)0x65); // gs: 2087 emit_int8((uint8_t)0x90); 2088 } 2089 } 2090 2091 #ifndef _LP64 2092 void MacroAssembler::fcmp(Register tmp) { 2093 fcmp(tmp, 1, true, true); 2094 } 2095 2096 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { 2097 assert(!pop_right || pop_left, "usage error"); 2098 if (VM_Version::supports_cmov()) { 2099 assert(tmp == noreg, "unneeded temp"); 2100 if (pop_left) { 2101 fucomip(index); 2102 } else { 2103 fucomi(index); 2104 } 2105 if (pop_right) { 2106 fpop(); 2107 } 2108 } else { 2109 assert(tmp != noreg, "need temp"); 2110 if (pop_left) { 2111 if (pop_right) { 2112 fcompp(); 2113 } else { 2114 fcomp(index); 2115 } 2116 } else { 2117 fcom(index); 2118 } 2119 // convert FPU condition into eflags condition via rax, 2120 save_rax(tmp); 2121 fwait(); fnstsw_ax(); 2122 sahf(); 2123 restore_rax(tmp); 2124 } 2125 // condition codes set as follows: 2126 // 2127 // CF (corresponds to C0) if x < y 2128 // PF (corresponds to C2) if unordered 2129 // ZF (corresponds to C3) if x = y 2130 } 2131 2132 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { 2133 fcmp2int(dst, unordered_is_less, 1, true, true); 2134 } 2135 2136 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { 2137 fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); 2138 Label L; 2139 if (unordered_is_less) { 2140 movl(dst, -1); 2141 jcc(Assembler::parity, L); 2142 jcc(Assembler::below , L); 2143 movl(dst, 0); 2144 jcc(Assembler::equal , L); 2145 increment(dst); 2146 } else { // unordered is greater 2147 movl(dst, 1); 2148 jcc(Assembler::parity, L); 2149 jcc(Assembler::above , L); 2150 movl(dst, 0); 2151 jcc(Assembler::equal , L); 2152 decrementl(dst); 2153 } 2154 bind(L); 2155 } 2156 2157 void MacroAssembler::fld_d(AddressLiteral src) { 2158 fld_d(as_Address(src)); 2159 } 2160 2161 void MacroAssembler::fld_s(AddressLiteral src) { 2162 fld_s(as_Address(src)); 2163 } 2164 2165 void MacroAssembler::fldcw(AddressLiteral src) { 2166 fldcw(as_Address(src)); 2167 } 2168 2169 void MacroAssembler::fpop() { 2170 ffree(); 2171 fincstp(); 2172 } 2173 2174 void MacroAssembler::fremr(Register tmp) { 2175 save_rax(tmp); 2176 { Label L; 2177 bind(L); 2178 fprem(); 2179 fwait(); fnstsw_ax(); 2180 sahf(); 2181 jcc(Assembler::parity, L); 2182 } 2183 restore_rax(tmp); 2184 // Result is in ST0. 2185 // Note: fxch & fpop to get rid of ST1 2186 // (otherwise FPU stack could overflow eventually) 2187 fxch(1); 2188 fpop(); 2189 } 2190 2191 void MacroAssembler::empty_FPU_stack() { 2192 if (VM_Version::supports_mmx()) { 2193 emms(); 2194 } else { 2195 for (int i = 8; i-- > 0; ) ffree(i); 2196 } 2197 } 2198 #endif // !LP64 2199 2200 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2201 assert(rscratch != noreg || always_reachable(src), "missing"); 2202 if (reachable(src)) { 2203 Assembler::mulpd(dst, as_Address(src)); 2204 } else { 2205 lea(rscratch, src); 2206 Assembler::mulpd(dst, Address(rscratch, 0)); 2207 } 2208 } 2209 2210 void MacroAssembler::load_float(Address src) { 2211 #ifdef _LP64 2212 movflt(xmm0, src); 2213 #else 2214 if (UseSSE >= 1) { 2215 movflt(xmm0, src); 2216 } else { 2217 fld_s(src); 2218 } 2219 #endif // LP64 2220 } 2221 2222 void MacroAssembler::store_float(Address dst) { 2223 #ifdef _LP64 2224 movflt(dst, xmm0); 2225 #else 2226 if (UseSSE >= 1) { 2227 movflt(dst, xmm0); 2228 } else { 2229 fstp_s(dst); 2230 } 2231 #endif // LP64 2232 } 2233 2234 void MacroAssembler::load_double(Address src) { 2235 #ifdef _LP64 2236 movdbl(xmm0, src); 2237 #else 2238 if (UseSSE >= 2) { 2239 movdbl(xmm0, src); 2240 } else { 2241 fld_d(src); 2242 } 2243 #endif // LP64 2244 } 2245 2246 void MacroAssembler::store_double(Address dst) { 2247 #ifdef _LP64 2248 movdbl(dst, xmm0); 2249 #else 2250 if (UseSSE >= 2) { 2251 movdbl(dst, xmm0); 2252 } else { 2253 fstp_d(dst); 2254 } 2255 #endif // LP64 2256 } 2257 2258 // dst = c = a * b + c 2259 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2260 Assembler::vfmadd231sd(c, a, b); 2261 if (dst != c) { 2262 movdbl(dst, c); 2263 } 2264 } 2265 2266 // dst = c = a * b + c 2267 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2268 Assembler::vfmadd231ss(c, a, b); 2269 if (dst != c) { 2270 movflt(dst, c); 2271 } 2272 } 2273 2274 // dst = c = a * b + c 2275 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2276 Assembler::vfmadd231pd(c, a, b, vector_len); 2277 if (dst != c) { 2278 vmovdqu(dst, c); 2279 } 2280 } 2281 2282 // dst = c = a * b + c 2283 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2284 Assembler::vfmadd231ps(c, a, b, vector_len); 2285 if (dst != c) { 2286 vmovdqu(dst, c); 2287 } 2288 } 2289 2290 // dst = c = a * b + c 2291 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2292 Assembler::vfmadd231pd(c, a, b, vector_len); 2293 if (dst != c) { 2294 vmovdqu(dst, c); 2295 } 2296 } 2297 2298 // dst = c = a * b + c 2299 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2300 Assembler::vfmadd231ps(c, a, b, vector_len); 2301 if (dst != c) { 2302 vmovdqu(dst, c); 2303 } 2304 } 2305 2306 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 2307 assert(rscratch != noreg || always_reachable(dst), "missing"); 2308 2309 if (reachable(dst)) { 2310 incrementl(as_Address(dst)); 2311 } else { 2312 lea(rscratch, dst); 2313 incrementl(Address(rscratch, 0)); 2314 } 2315 } 2316 2317 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 2318 incrementl(as_Address(dst, rscratch)); 2319 } 2320 2321 void MacroAssembler::incrementl(Register reg, int value) { 2322 if (value == min_jint) {addl(reg, value) ; return; } 2323 if (value < 0) { decrementl(reg, -value); return; } 2324 if (value == 0) { ; return; } 2325 if (value == 1 && UseIncDec) { incl(reg) ; return; } 2326 /* else */ { addl(reg, value) ; return; } 2327 } 2328 2329 void MacroAssembler::incrementl(Address dst, int value) { 2330 if (value == min_jint) {addl(dst, value) ; return; } 2331 if (value < 0) { decrementl(dst, -value); return; } 2332 if (value == 0) { ; return; } 2333 if (value == 1 && UseIncDec) { incl(dst) ; return; } 2334 /* else */ { addl(dst, value) ; return; } 2335 } 2336 2337 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 2338 assert(rscratch != noreg || always_reachable(dst), "missing"); 2339 2340 if (reachable(dst)) { 2341 jmp_literal(dst.target(), dst.rspec()); 2342 } else { 2343 lea(rscratch, dst); 2344 jmp(rscratch); 2345 } 2346 } 2347 2348 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 2349 assert(rscratch != noreg || always_reachable(dst), "missing"); 2350 2351 if (reachable(dst)) { 2352 InstructionMark im(this); 2353 relocate(dst.reloc()); 2354 const int short_size = 2; 2355 const int long_size = 6; 2356 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 2357 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 2358 // 0111 tttn #8-bit disp 2359 emit_int8(0x70 | cc); 2360 emit_int8((offs - short_size) & 0xFF); 2361 } else { 2362 // 0000 1111 1000 tttn #32-bit disp 2363 emit_int8(0x0F); 2364 emit_int8((unsigned char)(0x80 | cc)); 2365 emit_int32(offs - long_size); 2366 } 2367 } else { 2368 #ifdef ASSERT 2369 warning("reversing conditional branch"); 2370 #endif /* ASSERT */ 2371 Label skip; 2372 jccb(reverse[cc], skip); 2373 lea(rscratch, dst); 2374 Assembler::jmp(rscratch); 2375 bind(skip); 2376 } 2377 } 2378 2379 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 2380 assert(rscratch != noreg || always_reachable(src), "missing"); 2381 2382 if (reachable(src)) { 2383 Assembler::ldmxcsr(as_Address(src)); 2384 } else { 2385 lea(rscratch, src); 2386 Assembler::ldmxcsr(Address(rscratch, 0)); 2387 } 2388 } 2389 2390 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2391 int off; 2392 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2393 off = offset(); 2394 movsbl(dst, src); // movsxb 2395 } else { 2396 off = load_unsigned_byte(dst, src); 2397 shll(dst, 24); 2398 sarl(dst, 24); 2399 } 2400 return off; 2401 } 2402 2403 // Note: load_signed_short used to be called load_signed_word. 2404 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 2405 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 2406 // The term "word" in HotSpot means a 32- or 64-bit machine word. 2407 int MacroAssembler::load_signed_short(Register dst, Address src) { 2408 int off; 2409 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2410 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 2411 // version but this is what 64bit has always done. This seems to imply 2412 // that users are only using 32bits worth. 2413 off = offset(); 2414 movswl(dst, src); // movsxw 2415 } else { 2416 off = load_unsigned_short(dst, src); 2417 shll(dst, 16); 2418 sarl(dst, 16); 2419 } 2420 return off; 2421 } 2422 2423 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2424 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2425 // and "3.9 Partial Register Penalties", p. 22). 2426 int off; 2427 if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { 2428 off = offset(); 2429 movzbl(dst, src); // movzxb 2430 } else { 2431 xorl(dst, dst); 2432 off = offset(); 2433 movb(dst, src); 2434 } 2435 return off; 2436 } 2437 2438 // Note: load_unsigned_short used to be called load_unsigned_word. 2439 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2440 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2441 // and "3.9 Partial Register Penalties", p. 22). 2442 int off; 2443 if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { 2444 off = offset(); 2445 movzwl(dst, src); // movzxw 2446 } else { 2447 xorl(dst, dst); 2448 off = offset(); 2449 movw(dst, src); 2450 } 2451 return off; 2452 } 2453 2454 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 2455 switch (size_in_bytes) { 2456 #ifndef _LP64 2457 case 8: 2458 assert(dst2 != noreg, "second dest register required"); 2459 movl(dst, src); 2460 movl(dst2, src.plus_disp(BytesPerInt)); 2461 break; 2462 #else 2463 case 8: movq(dst, src); break; 2464 #endif 2465 case 4: movl(dst, src); break; 2466 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2467 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2468 default: ShouldNotReachHere(); 2469 } 2470 } 2471 2472 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 2473 switch (size_in_bytes) { 2474 #ifndef _LP64 2475 case 8: 2476 assert(src2 != noreg, "second source register required"); 2477 movl(dst, src); 2478 movl(dst.plus_disp(BytesPerInt), src2); 2479 break; 2480 #else 2481 case 8: movq(dst, src); break; 2482 #endif 2483 case 4: movl(dst, src); break; 2484 case 2: movw(dst, src); break; 2485 case 1: movb(dst, src); break; 2486 default: ShouldNotReachHere(); 2487 } 2488 } 2489 2490 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 2491 assert(rscratch != noreg || always_reachable(dst), "missing"); 2492 2493 if (reachable(dst)) { 2494 movl(as_Address(dst), src); 2495 } else { 2496 lea(rscratch, dst); 2497 movl(Address(rscratch, 0), src); 2498 } 2499 } 2500 2501 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 2502 if (reachable(src)) { 2503 movl(dst, as_Address(src)); 2504 } else { 2505 lea(dst, src); 2506 movl(dst, Address(dst, 0)); 2507 } 2508 } 2509 2510 // C++ bool manipulation 2511 2512 void MacroAssembler::movbool(Register dst, Address src) { 2513 if(sizeof(bool) == 1) 2514 movb(dst, src); 2515 else if(sizeof(bool) == 2) 2516 movw(dst, src); 2517 else if(sizeof(bool) == 4) 2518 movl(dst, src); 2519 else 2520 // unsupported 2521 ShouldNotReachHere(); 2522 } 2523 2524 void MacroAssembler::movbool(Address dst, bool boolconst) { 2525 if(sizeof(bool) == 1) 2526 movb(dst, (int) boolconst); 2527 else if(sizeof(bool) == 2) 2528 movw(dst, (int) boolconst); 2529 else if(sizeof(bool) == 4) 2530 movl(dst, (int) boolconst); 2531 else 2532 // unsupported 2533 ShouldNotReachHere(); 2534 } 2535 2536 void MacroAssembler::movbool(Address dst, Register src) { 2537 if(sizeof(bool) == 1) 2538 movb(dst, src); 2539 else if(sizeof(bool) == 2) 2540 movw(dst, src); 2541 else if(sizeof(bool) == 4) 2542 movl(dst, src); 2543 else 2544 // unsupported 2545 ShouldNotReachHere(); 2546 } 2547 2548 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2549 assert(rscratch != noreg || always_reachable(src), "missing"); 2550 2551 if (reachable(src)) { 2552 movdl(dst, as_Address(src)); 2553 } else { 2554 lea(rscratch, src); 2555 movdl(dst, Address(rscratch, 0)); 2556 } 2557 } 2558 2559 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 2560 assert(rscratch != noreg || always_reachable(src), "missing"); 2561 2562 if (reachable(src)) { 2563 movq(dst, as_Address(src)); 2564 } else { 2565 lea(rscratch, src); 2566 movq(dst, Address(rscratch, 0)); 2567 } 2568 } 2569 2570 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2571 assert(rscratch != noreg || always_reachable(src), "missing"); 2572 2573 if (reachable(src)) { 2574 if (UseXmmLoadAndClearUpper) { 2575 movsd (dst, as_Address(src)); 2576 } else { 2577 movlpd(dst, as_Address(src)); 2578 } 2579 } else { 2580 lea(rscratch, src); 2581 if (UseXmmLoadAndClearUpper) { 2582 movsd (dst, Address(rscratch, 0)); 2583 } else { 2584 movlpd(dst, Address(rscratch, 0)); 2585 } 2586 } 2587 } 2588 2589 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 2590 assert(rscratch != noreg || always_reachable(src), "missing"); 2591 2592 if (reachable(src)) { 2593 movss(dst, as_Address(src)); 2594 } else { 2595 lea(rscratch, src); 2596 movss(dst, Address(rscratch, 0)); 2597 } 2598 } 2599 2600 void MacroAssembler::movptr(Register dst, Register src) { 2601 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2602 } 2603 2604 void MacroAssembler::movptr(Register dst, Address src) { 2605 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2606 } 2607 2608 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 2609 void MacroAssembler::movptr(Register dst, intptr_t src) { 2610 #ifdef _LP64 2611 if (is_uimm32(src)) { 2612 movl(dst, checked_cast<uint32_t>(src)); 2613 } else if (is_simm32(src)) { 2614 movq(dst, checked_cast<int32_t>(src)); 2615 } else { 2616 mov64(dst, src); 2617 } 2618 #else 2619 movl(dst, src); 2620 #endif 2621 } 2622 2623 void MacroAssembler::movptr(Address dst, Register src) { 2624 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2625 } 2626 2627 void MacroAssembler::movptr(Address dst, int32_t src) { 2628 LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); 2629 } 2630 2631 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 2632 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2633 Assembler::movdqu(dst, src); 2634 } 2635 2636 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 2637 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2638 Assembler::movdqu(dst, src); 2639 } 2640 2641 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 2642 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2643 Assembler::movdqu(dst, src); 2644 } 2645 2646 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2647 assert(rscratch != noreg || always_reachable(src), "missing"); 2648 2649 if (reachable(src)) { 2650 movdqu(dst, as_Address(src)); 2651 } else { 2652 lea(rscratch, src); 2653 movdqu(dst, Address(rscratch, 0)); 2654 } 2655 } 2656 2657 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2658 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2659 Assembler::vmovdqu(dst, src); 2660 } 2661 2662 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2663 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2664 Assembler::vmovdqu(dst, src); 2665 } 2666 2667 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2668 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2669 Assembler::vmovdqu(dst, src); 2670 } 2671 2672 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2673 assert(rscratch != noreg || always_reachable(src), "missing"); 2674 2675 if (reachable(src)) { 2676 vmovdqu(dst, as_Address(src)); 2677 } 2678 else { 2679 lea(rscratch, src); 2680 vmovdqu(dst, Address(rscratch, 0)); 2681 } 2682 } 2683 2684 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2685 assert(rscratch != noreg || always_reachable(src), "missing"); 2686 2687 if (vector_len == AVX_512bit) { 2688 evmovdquq(dst, src, AVX_512bit, rscratch); 2689 } else if (vector_len == AVX_256bit) { 2690 vmovdqu(dst, src, rscratch); 2691 } else { 2692 movdqu(dst, src, rscratch); 2693 } 2694 } 2695 2696 void MacroAssembler::kmov(KRegister dst, Address src) { 2697 if (VM_Version::supports_avx512bw()) { 2698 kmovql(dst, src); 2699 } else { 2700 assert(VM_Version::supports_evex(), ""); 2701 kmovwl(dst, src); 2702 } 2703 } 2704 2705 void MacroAssembler::kmov(Address dst, KRegister src) { 2706 if (VM_Version::supports_avx512bw()) { 2707 kmovql(dst, src); 2708 } else { 2709 assert(VM_Version::supports_evex(), ""); 2710 kmovwl(dst, src); 2711 } 2712 } 2713 2714 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2715 if (VM_Version::supports_avx512bw()) { 2716 kmovql(dst, src); 2717 } else { 2718 assert(VM_Version::supports_evex(), ""); 2719 kmovwl(dst, src); 2720 } 2721 } 2722 2723 void MacroAssembler::kmov(Register dst, KRegister src) { 2724 if (VM_Version::supports_avx512bw()) { 2725 kmovql(dst, src); 2726 } else { 2727 assert(VM_Version::supports_evex(), ""); 2728 kmovwl(dst, src); 2729 } 2730 } 2731 2732 void MacroAssembler::kmov(KRegister dst, Register src) { 2733 if (VM_Version::supports_avx512bw()) { 2734 kmovql(dst, src); 2735 } else { 2736 assert(VM_Version::supports_evex(), ""); 2737 kmovwl(dst, src); 2738 } 2739 } 2740 2741 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2742 assert(rscratch != noreg || always_reachable(src), "missing"); 2743 2744 if (reachable(src)) { 2745 kmovql(dst, as_Address(src)); 2746 } else { 2747 lea(rscratch, src); 2748 kmovql(dst, Address(rscratch, 0)); 2749 } 2750 } 2751 2752 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2753 assert(rscratch != noreg || always_reachable(src), "missing"); 2754 2755 if (reachable(src)) { 2756 kmovwl(dst, as_Address(src)); 2757 } else { 2758 lea(rscratch, src); 2759 kmovwl(dst, Address(rscratch, 0)); 2760 } 2761 } 2762 2763 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2764 int vector_len, Register rscratch) { 2765 assert(rscratch != noreg || always_reachable(src), "missing"); 2766 2767 if (reachable(src)) { 2768 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2769 } else { 2770 lea(rscratch, src); 2771 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2772 } 2773 } 2774 2775 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2776 int vector_len, Register rscratch) { 2777 assert(rscratch != noreg || always_reachable(src), "missing"); 2778 2779 if (reachable(src)) { 2780 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2781 } else { 2782 lea(rscratch, src); 2783 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2784 } 2785 } 2786 2787 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2788 assert(rscratch != noreg || always_reachable(src), "missing"); 2789 2790 if (reachable(src)) { 2791 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2792 } else { 2793 lea(rscratch, src); 2794 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2795 } 2796 } 2797 2798 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2799 assert(rscratch != noreg || always_reachable(src), "missing"); 2800 2801 if (reachable(src)) { 2802 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2803 } else { 2804 lea(rscratch, src); 2805 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2806 } 2807 } 2808 2809 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2810 assert(rscratch != noreg || always_reachable(src), "missing"); 2811 2812 if (reachable(src)) { 2813 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2814 } else { 2815 lea(rscratch, src); 2816 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2817 } 2818 } 2819 2820 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2821 assert(rscratch != noreg || always_reachable(src), "missing"); 2822 2823 if (reachable(src)) { 2824 Assembler::movdqa(dst, as_Address(src)); 2825 } else { 2826 lea(rscratch, src); 2827 Assembler::movdqa(dst, Address(rscratch, 0)); 2828 } 2829 } 2830 2831 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2832 assert(rscratch != noreg || always_reachable(src), "missing"); 2833 2834 if (reachable(src)) { 2835 Assembler::movsd(dst, as_Address(src)); 2836 } else { 2837 lea(rscratch, src); 2838 Assembler::movsd(dst, Address(rscratch, 0)); 2839 } 2840 } 2841 2842 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2843 assert(rscratch != noreg || always_reachable(src), "missing"); 2844 2845 if (reachable(src)) { 2846 Assembler::movss(dst, as_Address(src)); 2847 } else { 2848 lea(rscratch, src); 2849 Assembler::movss(dst, Address(rscratch, 0)); 2850 } 2851 } 2852 2853 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2854 assert(rscratch != noreg || always_reachable(src), "missing"); 2855 2856 if (reachable(src)) { 2857 Assembler::movddup(dst, as_Address(src)); 2858 } else { 2859 lea(rscratch, src); 2860 Assembler::movddup(dst, Address(rscratch, 0)); 2861 } 2862 } 2863 2864 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2865 assert(rscratch != noreg || always_reachable(src), "missing"); 2866 2867 if (reachable(src)) { 2868 Assembler::vmovddup(dst, as_Address(src), vector_len); 2869 } else { 2870 lea(rscratch, src); 2871 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2872 } 2873 } 2874 2875 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2876 assert(rscratch != noreg || always_reachable(src), "missing"); 2877 2878 if (reachable(src)) { 2879 Assembler::mulsd(dst, as_Address(src)); 2880 } else { 2881 lea(rscratch, src); 2882 Assembler::mulsd(dst, Address(rscratch, 0)); 2883 } 2884 } 2885 2886 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2887 assert(rscratch != noreg || always_reachable(src), "missing"); 2888 2889 if (reachable(src)) { 2890 Assembler::mulss(dst, as_Address(src)); 2891 } else { 2892 lea(rscratch, src); 2893 Assembler::mulss(dst, Address(rscratch, 0)); 2894 } 2895 } 2896 2897 void MacroAssembler::null_check(Register reg, int offset) { 2898 if (needs_explicit_null_check(offset)) { 2899 // provoke OS null exception if reg is null by 2900 // accessing M[reg] w/o changing any (non-CC) registers 2901 // NOTE: cmpl is plenty here to provoke a segv 2902 cmpptr(rax, Address(reg, 0)); 2903 // Note: should probably use testl(rax, Address(reg, 0)); 2904 // may be shorter code (however, this version of 2905 // testl needs to be implemented first) 2906 } else { 2907 // nothing to do, (later) access of M[reg + offset] 2908 // will provoke OS null exception if reg is null 2909 } 2910 } 2911 2912 void MacroAssembler::os_breakpoint() { 2913 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2914 // (e.g., MSVC can't call ps() otherwise) 2915 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2916 } 2917 2918 void MacroAssembler::unimplemented(const char* what) { 2919 const char* buf = nullptr; 2920 { 2921 ResourceMark rm; 2922 stringStream ss; 2923 ss.print("unimplemented: %s", what); 2924 buf = code_string(ss.as_string()); 2925 } 2926 stop(buf); 2927 } 2928 2929 #ifdef _LP64 2930 #define XSTATE_BV 0x200 2931 #endif 2932 2933 void MacroAssembler::pop_CPU_state() { 2934 pop_FPU_state(); 2935 pop_IU_state(); 2936 } 2937 2938 void MacroAssembler::pop_FPU_state() { 2939 #ifndef _LP64 2940 frstor(Address(rsp, 0)); 2941 #else 2942 fxrstor(Address(rsp, 0)); 2943 #endif 2944 addptr(rsp, FPUStateSizeInWords * wordSize); 2945 } 2946 2947 void MacroAssembler::pop_IU_state() { 2948 popa(); 2949 LP64_ONLY(addq(rsp, 8)); 2950 popf(); 2951 } 2952 2953 // Save Integer and Float state 2954 // Warning: Stack must be 16 byte aligned (64bit) 2955 void MacroAssembler::push_CPU_state() { 2956 push_IU_state(); 2957 push_FPU_state(); 2958 } 2959 2960 void MacroAssembler::push_FPU_state() { 2961 subptr(rsp, FPUStateSizeInWords * wordSize); 2962 #ifndef _LP64 2963 fnsave(Address(rsp, 0)); 2964 fwait(); 2965 #else 2966 fxsave(Address(rsp, 0)); 2967 #endif // LP64 2968 } 2969 2970 void MacroAssembler::push_IU_state() { 2971 // Push flags first because pusha kills them 2972 pushf(); 2973 // Make sure rsp stays 16-byte aligned 2974 LP64_ONLY(subq(rsp, 8)); 2975 pusha(); 2976 } 2977 2978 void MacroAssembler::push_cont_fastpath() { 2979 if (!Continuations::enabled()) return; 2980 2981 #ifndef _LP64 2982 Register rthread = rax; 2983 Register rrealsp = rbx; 2984 push(rthread); 2985 push(rrealsp); 2986 2987 get_thread(rthread); 2988 2989 // The code below wants the original RSP. 2990 // Move it back after the pushes above. 2991 movptr(rrealsp, rsp); 2992 addptr(rrealsp, 2*wordSize); 2993 #else 2994 Register rthread = r15_thread; 2995 Register rrealsp = rsp; 2996 #endif 2997 2998 Label done; 2999 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3000 jccb(Assembler::belowEqual, done); 3001 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp); 3002 bind(done); 3003 3004 #ifndef _LP64 3005 pop(rrealsp); 3006 pop(rthread); 3007 #endif 3008 } 3009 3010 void MacroAssembler::pop_cont_fastpath() { 3011 if (!Continuations::enabled()) return; 3012 3013 #ifndef _LP64 3014 Register rthread = rax; 3015 Register rrealsp = rbx; 3016 push(rthread); 3017 push(rrealsp); 3018 3019 get_thread(rthread); 3020 3021 // The code below wants the original RSP. 3022 // Move it back after the pushes above. 3023 movptr(rrealsp, rsp); 3024 addptr(rrealsp, 2*wordSize); 3025 #else 3026 Register rthread = r15_thread; 3027 Register rrealsp = rsp; 3028 #endif 3029 3030 Label done; 3031 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3032 jccb(Assembler::below, done); 3033 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0); 3034 bind(done); 3035 3036 #ifndef _LP64 3037 pop(rrealsp); 3038 pop(rthread); 3039 #endif 3040 } 3041 3042 void MacroAssembler::inc_held_monitor_count() { 3043 #ifndef _LP64 3044 Register thread = rax; 3045 push(thread); 3046 get_thread(thread); 3047 incrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3048 pop(thread); 3049 #else // LP64 3050 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3051 #endif 3052 } 3053 3054 void MacroAssembler::dec_held_monitor_count() { 3055 #ifndef _LP64 3056 Register thread = rax; 3057 push(thread); 3058 get_thread(thread); 3059 decrementl(Address(thread, JavaThread::held_monitor_count_offset())); 3060 pop(thread); 3061 #else // LP64 3062 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3063 #endif 3064 } 3065 3066 #ifdef ASSERT 3067 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 3068 #ifdef _LP64 3069 Label no_cont; 3070 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 3071 testl(cont, cont); 3072 jcc(Assembler::zero, no_cont); 3073 stop(name); 3074 bind(no_cont); 3075 #else 3076 Unimplemented(); 3077 #endif 3078 } 3079 #endif 3080 3081 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register 3082 if (!java_thread->is_valid()) { 3083 java_thread = rdi; 3084 get_thread(java_thread); 3085 } 3086 // we must set sp to zero to clear frame 3087 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 3088 // must clear fp, so that compiled frames are not confused; it is 3089 // possible that we need it only for debugging 3090 if (clear_fp) { 3091 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 3092 } 3093 // Always clear the pc because it could have been set by make_walkable() 3094 movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 3095 vzeroupper(); 3096 } 3097 3098 void MacroAssembler::restore_rax(Register tmp) { 3099 if (tmp == noreg) pop(rax); 3100 else if (tmp != rax) mov(rax, tmp); 3101 } 3102 3103 void MacroAssembler::round_to(Register reg, int modulus) { 3104 addptr(reg, modulus - 1); 3105 andptr(reg, -modulus); 3106 } 3107 3108 void MacroAssembler::save_rax(Register tmp) { 3109 if (tmp == noreg) push(rax); 3110 else if (tmp != rax) mov(tmp, rax); 3111 } 3112 3113 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) { 3114 if (at_return) { 3115 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 3116 // we may safely use rsp instead to perform the stack watermark check. 3117 cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset())); 3118 jcc(Assembler::above, slow_path); 3119 return; 3120 } 3121 testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 3122 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 3123 } 3124 3125 // Calls to C land 3126 // 3127 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 3128 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 3129 // has to be reset to 0. This is required to allow proper stack traversal. 3130 void MacroAssembler::set_last_Java_frame(Register java_thread, 3131 Register last_java_sp, 3132 Register last_java_fp, 3133 address last_java_pc, 3134 Register rscratch) { 3135 vzeroupper(); 3136 // determine java_thread register 3137 if (!java_thread->is_valid()) { 3138 java_thread = rdi; 3139 get_thread(java_thread); 3140 } 3141 // determine last_java_sp register 3142 if (!last_java_sp->is_valid()) { 3143 last_java_sp = rsp; 3144 } 3145 // last_java_fp is optional 3146 if (last_java_fp->is_valid()) { 3147 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 3148 } 3149 // last_java_pc is optional 3150 if (last_java_pc != nullptr) { 3151 Address java_pc(java_thread, 3152 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 3153 lea(java_pc, InternalAddress(last_java_pc), rscratch); 3154 } 3155 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 3156 } 3157 3158 void MacroAssembler::shlptr(Register dst, int imm8) { 3159 LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); 3160 } 3161 3162 void MacroAssembler::shrptr(Register dst, int imm8) { 3163 LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); 3164 } 3165 3166 void MacroAssembler::sign_extend_byte(Register reg) { 3167 if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { 3168 movsbl(reg, reg); // movsxb 3169 } else { 3170 shll(reg, 24); 3171 sarl(reg, 24); 3172 } 3173 } 3174 3175 void MacroAssembler::sign_extend_short(Register reg) { 3176 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 3177 movswl(reg, reg); // movsxw 3178 } else { 3179 shll(reg, 16); 3180 sarl(reg, 16); 3181 } 3182 } 3183 3184 void MacroAssembler::testl(Address dst, int32_t imm32) { 3185 if (imm32 >= 0 && is8bit(imm32)) { 3186 testb(dst, imm32); 3187 } else { 3188 Assembler::testl(dst, imm32); 3189 } 3190 } 3191 3192 void MacroAssembler::testl(Register dst, int32_t imm32) { 3193 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 3194 testb(dst, imm32); 3195 } else { 3196 Assembler::testl(dst, imm32); 3197 } 3198 } 3199 3200 void MacroAssembler::testl(Register dst, AddressLiteral src) { 3201 assert(always_reachable(src), "Address should be reachable"); 3202 testl(dst, as_Address(src)); 3203 } 3204 3205 #ifdef _LP64 3206 3207 void MacroAssembler::testq(Address dst, int32_t imm32) { 3208 if (imm32 >= 0) { 3209 testl(dst, imm32); 3210 } else { 3211 Assembler::testq(dst, imm32); 3212 } 3213 } 3214 3215 void MacroAssembler::testq(Register dst, int32_t imm32) { 3216 if (imm32 >= 0) { 3217 testl(dst, imm32); 3218 } else { 3219 Assembler::testq(dst, imm32); 3220 } 3221 } 3222 3223 #endif 3224 3225 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 3226 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3227 Assembler::pcmpeqb(dst, src); 3228 } 3229 3230 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 3231 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3232 Assembler::pcmpeqw(dst, src); 3233 } 3234 3235 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 3236 assert((dst->encoding() < 16),"XMM register should be 0-15"); 3237 Assembler::pcmpestri(dst, src, imm8); 3238 } 3239 3240 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 3241 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3242 Assembler::pcmpestri(dst, src, imm8); 3243 } 3244 3245 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 3246 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3247 Assembler::pmovzxbw(dst, src); 3248 } 3249 3250 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 3251 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3252 Assembler::pmovzxbw(dst, src); 3253 } 3254 3255 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 3256 assert((src->encoding() < 16),"XMM register should be 0-15"); 3257 Assembler::pmovmskb(dst, src); 3258 } 3259 3260 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 3261 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3262 Assembler::ptest(dst, src); 3263 } 3264 3265 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3266 assert(rscratch != noreg || always_reachable(src), "missing"); 3267 3268 if (reachable(src)) { 3269 Assembler::sqrtss(dst, as_Address(src)); 3270 } else { 3271 lea(rscratch, src); 3272 Assembler::sqrtss(dst, Address(rscratch, 0)); 3273 } 3274 } 3275 3276 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3277 assert(rscratch != noreg || always_reachable(src), "missing"); 3278 3279 if (reachable(src)) { 3280 Assembler::subsd(dst, as_Address(src)); 3281 } else { 3282 lea(rscratch, src); 3283 Assembler::subsd(dst, Address(rscratch, 0)); 3284 } 3285 } 3286 3287 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 3288 assert(rscratch != noreg || always_reachable(src), "missing"); 3289 3290 if (reachable(src)) { 3291 Assembler::roundsd(dst, as_Address(src), rmode); 3292 } else { 3293 lea(rscratch, src); 3294 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 3295 } 3296 } 3297 3298 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3299 assert(rscratch != noreg || always_reachable(src), "missing"); 3300 3301 if (reachable(src)) { 3302 Assembler::subss(dst, as_Address(src)); 3303 } else { 3304 lea(rscratch, src); 3305 Assembler::subss(dst, Address(rscratch, 0)); 3306 } 3307 } 3308 3309 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3310 assert(rscratch != noreg || always_reachable(src), "missing"); 3311 3312 if (reachable(src)) { 3313 Assembler::ucomisd(dst, as_Address(src)); 3314 } else { 3315 lea(rscratch, src); 3316 Assembler::ucomisd(dst, Address(rscratch, 0)); 3317 } 3318 } 3319 3320 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3321 assert(rscratch != noreg || always_reachable(src), "missing"); 3322 3323 if (reachable(src)) { 3324 Assembler::ucomiss(dst, as_Address(src)); 3325 } else { 3326 lea(rscratch, src); 3327 Assembler::ucomiss(dst, Address(rscratch, 0)); 3328 } 3329 } 3330 3331 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3332 assert(rscratch != noreg || always_reachable(src), "missing"); 3333 3334 // Used in sign-bit flipping with aligned address. 3335 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3336 if (reachable(src)) { 3337 Assembler::xorpd(dst, as_Address(src)); 3338 } else { 3339 lea(rscratch, src); 3340 Assembler::xorpd(dst, Address(rscratch, 0)); 3341 } 3342 } 3343 3344 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3345 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3346 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3347 } 3348 else { 3349 Assembler::xorpd(dst, src); 3350 } 3351 } 3352 3353 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3354 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3355 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3356 } else { 3357 Assembler::xorps(dst, src); 3358 } 3359 } 3360 3361 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 3362 assert(rscratch != noreg || always_reachable(src), "missing"); 3363 3364 // Used in sign-bit flipping with aligned address. 3365 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3366 if (reachable(src)) { 3367 Assembler::xorps(dst, as_Address(src)); 3368 } else { 3369 lea(rscratch, src); 3370 Assembler::xorps(dst, Address(rscratch, 0)); 3371 } 3372 } 3373 3374 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 3375 assert(rscratch != noreg || always_reachable(src), "missing"); 3376 3377 // Used in sign-bit flipping with aligned address. 3378 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3379 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3380 if (reachable(src)) { 3381 Assembler::pshufb(dst, as_Address(src)); 3382 } else { 3383 lea(rscratch, src); 3384 Assembler::pshufb(dst, Address(rscratch, 0)); 3385 } 3386 } 3387 3388 // AVX 3-operands instructions 3389 3390 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3391 assert(rscratch != noreg || always_reachable(src), "missing"); 3392 3393 if (reachable(src)) { 3394 vaddsd(dst, nds, as_Address(src)); 3395 } else { 3396 lea(rscratch, src); 3397 vaddsd(dst, nds, Address(rscratch, 0)); 3398 } 3399 } 3400 3401 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3402 assert(rscratch != noreg || always_reachable(src), "missing"); 3403 3404 if (reachable(src)) { 3405 vaddss(dst, nds, as_Address(src)); 3406 } else { 3407 lea(rscratch, src); 3408 vaddss(dst, nds, Address(rscratch, 0)); 3409 } 3410 } 3411 3412 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3413 assert(UseAVX > 0, "requires some form of AVX"); 3414 assert(rscratch != noreg || always_reachable(src), "missing"); 3415 3416 if (reachable(src)) { 3417 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 3418 } else { 3419 lea(rscratch, src); 3420 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 3421 } 3422 } 3423 3424 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3425 assert(UseAVX > 0, "requires some form of AVX"); 3426 assert(rscratch != noreg || always_reachable(src), "missing"); 3427 3428 if (reachable(src)) { 3429 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 3430 } else { 3431 lea(rscratch, src); 3432 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 3433 } 3434 } 3435 3436 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3437 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3438 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3439 3440 vandps(dst, nds, negate_field, vector_len, rscratch); 3441 } 3442 3443 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3444 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3445 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3446 3447 vandpd(dst, nds, negate_field, vector_len, rscratch); 3448 } 3449 3450 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3451 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3452 Assembler::vpaddb(dst, nds, src, vector_len); 3453 } 3454 3455 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3456 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3457 Assembler::vpaddb(dst, nds, src, vector_len); 3458 } 3459 3460 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3461 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3462 Assembler::vpaddw(dst, nds, src, vector_len); 3463 } 3464 3465 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3466 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3467 Assembler::vpaddw(dst, nds, src, vector_len); 3468 } 3469 3470 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3471 assert(rscratch != noreg || always_reachable(src), "missing"); 3472 3473 if (reachable(src)) { 3474 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3475 } else { 3476 lea(rscratch, src); 3477 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 3478 } 3479 } 3480 3481 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3482 assert(rscratch != noreg || always_reachable(src), "missing"); 3483 3484 if (reachable(src)) { 3485 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 3486 } else { 3487 lea(rscratch, src); 3488 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 3489 } 3490 } 3491 3492 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3493 assert(rscratch != noreg || always_reachable(src), "missing"); 3494 3495 if (reachable(src)) { 3496 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 3497 } else { 3498 lea(rscratch, src); 3499 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 3500 } 3501 } 3502 3503 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3504 assert(rscratch != noreg || always_reachable(src), "missing"); 3505 3506 if (reachable(src)) { 3507 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 3508 } else { 3509 lea(rscratch, src); 3510 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 3511 } 3512 } 3513 3514 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3515 assert(rscratch != noreg || always_reachable(src), "missing"); 3516 3517 if (reachable(src)) { 3518 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 3519 } else { 3520 lea(rscratch, src); 3521 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 3522 } 3523 } 3524 3525 // Vector float blend 3526 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3527 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3528 // WARN: Allow dst == (src1|src2), mask == scratch 3529 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3530 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst; 3531 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3532 if (blend_emulation && scratch_available && dst_available) { 3533 if (compute_mask) { 3534 vpsrad(scratch, mask, 32, vector_len); 3535 mask = scratch; 3536 } 3537 if (dst == src1) { 3538 vpandn(dst, mask, src1, vector_len); // if mask == 0, src1 3539 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3540 } else { 3541 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3542 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1 3543 } 3544 vpor(dst, dst, scratch, vector_len); 3545 } else { 3546 Assembler::vblendvps(dst, src1, src2, mask, vector_len); 3547 } 3548 } 3549 3550 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3551 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3552 // WARN: Allow dst == (src1|src2), mask == scratch 3553 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3554 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask); 3555 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3556 if (blend_emulation && scratch_available && dst_available) { 3557 if (compute_mask) { 3558 vpxor(scratch, scratch, scratch, vector_len); 3559 vpcmpgtq(scratch, scratch, mask, vector_len); 3560 mask = scratch; 3561 } 3562 if (dst == src1) { 3563 vpandn(dst, mask, src1, vector_len); // if mask == 0, src 3564 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3565 } else { 3566 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3567 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src 3568 } 3569 vpor(dst, dst, scratch, vector_len); 3570 } else { 3571 Assembler::vblendvpd(dst, src1, src2, mask, vector_len); 3572 } 3573 } 3574 3575 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3576 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3577 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3578 } 3579 3580 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3581 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3582 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3583 } 3584 3585 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3586 assert(rscratch != noreg || always_reachable(src), "missing"); 3587 3588 if (reachable(src)) { 3589 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 3590 } else { 3591 lea(rscratch, src); 3592 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 3593 } 3594 } 3595 3596 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3597 int comparison, bool is_signed, int vector_len, Register rscratch) { 3598 assert(rscratch != noreg || always_reachable(src), "missing"); 3599 3600 if (reachable(src)) { 3601 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3602 } else { 3603 lea(rscratch, src); 3604 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3605 } 3606 } 3607 3608 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3609 int comparison, bool is_signed, int vector_len, Register rscratch) { 3610 assert(rscratch != noreg || always_reachable(src), "missing"); 3611 3612 if (reachable(src)) { 3613 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3614 } else { 3615 lea(rscratch, src); 3616 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3617 } 3618 } 3619 3620 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3621 int comparison, bool is_signed, int vector_len, Register rscratch) { 3622 assert(rscratch != noreg || always_reachable(src), "missing"); 3623 3624 if (reachable(src)) { 3625 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3626 } else { 3627 lea(rscratch, src); 3628 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3629 } 3630 } 3631 3632 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3633 int comparison, bool is_signed, int vector_len, Register rscratch) { 3634 assert(rscratch != noreg || always_reachable(src), "missing"); 3635 3636 if (reachable(src)) { 3637 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3638 } else { 3639 lea(rscratch, src); 3640 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3641 } 3642 } 3643 3644 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3645 if (width == Assembler::Q) { 3646 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3647 } else { 3648 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3649 } 3650 } 3651 3652 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3653 int eq_cond_enc = 0x29; 3654 int gt_cond_enc = 0x37; 3655 if (width != Assembler::Q) { 3656 eq_cond_enc = 0x74 + width; 3657 gt_cond_enc = 0x64 + width; 3658 } 3659 switch (cond) { 3660 case eq: 3661 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3662 break; 3663 case neq: 3664 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3665 vallones(xtmp, vector_len); 3666 vpxor(dst, xtmp, dst, vector_len); 3667 break; 3668 case le: 3669 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3670 vallones(xtmp, vector_len); 3671 vpxor(dst, xtmp, dst, vector_len); 3672 break; 3673 case nlt: 3674 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3675 vallones(xtmp, vector_len); 3676 vpxor(dst, xtmp, dst, vector_len); 3677 break; 3678 case lt: 3679 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3680 break; 3681 case nle: 3682 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3683 break; 3684 default: 3685 assert(false, "Should not reach here"); 3686 } 3687 } 3688 3689 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3690 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3691 Assembler::vpmovzxbw(dst, src, vector_len); 3692 } 3693 3694 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3695 assert((src->encoding() < 16),"XMM register should be 0-15"); 3696 Assembler::vpmovmskb(dst, src, vector_len); 3697 } 3698 3699 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3700 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3701 Assembler::vpmullw(dst, nds, src, vector_len); 3702 } 3703 3704 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3705 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3706 Assembler::vpmullw(dst, nds, src, vector_len); 3707 } 3708 3709 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3710 assert((UseAVX > 0), "AVX support is needed"); 3711 assert(rscratch != noreg || always_reachable(src), "missing"); 3712 3713 if (reachable(src)) { 3714 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3715 } else { 3716 lea(rscratch, src); 3717 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3718 } 3719 } 3720 3721 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3722 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3723 Assembler::vpsubb(dst, nds, src, vector_len); 3724 } 3725 3726 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3727 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3728 Assembler::vpsubb(dst, nds, src, vector_len); 3729 } 3730 3731 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3732 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3733 Assembler::vpsubw(dst, nds, src, vector_len); 3734 } 3735 3736 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3737 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3738 Assembler::vpsubw(dst, nds, src, vector_len); 3739 } 3740 3741 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3742 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3743 Assembler::vpsraw(dst, nds, shift, vector_len); 3744 } 3745 3746 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3747 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3748 Assembler::vpsraw(dst, nds, shift, vector_len); 3749 } 3750 3751 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3752 assert(UseAVX > 2,""); 3753 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3754 vector_len = 2; 3755 } 3756 Assembler::evpsraq(dst, nds, shift, vector_len); 3757 } 3758 3759 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3760 assert(UseAVX > 2,""); 3761 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3762 vector_len = 2; 3763 } 3764 Assembler::evpsraq(dst, nds, shift, vector_len); 3765 } 3766 3767 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3768 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3769 Assembler::vpsrlw(dst, nds, shift, vector_len); 3770 } 3771 3772 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3773 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3774 Assembler::vpsrlw(dst, nds, shift, vector_len); 3775 } 3776 3777 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3778 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3779 Assembler::vpsllw(dst, nds, shift, vector_len); 3780 } 3781 3782 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3783 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpsllw(dst, nds, shift, vector_len); 3785 } 3786 3787 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3788 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3789 Assembler::vptest(dst, src); 3790 } 3791 3792 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3793 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3794 Assembler::punpcklbw(dst, src); 3795 } 3796 3797 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3798 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3799 Assembler::pshufd(dst, src, mode); 3800 } 3801 3802 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3803 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3804 Assembler::pshuflw(dst, src, mode); 3805 } 3806 3807 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3808 assert(rscratch != noreg || always_reachable(src), "missing"); 3809 3810 if (reachable(src)) { 3811 vandpd(dst, nds, as_Address(src), vector_len); 3812 } else { 3813 lea(rscratch, src); 3814 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3815 } 3816 } 3817 3818 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3819 assert(rscratch != noreg || always_reachable(src), "missing"); 3820 3821 if (reachable(src)) { 3822 vandps(dst, nds, as_Address(src), vector_len); 3823 } else { 3824 lea(rscratch, src); 3825 vandps(dst, nds, Address(rscratch, 0), vector_len); 3826 } 3827 } 3828 3829 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3830 bool merge, int vector_len, Register rscratch) { 3831 assert(rscratch != noreg || always_reachable(src), "missing"); 3832 3833 if (reachable(src)) { 3834 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3835 } else { 3836 lea(rscratch, src); 3837 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3838 } 3839 } 3840 3841 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3842 assert(rscratch != noreg || always_reachable(src), "missing"); 3843 3844 if (reachable(src)) { 3845 vdivsd(dst, nds, as_Address(src)); 3846 } else { 3847 lea(rscratch, src); 3848 vdivsd(dst, nds, Address(rscratch, 0)); 3849 } 3850 } 3851 3852 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3853 assert(rscratch != noreg || always_reachable(src), "missing"); 3854 3855 if (reachable(src)) { 3856 vdivss(dst, nds, as_Address(src)); 3857 } else { 3858 lea(rscratch, src); 3859 vdivss(dst, nds, Address(rscratch, 0)); 3860 } 3861 } 3862 3863 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3864 assert(rscratch != noreg || always_reachable(src), "missing"); 3865 3866 if (reachable(src)) { 3867 vmulsd(dst, nds, as_Address(src)); 3868 } else { 3869 lea(rscratch, src); 3870 vmulsd(dst, nds, Address(rscratch, 0)); 3871 } 3872 } 3873 3874 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3875 assert(rscratch != noreg || always_reachable(src), "missing"); 3876 3877 if (reachable(src)) { 3878 vmulss(dst, nds, as_Address(src)); 3879 } else { 3880 lea(rscratch, src); 3881 vmulss(dst, nds, Address(rscratch, 0)); 3882 } 3883 } 3884 3885 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3886 assert(rscratch != noreg || always_reachable(src), "missing"); 3887 3888 if (reachable(src)) { 3889 vsubsd(dst, nds, as_Address(src)); 3890 } else { 3891 lea(rscratch, src); 3892 vsubsd(dst, nds, Address(rscratch, 0)); 3893 } 3894 } 3895 3896 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3897 assert(rscratch != noreg || always_reachable(src), "missing"); 3898 3899 if (reachable(src)) { 3900 vsubss(dst, nds, as_Address(src)); 3901 } else { 3902 lea(rscratch, src); 3903 vsubss(dst, nds, Address(rscratch, 0)); 3904 } 3905 } 3906 3907 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3908 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3909 assert(rscratch != noreg || always_reachable(src), "missing"); 3910 3911 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3912 } 3913 3914 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3915 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3916 assert(rscratch != noreg || always_reachable(src), "missing"); 3917 3918 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3919 } 3920 3921 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3922 assert(rscratch != noreg || always_reachable(src), "missing"); 3923 3924 if (reachable(src)) { 3925 vxorpd(dst, nds, as_Address(src), vector_len); 3926 } else { 3927 lea(rscratch, src); 3928 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3929 } 3930 } 3931 3932 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3933 assert(rscratch != noreg || always_reachable(src), "missing"); 3934 3935 if (reachable(src)) { 3936 vxorps(dst, nds, as_Address(src), vector_len); 3937 } else { 3938 lea(rscratch, src); 3939 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3940 } 3941 } 3942 3943 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3944 assert(rscratch != noreg || always_reachable(src), "missing"); 3945 3946 if (UseAVX > 1 || (vector_len < 1)) { 3947 if (reachable(src)) { 3948 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3949 } else { 3950 lea(rscratch, src); 3951 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3952 } 3953 } else { 3954 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3955 } 3956 } 3957 3958 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3959 assert(rscratch != noreg || always_reachable(src), "missing"); 3960 3961 if (reachable(src)) { 3962 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 3963 } else { 3964 lea(rscratch, src); 3965 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 3966 } 3967 } 3968 3969 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 3970 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 3971 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 3972 // The inverted mask is sign-extended 3973 andptr(possibly_non_local, inverted_mask); 3974 } 3975 3976 void MacroAssembler::resolve_jobject(Register value, 3977 Register thread, 3978 Register tmp) { 3979 assert_different_registers(value, thread, tmp); 3980 Label done, tagged, weak_tagged; 3981 testptr(value, value); 3982 jcc(Assembler::zero, done); // Use null as-is. 3983 testptr(value, JNIHandles::tag_mask); // Test for tag. 3984 jcc(Assembler::notZero, tagged); 3985 3986 // Resolve local handle 3987 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread); 3988 verify_oop(value); 3989 jmp(done); 3990 3991 bind(tagged); 3992 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 3993 jcc(Assembler::notZero, weak_tagged); 3994 3995 // Resolve global handle 3996 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 3997 verify_oop(value); 3998 jmp(done); 3999 4000 bind(weak_tagged); 4001 // Resolve jweak. 4002 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 4003 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread); 4004 verify_oop(value); 4005 4006 bind(done); 4007 } 4008 4009 void MacroAssembler::resolve_global_jobject(Register value, 4010 Register thread, 4011 Register tmp) { 4012 assert_different_registers(value, thread, tmp); 4013 Label done; 4014 4015 testptr(value, value); 4016 jcc(Assembler::zero, done); // Use null as-is. 4017 4018 #ifdef ASSERT 4019 { 4020 Label valid_global_tag; 4021 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 4022 jcc(Assembler::notZero, valid_global_tag); 4023 stop("non global jobject using resolve_global_jobject"); 4024 bind(valid_global_tag); 4025 } 4026 #endif 4027 4028 // Resolve global handle 4029 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4030 verify_oop(value); 4031 4032 bind(done); 4033 } 4034 4035 void MacroAssembler::subptr(Register dst, int32_t imm32) { 4036 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 4037 } 4038 4039 // Force generation of a 4 byte immediate value even if it fits into 8bit 4040 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 4041 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 4042 } 4043 4044 void MacroAssembler::subptr(Register dst, Register src) { 4045 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 4046 } 4047 4048 // C++ bool manipulation 4049 void MacroAssembler::testbool(Register dst) { 4050 if(sizeof(bool) == 1) 4051 testb(dst, 0xff); 4052 else if(sizeof(bool) == 2) { 4053 // testw implementation needed for two byte bools 4054 ShouldNotReachHere(); 4055 } else if(sizeof(bool) == 4) 4056 testl(dst, dst); 4057 else 4058 // unsupported 4059 ShouldNotReachHere(); 4060 } 4061 4062 void MacroAssembler::testptr(Register dst, Register src) { 4063 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); 4064 } 4065 4066 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 4067 void MacroAssembler::tlab_allocate(Register thread, Register obj, 4068 Register var_size_in_bytes, 4069 int con_size_in_bytes, 4070 Register t1, 4071 Register t2, 4072 Label& slow_case) { 4073 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 4074 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 4075 } 4076 4077 RegSet MacroAssembler::call_clobbered_gp_registers() { 4078 RegSet regs; 4079 #ifdef _LP64 4080 regs += RegSet::of(rax, rcx, rdx); 4081 #ifndef WINDOWS 4082 regs += RegSet::of(rsi, rdi); 4083 #endif 4084 regs += RegSet::range(r8, r11); 4085 #else 4086 regs += RegSet::of(rax, rcx, rdx); 4087 #endif 4088 return regs; 4089 } 4090 4091 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 4092 int num_xmm_registers = XMMRegister::available_xmm_registers(); 4093 #if defined(WINDOWS) && defined(_LP64) 4094 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 4095 if (num_xmm_registers > 16) { 4096 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 4097 } 4098 return result; 4099 #else 4100 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 4101 #endif 4102 } 4103 4104 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor 4105 4106 #ifndef _LP64 4107 static bool use_x87_registers() { return UseSSE < 2; } 4108 #endif 4109 static bool use_xmm_registers() { return UseSSE >= 1; } 4110 4111 // C1 only ever uses the first double/float of the XMM register. 4112 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); } 4113 4114 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4115 if (UseSSE == 1) { 4116 masm->movflt(Address(rsp, offset), reg); 4117 } else { 4118 masm->movdbl(Address(rsp, offset), reg); 4119 } 4120 } 4121 4122 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4123 if (UseSSE == 1) { 4124 masm->movflt(reg, Address(rsp, offset)); 4125 } else { 4126 masm->movdbl(reg, Address(rsp, offset)); 4127 } 4128 } 4129 4130 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, 4131 bool save_fpu, int& gp_area_size, 4132 int& fp_area_size, int& xmm_area_size) { 4133 4134 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 4135 StackAlignmentInBytes); 4136 #ifdef _LP64 4137 fp_area_size = 0; 4138 #else 4139 fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0; 4140 #endif 4141 xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0; 4142 4143 return gp_area_size + fp_area_size + xmm_area_size; 4144 } 4145 4146 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 4147 block_comment("push_call_clobbered_registers start"); 4148 // Regular registers 4149 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 4150 4151 int gp_area_size; 4152 int fp_area_size; 4153 int xmm_area_size; 4154 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 4155 gp_area_size, fp_area_size, xmm_area_size); 4156 subptr(rsp, total_save_size); 4157 4158 push_set(gp_registers_to_push, 0); 4159 4160 #ifndef _LP64 4161 if (save_fpu && use_x87_registers()) { 4162 fnsave(Address(rsp, gp_area_size)); 4163 fwait(); 4164 } 4165 #endif 4166 if (save_fpu && use_xmm_registers()) { 4167 push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4168 } 4169 4170 block_comment("push_call_clobbered_registers end"); 4171 } 4172 4173 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 4174 block_comment("pop_call_clobbered_registers start"); 4175 4176 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 4177 4178 int gp_area_size; 4179 int fp_area_size; 4180 int xmm_area_size; 4181 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 4182 gp_area_size, fp_area_size, xmm_area_size); 4183 4184 if (restore_fpu && use_xmm_registers()) { 4185 pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4186 } 4187 #ifndef _LP64 4188 if (restore_fpu && use_x87_registers()) { 4189 frstor(Address(rsp, gp_area_size)); 4190 } 4191 #endif 4192 4193 pop_set(gp_registers_to_pop, 0); 4194 4195 addptr(rsp, total_save_size); 4196 4197 vzeroupper(); 4198 4199 block_comment("pop_call_clobbered_registers end"); 4200 } 4201 4202 void MacroAssembler::push_set(XMMRegSet set, int offset) { 4203 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 4204 int spill_offset = offset; 4205 4206 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 4207 save_xmm_register(this, spill_offset, *it); 4208 spill_offset += xmm_save_size(); 4209 } 4210 } 4211 4212 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 4213 int restore_size = set.size() * xmm_save_size(); 4214 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 4215 4216 int restore_offset = offset + restore_size - xmm_save_size(); 4217 4218 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 4219 restore_xmm_register(this, restore_offset, *it); 4220 restore_offset -= xmm_save_size(); 4221 } 4222 } 4223 4224 void MacroAssembler::push_set(RegSet set, int offset) { 4225 int spill_offset; 4226 if (offset == -1) { 4227 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4228 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4229 subptr(rsp, aligned_size); 4230 spill_offset = 0; 4231 } else { 4232 spill_offset = offset; 4233 } 4234 4235 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 4236 movptr(Address(rsp, spill_offset), *it); 4237 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4238 } 4239 } 4240 4241 void MacroAssembler::pop_set(RegSet set, int offset) { 4242 4243 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4244 int restore_size = set.size() * gp_reg_size; 4245 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 4246 4247 int restore_offset; 4248 if (offset == -1) { 4249 restore_offset = restore_size - gp_reg_size; 4250 } else { 4251 restore_offset = offset + restore_size - gp_reg_size; 4252 } 4253 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 4254 movptr(*it, Address(rsp, restore_offset)); 4255 restore_offset -= gp_reg_size; 4256 } 4257 4258 if (offset == -1) { 4259 addptr(rsp, aligned_size); 4260 } 4261 } 4262 4263 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 4264 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 4265 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 4266 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 4267 Label done; 4268 4269 testptr(length_in_bytes, length_in_bytes); 4270 jcc(Assembler::zero, done); 4271 4272 // initialize topmost word, divide index by 2, check if odd and test if zero 4273 // note: for the remaining code to work, index must be a multiple of BytesPerWord 4274 #ifdef ASSERT 4275 { 4276 Label L; 4277 testptr(length_in_bytes, BytesPerWord - 1); 4278 jcc(Assembler::zero, L); 4279 stop("length must be a multiple of BytesPerWord"); 4280 bind(L); 4281 } 4282 #endif 4283 Register index = length_in_bytes; 4284 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 4285 if (UseIncDec) { 4286 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 4287 } else { 4288 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 4289 shrptr(index, 1); 4290 } 4291 #ifndef _LP64 4292 // index could have not been a multiple of 8 (i.e., bit 2 was set) 4293 { 4294 Label even; 4295 // note: if index was a multiple of 8, then it cannot 4296 // be 0 now otherwise it must have been 0 before 4297 // => if it is even, we don't need to check for 0 again 4298 jcc(Assembler::carryClear, even); 4299 // clear topmost word (no jump would be needed if conditional assignment worked here) 4300 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); 4301 // index could be 0 now, must check again 4302 jcc(Assembler::zero, done); 4303 bind(even); 4304 } 4305 #endif // !_LP64 4306 // initialize remaining object fields: index is a multiple of 2 now 4307 { 4308 Label loop; 4309 bind(loop); 4310 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 4311 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) 4312 decrement(index); 4313 jcc(Assembler::notZero, loop); 4314 } 4315 4316 bind(done); 4317 } 4318 4319 // Look up the method for a megamorphic invokeinterface call. 4320 // The target method is determined by <intf_klass, itable_index>. 4321 // The receiver klass is in recv_klass. 4322 // On success, the result will be in method_result, and execution falls through. 4323 // On failure, execution transfers to the given label. 4324 void MacroAssembler::lookup_interface_method(Register recv_klass, 4325 Register intf_klass, 4326 RegisterOrConstant itable_index, 4327 Register method_result, 4328 Register scan_temp, 4329 Label& L_no_such_interface, 4330 bool return_method) { 4331 assert_different_registers(recv_klass, intf_klass, scan_temp); 4332 assert_different_registers(method_result, intf_klass, scan_temp); 4333 assert(recv_klass != method_result || !return_method, 4334 "recv_klass can be destroyed when method isn't needed"); 4335 4336 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 4337 "caller must use same register for non-constant itable index as for method"); 4338 4339 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 4340 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4341 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4342 int scan_step = itableOffsetEntry::size() * wordSize; 4343 int vte_size = vtableEntry::size_in_bytes(); 4344 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4345 assert(vte_size == wordSize, "else adjust times_vte_scale"); 4346 4347 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4348 4349 // Could store the aligned, prescaled offset in the klass. 4350 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 4351 4352 if (return_method) { 4353 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 4354 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4355 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 4356 } 4357 4358 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 4359 // if (scan->interface() == intf) { 4360 // result = (klass + scan->offset() + itable_index); 4361 // } 4362 // } 4363 Label search, found_method; 4364 4365 for (int peel = 1; peel >= 0; peel--) { 4366 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 4367 cmpptr(intf_klass, method_result); 4368 4369 if (peel) { 4370 jccb(Assembler::equal, found_method); 4371 } else { 4372 jccb(Assembler::notEqual, search); 4373 // (invert the test to fall through to found_method...) 4374 } 4375 4376 if (!peel) break; 4377 4378 bind(search); 4379 4380 // Check that the previous entry is non-null. A null entry means that 4381 // the receiver class doesn't implement the interface, and wasn't the 4382 // same as when the caller was compiled. 4383 testptr(method_result, method_result); 4384 jcc(Assembler::zero, L_no_such_interface); 4385 addptr(scan_temp, scan_step); 4386 } 4387 4388 bind(found_method); 4389 4390 if (return_method) { 4391 // Got a hit. 4392 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 4393 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 4394 } 4395 } 4396 4397 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 4398 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 4399 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 4400 // The target method is determined by <holder_klass, itable_index>. 4401 // The receiver klass is in recv_klass. 4402 // On success, the result will be in method_result, and execution falls through. 4403 // On failure, execution transfers to the given label. 4404 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 4405 Register holder_klass, 4406 Register resolved_klass, 4407 Register method_result, 4408 Register scan_temp, 4409 Register temp_reg2, 4410 Register receiver, 4411 int itable_index, 4412 Label& L_no_such_interface) { 4413 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 4414 Register temp_itbl_klass = method_result; 4415 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 4416 4417 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4418 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4419 int scan_step = itableOffsetEntry::size() * wordSize; 4420 int vte_size = vtableEntry::size_in_bytes(); 4421 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 4422 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 4423 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4424 assert(vte_size == wordSize, "adjust times_vte_scale"); 4425 4426 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 4427 4428 // temp_itbl_klass = recv_klass.itable[0] 4429 // scan_temp = &recv_klass.itable[0] + step 4430 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4431 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 4432 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 4433 xorptr(temp_reg, temp_reg); 4434 4435 // Initial checks: 4436 // - if (holder_klass != resolved_klass), go to "scan for resolved" 4437 // - if (itable[0] == 0), no such interface 4438 // - if (itable[0] == holder_klass), shortcut to "holder found" 4439 cmpptr(holder_klass, resolved_klass); 4440 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 4441 testptr(temp_itbl_klass, temp_itbl_klass); 4442 jccb(Assembler::zero, L_no_such_interface); 4443 cmpptr(holder_klass, temp_itbl_klass); 4444 jccb(Assembler::equal, L_holder_found); 4445 4446 // Loop: Look for holder_klass record in itable 4447 // do { 4448 // tmp = itable[index]; 4449 // index += step; 4450 // if (tmp == holder_klass) { 4451 // goto L_holder_found; // Found! 4452 // } 4453 // } while (tmp != 0); 4454 // goto L_no_such_interface // Not found. 4455 Label L_scan_holder; 4456 bind(L_scan_holder); 4457 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4458 addptr(scan_temp, scan_step); 4459 cmpptr(holder_klass, temp_itbl_klass); 4460 jccb(Assembler::equal, L_holder_found); 4461 testptr(temp_itbl_klass, temp_itbl_klass); 4462 jccb(Assembler::notZero, L_scan_holder); 4463 4464 jmpb(L_no_such_interface); 4465 4466 // Loop: Look for resolved_class record in itable 4467 // do { 4468 // tmp = itable[index]; 4469 // index += step; 4470 // if (tmp == holder_klass) { 4471 // // Also check if we have met a holder klass 4472 // holder_tmp = itable[index-step-ioffset]; 4473 // } 4474 // if (tmp == resolved_klass) { 4475 // goto L_resolved_found; // Found! 4476 // } 4477 // } while (tmp != 0); 4478 // goto L_no_such_interface // Not found. 4479 // 4480 Label L_loop_scan_resolved; 4481 bind(L_loop_scan_resolved); 4482 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4483 addptr(scan_temp, scan_step); 4484 bind(L_loop_scan_resolved_entry); 4485 cmpptr(holder_klass, temp_itbl_klass); 4486 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4487 cmpptr(resolved_klass, temp_itbl_klass); 4488 jccb(Assembler::equal, L_resolved_found); 4489 testptr(temp_itbl_klass, temp_itbl_klass); 4490 jccb(Assembler::notZero, L_loop_scan_resolved); 4491 4492 jmpb(L_no_such_interface); 4493 4494 Label L_ready; 4495 4496 // See if we already have a holder klass. If not, go and scan for it. 4497 bind(L_resolved_found); 4498 testptr(temp_reg, temp_reg); 4499 jccb(Assembler::zero, L_scan_holder); 4500 jmpb(L_ready); 4501 4502 bind(L_holder_found); 4503 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4504 4505 // Finally, temp_reg contains holder_klass vtable offset 4506 bind(L_ready); 4507 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4508 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 4509 load_klass(scan_temp, receiver, noreg); 4510 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4511 } else { 4512 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4513 } 4514 } 4515 4516 4517 // virtual method calling 4518 void MacroAssembler::lookup_virtual_method(Register recv_klass, 4519 RegisterOrConstant vtable_index, 4520 Register method_result) { 4521 const ByteSize base = Klass::vtable_start_offset(); 4522 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 4523 Address vtable_entry_addr(recv_klass, 4524 vtable_index, Address::times_ptr, 4525 base + vtableEntry::method_offset()); 4526 movptr(method_result, vtable_entry_addr); 4527 } 4528 4529 4530 void MacroAssembler::check_klass_subtype(Register sub_klass, 4531 Register super_klass, 4532 Register temp_reg, 4533 Label& L_success) { 4534 Label L_failure; 4535 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 4536 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 4537 bind(L_failure); 4538 } 4539 4540 4541 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 4542 Register super_klass, 4543 Register temp_reg, 4544 Label* L_success, 4545 Label* L_failure, 4546 Label* L_slow_path, 4547 RegisterOrConstant super_check_offset) { 4548 assert_different_registers(sub_klass, super_klass, temp_reg); 4549 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 4550 if (super_check_offset.is_register()) { 4551 assert_different_registers(sub_klass, super_klass, 4552 super_check_offset.as_register()); 4553 } else if (must_load_sco) { 4554 assert(temp_reg != noreg, "supply either a temp or a register offset"); 4555 } 4556 4557 Label L_fallthrough; 4558 int label_nulls = 0; 4559 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4560 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4561 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 4562 assert(label_nulls <= 1, "at most one null in the batch"); 4563 4564 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4565 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 4566 Address super_check_offset_addr(super_klass, sco_offset); 4567 4568 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 4569 // range of a jccb. If this routine grows larger, reconsider at 4570 // least some of these. 4571 #define local_jcc(assembler_cond, label) \ 4572 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 4573 else jcc( assembler_cond, label) /*omit semi*/ 4574 4575 // Hacked jmp, which may only be used just before L_fallthrough. 4576 #define final_jmp(label) \ 4577 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 4578 else jmp(label) /*omit semi*/ 4579 4580 // If the pointers are equal, we are done (e.g., String[] elements). 4581 // This self-check enables sharing of secondary supertype arrays among 4582 // non-primary types such as array-of-interface. Otherwise, each such 4583 // type would need its own customized SSA. 4584 // We move this check to the front of the fast path because many 4585 // type checks are in fact trivially successful in this manner, 4586 // so we get a nicely predicted branch right at the start of the check. 4587 cmpptr(sub_klass, super_klass); 4588 local_jcc(Assembler::equal, *L_success); 4589 4590 // Check the supertype display: 4591 if (must_load_sco) { 4592 // Positive movl does right thing on LP64. 4593 movl(temp_reg, super_check_offset_addr); 4594 super_check_offset = RegisterOrConstant(temp_reg); 4595 } 4596 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 4597 cmpptr(super_klass, super_check_addr); // load displayed supertype 4598 4599 // This check has worked decisively for primary supers. 4600 // Secondary supers are sought in the super_cache ('super_cache_addr'). 4601 // (Secondary supers are interfaces and very deeply nested subtypes.) 4602 // This works in the same check above because of a tricky aliasing 4603 // between the super_cache and the primary super display elements. 4604 // (The 'super_check_addr' can address either, as the case requires.) 4605 // Note that the cache is updated below if it does not help us find 4606 // what we need immediately. 4607 // So if it was a primary super, we can just fail immediately. 4608 // Otherwise, it's the slow path for us (no success at this point). 4609 4610 if (super_check_offset.is_register()) { 4611 local_jcc(Assembler::equal, *L_success); 4612 cmpl(super_check_offset.as_register(), sc_offset); 4613 if (L_failure == &L_fallthrough) { 4614 local_jcc(Assembler::equal, *L_slow_path); 4615 } else { 4616 local_jcc(Assembler::notEqual, *L_failure); 4617 final_jmp(*L_slow_path); 4618 } 4619 } else if (super_check_offset.as_constant() == sc_offset) { 4620 // Need a slow path; fast failure is impossible. 4621 if (L_slow_path == &L_fallthrough) { 4622 local_jcc(Assembler::equal, *L_success); 4623 } else { 4624 local_jcc(Assembler::notEqual, *L_slow_path); 4625 final_jmp(*L_success); 4626 } 4627 } else { 4628 // No slow path; it's a fast decision. 4629 if (L_failure == &L_fallthrough) { 4630 local_jcc(Assembler::equal, *L_success); 4631 } else { 4632 local_jcc(Assembler::notEqual, *L_failure); 4633 final_jmp(*L_success); 4634 } 4635 } 4636 4637 bind(L_fallthrough); 4638 4639 #undef local_jcc 4640 #undef final_jmp 4641 } 4642 4643 4644 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4645 Register super_klass, 4646 Register temp_reg, 4647 Register temp2_reg, 4648 Label* L_success, 4649 Label* L_failure, 4650 bool set_cond_codes) { 4651 assert_different_registers(sub_klass, super_klass, temp_reg); 4652 if (temp2_reg != noreg) 4653 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 4654 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 4655 4656 Label L_fallthrough; 4657 int label_nulls = 0; 4658 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4659 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4660 assert(label_nulls <= 1, "at most one null in the batch"); 4661 4662 // a couple of useful fields in sub_klass: 4663 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 4664 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4665 Address secondary_supers_addr(sub_klass, ss_offset); 4666 Address super_cache_addr( sub_klass, sc_offset); 4667 4668 // Do a linear scan of the secondary super-klass chain. 4669 // This code is rarely used, so simplicity is a virtue here. 4670 // The repne_scan instruction uses fixed registers, which we must spill. 4671 // Don't worry too much about pre-existing connections with the input regs. 4672 4673 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 4674 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 4675 4676 // Get super_klass value into rax (even if it was in rdi or rcx). 4677 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 4678 if (super_klass != rax) { 4679 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 4680 mov(rax, super_klass); 4681 } 4682 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 4683 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 4684 4685 #ifndef PRODUCT 4686 uint* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4687 ExternalAddress pst_counter_addr((address) pst_counter); 4688 NOT_LP64( incrementl(pst_counter_addr) ); 4689 LP64_ONLY( lea(rcx, pst_counter_addr) ); 4690 LP64_ONLY( incrementl(Address(rcx, 0)) ); 4691 #endif //PRODUCT 4692 4693 // We will consult the secondary-super array. 4694 movptr(rdi, secondary_supers_addr); 4695 // Load the array length. (Positive movl does right thing on LP64.) 4696 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4697 // Skip to start of data. 4698 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4699 4700 // Scan RCX words at [RDI] for an occurrence of RAX. 4701 // Set NZ/Z based on last compare. 4702 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4703 // not change flags (only scas instruction which is repeated sets flags). 4704 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4705 4706 testptr(rax,rax); // Set Z = 0 4707 repne_scan(); 4708 4709 // Unspill the temp. registers: 4710 if (pushed_rdi) pop(rdi); 4711 if (pushed_rcx) pop(rcx); 4712 if (pushed_rax) pop(rax); 4713 4714 if (set_cond_codes) { 4715 // Special hack for the AD files: rdi is guaranteed non-zero. 4716 assert(!pushed_rdi, "rdi must be left non-null"); 4717 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4718 } 4719 4720 if (L_failure == &L_fallthrough) 4721 jccb(Assembler::notEqual, *L_failure); 4722 else jcc(Assembler::notEqual, *L_failure); 4723 4724 // Success. Cache the super we found and proceed in triumph. 4725 movptr(super_cache_addr, super_klass); 4726 4727 if (L_success != &L_fallthrough) { 4728 jmp(*L_success); 4729 } 4730 4731 #undef IS_A_TEMP 4732 4733 bind(L_fallthrough); 4734 } 4735 4736 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { 4737 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 4738 4739 Label L_fallthrough; 4740 if (L_fast_path == nullptr) { 4741 L_fast_path = &L_fallthrough; 4742 } else if (L_slow_path == nullptr) { 4743 L_slow_path = &L_fallthrough; 4744 } 4745 4746 // Fast path check: class is fully initialized 4747 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 4748 jcc(Assembler::equal, *L_fast_path); 4749 4750 // Fast path check: current thread is initializer thread 4751 cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); 4752 if (L_slow_path == &L_fallthrough) { 4753 jcc(Assembler::equal, *L_fast_path); 4754 bind(*L_slow_path); 4755 } else if (L_fast_path == &L_fallthrough) { 4756 jcc(Assembler::notEqual, *L_slow_path); 4757 bind(*L_fast_path); 4758 } else { 4759 Unimplemented(); 4760 } 4761 } 4762 4763 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 4764 if (VM_Version::supports_cmov()) { 4765 cmovl(cc, dst, src); 4766 } else { 4767 Label L; 4768 jccb(negate_condition(cc), L); 4769 movl(dst, src); 4770 bind(L); 4771 } 4772 } 4773 4774 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 4775 if (VM_Version::supports_cmov()) { 4776 cmovl(cc, dst, src); 4777 } else { 4778 Label L; 4779 jccb(negate_condition(cc), L); 4780 movl(dst, src); 4781 bind(L); 4782 } 4783 } 4784 4785 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 4786 if (!VerifyOops) return; 4787 4788 BLOCK_COMMENT("verify_oop {"); 4789 #ifdef _LP64 4790 push(rscratch1); 4791 #endif 4792 push(rax); // save rax 4793 push(reg); // pass register argument 4794 4795 // Pass register number to verify_oop_subroutine 4796 const char* b = nullptr; 4797 { 4798 ResourceMark rm; 4799 stringStream ss; 4800 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 4801 b = code_string(ss.as_string()); 4802 } 4803 ExternalAddress buffer((address) b); 4804 pushptr(buffer.addr(), rscratch1); 4805 4806 // call indirectly to solve generation ordering problem 4807 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4808 call(rax); 4809 // Caller pops the arguments (oop, message) and restores rax, r10 4810 BLOCK_COMMENT("} verify_oop"); 4811 } 4812 4813 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 4814 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 4815 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 4816 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 4817 vpternlogd(dst, 0xFF, dst, dst, vector_len); 4818 } else if (VM_Version::supports_avx()) { 4819 vpcmpeqd(dst, dst, dst, vector_len); 4820 } else { 4821 assert(VM_Version::supports_sse2(), ""); 4822 pcmpeqd(dst, dst); 4823 } 4824 } 4825 4826 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 4827 int extra_slot_offset) { 4828 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 4829 int stackElementSize = Interpreter::stackElementSize; 4830 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 4831 #ifdef ASSERT 4832 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 4833 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 4834 #endif 4835 Register scale_reg = noreg; 4836 Address::ScaleFactor scale_factor = Address::no_scale; 4837 if (arg_slot.is_constant()) { 4838 offset += arg_slot.as_constant() * stackElementSize; 4839 } else { 4840 scale_reg = arg_slot.as_register(); 4841 scale_factor = Address::times(stackElementSize); 4842 } 4843 offset += wordSize; // return PC is on stack 4844 return Address(rsp, scale_reg, scale_factor, offset); 4845 } 4846 4847 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 4848 if (!VerifyOops) return; 4849 4850 #ifdef _LP64 4851 push(rscratch1); 4852 #endif 4853 push(rax); // save rax, 4854 // addr may contain rsp so we will have to adjust it based on the push 4855 // we just did (and on 64 bit we do two pushes) 4856 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 4857 // stores rax into addr which is backwards of what was intended. 4858 if (addr.uses(rsp)) { 4859 lea(rax, addr); 4860 pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); 4861 } else { 4862 pushptr(addr); 4863 } 4864 4865 // Pass register number to verify_oop_subroutine 4866 const char* b = nullptr; 4867 { 4868 ResourceMark rm; 4869 stringStream ss; 4870 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 4871 b = code_string(ss.as_string()); 4872 } 4873 ExternalAddress buffer((address) b); 4874 pushptr(buffer.addr(), rscratch1); 4875 4876 // call indirectly to solve generation ordering problem 4877 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4878 call(rax); 4879 // Caller pops the arguments (addr, message) and restores rax, r10. 4880 } 4881 4882 void MacroAssembler::verify_tlab() { 4883 #ifdef ASSERT 4884 if (UseTLAB && VerifyOops) { 4885 Label next, ok; 4886 Register t1 = rsi; 4887 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); 4888 4889 push(t1); 4890 NOT_LP64(push(thread_reg)); 4891 NOT_LP64(get_thread(thread_reg)); 4892 4893 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4894 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); 4895 jcc(Assembler::aboveEqual, next); 4896 STOP("assert(top >= start)"); 4897 should_not_reach_here(); 4898 4899 bind(next); 4900 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); 4901 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 4902 jcc(Assembler::aboveEqual, ok); 4903 STOP("assert(top <= end)"); 4904 should_not_reach_here(); 4905 4906 bind(ok); 4907 NOT_LP64(pop(thread_reg)); 4908 pop(t1); 4909 } 4910 #endif 4911 } 4912 4913 class ControlWord { 4914 public: 4915 int32_t _value; 4916 4917 int rounding_control() const { return (_value >> 10) & 3 ; } 4918 int precision_control() const { return (_value >> 8) & 3 ; } 4919 bool precision() const { return ((_value >> 5) & 1) != 0; } 4920 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4921 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4922 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4923 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4924 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4925 4926 void print() const { 4927 // rounding control 4928 const char* rc; 4929 switch (rounding_control()) { 4930 case 0: rc = "round near"; break; 4931 case 1: rc = "round down"; break; 4932 case 2: rc = "round up "; break; 4933 case 3: rc = "chop "; break; 4934 default: 4935 rc = nullptr; // silence compiler warnings 4936 fatal("Unknown rounding control: %d", rounding_control()); 4937 }; 4938 // precision control 4939 const char* pc; 4940 switch (precision_control()) { 4941 case 0: pc = "24 bits "; break; 4942 case 1: pc = "reserved"; break; 4943 case 2: pc = "53 bits "; break; 4944 case 3: pc = "64 bits "; break; 4945 default: 4946 pc = nullptr; // silence compiler warnings 4947 fatal("Unknown precision control: %d", precision_control()); 4948 }; 4949 // flags 4950 char f[9]; 4951 f[0] = ' '; 4952 f[1] = ' '; 4953 f[2] = (precision ()) ? 'P' : 'p'; 4954 f[3] = (underflow ()) ? 'U' : 'u'; 4955 f[4] = (overflow ()) ? 'O' : 'o'; 4956 f[5] = (zero_divide ()) ? 'Z' : 'z'; 4957 f[6] = (denormalized()) ? 'D' : 'd'; 4958 f[7] = (invalid ()) ? 'I' : 'i'; 4959 f[8] = '\x0'; 4960 // output 4961 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 4962 } 4963 4964 }; 4965 4966 class StatusWord { 4967 public: 4968 int32_t _value; 4969 4970 bool busy() const { return ((_value >> 15) & 1) != 0; } 4971 bool C3() const { return ((_value >> 14) & 1) != 0; } 4972 bool C2() const { return ((_value >> 10) & 1) != 0; } 4973 bool C1() const { return ((_value >> 9) & 1) != 0; } 4974 bool C0() const { return ((_value >> 8) & 1) != 0; } 4975 int top() const { return (_value >> 11) & 7 ; } 4976 bool error_status() const { return ((_value >> 7) & 1) != 0; } 4977 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 4978 bool precision() const { return ((_value >> 5) & 1) != 0; } 4979 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4980 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4981 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4982 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4983 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4984 4985 void print() const { 4986 // condition codes 4987 char c[5]; 4988 c[0] = (C3()) ? '3' : '-'; 4989 c[1] = (C2()) ? '2' : '-'; 4990 c[2] = (C1()) ? '1' : '-'; 4991 c[3] = (C0()) ? '0' : '-'; 4992 c[4] = '\x0'; 4993 // flags 4994 char f[9]; 4995 f[0] = (error_status()) ? 'E' : '-'; 4996 f[1] = (stack_fault ()) ? 'S' : '-'; 4997 f[2] = (precision ()) ? 'P' : '-'; 4998 f[3] = (underflow ()) ? 'U' : '-'; 4999 f[4] = (overflow ()) ? 'O' : '-'; 5000 f[5] = (zero_divide ()) ? 'Z' : '-'; 5001 f[6] = (denormalized()) ? 'D' : '-'; 5002 f[7] = (invalid ()) ? 'I' : '-'; 5003 f[8] = '\x0'; 5004 // output 5005 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 5006 } 5007 5008 }; 5009 5010 class TagWord { 5011 public: 5012 int32_t _value; 5013 5014 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 5015 5016 void print() const { 5017 printf("%04x", _value & 0xFFFF); 5018 } 5019 5020 }; 5021 5022 class FPU_Register { 5023 public: 5024 int32_t _m0; 5025 int32_t _m1; 5026 int16_t _ex; 5027 5028 bool is_indefinite() const { 5029 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 5030 } 5031 5032 void print() const { 5033 char sign = (_ex < 0) ? '-' : '+'; 5034 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 5035 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 5036 }; 5037 5038 }; 5039 5040 class FPU_State { 5041 public: 5042 enum { 5043 register_size = 10, 5044 number_of_registers = 8, 5045 register_mask = 7 5046 }; 5047 5048 ControlWord _control_word; 5049 StatusWord _status_word; 5050 TagWord _tag_word; 5051 int32_t _error_offset; 5052 int32_t _error_selector; 5053 int32_t _data_offset; 5054 int32_t _data_selector; 5055 int8_t _register[register_size * number_of_registers]; 5056 5057 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 5058 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 5059 5060 const char* tag_as_string(int tag) const { 5061 switch (tag) { 5062 case 0: return "valid"; 5063 case 1: return "zero"; 5064 case 2: return "special"; 5065 case 3: return "empty"; 5066 } 5067 ShouldNotReachHere(); 5068 return nullptr; 5069 } 5070 5071 void print() const { 5072 // print computation registers 5073 { int t = _status_word.top(); 5074 for (int i = 0; i < number_of_registers; i++) { 5075 int j = (i - t) & register_mask; 5076 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 5077 st(j)->print(); 5078 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 5079 } 5080 } 5081 printf("\n"); 5082 // print control registers 5083 printf("ctrl = "); _control_word.print(); printf("\n"); 5084 printf("stat = "); _status_word .print(); printf("\n"); 5085 printf("tags = "); _tag_word .print(); printf("\n"); 5086 } 5087 5088 }; 5089 5090 class Flag_Register { 5091 public: 5092 int32_t _value; 5093 5094 bool overflow() const { return ((_value >> 11) & 1) != 0; } 5095 bool direction() const { return ((_value >> 10) & 1) != 0; } 5096 bool sign() const { return ((_value >> 7) & 1) != 0; } 5097 bool zero() const { return ((_value >> 6) & 1) != 0; } 5098 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 5099 bool parity() const { return ((_value >> 2) & 1) != 0; } 5100 bool carry() const { return ((_value >> 0) & 1) != 0; } 5101 5102 void print() const { 5103 // flags 5104 char f[8]; 5105 f[0] = (overflow ()) ? 'O' : '-'; 5106 f[1] = (direction ()) ? 'D' : '-'; 5107 f[2] = (sign ()) ? 'S' : '-'; 5108 f[3] = (zero ()) ? 'Z' : '-'; 5109 f[4] = (auxiliary_carry()) ? 'A' : '-'; 5110 f[5] = (parity ()) ? 'P' : '-'; 5111 f[6] = (carry ()) ? 'C' : '-'; 5112 f[7] = '\x0'; 5113 // output 5114 printf("%08x flags = %s", _value, f); 5115 } 5116 5117 }; 5118 5119 class IU_Register { 5120 public: 5121 int32_t _value; 5122 5123 void print() const { 5124 printf("%08x %11d", _value, _value); 5125 } 5126 5127 }; 5128 5129 class IU_State { 5130 public: 5131 Flag_Register _eflags; 5132 IU_Register _rdi; 5133 IU_Register _rsi; 5134 IU_Register _rbp; 5135 IU_Register _rsp; 5136 IU_Register _rbx; 5137 IU_Register _rdx; 5138 IU_Register _rcx; 5139 IU_Register _rax; 5140 5141 void print() const { 5142 // computation registers 5143 printf("rax, = "); _rax.print(); printf("\n"); 5144 printf("rbx, = "); _rbx.print(); printf("\n"); 5145 printf("rcx = "); _rcx.print(); printf("\n"); 5146 printf("rdx = "); _rdx.print(); printf("\n"); 5147 printf("rdi = "); _rdi.print(); printf("\n"); 5148 printf("rsi = "); _rsi.print(); printf("\n"); 5149 printf("rbp, = "); _rbp.print(); printf("\n"); 5150 printf("rsp = "); _rsp.print(); printf("\n"); 5151 printf("\n"); 5152 // control registers 5153 printf("flgs = "); _eflags.print(); printf("\n"); 5154 } 5155 }; 5156 5157 5158 class CPU_State { 5159 public: 5160 FPU_State _fpu_state; 5161 IU_State _iu_state; 5162 5163 void print() const { 5164 printf("--------------------------------------------------\n"); 5165 _iu_state .print(); 5166 printf("\n"); 5167 _fpu_state.print(); 5168 printf("--------------------------------------------------\n"); 5169 } 5170 5171 }; 5172 5173 5174 static void _print_CPU_state(CPU_State* state) { 5175 state->print(); 5176 }; 5177 5178 5179 void MacroAssembler::print_CPU_state() { 5180 push_CPU_state(); 5181 push(rsp); // pass CPU state 5182 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5183 addptr(rsp, wordSize); // discard argument 5184 pop_CPU_state(); 5185 } 5186 5187 5188 #ifndef _LP64 5189 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { 5190 static int counter = 0; 5191 FPU_State* fs = &state->_fpu_state; 5192 counter++; 5193 // For leaf calls, only verify that the top few elements remain empty. 5194 // We only need 1 empty at the top for C2 code. 5195 if( stack_depth < 0 ) { 5196 if( fs->tag_for_st(7) != 3 ) { 5197 printf("FPR7 not empty\n"); 5198 state->print(); 5199 assert(false, "error"); 5200 return false; 5201 } 5202 return true; // All other stack states do not matter 5203 } 5204 5205 assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(), 5206 "bad FPU control word"); 5207 5208 // compute stack depth 5209 int i = 0; 5210 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; 5211 int d = i; 5212 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; 5213 // verify findings 5214 if (i != FPU_State::number_of_registers) { 5215 // stack not contiguous 5216 printf("%s: stack not contiguous at ST%d\n", s, i); 5217 state->print(); 5218 assert(false, "error"); 5219 return false; 5220 } 5221 // check if computed stack depth corresponds to expected stack depth 5222 if (stack_depth < 0) { 5223 // expected stack depth is -stack_depth or less 5224 if (d > -stack_depth) { 5225 // too many elements on the stack 5226 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); 5227 state->print(); 5228 assert(false, "error"); 5229 return false; 5230 } 5231 } else { 5232 // expected stack depth is stack_depth 5233 if (d != stack_depth) { 5234 // wrong stack depth 5235 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); 5236 state->print(); 5237 assert(false, "error"); 5238 return false; 5239 } 5240 } 5241 // everything is cool 5242 return true; 5243 } 5244 5245 void MacroAssembler::verify_FPU(int stack_depth, const char* s) { 5246 if (!VerifyFPU) return; 5247 push_CPU_state(); 5248 push(rsp); // pass CPU state 5249 ExternalAddress msg((address) s); 5250 // pass message string s 5251 pushptr(msg.addr(), noreg); 5252 push(stack_depth); // pass stack depth 5253 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); 5254 addptr(rsp, 3 * wordSize); // discard arguments 5255 // check for error 5256 { Label L; 5257 testl(rax, rax); 5258 jcc(Assembler::notZero, L); 5259 int3(); // break if error condition 5260 bind(L); 5261 } 5262 pop_CPU_state(); 5263 } 5264 #endif // _LP64 5265 5266 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5267 // Either restore the MXCSR register after returning from the JNI Call 5268 // or verify that it wasn't changed (with -Xcheck:jni flag). 5269 if (VM_Version::supports_sse()) { 5270 if (RestoreMXCSROnJNICalls) { 5271 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5272 } else if (CheckJNICalls) { 5273 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5274 } 5275 } 5276 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5277 vzeroupper(); 5278 5279 #ifndef _LP64 5280 // Either restore the x87 floating pointer control word after returning 5281 // from the JNI call or verify that it wasn't changed. 5282 if (CheckJNICalls) { 5283 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); 5284 } 5285 #endif // _LP64 5286 } 5287 5288 // ((OopHandle)result).resolve(); 5289 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5290 assert_different_registers(result, tmp); 5291 5292 // Only 64 bit platforms support GCs that require a tmp register 5293 // Only IN_HEAP loads require a thread_tmp register 5294 // OopHandle::resolve is an indirection like jobject. 5295 access_load_at(T_OBJECT, IN_NATIVE, 5296 result, Address(result, 0), tmp, /*tmp_thread*/noreg); 5297 } 5298 5299 // ((WeakHandle)result).resolve(); 5300 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5301 assert_different_registers(rresult, rtmp); 5302 Label resolved; 5303 5304 // A null weak handle resolves to null. 5305 cmpptr(rresult, 0); 5306 jcc(Assembler::equal, resolved); 5307 5308 // Only 64 bit platforms support GCs that require a tmp register 5309 // Only IN_HEAP loads require a thread_tmp register 5310 // WeakHandle::resolve is an indirection like jweak. 5311 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5312 rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); 5313 bind(resolved); 5314 } 5315 5316 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5317 // get mirror 5318 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5319 load_method_holder(mirror, method); 5320 movptr(mirror, Address(mirror, mirror_offset)); 5321 resolve_oop_handle(mirror, tmp); 5322 } 5323 5324 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5325 load_method_holder(rresult, rmethod); 5326 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5327 } 5328 5329 void MacroAssembler::load_method_holder(Register holder, Register method) { 5330 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5331 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5332 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5333 } 5334 5335 #ifdef _LP64 5336 void MacroAssembler::load_nklass_compact(Register dst, Register src) { 5337 assert(UseCompactObjectHeaders, "expect compact object headers"); 5338 movq(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5339 shrq(dst, markWord::klass_shift); 5340 } 5341 #endif 5342 5343 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5344 BLOCK_COMMENT("load_klass"); 5345 assert_different_registers(src, tmp); 5346 assert_different_registers(dst, tmp); 5347 #ifdef _LP64 5348 if (UseCompactObjectHeaders) { 5349 load_nklass_compact(dst, src); 5350 decode_klass_not_null(dst, tmp); 5351 } else if (UseCompressedClassPointers) { 5352 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5353 decode_klass_not_null(dst, tmp); 5354 } else 5355 #endif 5356 { 5357 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5358 } 5359 } 5360 5361 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 5362 assert(!UseCompactObjectHeaders, "not with compact headers"); 5363 assert_different_registers(src, tmp); 5364 assert_different_registers(dst, tmp); 5365 #ifdef _LP64 5366 if (UseCompressedClassPointers) { 5367 encode_klass_not_null(src, tmp); 5368 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5369 } else 5370 #endif 5371 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5372 } 5373 5374 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { 5375 BLOCK_COMMENT("cmp_klass 1"); 5376 #ifdef _LP64 5377 if (UseCompactObjectHeaders) { 5378 load_nklass_compact(tmp, obj); 5379 cmpl(klass, tmp); 5380 } else if (UseCompressedClassPointers) { 5381 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5382 } else 5383 #endif 5384 { 5385 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5386 } 5387 } 5388 5389 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) { 5390 BLOCK_COMMENT("cmp_klass 2"); 5391 #ifdef _LP64 5392 if (UseCompactObjectHeaders) { 5393 assert(tmp2 != noreg, "need tmp2"); 5394 assert_different_registers(src, dst, tmp1, tmp2); 5395 load_nklass_compact(tmp1, src); 5396 load_nklass_compact(tmp2, dst); 5397 cmpl(tmp1, tmp2); 5398 } else if (UseCompressedClassPointers) { 5399 movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes())); 5400 cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes())); 5401 } else 5402 #endif 5403 { 5404 movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes())); 5405 cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes())); 5406 } 5407 } 5408 5409 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 5410 Register tmp1, Register thread_tmp) { 5411 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5412 decorators = AccessInternal::decorator_fixup(decorators, type); 5413 bool as_raw = (decorators & AS_RAW) != 0; 5414 if (as_raw) { 5415 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5416 } else { 5417 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 5418 } 5419 } 5420 5421 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 5422 Register tmp1, Register tmp2, Register tmp3) { 5423 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5424 decorators = AccessInternal::decorator_fixup(decorators, type); 5425 bool as_raw = (decorators & AS_RAW) != 0; 5426 if (as_raw) { 5427 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5428 } else { 5429 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5430 } 5431 } 5432 5433 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5434 Register thread_tmp, DecoratorSet decorators) { 5435 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); 5436 } 5437 5438 // Doesn't do verification, generates fixed size code 5439 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5440 Register thread_tmp, DecoratorSet decorators) { 5441 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); 5442 } 5443 5444 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5445 Register tmp2, Register tmp3, DecoratorSet decorators) { 5446 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5447 } 5448 5449 // Used for storing nulls. 5450 void MacroAssembler::store_heap_oop_null(Address dst) { 5451 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5452 } 5453 5454 #ifdef _LP64 5455 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5456 assert(!UseCompactObjectHeaders, "Don't use with compact headers"); 5457 if (UseCompressedClassPointers) { 5458 // Store to klass gap in destination 5459 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 5460 } 5461 } 5462 5463 #ifdef ASSERT 5464 void MacroAssembler::verify_heapbase(const char* msg) { 5465 assert (UseCompressedOops, "should be compressed"); 5466 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5467 if (CheckCompressedOops) { 5468 Label ok; 5469 ExternalAddress src2(CompressedOops::ptrs_base_addr()); 5470 const bool is_src2_reachable = reachable(src2); 5471 if (!is_src2_reachable) { 5472 push(rscratch1); // cmpptr trashes rscratch1 5473 } 5474 cmpptr(r12_heapbase, src2, rscratch1); 5475 jcc(Assembler::equal, ok); 5476 STOP(msg); 5477 bind(ok); 5478 if (!is_src2_reachable) { 5479 pop(rscratch1); 5480 } 5481 } 5482 } 5483 #endif 5484 5485 // Algorithm must match oop.inline.hpp encode_heap_oop. 5486 void MacroAssembler::encode_heap_oop(Register r) { 5487 #ifdef ASSERT 5488 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5489 #endif 5490 verify_oop_msg(r, "broken oop in encode_heap_oop"); 5491 if (CompressedOops::base() == nullptr) { 5492 if (CompressedOops::shift() != 0) { 5493 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5494 shrq(r, LogMinObjAlignmentInBytes); 5495 } 5496 return; 5497 } 5498 testq(r, r); 5499 cmovq(Assembler::equal, r, r12_heapbase); 5500 subq(r, r12_heapbase); 5501 shrq(r, LogMinObjAlignmentInBytes); 5502 } 5503 5504 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5505 #ifdef ASSERT 5506 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5507 if (CheckCompressedOops) { 5508 Label ok; 5509 testq(r, r); 5510 jcc(Assembler::notEqual, ok); 5511 STOP("null oop passed to encode_heap_oop_not_null"); 5512 bind(ok); 5513 } 5514 #endif 5515 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5516 if (CompressedOops::base() != nullptr) { 5517 subq(r, r12_heapbase); 5518 } 5519 if (CompressedOops::shift() != 0) { 5520 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5521 shrq(r, LogMinObjAlignmentInBytes); 5522 } 5523 } 5524 5525 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5526 #ifdef ASSERT 5527 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5528 if (CheckCompressedOops) { 5529 Label ok; 5530 testq(src, src); 5531 jcc(Assembler::notEqual, ok); 5532 STOP("null oop passed to encode_heap_oop_not_null2"); 5533 bind(ok); 5534 } 5535 #endif 5536 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5537 if (dst != src) { 5538 movq(dst, src); 5539 } 5540 if (CompressedOops::base() != nullptr) { 5541 subq(dst, r12_heapbase); 5542 } 5543 if (CompressedOops::shift() != 0) { 5544 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5545 shrq(dst, LogMinObjAlignmentInBytes); 5546 } 5547 } 5548 5549 void MacroAssembler::decode_heap_oop(Register r) { 5550 #ifdef ASSERT 5551 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5552 #endif 5553 if (CompressedOops::base() == nullptr) { 5554 if (CompressedOops::shift() != 0) { 5555 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5556 shlq(r, LogMinObjAlignmentInBytes); 5557 } 5558 } else { 5559 Label done; 5560 shlq(r, LogMinObjAlignmentInBytes); 5561 jccb(Assembler::equal, done); 5562 addq(r, r12_heapbase); 5563 bind(done); 5564 } 5565 verify_oop_msg(r, "broken oop in decode_heap_oop"); 5566 } 5567 5568 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5569 // Note: it will change flags 5570 assert (UseCompressedOops, "should only be used for compressed headers"); 5571 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5572 // Cannot assert, unverified entry point counts instructions (see .ad file) 5573 // vtableStubs also counts instructions in pd_code_size_limit. 5574 // Also do not verify_oop as this is called by verify_oop. 5575 if (CompressedOops::shift() != 0) { 5576 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5577 shlq(r, LogMinObjAlignmentInBytes); 5578 if (CompressedOops::base() != nullptr) { 5579 addq(r, r12_heapbase); 5580 } 5581 } else { 5582 assert (CompressedOops::base() == nullptr, "sanity"); 5583 } 5584 } 5585 5586 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5587 // Note: it will change flags 5588 assert (UseCompressedOops, "should only be used for compressed headers"); 5589 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5590 // Cannot assert, unverified entry point counts instructions (see .ad file) 5591 // vtableStubs also counts instructions in pd_code_size_limit. 5592 // Also do not verify_oop as this is called by verify_oop. 5593 if (CompressedOops::shift() != 0) { 5594 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5595 if (LogMinObjAlignmentInBytes == Address::times_8) { 5596 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 5597 } else { 5598 if (dst != src) { 5599 movq(dst, src); 5600 } 5601 shlq(dst, LogMinObjAlignmentInBytes); 5602 if (CompressedOops::base() != nullptr) { 5603 addq(dst, r12_heapbase); 5604 } 5605 } 5606 } else { 5607 assert (CompressedOops::base() == nullptr, "sanity"); 5608 if (dst != src) { 5609 movq(dst, src); 5610 } 5611 } 5612 } 5613 5614 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 5615 assert_different_registers(r, tmp); 5616 if (CompressedKlassPointers::base() != nullptr) { 5617 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5618 subq(r, tmp); 5619 } 5620 if (CompressedKlassPointers::shift() != 0) { 5621 shrq(r, CompressedKlassPointers::shift()); 5622 } 5623 } 5624 5625 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 5626 assert_different_registers(src, dst); 5627 if (CompressedKlassPointers::base() != nullptr) { 5628 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 5629 addq(dst, src); 5630 } else { 5631 movptr(dst, src); 5632 } 5633 if (CompressedKlassPointers::shift() != 0) { 5634 shrq(dst, CompressedKlassPointers::shift()); 5635 } 5636 } 5637 5638 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 5639 assert_different_registers(r, tmp); 5640 // Note: it will change flags 5641 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 5642 // Cannot assert, unverified entry point counts instructions (see .ad file) 5643 // vtableStubs also counts instructions in pd_code_size_limit. 5644 // Also do not verify_oop as this is called by verify_oop. 5645 if (CompressedKlassPointers::shift() != 0) { 5646 shlq(r, CompressedKlassPointers::shift()); 5647 } 5648 if (CompressedKlassPointers::base() != nullptr) { 5649 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5650 addq(r, tmp); 5651 } 5652 } 5653 5654 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 5655 assert_different_registers(src, dst); 5656 // Note: it will change flags 5657 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5658 // Cannot assert, unverified entry point counts instructions (see .ad file) 5659 // vtableStubs also counts instructions in pd_code_size_limit. 5660 // Also do not verify_oop as this is called by verify_oop. 5661 5662 if (CompressedKlassPointers::base() == nullptr && 5663 CompressedKlassPointers::shift() == 0) { 5664 // The best case scenario is that there is no base or shift. Then it is already 5665 // a pointer that needs nothing but a register rename. 5666 movl(dst, src); 5667 } else { 5668 if (CompressedKlassPointers::shift() <= Address::times_8) { 5669 if (CompressedKlassPointers::base() != nullptr) { 5670 mov64(dst, (int64_t)CompressedKlassPointers::base()); 5671 } else { 5672 xorq(dst, dst); 5673 } 5674 if (CompressedKlassPointers::shift() != 0) { 5675 assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?"); 5676 leaq(dst, Address(dst, src, Address::times_8, 0)); 5677 } else { 5678 addq(dst, src); 5679 } 5680 } else { 5681 if (CompressedKlassPointers::base() != nullptr) { 5682 const uint64_t base_right_shifted = 5683 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); 5684 mov64(dst, base_right_shifted); 5685 } else { 5686 xorq(dst, dst); 5687 } 5688 addq(dst, src); 5689 shlq(dst, CompressedKlassPointers::shift()); 5690 } 5691 } 5692 } 5693 5694 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5695 assert (UseCompressedOops, "should only be used for compressed headers"); 5696 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5697 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5698 int oop_index = oop_recorder()->find_index(obj); 5699 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5700 mov_narrow_oop(dst, oop_index, rspec); 5701 } 5702 5703 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 5704 assert (UseCompressedOops, "should only be used for compressed headers"); 5705 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5706 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5707 int oop_index = oop_recorder()->find_index(obj); 5708 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5709 mov_narrow_oop(dst, oop_index, rspec); 5710 } 5711 5712 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5713 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5714 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5715 int klass_index = oop_recorder()->find_index(k); 5716 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5717 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5718 } 5719 5720 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 5721 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5722 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5723 int klass_index = oop_recorder()->find_index(k); 5724 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5725 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5726 } 5727 5728 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 5729 assert (UseCompressedOops, "should only be used for compressed headers"); 5730 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5731 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5732 int oop_index = oop_recorder()->find_index(obj); 5733 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5734 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5735 } 5736 5737 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 5738 assert (UseCompressedOops, "should only be used for compressed headers"); 5739 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5740 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5741 int oop_index = oop_recorder()->find_index(obj); 5742 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5743 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5744 } 5745 5746 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 5747 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5748 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5749 int klass_index = oop_recorder()->find_index(k); 5750 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5751 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5752 } 5753 5754 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 5755 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5756 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5757 int klass_index = oop_recorder()->find_index(k); 5758 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5759 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5760 } 5761 5762 void MacroAssembler::reinit_heapbase() { 5763 if (UseCompressedOops) { 5764 if (Universe::heap() != nullptr) { 5765 if (CompressedOops::base() == nullptr) { 5766 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 5767 } else { 5768 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base()); 5769 } 5770 } else { 5771 movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr())); 5772 } 5773 } 5774 } 5775 5776 #endif // _LP64 5777 5778 #if COMPILER2_OR_JVMCI 5779 5780 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 5781 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5782 // cnt - number of qwords (8-byte words). 5783 // base - start address, qword aligned. 5784 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 5785 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 5786 if (use64byteVector) { 5787 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 5788 } else if (MaxVectorSize >= 32) { 5789 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 5790 } else { 5791 pxor(xtmp, xtmp); 5792 } 5793 jmp(L_zero_64_bytes); 5794 5795 BIND(L_loop); 5796 if (MaxVectorSize >= 32) { 5797 fill64(base, 0, xtmp, use64byteVector); 5798 } else { 5799 movdqu(Address(base, 0), xtmp); 5800 movdqu(Address(base, 16), xtmp); 5801 movdqu(Address(base, 32), xtmp); 5802 movdqu(Address(base, 48), xtmp); 5803 } 5804 addptr(base, 64); 5805 5806 BIND(L_zero_64_bytes); 5807 subptr(cnt, 8); 5808 jccb(Assembler::greaterEqual, L_loop); 5809 5810 // Copy trailing 64 bytes 5811 if (use64byteVector) { 5812 addptr(cnt, 8); 5813 jccb(Assembler::equal, L_end); 5814 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 5815 jmp(L_end); 5816 } else { 5817 addptr(cnt, 4); 5818 jccb(Assembler::less, L_tail); 5819 if (MaxVectorSize >= 32) { 5820 vmovdqu(Address(base, 0), xtmp); 5821 } else { 5822 movdqu(Address(base, 0), xtmp); 5823 movdqu(Address(base, 16), xtmp); 5824 } 5825 } 5826 addptr(base, 32); 5827 subptr(cnt, 4); 5828 5829 BIND(L_tail); 5830 addptr(cnt, 4); 5831 jccb(Assembler::lessEqual, L_end); 5832 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 5833 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 5834 } else { 5835 decrement(cnt); 5836 5837 BIND(L_sloop); 5838 movq(Address(base, 0), xtmp); 5839 addptr(base, 8); 5840 decrement(cnt); 5841 jccb(Assembler::greaterEqual, L_sloop); 5842 } 5843 BIND(L_end); 5844 } 5845 5846 // Clearing constant sized memory using YMM/ZMM registers. 5847 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5848 assert(UseAVX > 2 && VM_Version::supports_avx512vl(), ""); 5849 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 5850 5851 int vector64_count = (cnt & (~0x7)) >> 3; 5852 cnt = cnt & 0x7; 5853 const int fill64_per_loop = 4; 5854 const int max_unrolled_fill64 = 8; 5855 5856 // 64 byte initialization loop. 5857 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 5858 int start64 = 0; 5859 if (vector64_count > max_unrolled_fill64) { 5860 Label LOOP; 5861 Register index = rtmp; 5862 5863 start64 = vector64_count - (vector64_count % fill64_per_loop); 5864 5865 movl(index, 0); 5866 BIND(LOOP); 5867 for (int i = 0; i < fill64_per_loop; i++) { 5868 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 5869 } 5870 addl(index, fill64_per_loop * 64); 5871 cmpl(index, start64 * 64); 5872 jccb(Assembler::less, LOOP); 5873 } 5874 for (int i = start64; i < vector64_count; i++) { 5875 fill64(base, i * 64, xtmp, use64byteVector); 5876 } 5877 5878 // Clear remaining 64 byte tail. 5879 int disp = vector64_count * 64; 5880 if (cnt) { 5881 switch (cnt) { 5882 case 1: 5883 movq(Address(base, disp), xtmp); 5884 break; 5885 case 2: 5886 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 5887 break; 5888 case 3: 5889 movl(rtmp, 0x7); 5890 kmovwl(mask, rtmp); 5891 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 5892 break; 5893 case 4: 5894 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5895 break; 5896 case 5: 5897 if (use64byteVector) { 5898 movl(rtmp, 0x1F); 5899 kmovwl(mask, rtmp); 5900 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5901 } else { 5902 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5903 movq(Address(base, disp + 32), xtmp); 5904 } 5905 break; 5906 case 6: 5907 if (use64byteVector) { 5908 movl(rtmp, 0x3F); 5909 kmovwl(mask, rtmp); 5910 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5911 } else { 5912 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5913 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 5914 } 5915 break; 5916 case 7: 5917 if (use64byteVector) { 5918 movl(rtmp, 0x7F); 5919 kmovwl(mask, rtmp); 5920 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5921 } else { 5922 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5923 movl(rtmp, 0x7); 5924 kmovwl(mask, rtmp); 5925 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 5926 } 5927 break; 5928 default: 5929 fatal("Unexpected length : %d\n",cnt); 5930 break; 5931 } 5932 } 5933 } 5934 5935 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 5936 bool is_large, KRegister mask) { 5937 // cnt - number of qwords (8-byte words). 5938 // base - start address, qword aligned. 5939 // is_large - if optimizers know cnt is larger than InitArrayShortSize 5940 assert(base==rdi, "base register must be edi for rep stos"); 5941 assert(tmp==rax, "tmp register must be eax for rep stos"); 5942 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 5943 assert(InitArrayShortSize % BytesPerLong == 0, 5944 "InitArrayShortSize should be the multiple of BytesPerLong"); 5945 5946 Label DONE; 5947 if (!is_large || !UseXMMForObjInit) { 5948 xorptr(tmp, tmp); 5949 } 5950 5951 if (!is_large) { 5952 Label LOOP, LONG; 5953 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 5954 jccb(Assembler::greater, LONG); 5955 5956 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5957 5958 decrement(cnt); 5959 jccb(Assembler::negative, DONE); // Zero length 5960 5961 // Use individual pointer-sized stores for small counts: 5962 BIND(LOOP); 5963 movptr(Address(base, cnt, Address::times_ptr), tmp); 5964 decrement(cnt); 5965 jccb(Assembler::greaterEqual, LOOP); 5966 jmpb(DONE); 5967 5968 BIND(LONG); 5969 } 5970 5971 // Use longer rep-prefixed ops for non-small counts: 5972 if (UseFastStosb) { 5973 shlptr(cnt, 3); // convert to number of bytes 5974 rep_stosb(); 5975 } else if (UseXMMForObjInit) { 5976 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 5977 } else { 5978 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 5979 rep_stos(); 5980 } 5981 5982 BIND(DONE); 5983 } 5984 5985 #endif //COMPILER2_OR_JVMCI 5986 5987 5988 void MacroAssembler::generate_fill(BasicType t, bool aligned, 5989 Register to, Register value, Register count, 5990 Register rtmp, XMMRegister xtmp) { 5991 ShortBranchVerifier sbv(this); 5992 assert_different_registers(to, value, count, rtmp); 5993 Label L_exit; 5994 Label L_fill_2_bytes, L_fill_4_bytes; 5995 5996 #if defined(COMPILER2) && defined(_LP64) 5997 if(MaxVectorSize >=32 && 5998 VM_Version::supports_avx512vlbw() && 5999 VM_Version::supports_bmi2()) { 6000 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 6001 return; 6002 } 6003 #endif 6004 6005 int shift = -1; 6006 switch (t) { 6007 case T_BYTE: 6008 shift = 2; 6009 break; 6010 case T_SHORT: 6011 shift = 1; 6012 break; 6013 case T_INT: 6014 shift = 0; 6015 break; 6016 default: ShouldNotReachHere(); 6017 } 6018 6019 if (t == T_BYTE) { 6020 andl(value, 0xff); 6021 movl(rtmp, value); 6022 shll(rtmp, 8); 6023 orl(value, rtmp); 6024 } 6025 if (t == T_SHORT) { 6026 andl(value, 0xffff); 6027 } 6028 if (t == T_BYTE || t == T_SHORT) { 6029 movl(rtmp, value); 6030 shll(rtmp, 16); 6031 orl(value, rtmp); 6032 } 6033 6034 cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 6035 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 6036 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 6037 Label L_skip_align2; 6038 // align source address at 4 bytes address boundary 6039 if (t == T_BYTE) { 6040 Label L_skip_align1; 6041 // One byte misalignment happens only for byte arrays 6042 testptr(to, 1); 6043 jccb(Assembler::zero, L_skip_align1); 6044 movb(Address(to, 0), value); 6045 increment(to); 6046 decrement(count); 6047 BIND(L_skip_align1); 6048 } 6049 // Two bytes misalignment happens only for byte and short (char) arrays 6050 testptr(to, 2); 6051 jccb(Assembler::zero, L_skip_align2); 6052 movw(Address(to, 0), value); 6053 addptr(to, 2); 6054 subl(count, 1<<(shift-1)); 6055 BIND(L_skip_align2); 6056 } 6057 if (UseSSE < 2) { 6058 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6059 // Fill 32-byte chunks 6060 subl(count, 8 << shift); 6061 jcc(Assembler::less, L_check_fill_8_bytes); 6062 align(16); 6063 6064 BIND(L_fill_32_bytes_loop); 6065 6066 for (int i = 0; i < 32; i += 4) { 6067 movl(Address(to, i), value); 6068 } 6069 6070 addptr(to, 32); 6071 subl(count, 8 << shift); 6072 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6073 BIND(L_check_fill_8_bytes); 6074 addl(count, 8 << shift); 6075 jccb(Assembler::zero, L_exit); 6076 jmpb(L_fill_8_bytes); 6077 6078 // 6079 // length is too short, just fill qwords 6080 // 6081 BIND(L_fill_8_bytes_loop); 6082 movl(Address(to, 0), value); 6083 movl(Address(to, 4), value); 6084 addptr(to, 8); 6085 BIND(L_fill_8_bytes); 6086 subl(count, 1 << (shift + 1)); 6087 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6088 // fall through to fill 4 bytes 6089 } else { 6090 Label L_fill_32_bytes; 6091 if (!UseUnalignedLoadStores) { 6092 // align to 8 bytes, we know we are 4 byte aligned to start 6093 testptr(to, 4); 6094 jccb(Assembler::zero, L_fill_32_bytes); 6095 movl(Address(to, 0), value); 6096 addptr(to, 4); 6097 subl(count, 1<<shift); 6098 } 6099 BIND(L_fill_32_bytes); 6100 { 6101 assert( UseSSE >= 2, "supported cpu only" ); 6102 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6103 movdl(xtmp, value); 6104 if (UseAVX >= 2 && UseUnalignedLoadStores) { 6105 Label L_check_fill_32_bytes; 6106 if (UseAVX > 2) { 6107 // Fill 64-byte chunks 6108 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 6109 6110 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 6111 cmpl(count, VM_Version::avx3_threshold()); 6112 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 6113 6114 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 6115 6116 subl(count, 16 << shift); 6117 jccb(Assembler::less, L_check_fill_32_bytes); 6118 align(16); 6119 6120 BIND(L_fill_64_bytes_loop_avx3); 6121 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 6122 addptr(to, 64); 6123 subl(count, 16 << shift); 6124 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 6125 jmpb(L_check_fill_32_bytes); 6126 6127 BIND(L_check_fill_64_bytes_avx2); 6128 } 6129 // Fill 64-byte chunks 6130 Label L_fill_64_bytes_loop; 6131 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 6132 6133 subl(count, 16 << shift); 6134 jcc(Assembler::less, L_check_fill_32_bytes); 6135 align(16); 6136 6137 BIND(L_fill_64_bytes_loop); 6138 vmovdqu(Address(to, 0), xtmp); 6139 vmovdqu(Address(to, 32), xtmp); 6140 addptr(to, 64); 6141 subl(count, 16 << shift); 6142 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 6143 6144 BIND(L_check_fill_32_bytes); 6145 addl(count, 8 << shift); 6146 jccb(Assembler::less, L_check_fill_8_bytes); 6147 vmovdqu(Address(to, 0), xtmp); 6148 addptr(to, 32); 6149 subl(count, 8 << shift); 6150 6151 BIND(L_check_fill_8_bytes); 6152 // clean upper bits of YMM registers 6153 movdl(xtmp, value); 6154 pshufd(xtmp, xtmp, 0); 6155 } else { 6156 // Fill 32-byte chunks 6157 pshufd(xtmp, xtmp, 0); 6158 6159 subl(count, 8 << shift); 6160 jcc(Assembler::less, L_check_fill_8_bytes); 6161 align(16); 6162 6163 BIND(L_fill_32_bytes_loop); 6164 6165 if (UseUnalignedLoadStores) { 6166 movdqu(Address(to, 0), xtmp); 6167 movdqu(Address(to, 16), xtmp); 6168 } else { 6169 movq(Address(to, 0), xtmp); 6170 movq(Address(to, 8), xtmp); 6171 movq(Address(to, 16), xtmp); 6172 movq(Address(to, 24), xtmp); 6173 } 6174 6175 addptr(to, 32); 6176 subl(count, 8 << shift); 6177 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6178 6179 BIND(L_check_fill_8_bytes); 6180 } 6181 addl(count, 8 << shift); 6182 jccb(Assembler::zero, L_exit); 6183 jmpb(L_fill_8_bytes); 6184 6185 // 6186 // length is too short, just fill qwords 6187 // 6188 BIND(L_fill_8_bytes_loop); 6189 movq(Address(to, 0), xtmp); 6190 addptr(to, 8); 6191 BIND(L_fill_8_bytes); 6192 subl(count, 1 << (shift + 1)); 6193 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6194 } 6195 } 6196 // fill trailing 4 bytes 6197 BIND(L_fill_4_bytes); 6198 testl(count, 1<<shift); 6199 jccb(Assembler::zero, L_fill_2_bytes); 6200 movl(Address(to, 0), value); 6201 if (t == T_BYTE || t == T_SHORT) { 6202 Label L_fill_byte; 6203 addptr(to, 4); 6204 BIND(L_fill_2_bytes); 6205 // fill trailing 2 bytes 6206 testl(count, 1<<(shift-1)); 6207 jccb(Assembler::zero, L_fill_byte); 6208 movw(Address(to, 0), value); 6209 if (t == T_BYTE) { 6210 addptr(to, 2); 6211 BIND(L_fill_byte); 6212 // fill trailing byte 6213 testl(count, 1); 6214 jccb(Assembler::zero, L_exit); 6215 movb(Address(to, 0), value); 6216 } else { 6217 BIND(L_fill_byte); 6218 } 6219 } else { 6220 BIND(L_fill_2_bytes); 6221 } 6222 BIND(L_exit); 6223 } 6224 6225 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 6226 switch(type) { 6227 case T_BYTE: 6228 case T_BOOLEAN: 6229 evpbroadcastb(dst, src, vector_len); 6230 break; 6231 case T_SHORT: 6232 case T_CHAR: 6233 evpbroadcastw(dst, src, vector_len); 6234 break; 6235 case T_INT: 6236 case T_FLOAT: 6237 evpbroadcastd(dst, src, vector_len); 6238 break; 6239 case T_LONG: 6240 case T_DOUBLE: 6241 evpbroadcastq(dst, src, vector_len); 6242 break; 6243 default: 6244 fatal("Unhandled type : %s", type2name(type)); 6245 break; 6246 } 6247 } 6248 6249 // encode char[] to byte[] in ISO_8859_1 or ASCII 6250 //@IntrinsicCandidate 6251 //private static int implEncodeISOArray(byte[] sa, int sp, 6252 //byte[] da, int dp, int len) { 6253 // int i = 0; 6254 // for (; i < len; i++) { 6255 // char c = StringUTF16.getChar(sa, sp++); 6256 // if (c > '\u00FF') 6257 // break; 6258 // da[dp++] = (byte)c; 6259 // } 6260 // return i; 6261 //} 6262 // 6263 //@IntrinsicCandidate 6264 //private static int implEncodeAsciiArray(char[] sa, int sp, 6265 // byte[] da, int dp, int len) { 6266 // int i = 0; 6267 // for (; i < len; i++) { 6268 // char c = sa[sp++]; 6269 // if (c >= '\u0080') 6270 // break; 6271 // da[dp++] = (byte)c; 6272 // } 6273 // return i; 6274 //} 6275 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6276 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6277 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6278 Register tmp5, Register result, bool ascii) { 6279 6280 // rsi: src 6281 // rdi: dst 6282 // rdx: len 6283 // rcx: tmp5 6284 // rax: result 6285 ShortBranchVerifier sbv(this); 6286 assert_different_registers(src, dst, len, tmp5, result); 6287 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6288 6289 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6290 int short_mask = ascii ? 0xff80 : 0xff00; 6291 6292 // set result 6293 xorl(result, result); 6294 // check for zero length 6295 testl(len, len); 6296 jcc(Assembler::zero, L_done); 6297 6298 movl(result, len); 6299 6300 // Setup pointers 6301 lea(src, Address(src, len, Address::times_2)); // char[] 6302 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6303 negptr(len); 6304 6305 if (UseSSE42Intrinsics || UseAVX >= 2) { 6306 Label L_copy_8_chars, L_copy_8_chars_exit; 6307 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6308 6309 if (UseAVX >= 2) { 6310 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6311 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6312 movdl(tmp1Reg, tmp5); 6313 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6314 jmp(L_chars_32_check); 6315 6316 bind(L_copy_32_chars); 6317 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6318 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6319 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6320 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6321 jccb(Assembler::notZero, L_copy_32_chars_exit); 6322 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6323 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6324 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6325 6326 bind(L_chars_32_check); 6327 addptr(len, 32); 6328 jcc(Assembler::lessEqual, L_copy_32_chars); 6329 6330 bind(L_copy_32_chars_exit); 6331 subptr(len, 16); 6332 jccb(Assembler::greater, L_copy_16_chars_exit); 6333 6334 } else if (UseSSE42Intrinsics) { 6335 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6336 movdl(tmp1Reg, tmp5); 6337 pshufd(tmp1Reg, tmp1Reg, 0); 6338 jmpb(L_chars_16_check); 6339 } 6340 6341 bind(L_copy_16_chars); 6342 if (UseAVX >= 2) { 6343 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6344 vptest(tmp2Reg, tmp1Reg); 6345 jcc(Assembler::notZero, L_copy_16_chars_exit); 6346 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6347 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6348 } else { 6349 if (UseAVX > 0) { 6350 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6351 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6352 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6353 } else { 6354 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6355 por(tmp2Reg, tmp3Reg); 6356 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6357 por(tmp2Reg, tmp4Reg); 6358 } 6359 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6360 jccb(Assembler::notZero, L_copy_16_chars_exit); 6361 packuswb(tmp3Reg, tmp4Reg); 6362 } 6363 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 6364 6365 bind(L_chars_16_check); 6366 addptr(len, 16); 6367 jcc(Assembler::lessEqual, L_copy_16_chars); 6368 6369 bind(L_copy_16_chars_exit); 6370 if (UseAVX >= 2) { 6371 // clean upper bits of YMM registers 6372 vpxor(tmp2Reg, tmp2Reg); 6373 vpxor(tmp3Reg, tmp3Reg); 6374 vpxor(tmp4Reg, tmp4Reg); 6375 movdl(tmp1Reg, tmp5); 6376 pshufd(tmp1Reg, tmp1Reg, 0); 6377 } 6378 subptr(len, 8); 6379 jccb(Assembler::greater, L_copy_8_chars_exit); 6380 6381 bind(L_copy_8_chars); 6382 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 6383 ptest(tmp3Reg, tmp1Reg); 6384 jccb(Assembler::notZero, L_copy_8_chars_exit); 6385 packuswb(tmp3Reg, tmp1Reg); 6386 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 6387 addptr(len, 8); 6388 jccb(Assembler::lessEqual, L_copy_8_chars); 6389 6390 bind(L_copy_8_chars_exit); 6391 subptr(len, 8); 6392 jccb(Assembler::zero, L_done); 6393 } 6394 6395 bind(L_copy_1_char); 6396 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 6397 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 6398 jccb(Assembler::notZero, L_copy_1_char_exit); 6399 movb(Address(dst, len, Address::times_1, 0), tmp5); 6400 addptr(len, 1); 6401 jccb(Assembler::less, L_copy_1_char); 6402 6403 bind(L_copy_1_char_exit); 6404 addptr(result, len); // len is negative count of not processed elements 6405 6406 bind(L_done); 6407 } 6408 6409 #ifdef _LP64 6410 /** 6411 * Helper for multiply_to_len(). 6412 */ 6413 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 6414 addq(dest_lo, src1); 6415 adcq(dest_hi, 0); 6416 addq(dest_lo, src2); 6417 adcq(dest_hi, 0); 6418 } 6419 6420 /** 6421 * Multiply 64 bit by 64 bit first loop. 6422 */ 6423 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 6424 Register y, Register y_idx, Register z, 6425 Register carry, Register product, 6426 Register idx, Register kdx) { 6427 // 6428 // jlong carry, x[], y[], z[]; 6429 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6430 // huge_128 product = y[idx] * x[xstart] + carry; 6431 // z[kdx] = (jlong)product; 6432 // carry = (jlong)(product >>> 64); 6433 // } 6434 // z[xstart] = carry; 6435 // 6436 6437 Label L_first_loop, L_first_loop_exit; 6438 Label L_one_x, L_one_y, L_multiply; 6439 6440 decrementl(xstart); 6441 jcc(Assembler::negative, L_one_x); 6442 6443 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6444 rorq(x_xstart, 32); // convert big-endian to little-endian 6445 6446 bind(L_first_loop); 6447 decrementl(idx); 6448 jcc(Assembler::negative, L_first_loop_exit); 6449 decrementl(idx); 6450 jcc(Assembler::negative, L_one_y); 6451 movq(y_idx, Address(y, idx, Address::times_4, 0)); 6452 rorq(y_idx, 32); // convert big-endian to little-endian 6453 bind(L_multiply); 6454 movq(product, x_xstart); 6455 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 6456 addq(product, carry); 6457 adcq(rdx, 0); 6458 subl(kdx, 2); 6459 movl(Address(z, kdx, Address::times_4, 4), product); 6460 shrq(product, 32); 6461 movl(Address(z, kdx, Address::times_4, 0), product); 6462 movq(carry, rdx); 6463 jmp(L_first_loop); 6464 6465 bind(L_one_y); 6466 movl(y_idx, Address(y, 0)); 6467 jmp(L_multiply); 6468 6469 bind(L_one_x); 6470 movl(x_xstart, Address(x, 0)); 6471 jmp(L_first_loop); 6472 6473 bind(L_first_loop_exit); 6474 } 6475 6476 /** 6477 * Multiply 64 bit by 64 bit and add 128 bit. 6478 */ 6479 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 6480 Register yz_idx, Register idx, 6481 Register carry, Register product, int offset) { 6482 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 6483 // z[kdx] = (jlong)product; 6484 6485 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 6486 rorq(yz_idx, 32); // convert big-endian to little-endian 6487 movq(product, x_xstart); 6488 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6489 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 6490 rorq(yz_idx, 32); // convert big-endian to little-endian 6491 6492 add2_with_carry(rdx, product, carry, yz_idx); 6493 6494 movl(Address(z, idx, Address::times_4, offset+4), product); 6495 shrq(product, 32); 6496 movl(Address(z, idx, Address::times_4, offset), product); 6497 6498 } 6499 6500 /** 6501 * Multiply 128 bit by 128 bit. Unrolled inner loop. 6502 */ 6503 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 6504 Register yz_idx, Register idx, Register jdx, 6505 Register carry, Register product, 6506 Register carry2) { 6507 // jlong carry, x[], y[], z[]; 6508 // int kdx = ystart+1; 6509 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6510 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 6511 // z[kdx+idx+1] = (jlong)product; 6512 // jlong carry2 = (jlong)(product >>> 64); 6513 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 6514 // z[kdx+idx] = (jlong)product; 6515 // carry = (jlong)(product >>> 64); 6516 // } 6517 // idx += 2; 6518 // if (idx > 0) { 6519 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 6520 // z[kdx+idx] = (jlong)product; 6521 // carry = (jlong)(product >>> 64); 6522 // } 6523 // 6524 6525 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6526 6527 movl(jdx, idx); 6528 andl(jdx, 0xFFFFFFFC); 6529 shrl(jdx, 2); 6530 6531 bind(L_third_loop); 6532 subl(jdx, 1); 6533 jcc(Assembler::negative, L_third_loop_exit); 6534 subl(idx, 4); 6535 6536 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 6537 movq(carry2, rdx); 6538 6539 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 6540 movq(carry, rdx); 6541 jmp(L_third_loop); 6542 6543 bind (L_third_loop_exit); 6544 6545 andl (idx, 0x3); 6546 jcc(Assembler::zero, L_post_third_loop_done); 6547 6548 Label L_check_1; 6549 subl(idx, 2); 6550 jcc(Assembler::negative, L_check_1); 6551 6552 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 6553 movq(carry, rdx); 6554 6555 bind (L_check_1); 6556 addl (idx, 0x2); 6557 andl (idx, 0x1); 6558 subl(idx, 1); 6559 jcc(Assembler::negative, L_post_third_loop_done); 6560 6561 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 6562 movq(product, x_xstart); 6563 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6564 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 6565 6566 add2_with_carry(rdx, product, yz_idx, carry); 6567 6568 movl(Address(z, idx, Address::times_4, 0), product); 6569 shrq(product, 32); 6570 6571 shlq(rdx, 32); 6572 orq(product, rdx); 6573 movq(carry, product); 6574 6575 bind(L_post_third_loop_done); 6576 } 6577 6578 /** 6579 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 6580 * 6581 */ 6582 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 6583 Register carry, Register carry2, 6584 Register idx, Register jdx, 6585 Register yz_idx1, Register yz_idx2, 6586 Register tmp, Register tmp3, Register tmp4) { 6587 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 6588 6589 // jlong carry, x[], y[], z[]; 6590 // int kdx = ystart+1; 6591 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6592 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 6593 // jlong carry2 = (jlong)(tmp3 >>> 64); 6594 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 6595 // carry = (jlong)(tmp4 >>> 64); 6596 // z[kdx+idx+1] = (jlong)tmp3; 6597 // z[kdx+idx] = (jlong)tmp4; 6598 // } 6599 // idx += 2; 6600 // if (idx > 0) { 6601 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 6602 // z[kdx+idx] = (jlong)yz_idx1; 6603 // carry = (jlong)(yz_idx1 >>> 64); 6604 // } 6605 // 6606 6607 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6608 6609 movl(jdx, idx); 6610 andl(jdx, 0xFFFFFFFC); 6611 shrl(jdx, 2); 6612 6613 bind(L_third_loop); 6614 subl(jdx, 1); 6615 jcc(Assembler::negative, L_third_loop_exit); 6616 subl(idx, 4); 6617 6618 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 6619 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 6620 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 6621 rorxq(yz_idx2, yz_idx2, 32); 6622 6623 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6624 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 6625 6626 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 6627 rorxq(yz_idx1, yz_idx1, 32); 6628 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6629 rorxq(yz_idx2, yz_idx2, 32); 6630 6631 if (VM_Version::supports_adx()) { 6632 adcxq(tmp3, carry); 6633 adoxq(tmp3, yz_idx1); 6634 6635 adcxq(tmp4, tmp); 6636 adoxq(tmp4, yz_idx2); 6637 6638 movl(carry, 0); // does not affect flags 6639 adcxq(carry2, carry); 6640 adoxq(carry2, carry); 6641 } else { 6642 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 6643 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 6644 } 6645 movq(carry, carry2); 6646 6647 movl(Address(z, idx, Address::times_4, 12), tmp3); 6648 shrq(tmp3, 32); 6649 movl(Address(z, idx, Address::times_4, 8), tmp3); 6650 6651 movl(Address(z, idx, Address::times_4, 4), tmp4); 6652 shrq(tmp4, 32); 6653 movl(Address(z, idx, Address::times_4, 0), tmp4); 6654 6655 jmp(L_third_loop); 6656 6657 bind (L_third_loop_exit); 6658 6659 andl (idx, 0x3); 6660 jcc(Assembler::zero, L_post_third_loop_done); 6661 6662 Label L_check_1; 6663 subl(idx, 2); 6664 jcc(Assembler::negative, L_check_1); 6665 6666 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 6667 rorxq(yz_idx1, yz_idx1, 32); 6668 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6669 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6670 rorxq(yz_idx2, yz_idx2, 32); 6671 6672 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 6673 6674 movl(Address(z, idx, Address::times_4, 4), tmp3); 6675 shrq(tmp3, 32); 6676 movl(Address(z, idx, Address::times_4, 0), tmp3); 6677 movq(carry, tmp4); 6678 6679 bind (L_check_1); 6680 addl (idx, 0x2); 6681 andl (idx, 0x1); 6682 subl(idx, 1); 6683 jcc(Assembler::negative, L_post_third_loop_done); 6684 movl(tmp4, Address(y, idx, Address::times_4, 0)); 6685 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 6686 movl(tmp4, Address(z, idx, Address::times_4, 0)); 6687 6688 add2_with_carry(carry2, tmp3, tmp4, carry); 6689 6690 movl(Address(z, idx, Address::times_4, 0), tmp3); 6691 shrq(tmp3, 32); 6692 6693 shlq(carry2, 32); 6694 orq(tmp3, carry2); 6695 movq(carry, tmp3); 6696 6697 bind(L_post_third_loop_done); 6698 } 6699 6700 /** 6701 * Code for BigInteger::multiplyToLen() intrinsic. 6702 * 6703 * rdi: x 6704 * rax: xlen 6705 * rsi: y 6706 * rcx: ylen 6707 * r8: z 6708 * r11: zlen 6709 * r12: tmp1 6710 * r13: tmp2 6711 * r14: tmp3 6712 * r15: tmp4 6713 * rbx: tmp5 6714 * 6715 */ 6716 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, 6717 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 6718 ShortBranchVerifier sbv(this); 6719 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 6720 6721 push(tmp1); 6722 push(tmp2); 6723 push(tmp3); 6724 push(tmp4); 6725 push(tmp5); 6726 6727 push(xlen); 6728 push(zlen); 6729 6730 const Register idx = tmp1; 6731 const Register kdx = tmp2; 6732 const Register xstart = tmp3; 6733 6734 const Register y_idx = tmp4; 6735 const Register carry = tmp5; 6736 const Register product = xlen; 6737 const Register x_xstart = zlen; // reuse register 6738 6739 // First Loop. 6740 // 6741 // final static long LONG_MASK = 0xffffffffL; 6742 // int xstart = xlen - 1; 6743 // int ystart = ylen - 1; 6744 // long carry = 0; 6745 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6746 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 6747 // z[kdx] = (int)product; 6748 // carry = product >>> 32; 6749 // } 6750 // z[xstart] = (int)carry; 6751 // 6752 6753 movl(idx, ylen); // idx = ylen; 6754 movl(kdx, zlen); // kdx = xlen+ylen; 6755 xorq(carry, carry); // carry = 0; 6756 6757 Label L_done; 6758 6759 movl(xstart, xlen); 6760 decrementl(xstart); 6761 jcc(Assembler::negative, L_done); 6762 6763 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 6764 6765 Label L_second_loop; 6766 testl(kdx, kdx); 6767 jcc(Assembler::zero, L_second_loop); 6768 6769 Label L_carry; 6770 subl(kdx, 1); 6771 jcc(Assembler::zero, L_carry); 6772 6773 movl(Address(z, kdx, Address::times_4, 0), carry); 6774 shrq(carry, 32); 6775 subl(kdx, 1); 6776 6777 bind(L_carry); 6778 movl(Address(z, kdx, Address::times_4, 0), carry); 6779 6780 // Second and third (nested) loops. 6781 // 6782 // for (int i = xstart-1; i >= 0; i--) { // Second loop 6783 // carry = 0; 6784 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 6785 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 6786 // (z[k] & LONG_MASK) + carry; 6787 // z[k] = (int)product; 6788 // carry = product >>> 32; 6789 // } 6790 // z[i] = (int)carry; 6791 // } 6792 // 6793 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 6794 6795 const Register jdx = tmp1; 6796 6797 bind(L_second_loop); 6798 xorl(carry, carry); // carry = 0; 6799 movl(jdx, ylen); // j = ystart+1 6800 6801 subl(xstart, 1); // i = xstart-1; 6802 jcc(Assembler::negative, L_done); 6803 6804 push (z); 6805 6806 Label L_last_x; 6807 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 6808 subl(xstart, 1); // i = xstart-1; 6809 jcc(Assembler::negative, L_last_x); 6810 6811 if (UseBMI2Instructions) { 6812 movq(rdx, Address(x, xstart, Address::times_4, 0)); 6813 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 6814 } else { 6815 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6816 rorq(x_xstart, 32); // convert big-endian to little-endian 6817 } 6818 6819 Label L_third_loop_prologue; 6820 bind(L_third_loop_prologue); 6821 6822 push (x); 6823 push (xstart); 6824 push (ylen); 6825 6826 6827 if (UseBMI2Instructions) { 6828 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 6829 } else { // !UseBMI2Instructions 6830 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 6831 } 6832 6833 pop(ylen); 6834 pop(xlen); 6835 pop(x); 6836 pop(z); 6837 6838 movl(tmp3, xlen); 6839 addl(tmp3, 1); 6840 movl(Address(z, tmp3, Address::times_4, 0), carry); 6841 subl(tmp3, 1); 6842 jccb(Assembler::negative, L_done); 6843 6844 shrq(carry, 32); 6845 movl(Address(z, tmp3, Address::times_4, 0), carry); 6846 jmp(L_second_loop); 6847 6848 // Next infrequent code is moved outside loops. 6849 bind(L_last_x); 6850 if (UseBMI2Instructions) { 6851 movl(rdx, Address(x, 0)); 6852 } else { 6853 movl(x_xstart, Address(x, 0)); 6854 } 6855 jmp(L_third_loop_prologue); 6856 6857 bind(L_done); 6858 6859 pop(zlen); 6860 pop(xlen); 6861 6862 pop(tmp5); 6863 pop(tmp4); 6864 pop(tmp3); 6865 pop(tmp2); 6866 pop(tmp1); 6867 } 6868 6869 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 6870 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 6871 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 6872 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 6873 Label VECTOR8_TAIL, VECTOR4_TAIL; 6874 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 6875 Label SAME_TILL_END, DONE; 6876 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 6877 6878 //scale is in rcx in both Win64 and Unix 6879 ShortBranchVerifier sbv(this); 6880 6881 shlq(length); 6882 xorq(result, result); 6883 6884 if ((AVX3Threshold == 0) && (UseAVX > 2) && 6885 VM_Version::supports_avx512vlbw()) { 6886 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 6887 6888 cmpq(length, 64); 6889 jcc(Assembler::less, VECTOR32_TAIL); 6890 6891 movq(tmp1, length); 6892 andq(tmp1, 0x3F); // tail count 6893 andq(length, ~(0x3F)); //vector count 6894 6895 bind(VECTOR64_LOOP); 6896 // AVX512 code to compare 64 byte vectors. 6897 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 6898 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 6899 kortestql(k7, k7); 6900 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 6901 addq(result, 64); 6902 subq(length, 64); 6903 jccb(Assembler::notZero, VECTOR64_LOOP); 6904 6905 //bind(VECTOR64_TAIL); 6906 testq(tmp1, tmp1); 6907 jcc(Assembler::zero, SAME_TILL_END); 6908 6909 //bind(VECTOR64_TAIL); 6910 // AVX512 code to compare up to 63 byte vectors. 6911 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 6912 shlxq(tmp2, tmp2, tmp1); 6913 notq(tmp2); 6914 kmovql(k3, tmp2); 6915 6916 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 6917 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 6918 6919 ktestql(k7, k3); 6920 jcc(Assembler::below, SAME_TILL_END); // not mismatch 6921 6922 bind(VECTOR64_NOT_EQUAL); 6923 kmovql(tmp1, k7); 6924 notq(tmp1); 6925 tzcntq(tmp1, tmp1); 6926 addq(result, tmp1); 6927 shrq(result); 6928 jmp(DONE); 6929 bind(VECTOR32_TAIL); 6930 } 6931 6932 cmpq(length, 8); 6933 jcc(Assembler::equal, VECTOR8_LOOP); 6934 jcc(Assembler::less, VECTOR4_TAIL); 6935 6936 if (UseAVX >= 2) { 6937 Label VECTOR16_TAIL, VECTOR32_LOOP; 6938 6939 cmpq(length, 16); 6940 jcc(Assembler::equal, VECTOR16_LOOP); 6941 jcc(Assembler::less, VECTOR8_LOOP); 6942 6943 cmpq(length, 32); 6944 jccb(Assembler::less, VECTOR16_TAIL); 6945 6946 subq(length, 32); 6947 bind(VECTOR32_LOOP); 6948 vmovdqu(rymm0, Address(obja, result)); 6949 vmovdqu(rymm1, Address(objb, result)); 6950 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 6951 vptest(rymm2, rymm2); 6952 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 6953 addq(result, 32); 6954 subq(length, 32); 6955 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 6956 addq(length, 32); 6957 jcc(Assembler::equal, SAME_TILL_END); 6958 //falling through if less than 32 bytes left //close the branch here. 6959 6960 bind(VECTOR16_TAIL); 6961 cmpq(length, 16); 6962 jccb(Assembler::less, VECTOR8_TAIL); 6963 bind(VECTOR16_LOOP); 6964 movdqu(rymm0, Address(obja, result)); 6965 movdqu(rymm1, Address(objb, result)); 6966 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 6967 ptest(rymm2, rymm2); 6968 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6969 addq(result, 16); 6970 subq(length, 16); 6971 jcc(Assembler::equal, SAME_TILL_END); 6972 //falling through if less than 16 bytes left 6973 } else {//regular intrinsics 6974 6975 cmpq(length, 16); 6976 jccb(Assembler::less, VECTOR8_TAIL); 6977 6978 subq(length, 16); 6979 bind(VECTOR16_LOOP); 6980 movdqu(rymm0, Address(obja, result)); 6981 movdqu(rymm1, Address(objb, result)); 6982 pxor(rymm0, rymm1); 6983 ptest(rymm0, rymm0); 6984 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6985 addq(result, 16); 6986 subq(length, 16); 6987 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 6988 addq(length, 16); 6989 jcc(Assembler::equal, SAME_TILL_END); 6990 //falling through if less than 16 bytes left 6991 } 6992 6993 bind(VECTOR8_TAIL); 6994 cmpq(length, 8); 6995 jccb(Assembler::less, VECTOR4_TAIL); 6996 bind(VECTOR8_LOOP); 6997 movq(tmp1, Address(obja, result)); 6998 movq(tmp2, Address(objb, result)); 6999 xorq(tmp1, tmp2); 7000 testq(tmp1, tmp1); 7001 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 7002 addq(result, 8); 7003 subq(length, 8); 7004 jcc(Assembler::equal, SAME_TILL_END); 7005 //falling through if less than 8 bytes left 7006 7007 bind(VECTOR4_TAIL); 7008 cmpq(length, 4); 7009 jccb(Assembler::less, BYTES_TAIL); 7010 bind(VECTOR4_LOOP); 7011 movl(tmp1, Address(obja, result)); 7012 xorl(tmp1, Address(objb, result)); 7013 testl(tmp1, tmp1); 7014 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 7015 addq(result, 4); 7016 subq(length, 4); 7017 jcc(Assembler::equal, SAME_TILL_END); 7018 //falling through if less than 4 bytes left 7019 7020 bind(BYTES_TAIL); 7021 bind(BYTES_LOOP); 7022 load_unsigned_byte(tmp1, Address(obja, result)); 7023 load_unsigned_byte(tmp2, Address(objb, result)); 7024 xorl(tmp1, tmp2); 7025 testl(tmp1, tmp1); 7026 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7027 decq(length); 7028 jcc(Assembler::zero, SAME_TILL_END); 7029 incq(result); 7030 load_unsigned_byte(tmp1, Address(obja, result)); 7031 load_unsigned_byte(tmp2, Address(objb, result)); 7032 xorl(tmp1, tmp2); 7033 testl(tmp1, tmp1); 7034 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7035 decq(length); 7036 jcc(Assembler::zero, SAME_TILL_END); 7037 incq(result); 7038 load_unsigned_byte(tmp1, Address(obja, result)); 7039 load_unsigned_byte(tmp2, Address(objb, result)); 7040 xorl(tmp1, tmp2); 7041 testl(tmp1, tmp1); 7042 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7043 jmp(SAME_TILL_END); 7044 7045 if (UseAVX >= 2) { 7046 bind(VECTOR32_NOT_EQUAL); 7047 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 7048 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 7049 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 7050 vpmovmskb(tmp1, rymm0); 7051 bsfq(tmp1, tmp1); 7052 addq(result, tmp1); 7053 shrq(result); 7054 jmp(DONE); 7055 } 7056 7057 bind(VECTOR16_NOT_EQUAL); 7058 if (UseAVX >= 2) { 7059 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 7060 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 7061 pxor(rymm0, rymm2); 7062 } else { 7063 pcmpeqb(rymm2, rymm2); 7064 pxor(rymm0, rymm1); 7065 pcmpeqb(rymm0, rymm1); 7066 pxor(rymm0, rymm2); 7067 } 7068 pmovmskb(tmp1, rymm0); 7069 bsfq(tmp1, tmp1); 7070 addq(result, tmp1); 7071 shrq(result); 7072 jmpb(DONE); 7073 7074 bind(VECTOR8_NOT_EQUAL); 7075 bind(VECTOR4_NOT_EQUAL); 7076 bsfq(tmp1, tmp1); 7077 shrq(tmp1, 3); 7078 addq(result, tmp1); 7079 bind(BYTES_NOT_EQUAL); 7080 shrq(result); 7081 jmpb(DONE); 7082 7083 bind(SAME_TILL_END); 7084 mov64(result, -1); 7085 7086 bind(DONE); 7087 } 7088 7089 //Helper functions for square_to_len() 7090 7091 /** 7092 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 7093 * Preserves x and z and modifies rest of the registers. 7094 */ 7095 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7096 // Perform square and right shift by 1 7097 // Handle odd xlen case first, then for even xlen do the following 7098 // jlong carry = 0; 7099 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 7100 // huge_128 product = x[j:j+1] * x[j:j+1]; 7101 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 7102 // z[i+2:i+3] = (jlong)(product >>> 1); 7103 // carry = (jlong)product; 7104 // } 7105 7106 xorq(tmp5, tmp5); // carry 7107 xorq(rdxReg, rdxReg); 7108 xorl(tmp1, tmp1); // index for x 7109 xorl(tmp4, tmp4); // index for z 7110 7111 Label L_first_loop, L_first_loop_exit; 7112 7113 testl(xlen, 1); 7114 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 7115 7116 // Square and right shift by 1 the odd element using 32 bit multiply 7117 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 7118 imulq(raxReg, raxReg); 7119 shrq(raxReg, 1); 7120 adcq(tmp5, 0); 7121 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 7122 incrementl(tmp1); 7123 addl(tmp4, 2); 7124 7125 // Square and right shift by 1 the rest using 64 bit multiply 7126 bind(L_first_loop); 7127 cmpptr(tmp1, xlen); 7128 jccb(Assembler::equal, L_first_loop_exit); 7129 7130 // Square 7131 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 7132 rorq(raxReg, 32); // convert big-endian to little-endian 7133 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 7134 7135 // Right shift by 1 and save carry 7136 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 7137 rcrq(rdxReg, 1); 7138 rcrq(raxReg, 1); 7139 adcq(tmp5, 0); 7140 7141 // Store result in z 7142 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 7143 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 7144 7145 // Update indices for x and z 7146 addl(tmp1, 2); 7147 addl(tmp4, 4); 7148 jmp(L_first_loop); 7149 7150 bind(L_first_loop_exit); 7151 } 7152 7153 7154 /** 7155 * Perform the following multiply add operation using BMI2 instructions 7156 * carry:sum = sum + op1*op2 + carry 7157 * op2 should be in rdx 7158 * op2 is preserved, all other registers are modified 7159 */ 7160 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 7161 // assert op2 is rdx 7162 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 7163 addq(sum, carry); 7164 adcq(tmp2, 0); 7165 addq(sum, op1); 7166 adcq(tmp2, 0); 7167 movq(carry, tmp2); 7168 } 7169 7170 /** 7171 * Perform the following multiply add operation: 7172 * carry:sum = sum + op1*op2 + carry 7173 * Preserves op1, op2 and modifies rest of registers 7174 */ 7175 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 7176 // rdx:rax = op1 * op2 7177 movq(raxReg, op2); 7178 mulq(op1); 7179 7180 // rdx:rax = sum + carry + rdx:rax 7181 addq(sum, carry); 7182 adcq(rdxReg, 0); 7183 addq(sum, raxReg); 7184 adcq(rdxReg, 0); 7185 7186 // carry:sum = rdx:sum 7187 movq(carry, rdxReg); 7188 } 7189 7190 /** 7191 * Add 64 bit long carry into z[] with carry propagation. 7192 * Preserves z and carry register values and modifies rest of registers. 7193 * 7194 */ 7195 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 7196 Label L_fourth_loop, L_fourth_loop_exit; 7197 7198 movl(tmp1, 1); 7199 subl(zlen, 2); 7200 addq(Address(z, zlen, Address::times_4, 0), carry); 7201 7202 bind(L_fourth_loop); 7203 jccb(Assembler::carryClear, L_fourth_loop_exit); 7204 subl(zlen, 2); 7205 jccb(Assembler::negative, L_fourth_loop_exit); 7206 addq(Address(z, zlen, Address::times_4, 0), tmp1); 7207 jmp(L_fourth_loop); 7208 bind(L_fourth_loop_exit); 7209 } 7210 7211 /** 7212 * Shift z[] left by 1 bit. 7213 * Preserves x, len, z and zlen registers and modifies rest of the registers. 7214 * 7215 */ 7216 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 7217 7218 Label L_fifth_loop, L_fifth_loop_exit; 7219 7220 // Fifth loop 7221 // Perform primitiveLeftShift(z, zlen, 1) 7222 7223 const Register prev_carry = tmp1; 7224 const Register new_carry = tmp4; 7225 const Register value = tmp2; 7226 const Register zidx = tmp3; 7227 7228 // int zidx, carry; 7229 // long value; 7230 // carry = 0; 7231 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 7232 // (carry:value) = (z[i] << 1) | carry ; 7233 // z[i] = value; 7234 // } 7235 7236 movl(zidx, zlen); 7237 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 7238 7239 bind(L_fifth_loop); 7240 decl(zidx); // Use decl to preserve carry flag 7241 decl(zidx); 7242 jccb(Assembler::negative, L_fifth_loop_exit); 7243 7244 if (UseBMI2Instructions) { 7245 movq(value, Address(z, zidx, Address::times_4, 0)); 7246 rclq(value, 1); 7247 rorxq(value, value, 32); 7248 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7249 } 7250 else { 7251 // clear new_carry 7252 xorl(new_carry, new_carry); 7253 7254 // Shift z[i] by 1, or in previous carry and save new carry 7255 movq(value, Address(z, zidx, Address::times_4, 0)); 7256 shlq(value, 1); 7257 adcl(new_carry, 0); 7258 7259 orq(value, prev_carry); 7260 rorq(value, 0x20); 7261 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7262 7263 // Set previous carry = new carry 7264 movl(prev_carry, new_carry); 7265 } 7266 jmp(L_fifth_loop); 7267 7268 bind(L_fifth_loop_exit); 7269 } 7270 7271 7272 /** 7273 * Code for BigInteger::squareToLen() intrinsic 7274 * 7275 * rdi: x 7276 * rsi: len 7277 * r8: z 7278 * rcx: zlen 7279 * r12: tmp1 7280 * r13: tmp2 7281 * r14: tmp3 7282 * r15: tmp4 7283 * rbx: tmp5 7284 * 7285 */ 7286 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7287 7288 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7289 push(tmp1); 7290 push(tmp2); 7291 push(tmp3); 7292 push(tmp4); 7293 push(tmp5); 7294 7295 // First loop 7296 // Store the squares, right shifted one bit (i.e., divided by 2). 7297 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7298 7299 // Add in off-diagonal sums. 7300 // 7301 // Second, third (nested) and fourth loops. 7302 // zlen +=2; 7303 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7304 // carry = 0; 7305 // long op2 = x[xidx:xidx+1]; 7306 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7307 // k -= 2; 7308 // long op1 = x[j:j+1]; 7309 // long sum = z[k:k+1]; 7310 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7311 // z[k:k+1] = sum; 7312 // } 7313 // add_one_64(z, k, carry, tmp_regs); 7314 // } 7315 7316 const Register carry = tmp5; 7317 const Register sum = tmp3; 7318 const Register op1 = tmp4; 7319 Register op2 = tmp2; 7320 7321 push(zlen); 7322 push(len); 7323 addl(zlen,2); 7324 bind(L_second_loop); 7325 xorq(carry, carry); 7326 subl(zlen, 4); 7327 subl(len, 2); 7328 push(zlen); 7329 push(len); 7330 cmpl(len, 0); 7331 jccb(Assembler::lessEqual, L_second_loop_exit); 7332 7333 // Multiply an array by one 64 bit long. 7334 if (UseBMI2Instructions) { 7335 op2 = rdxReg; 7336 movq(op2, Address(x, len, Address::times_4, 0)); 7337 rorxq(op2, op2, 32); 7338 } 7339 else { 7340 movq(op2, Address(x, len, Address::times_4, 0)); 7341 rorq(op2, 32); 7342 } 7343 7344 bind(L_third_loop); 7345 decrementl(len); 7346 jccb(Assembler::negative, L_third_loop_exit); 7347 decrementl(len); 7348 jccb(Assembler::negative, L_last_x); 7349 7350 movq(op1, Address(x, len, Address::times_4, 0)); 7351 rorq(op1, 32); 7352 7353 bind(L_multiply); 7354 subl(zlen, 2); 7355 movq(sum, Address(z, zlen, Address::times_4, 0)); 7356 7357 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7358 if (UseBMI2Instructions) { 7359 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 7360 } 7361 else { 7362 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7363 } 7364 7365 movq(Address(z, zlen, Address::times_4, 0), sum); 7366 7367 jmp(L_third_loop); 7368 bind(L_third_loop_exit); 7369 7370 // Fourth loop 7371 // Add 64 bit long carry into z with carry propagation. 7372 // Uses offsetted zlen. 7373 add_one_64(z, zlen, carry, tmp1); 7374 7375 pop(len); 7376 pop(zlen); 7377 jmp(L_second_loop); 7378 7379 // Next infrequent code is moved outside loops. 7380 bind(L_last_x); 7381 movl(op1, Address(x, 0)); 7382 jmp(L_multiply); 7383 7384 bind(L_second_loop_exit); 7385 pop(len); 7386 pop(zlen); 7387 pop(len); 7388 pop(zlen); 7389 7390 // Fifth loop 7391 // Shift z left 1 bit. 7392 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 7393 7394 // z[zlen-1] |= x[len-1] & 1; 7395 movl(tmp3, Address(x, len, Address::times_4, -4)); 7396 andl(tmp3, 1); 7397 orl(Address(z, zlen, Address::times_4, -4), tmp3); 7398 7399 pop(tmp5); 7400 pop(tmp4); 7401 pop(tmp3); 7402 pop(tmp2); 7403 pop(tmp1); 7404 } 7405 7406 /** 7407 * Helper function for mul_add() 7408 * Multiply the in[] by int k and add to out[] starting at offset offs using 7409 * 128 bit by 32 bit multiply and return the carry in tmp5. 7410 * Only quad int aligned length of in[] is operated on in this function. 7411 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 7412 * This function preserves out, in and k registers. 7413 * len and offset point to the appropriate index in "in" & "out" correspondingly 7414 * tmp5 has the carry. 7415 * other registers are temporary and are modified. 7416 * 7417 */ 7418 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 7419 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 7420 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7421 7422 Label L_first_loop, L_first_loop_exit; 7423 7424 movl(tmp1, len); 7425 shrl(tmp1, 2); 7426 7427 bind(L_first_loop); 7428 subl(tmp1, 1); 7429 jccb(Assembler::negative, L_first_loop_exit); 7430 7431 subl(len, 4); 7432 subl(offset, 4); 7433 7434 Register op2 = tmp2; 7435 const Register sum = tmp3; 7436 const Register op1 = tmp4; 7437 const Register carry = tmp5; 7438 7439 if (UseBMI2Instructions) { 7440 op2 = rdxReg; 7441 } 7442 7443 movq(op1, Address(in, len, Address::times_4, 8)); 7444 rorq(op1, 32); 7445 movq(sum, Address(out, offset, Address::times_4, 8)); 7446 rorq(sum, 32); 7447 if (UseBMI2Instructions) { 7448 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7449 } 7450 else { 7451 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7452 } 7453 // Store back in big endian from little endian 7454 rorq(sum, 0x20); 7455 movq(Address(out, offset, Address::times_4, 8), sum); 7456 7457 movq(op1, Address(in, len, Address::times_4, 0)); 7458 rorq(op1, 32); 7459 movq(sum, Address(out, offset, Address::times_4, 0)); 7460 rorq(sum, 32); 7461 if (UseBMI2Instructions) { 7462 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7463 } 7464 else { 7465 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7466 } 7467 // Store back in big endian from little endian 7468 rorq(sum, 0x20); 7469 movq(Address(out, offset, Address::times_4, 0), sum); 7470 7471 jmp(L_first_loop); 7472 bind(L_first_loop_exit); 7473 } 7474 7475 /** 7476 * Code for BigInteger::mulAdd() intrinsic 7477 * 7478 * rdi: out 7479 * rsi: in 7480 * r11: offs (out.length - offset) 7481 * rcx: len 7482 * r8: k 7483 * r12: tmp1 7484 * r13: tmp2 7485 * r14: tmp3 7486 * r15: tmp4 7487 * rbx: tmp5 7488 * Multiply the in[] by word k and add to out[], return the carry in rax 7489 */ 7490 void MacroAssembler::mul_add(Register out, Register in, Register offs, 7491 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 7492 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7493 7494 Label L_carry, L_last_in, L_done; 7495 7496 // carry = 0; 7497 // for (int j=len-1; j >= 0; j--) { 7498 // long product = (in[j] & LONG_MASK) * kLong + 7499 // (out[offs] & LONG_MASK) + carry; 7500 // out[offs--] = (int)product; 7501 // carry = product >>> 32; 7502 // } 7503 // 7504 push(tmp1); 7505 push(tmp2); 7506 push(tmp3); 7507 push(tmp4); 7508 push(tmp5); 7509 7510 Register op2 = tmp2; 7511 const Register sum = tmp3; 7512 const Register op1 = tmp4; 7513 const Register carry = tmp5; 7514 7515 if (UseBMI2Instructions) { 7516 op2 = rdxReg; 7517 movl(op2, k); 7518 } 7519 else { 7520 movl(op2, k); 7521 } 7522 7523 xorq(carry, carry); 7524 7525 //First loop 7526 7527 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 7528 //The carry is in tmp5 7529 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 7530 7531 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 7532 decrementl(len); 7533 jccb(Assembler::negative, L_carry); 7534 decrementl(len); 7535 jccb(Assembler::negative, L_last_in); 7536 7537 movq(op1, Address(in, len, Address::times_4, 0)); 7538 rorq(op1, 32); 7539 7540 subl(offs, 2); 7541 movq(sum, Address(out, offs, Address::times_4, 0)); 7542 rorq(sum, 32); 7543 7544 if (UseBMI2Instructions) { 7545 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7546 } 7547 else { 7548 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7549 } 7550 7551 // Store back in big endian from little endian 7552 rorq(sum, 0x20); 7553 movq(Address(out, offs, Address::times_4, 0), sum); 7554 7555 testl(len, len); 7556 jccb(Assembler::zero, L_carry); 7557 7558 //Multiply the last in[] entry, if any 7559 bind(L_last_in); 7560 movl(op1, Address(in, 0)); 7561 movl(sum, Address(out, offs, Address::times_4, -4)); 7562 7563 movl(raxReg, k); 7564 mull(op1); //tmp4 * eax -> edx:eax 7565 addl(sum, carry); 7566 adcl(rdxReg, 0); 7567 addl(sum, raxReg); 7568 adcl(rdxReg, 0); 7569 movl(carry, rdxReg); 7570 7571 movl(Address(out, offs, Address::times_4, -4), sum); 7572 7573 bind(L_carry); 7574 //return tmp5/carry as carry in rax 7575 movl(rax, carry); 7576 7577 bind(L_done); 7578 pop(tmp5); 7579 pop(tmp4); 7580 pop(tmp3); 7581 pop(tmp2); 7582 pop(tmp1); 7583 } 7584 #endif 7585 7586 /** 7587 * Emits code to update CRC-32 with a byte value according to constants in table 7588 * 7589 * @param [in,out]crc Register containing the crc. 7590 * @param [in]val Register containing the byte to fold into the CRC. 7591 * @param [in]table Register containing the table of crc constants. 7592 * 7593 * uint32_t crc; 7594 * val = crc_table[(val ^ crc) & 0xFF]; 7595 * crc = val ^ (crc >> 8); 7596 * 7597 */ 7598 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 7599 xorl(val, crc); 7600 andl(val, 0xFF); 7601 shrl(crc, 8); // unsigned shift 7602 xorl(crc, Address(table, val, Address::times_4, 0)); 7603 } 7604 7605 /** 7606 * Fold 128-bit data chunk 7607 */ 7608 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 7609 if (UseAVX > 0) { 7610 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 7611 vpclmulldq(xcrc, xK, xcrc); // [63:0] 7612 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 7613 pxor(xcrc, xtmp); 7614 } else { 7615 movdqa(xtmp, xcrc); 7616 pclmulhdq(xtmp, xK); // [123:64] 7617 pclmulldq(xcrc, xK); // [63:0] 7618 pxor(xcrc, xtmp); 7619 movdqu(xtmp, Address(buf, offset)); 7620 pxor(xcrc, xtmp); 7621 } 7622 } 7623 7624 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 7625 if (UseAVX > 0) { 7626 vpclmulhdq(xtmp, xK, xcrc); 7627 vpclmulldq(xcrc, xK, xcrc); 7628 pxor(xcrc, xbuf); 7629 pxor(xcrc, xtmp); 7630 } else { 7631 movdqa(xtmp, xcrc); 7632 pclmulhdq(xtmp, xK); 7633 pclmulldq(xcrc, xK); 7634 pxor(xcrc, xbuf); 7635 pxor(xcrc, xtmp); 7636 } 7637 } 7638 7639 /** 7640 * 8-bit folds to compute 32-bit CRC 7641 * 7642 * uint64_t xcrc; 7643 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 7644 */ 7645 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 7646 movdl(tmp, xcrc); 7647 andl(tmp, 0xFF); 7648 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 7649 psrldq(xcrc, 1); // unsigned shift one byte 7650 pxor(xcrc, xtmp); 7651 } 7652 7653 /** 7654 * uint32_t crc; 7655 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 7656 */ 7657 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 7658 movl(tmp, crc); 7659 andl(tmp, 0xFF); 7660 shrl(crc, 8); 7661 xorl(crc, Address(table, tmp, Address::times_4, 0)); 7662 } 7663 7664 /** 7665 * @param crc register containing existing CRC (32-bit) 7666 * @param buf register pointing to input byte buffer (byte*) 7667 * @param len register containing number of bytes 7668 * @param table register that will contain address of CRC table 7669 * @param tmp scratch register 7670 */ 7671 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 7672 assert_different_registers(crc, buf, len, table, tmp, rax); 7673 7674 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7675 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7676 7677 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7678 // context for the registers used, where all instructions below are using 128-bit mode 7679 // On EVEX without VL and BW, these instructions will all be AVX. 7680 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 7681 notl(crc); // ~crc 7682 cmpl(len, 16); 7683 jcc(Assembler::less, L_tail); 7684 7685 // Align buffer to 16 bytes 7686 movl(tmp, buf); 7687 andl(tmp, 0xF); 7688 jccb(Assembler::zero, L_aligned); 7689 subl(tmp, 16); 7690 addl(len, tmp); 7691 7692 align(4); 7693 BIND(L_align_loop); 7694 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7695 update_byte_crc32(crc, rax, table); 7696 increment(buf); 7697 incrementl(tmp); 7698 jccb(Assembler::less, L_align_loop); 7699 7700 BIND(L_aligned); 7701 movl(tmp, len); // save 7702 shrl(len, 4); 7703 jcc(Assembler::zero, L_tail_restore); 7704 7705 // Fold crc into first bytes of vector 7706 movdqa(xmm1, Address(buf, 0)); 7707 movdl(rax, xmm1); 7708 xorl(crc, rax); 7709 if (VM_Version::supports_sse4_1()) { 7710 pinsrd(xmm1, crc, 0); 7711 } else { 7712 pinsrw(xmm1, crc, 0); 7713 shrl(crc, 16); 7714 pinsrw(xmm1, crc, 1); 7715 } 7716 addptr(buf, 16); 7717 subl(len, 4); // len > 0 7718 jcc(Assembler::less, L_fold_tail); 7719 7720 movdqa(xmm2, Address(buf, 0)); 7721 movdqa(xmm3, Address(buf, 16)); 7722 movdqa(xmm4, Address(buf, 32)); 7723 addptr(buf, 48); 7724 subl(len, 3); 7725 jcc(Assembler::lessEqual, L_fold_512b); 7726 7727 // Fold total 512 bits of polynomial on each iteration, 7728 // 128 bits per each of 4 parallel streams. 7729 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 7730 7731 align32(); 7732 BIND(L_fold_512b_loop); 7733 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7734 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 7735 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 7736 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 7737 addptr(buf, 64); 7738 subl(len, 4); 7739 jcc(Assembler::greater, L_fold_512b_loop); 7740 7741 // Fold 512 bits to 128 bits. 7742 BIND(L_fold_512b); 7743 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7744 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 7745 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 7746 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 7747 7748 // Fold the rest of 128 bits data chunks 7749 BIND(L_fold_tail); 7750 addl(len, 3); 7751 jccb(Assembler::lessEqual, L_fold_128b); 7752 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7753 7754 BIND(L_fold_tail_loop); 7755 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7756 addptr(buf, 16); 7757 decrementl(len); 7758 jccb(Assembler::greater, L_fold_tail_loop); 7759 7760 // Fold 128 bits in xmm1 down into 32 bits in crc register. 7761 BIND(L_fold_128b); 7762 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 7763 if (UseAVX > 0) { 7764 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 7765 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 7766 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 7767 } else { 7768 movdqa(xmm2, xmm0); 7769 pclmulqdq(xmm2, xmm1, 0x1); 7770 movdqa(xmm3, xmm0); 7771 pand(xmm3, xmm2); 7772 pclmulqdq(xmm0, xmm3, 0x1); 7773 } 7774 psrldq(xmm1, 8); 7775 psrldq(xmm2, 4); 7776 pxor(xmm0, xmm1); 7777 pxor(xmm0, xmm2); 7778 7779 // 8 8-bit folds to compute 32-bit CRC. 7780 for (int j = 0; j < 4; j++) { 7781 fold_8bit_crc32(xmm0, table, xmm1, rax); 7782 } 7783 movdl(crc, xmm0); // mov 32 bits to general register 7784 for (int j = 0; j < 4; j++) { 7785 fold_8bit_crc32(crc, table, rax); 7786 } 7787 7788 BIND(L_tail_restore); 7789 movl(len, tmp); // restore 7790 BIND(L_tail); 7791 andl(len, 0xf); 7792 jccb(Assembler::zero, L_exit); 7793 7794 // Fold the rest of bytes 7795 align(4); 7796 BIND(L_tail_loop); 7797 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7798 update_byte_crc32(crc, rax, table); 7799 increment(buf); 7800 decrementl(len); 7801 jccb(Assembler::greater, L_tail_loop); 7802 7803 BIND(L_exit); 7804 notl(crc); // ~c 7805 } 7806 7807 #ifdef _LP64 7808 // Helper function for AVX 512 CRC32 7809 // Fold 512-bit data chunks 7810 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 7811 Register pos, int offset) { 7812 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 7813 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 7814 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 7815 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 7816 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 7817 } 7818 7819 // Helper function for AVX 512 CRC32 7820 // Compute CRC32 for < 256B buffers 7821 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 7822 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 7823 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 7824 7825 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 7826 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 7827 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 7828 7829 // check if there is enough buffer to be able to fold 16B at a time 7830 cmpl(len, 32); 7831 jcc(Assembler::less, L_less_than_32); 7832 7833 // if there is, load the constants 7834 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 7835 movdl(xmm0, crc); // get the initial crc value 7836 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7837 pxor(xmm7, xmm0); 7838 7839 // update the buffer pointer 7840 addl(pos, 16); 7841 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 7842 subl(len, 32); 7843 jmp(L_16B_reduction_loop); 7844 7845 bind(L_less_than_32); 7846 //mov initial crc to the return value. this is necessary for zero - length buffers. 7847 movl(rax, crc); 7848 testl(len, len); 7849 jcc(Assembler::equal, L_cleanup); 7850 7851 movdl(xmm0, crc); //get the initial crc value 7852 7853 cmpl(len, 16); 7854 jcc(Assembler::equal, L_exact_16_left); 7855 jcc(Assembler::less, L_less_than_16_left); 7856 7857 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7858 pxor(xmm7, xmm0); //xor the initial crc value 7859 addl(pos, 16); 7860 subl(len, 16); 7861 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 7862 jmp(L_get_last_two_xmms); 7863 7864 bind(L_less_than_16_left); 7865 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 7866 pxor(xmm1, xmm1); 7867 movptr(tmp1, rsp); 7868 movdqu(Address(tmp1, 0 * 16), xmm1); 7869 7870 cmpl(len, 4); 7871 jcc(Assembler::less, L_only_less_than_4); 7872 7873 //backup the counter value 7874 movl(tmp2, len); 7875 cmpl(len, 8); 7876 jcc(Assembler::less, L_less_than_8_left); 7877 7878 //load 8 Bytes 7879 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 7880 movq(Address(tmp1, 0 * 16), rax); 7881 addptr(tmp1, 8); 7882 subl(len, 8); 7883 addl(pos, 8); 7884 7885 bind(L_less_than_8_left); 7886 cmpl(len, 4); 7887 jcc(Assembler::less, L_less_than_4_left); 7888 7889 //load 4 Bytes 7890 movl(rax, Address(buf, pos, Address::times_1, 0)); 7891 movl(Address(tmp1, 0 * 16), rax); 7892 addptr(tmp1, 4); 7893 subl(len, 4); 7894 addl(pos, 4); 7895 7896 bind(L_less_than_4_left); 7897 cmpl(len, 2); 7898 jcc(Assembler::less, L_less_than_2_left); 7899 7900 // load 2 Bytes 7901 movw(rax, Address(buf, pos, Address::times_1, 0)); 7902 movl(Address(tmp1, 0 * 16), rax); 7903 addptr(tmp1, 2); 7904 subl(len, 2); 7905 addl(pos, 2); 7906 7907 bind(L_less_than_2_left); 7908 cmpl(len, 1); 7909 jcc(Assembler::less, L_zero_left); 7910 7911 // load 1 Byte 7912 movb(rax, Address(buf, pos, Address::times_1, 0)); 7913 movb(Address(tmp1, 0 * 16), rax); 7914 7915 bind(L_zero_left); 7916 movdqu(xmm7, Address(rsp, 0)); 7917 pxor(xmm7, xmm0); //xor the initial crc value 7918 7919 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7920 movdqu(xmm0, Address(rax, tmp2)); 7921 pshufb(xmm7, xmm0); 7922 jmp(L_128_done); 7923 7924 bind(L_exact_16_left); 7925 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 7926 pxor(xmm7, xmm0); //xor the initial crc value 7927 jmp(L_128_done); 7928 7929 bind(L_only_less_than_4); 7930 cmpl(len, 3); 7931 jcc(Assembler::less, L_only_less_than_3); 7932 7933 // load 3 Bytes 7934 movb(rax, Address(buf, pos, Address::times_1, 0)); 7935 movb(Address(tmp1, 0), rax); 7936 7937 movb(rax, Address(buf, pos, Address::times_1, 1)); 7938 movb(Address(tmp1, 1), rax); 7939 7940 movb(rax, Address(buf, pos, Address::times_1, 2)); 7941 movb(Address(tmp1, 2), rax); 7942 7943 movdqu(xmm7, Address(rsp, 0)); 7944 pxor(xmm7, xmm0); //xor the initial crc value 7945 7946 pslldq(xmm7, 0x5); 7947 jmp(L_barrett); 7948 bind(L_only_less_than_3); 7949 cmpl(len, 2); 7950 jcc(Assembler::less, L_only_less_than_2); 7951 7952 // load 2 Bytes 7953 movb(rax, Address(buf, pos, Address::times_1, 0)); 7954 movb(Address(tmp1, 0), rax); 7955 7956 movb(rax, Address(buf, pos, Address::times_1, 1)); 7957 movb(Address(tmp1, 1), rax); 7958 7959 movdqu(xmm7, Address(rsp, 0)); 7960 pxor(xmm7, xmm0); //xor the initial crc value 7961 7962 pslldq(xmm7, 0x6); 7963 jmp(L_barrett); 7964 7965 bind(L_only_less_than_2); 7966 //load 1 Byte 7967 movb(rax, Address(buf, pos, Address::times_1, 0)); 7968 movb(Address(tmp1, 0), rax); 7969 7970 movdqu(xmm7, Address(rsp, 0)); 7971 pxor(xmm7, xmm0); //xor the initial crc value 7972 7973 pslldq(xmm7, 0x7); 7974 } 7975 7976 /** 7977 * Compute CRC32 using AVX512 instructions 7978 * param crc register containing existing CRC (32-bit) 7979 * param buf register pointing to input byte buffer (byte*) 7980 * param len register containing number of bytes 7981 * param table address of crc or crc32c table 7982 * param tmp1 scratch register 7983 * param tmp2 scratch register 7984 * return rax result register 7985 * 7986 * This routine is identical for crc32c with the exception of the precomputed constant 7987 * table which will be passed as the table argument. The calculation steps are 7988 * the same for both variants. 7989 */ 7990 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 7991 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 7992 7993 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7994 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7995 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 7996 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 7997 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 7998 7999 const Register pos = r12; 8000 push(r12); 8001 subptr(rsp, 16 * 2 + 8); 8002 8003 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 8004 // context for the registers used, where all instructions below are using 128-bit mode 8005 // On EVEX without VL and BW, these instructions will all be AVX. 8006 movl(pos, 0); 8007 8008 // check if smaller than 256B 8009 cmpl(len, 256); 8010 jcc(Assembler::less, L_less_than_256); 8011 8012 // load the initial crc value 8013 movdl(xmm10, crc); 8014 8015 // receive the initial 64B data, xor the initial crc value 8016 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 8017 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 8018 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 8019 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 8020 8021 subl(len, 256); 8022 cmpl(len, 256); 8023 jcc(Assembler::less, L_fold_128_B_loop); 8024 8025 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 8026 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 8027 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 8028 subl(len, 256); 8029 8030 bind(L_fold_256_B_loop); 8031 addl(pos, 256); 8032 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 8033 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 8034 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 8035 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 8036 8037 subl(len, 256); 8038 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 8039 8040 // Fold 256 into 128 8041 addl(pos, 256); 8042 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 8043 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 8044 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 8045 8046 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 8047 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 8048 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 8049 8050 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 8051 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 8052 8053 addl(len, 128); 8054 jmp(L_fold_128_B_register); 8055 8056 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 8057 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 8058 8059 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 8060 bind(L_fold_128_B_loop); 8061 addl(pos, 128); 8062 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 8063 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 8064 8065 subl(len, 128); 8066 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 8067 8068 addl(pos, 128); 8069 8070 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 8071 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 8072 bind(L_fold_128_B_register); 8073 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 8074 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 8075 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 8076 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 8077 // save last that has no multiplicand 8078 vextracti64x2(xmm7, xmm4, 3); 8079 8080 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 8081 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 8082 // Needed later in reduction loop 8083 movdqu(xmm10, Address(table, 1 * 16)); 8084 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 8085 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 8086 8087 // Swap 1,0,3,2 - 01 00 11 10 8088 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 8089 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 8090 vextracti128(xmm5, xmm8, 1); 8091 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 8092 8093 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 8094 // instead of a cmp instruction, we use the negative flag with the jl instruction 8095 addl(len, 128 - 16); 8096 jcc(Assembler::less, L_final_reduction_for_128); 8097 8098 bind(L_16B_reduction_loop); 8099 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8100 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8101 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8102 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 8103 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8104 addl(pos, 16); 8105 subl(len, 16); 8106 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 8107 8108 bind(L_final_reduction_for_128); 8109 addl(len, 16); 8110 jcc(Assembler::equal, L_128_done); 8111 8112 bind(L_get_last_two_xmms); 8113 movdqu(xmm2, xmm7); 8114 addl(pos, len); 8115 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 8116 subl(pos, len); 8117 8118 // get rid of the extra data that was loaded before 8119 // load the shift constant 8120 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8121 movdqu(xmm0, Address(rax, len)); 8122 addl(rax, len); 8123 8124 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8125 //Change mask to 512 8126 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 8127 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 8128 8129 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 8130 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8131 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8132 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8133 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 8134 8135 bind(L_128_done); 8136 // compute crc of a 128-bit value 8137 movdqu(xmm10, Address(table, 3 * 16)); 8138 movdqu(xmm0, xmm7); 8139 8140 // 64b fold 8141 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 8142 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 8143 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8144 8145 // 32b fold 8146 movdqu(xmm0, xmm7); 8147 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 8148 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8149 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8150 jmp(L_barrett); 8151 8152 bind(L_less_than_256); 8153 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 8154 8155 //barrett reduction 8156 bind(L_barrett); 8157 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 8158 movdqu(xmm1, xmm7); 8159 movdqu(xmm2, xmm7); 8160 movdqu(xmm10, Address(table, 4 * 16)); 8161 8162 pclmulqdq(xmm7, xmm10, 0x0); 8163 pxor(xmm7, xmm2); 8164 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 8165 movdqu(xmm2, xmm7); 8166 pclmulqdq(xmm7, xmm10, 0x10); 8167 pxor(xmm7, xmm2); 8168 pxor(xmm7, xmm1); 8169 pextrd(crc, xmm7, 2); 8170 8171 bind(L_cleanup); 8172 addptr(rsp, 16 * 2 + 8); 8173 pop(r12); 8174 } 8175 8176 // S. Gueron / Information Processing Letters 112 (2012) 184 8177 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 8178 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 8179 // Output: the 64-bit carry-less product of B * CONST 8180 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 8181 Register tmp1, Register tmp2, Register tmp3) { 8182 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8183 if (n > 0) { 8184 addq(tmp3, n * 256 * 8); 8185 } 8186 // Q1 = TABLEExt[n][B & 0xFF]; 8187 movl(tmp1, in); 8188 andl(tmp1, 0x000000FF); 8189 shll(tmp1, 3); 8190 addq(tmp1, tmp3); 8191 movq(tmp1, Address(tmp1, 0)); 8192 8193 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8194 movl(tmp2, in); 8195 shrl(tmp2, 8); 8196 andl(tmp2, 0x000000FF); 8197 shll(tmp2, 3); 8198 addq(tmp2, tmp3); 8199 movq(tmp2, Address(tmp2, 0)); 8200 8201 shlq(tmp2, 8); 8202 xorq(tmp1, tmp2); 8203 8204 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8205 movl(tmp2, in); 8206 shrl(tmp2, 16); 8207 andl(tmp2, 0x000000FF); 8208 shll(tmp2, 3); 8209 addq(tmp2, tmp3); 8210 movq(tmp2, Address(tmp2, 0)); 8211 8212 shlq(tmp2, 16); 8213 xorq(tmp1, tmp2); 8214 8215 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8216 shrl(in, 24); 8217 andl(in, 0x000000FF); 8218 shll(in, 3); 8219 addq(in, tmp3); 8220 movq(in, Address(in, 0)); 8221 8222 shlq(in, 24); 8223 xorq(in, tmp1); 8224 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8225 } 8226 8227 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8228 Register in_out, 8229 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8230 XMMRegister w_xtmp2, 8231 Register tmp1, 8232 Register n_tmp2, Register n_tmp3) { 8233 if (is_pclmulqdq_supported) { 8234 movdl(w_xtmp1, in_out); // modified blindly 8235 8236 movl(tmp1, const_or_pre_comp_const_index); 8237 movdl(w_xtmp2, tmp1); 8238 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8239 8240 movdq(in_out, w_xtmp1); 8241 } else { 8242 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 8243 } 8244 } 8245 8246 // Recombination Alternative 2: No bit-reflections 8247 // T1 = (CRC_A * U1) << 1 8248 // T2 = (CRC_B * U2) << 1 8249 // C1 = T1 >> 32 8250 // C2 = T2 >> 32 8251 // T1 = T1 & 0xFFFFFFFF 8252 // T2 = T2 & 0xFFFFFFFF 8253 // T1 = CRC32(0, T1) 8254 // T2 = CRC32(0, T2) 8255 // C1 = C1 ^ T1 8256 // C2 = C2 ^ T2 8257 // CRC = C1 ^ C2 ^ CRC_C 8258 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8259 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8260 Register tmp1, Register tmp2, 8261 Register n_tmp3) { 8262 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8263 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8264 shlq(in_out, 1); 8265 movl(tmp1, in_out); 8266 shrq(in_out, 32); 8267 xorl(tmp2, tmp2); 8268 crc32(tmp2, tmp1, 4); 8269 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8270 shlq(in1, 1); 8271 movl(tmp1, in1); 8272 shrq(in1, 32); 8273 xorl(tmp2, tmp2); 8274 crc32(tmp2, tmp1, 4); 8275 xorl(in1, tmp2); 8276 xorl(in_out, in1); 8277 xorl(in_out, in2); 8278 } 8279 8280 // Set N to predefined value 8281 // Subtract from a length of a buffer 8282 // execute in a loop: 8283 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8284 // for i = 1 to N do 8285 // CRC_A = CRC32(CRC_A, A[i]) 8286 // CRC_B = CRC32(CRC_B, B[i]) 8287 // CRC_C = CRC32(CRC_C, C[i]) 8288 // end for 8289 // Recombine 8290 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8291 Register in_out1, Register in_out2, Register in_out3, 8292 Register tmp1, Register tmp2, Register tmp3, 8293 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8294 Register tmp4, Register tmp5, 8295 Register n_tmp6) { 8296 Label L_processPartitions; 8297 Label L_processPartition; 8298 Label L_exit; 8299 8300 bind(L_processPartitions); 8301 cmpl(in_out1, 3 * size); 8302 jcc(Assembler::less, L_exit); 8303 xorl(tmp1, tmp1); 8304 xorl(tmp2, tmp2); 8305 movq(tmp3, in_out2); 8306 addq(tmp3, size); 8307 8308 bind(L_processPartition); 8309 crc32(in_out3, Address(in_out2, 0), 8); 8310 crc32(tmp1, Address(in_out2, size), 8); 8311 crc32(tmp2, Address(in_out2, size * 2), 8); 8312 addq(in_out2, 8); 8313 cmpq(in_out2, tmp3); 8314 jcc(Assembler::less, L_processPartition); 8315 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8316 w_xtmp1, w_xtmp2, w_xtmp3, 8317 tmp4, tmp5, 8318 n_tmp6); 8319 addq(in_out2, 2 * size); 8320 subl(in_out1, 3 * size); 8321 jmp(L_processPartitions); 8322 8323 bind(L_exit); 8324 } 8325 #else 8326 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, 8327 Register tmp1, Register tmp2, Register tmp3, 8328 XMMRegister xtmp1, XMMRegister xtmp2) { 8329 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8330 if (n > 0) { 8331 addl(tmp3, n * 256 * 8); 8332 } 8333 // Q1 = TABLEExt[n][B & 0xFF]; 8334 movl(tmp1, in_out); 8335 andl(tmp1, 0x000000FF); 8336 shll(tmp1, 3); 8337 addl(tmp1, tmp3); 8338 movq(xtmp1, Address(tmp1, 0)); 8339 8340 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8341 movl(tmp2, in_out); 8342 shrl(tmp2, 8); 8343 andl(tmp2, 0x000000FF); 8344 shll(tmp2, 3); 8345 addl(tmp2, tmp3); 8346 movq(xtmp2, Address(tmp2, 0)); 8347 8348 psllq(xtmp2, 8); 8349 pxor(xtmp1, xtmp2); 8350 8351 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8352 movl(tmp2, in_out); 8353 shrl(tmp2, 16); 8354 andl(tmp2, 0x000000FF); 8355 shll(tmp2, 3); 8356 addl(tmp2, tmp3); 8357 movq(xtmp2, Address(tmp2, 0)); 8358 8359 psllq(xtmp2, 16); 8360 pxor(xtmp1, xtmp2); 8361 8362 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8363 shrl(in_out, 24); 8364 andl(in_out, 0x000000FF); 8365 shll(in_out, 3); 8366 addl(in_out, tmp3); 8367 movq(xtmp2, Address(in_out, 0)); 8368 8369 psllq(xtmp2, 24); 8370 pxor(xtmp1, xtmp2); // Result in CXMM 8371 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8372 } 8373 8374 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8375 Register in_out, 8376 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8377 XMMRegister w_xtmp2, 8378 Register tmp1, 8379 Register n_tmp2, Register n_tmp3) { 8380 if (is_pclmulqdq_supported) { 8381 movdl(w_xtmp1, in_out); 8382 8383 movl(tmp1, const_or_pre_comp_const_index); 8384 movdl(w_xtmp2, tmp1); 8385 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8386 // Keep result in XMM since GPR is 32 bit in length 8387 } else { 8388 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); 8389 } 8390 } 8391 8392 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8393 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8394 Register tmp1, Register tmp2, 8395 Register n_tmp3) { 8396 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8397 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8398 8399 psllq(w_xtmp1, 1); 8400 movdl(tmp1, w_xtmp1); 8401 psrlq(w_xtmp1, 32); 8402 movdl(in_out, w_xtmp1); 8403 8404 xorl(tmp2, tmp2); 8405 crc32(tmp2, tmp1, 4); 8406 xorl(in_out, tmp2); 8407 8408 psllq(w_xtmp2, 1); 8409 movdl(tmp1, w_xtmp2); 8410 psrlq(w_xtmp2, 32); 8411 movdl(in1, w_xtmp2); 8412 8413 xorl(tmp2, tmp2); 8414 crc32(tmp2, tmp1, 4); 8415 xorl(in1, tmp2); 8416 xorl(in_out, in1); 8417 xorl(in_out, in2); 8418 } 8419 8420 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8421 Register in_out1, Register in_out2, Register in_out3, 8422 Register tmp1, Register tmp2, Register tmp3, 8423 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8424 Register tmp4, Register tmp5, 8425 Register n_tmp6) { 8426 Label L_processPartitions; 8427 Label L_processPartition; 8428 Label L_exit; 8429 8430 bind(L_processPartitions); 8431 cmpl(in_out1, 3 * size); 8432 jcc(Assembler::less, L_exit); 8433 xorl(tmp1, tmp1); 8434 xorl(tmp2, tmp2); 8435 movl(tmp3, in_out2); 8436 addl(tmp3, size); 8437 8438 bind(L_processPartition); 8439 crc32(in_out3, Address(in_out2, 0), 4); 8440 crc32(tmp1, Address(in_out2, size), 4); 8441 crc32(tmp2, Address(in_out2, size*2), 4); 8442 crc32(in_out3, Address(in_out2, 0+4), 4); 8443 crc32(tmp1, Address(in_out2, size+4), 4); 8444 crc32(tmp2, Address(in_out2, size*2+4), 4); 8445 addl(in_out2, 8); 8446 cmpl(in_out2, tmp3); 8447 jcc(Assembler::less, L_processPartition); 8448 8449 push(tmp3); 8450 push(in_out1); 8451 push(in_out2); 8452 tmp4 = tmp3; 8453 tmp5 = in_out1; 8454 n_tmp6 = in_out2; 8455 8456 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8457 w_xtmp1, w_xtmp2, w_xtmp3, 8458 tmp4, tmp5, 8459 n_tmp6); 8460 8461 pop(in_out2); 8462 pop(in_out1); 8463 pop(tmp3); 8464 8465 addl(in_out2, 2 * size); 8466 subl(in_out1, 3 * size); 8467 jmp(L_processPartitions); 8468 8469 bind(L_exit); 8470 } 8471 #endif //LP64 8472 8473 #ifdef _LP64 8474 // Algorithm 2: Pipelined usage of the CRC32 instruction. 8475 // Input: A buffer I of L bytes. 8476 // Output: the CRC32C value of the buffer. 8477 // Notations: 8478 // Write L = 24N + r, with N = floor (L/24). 8479 // r = L mod 24 (0 <= r < 24). 8480 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 8481 // N quadwords, and R consists of r bytes. 8482 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 8483 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 8484 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 8485 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 8486 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8487 Register tmp1, Register tmp2, Register tmp3, 8488 Register tmp4, Register tmp5, Register tmp6, 8489 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8490 bool is_pclmulqdq_supported) { 8491 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8492 Label L_wordByWord; 8493 Label L_byteByByteProlog; 8494 Label L_byteByByte; 8495 Label L_exit; 8496 8497 if (is_pclmulqdq_supported ) { 8498 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8499 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); 8500 8501 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8502 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8503 8504 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8505 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8506 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 8507 } else { 8508 const_or_pre_comp_const_index[0] = 1; 8509 const_or_pre_comp_const_index[1] = 0; 8510 8511 const_or_pre_comp_const_index[2] = 3; 8512 const_or_pre_comp_const_index[3] = 2; 8513 8514 const_or_pre_comp_const_index[4] = 5; 8515 const_or_pre_comp_const_index[5] = 4; 8516 } 8517 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8518 in2, in1, in_out, 8519 tmp1, tmp2, tmp3, 8520 w_xtmp1, w_xtmp2, w_xtmp3, 8521 tmp4, tmp5, 8522 tmp6); 8523 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8524 in2, in1, in_out, 8525 tmp1, tmp2, tmp3, 8526 w_xtmp1, w_xtmp2, w_xtmp3, 8527 tmp4, tmp5, 8528 tmp6); 8529 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8530 in2, in1, in_out, 8531 tmp1, tmp2, tmp3, 8532 w_xtmp1, w_xtmp2, w_xtmp3, 8533 tmp4, tmp5, 8534 tmp6); 8535 movl(tmp1, in2); 8536 andl(tmp1, 0x00000007); 8537 negl(tmp1); 8538 addl(tmp1, in2); 8539 addq(tmp1, in1); 8540 8541 cmpq(in1, tmp1); 8542 jccb(Assembler::greaterEqual, L_byteByByteProlog); 8543 align(16); 8544 BIND(L_wordByWord); 8545 crc32(in_out, Address(in1, 0), 8); 8546 addq(in1, 8); 8547 cmpq(in1, tmp1); 8548 jcc(Assembler::less, L_wordByWord); 8549 8550 BIND(L_byteByByteProlog); 8551 andl(in2, 0x00000007); 8552 movl(tmp2, 1); 8553 8554 cmpl(tmp2, in2); 8555 jccb(Assembler::greater, L_exit); 8556 BIND(L_byteByByte); 8557 crc32(in_out, Address(in1, 0), 1); 8558 incq(in1); 8559 incl(tmp2); 8560 cmpl(tmp2, in2); 8561 jcc(Assembler::lessEqual, L_byteByByte); 8562 8563 BIND(L_exit); 8564 } 8565 #else 8566 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8567 Register tmp1, Register tmp2, Register tmp3, 8568 Register tmp4, Register tmp5, Register tmp6, 8569 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8570 bool is_pclmulqdq_supported) { 8571 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8572 Label L_wordByWord; 8573 Label L_byteByByteProlog; 8574 Label L_byteByByte; 8575 Label L_exit; 8576 8577 if (is_pclmulqdq_supported) { 8578 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; 8579 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); 8580 8581 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); 8582 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); 8583 8584 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); 8585 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); 8586 } else { 8587 const_or_pre_comp_const_index[0] = 1; 8588 const_or_pre_comp_const_index[1] = 0; 8589 8590 const_or_pre_comp_const_index[2] = 3; 8591 const_or_pre_comp_const_index[3] = 2; 8592 8593 const_or_pre_comp_const_index[4] = 5; 8594 const_or_pre_comp_const_index[5] = 4; 8595 } 8596 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8597 in2, in1, in_out, 8598 tmp1, tmp2, tmp3, 8599 w_xtmp1, w_xtmp2, w_xtmp3, 8600 tmp4, tmp5, 8601 tmp6); 8602 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8603 in2, in1, in_out, 8604 tmp1, tmp2, tmp3, 8605 w_xtmp1, w_xtmp2, w_xtmp3, 8606 tmp4, tmp5, 8607 tmp6); 8608 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8609 in2, in1, in_out, 8610 tmp1, tmp2, tmp3, 8611 w_xtmp1, w_xtmp2, w_xtmp3, 8612 tmp4, tmp5, 8613 tmp6); 8614 movl(tmp1, in2); 8615 andl(tmp1, 0x00000007); 8616 negl(tmp1); 8617 addl(tmp1, in2); 8618 addl(tmp1, in1); 8619 8620 BIND(L_wordByWord); 8621 cmpl(in1, tmp1); 8622 jcc(Assembler::greaterEqual, L_byteByByteProlog); 8623 crc32(in_out, Address(in1,0), 4); 8624 addl(in1, 4); 8625 jmp(L_wordByWord); 8626 8627 BIND(L_byteByByteProlog); 8628 andl(in2, 0x00000007); 8629 movl(tmp2, 1); 8630 8631 BIND(L_byteByByte); 8632 cmpl(tmp2, in2); 8633 jccb(Assembler::greater, L_exit); 8634 movb(tmp1, Address(in1, 0)); 8635 crc32(in_out, tmp1, 1); 8636 incl(in1); 8637 incl(tmp2); 8638 jmp(L_byteByByte); 8639 8640 BIND(L_exit); 8641 } 8642 #endif // LP64 8643 #undef BIND 8644 #undef BLOCK_COMMENT 8645 8646 // Compress char[] array to byte[]. 8647 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 8648 // Return the array length if every element in array can be encoded, 8649 // otherwise, the index of first non-latin1 (> 0xff) character. 8650 // @IntrinsicCandidate 8651 // public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 8652 // for (int i = 0; i < len; i++) { 8653 // char c = src[srcOff]; 8654 // if (c > 0xff) { 8655 // return i; // return index of non-latin1 char 8656 // } 8657 // dst[dstOff] = (byte)c; 8658 // srcOff++; 8659 // dstOff++; 8660 // } 8661 // return len; 8662 // } 8663 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 8664 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 8665 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 8666 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 8667 Label copy_chars_loop, done, reset_sp, copy_tail; 8668 8669 // rsi: src 8670 // rdi: dst 8671 // rdx: len 8672 // rcx: tmp5 8673 // rax: result 8674 8675 // rsi holds start addr of source char[] to be compressed 8676 // rdi holds start addr of destination byte[] 8677 // rdx holds length 8678 8679 assert(len != result, ""); 8680 8681 // save length for return 8682 movl(result, len); 8683 8684 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 8685 VM_Version::supports_avx512vlbw() && 8686 VM_Version::supports_bmi2()) { 8687 8688 Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail; 8689 8690 // alignment 8691 Label post_alignment; 8692 8693 // if length of the string is less than 32, handle it the old fashioned way 8694 testl(len, -32); 8695 jcc(Assembler::zero, below_threshold); 8696 8697 // First check whether a character is compressible ( <= 0xFF). 8698 // Create mask to test for Unicode chars inside zmm vector 8699 movl(tmp5, 0x00FF); 8700 evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit); 8701 8702 testl(len, -64); 8703 jccb(Assembler::zero, post_alignment); 8704 8705 movl(tmp5, dst); 8706 andl(tmp5, (32 - 1)); 8707 negl(tmp5); 8708 andl(tmp5, (32 - 1)); 8709 8710 // bail out when there is nothing to be done 8711 testl(tmp5, 0xFFFFFFFF); 8712 jccb(Assembler::zero, post_alignment); 8713 8714 // ~(~0 << len), where len is the # of remaining elements to process 8715 movl(len, 0xFFFFFFFF); 8716 shlxl(len, len, tmp5); 8717 notl(len); 8718 kmovdl(mask2, len); 8719 movl(len, result); 8720 8721 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8722 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8723 ktestd(mask1, mask2); 8724 jcc(Assembler::carryClear, copy_tail); 8725 8726 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8727 8728 addptr(src, tmp5); 8729 addptr(src, tmp5); 8730 addptr(dst, tmp5); 8731 subl(len, tmp5); 8732 8733 bind(post_alignment); 8734 // end of alignment 8735 8736 movl(tmp5, len); 8737 andl(tmp5, (32 - 1)); // tail count (in chars) 8738 andl(len, ~(32 - 1)); // vector count (in chars) 8739 jccb(Assembler::zero, copy_loop_tail); 8740 8741 lea(src, Address(src, len, Address::times_2)); 8742 lea(dst, Address(dst, len, Address::times_1)); 8743 negptr(len); 8744 8745 bind(copy_32_loop); 8746 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 8747 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 8748 kortestdl(mask1, mask1); 8749 jccb(Assembler::carryClear, reset_for_copy_tail); 8750 8751 // All elements in current processed chunk are valid candidates for 8752 // compression. Write a truncated byte elements to the memory. 8753 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 8754 addptr(len, 32); 8755 jccb(Assembler::notZero, copy_32_loop); 8756 8757 bind(copy_loop_tail); 8758 // bail out when there is nothing to be done 8759 testl(tmp5, 0xFFFFFFFF); 8760 jcc(Assembler::zero, done); 8761 8762 movl(len, tmp5); 8763 8764 // ~(~0 << len), where len is the # of remaining elements to process 8765 movl(tmp5, 0xFFFFFFFF); 8766 shlxl(tmp5, tmp5, len); 8767 notl(tmp5); 8768 8769 kmovdl(mask2, tmp5); 8770 8771 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8772 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8773 ktestd(mask1, mask2); 8774 jcc(Assembler::carryClear, copy_tail); 8775 8776 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8777 jmp(done); 8778 8779 bind(reset_for_copy_tail); 8780 lea(src, Address(src, tmp5, Address::times_2)); 8781 lea(dst, Address(dst, tmp5, Address::times_1)); 8782 subptr(len, tmp5); 8783 jmp(copy_chars_loop); 8784 8785 bind(below_threshold); 8786 } 8787 8788 if (UseSSE42Intrinsics) { 8789 Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail; 8790 8791 // vectored compression 8792 testl(len, 0xfffffff8); 8793 jcc(Assembler::zero, copy_tail); 8794 8795 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 8796 movdl(tmp1Reg, tmp5); 8797 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8798 8799 andl(len, 0xfffffff0); 8800 jccb(Assembler::zero, copy_16); 8801 8802 // compress 16 chars per iter 8803 pxor(tmp4Reg, tmp4Reg); 8804 8805 lea(src, Address(src, len, Address::times_2)); 8806 lea(dst, Address(dst, len, Address::times_1)); 8807 negptr(len); 8808 8809 bind(copy_32_loop); 8810 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 8811 por(tmp4Reg, tmp2Reg); 8812 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 8813 por(tmp4Reg, tmp3Reg); 8814 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 8815 jccb(Assembler::notZero, reset_for_copy_tail); 8816 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 8817 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 8818 addptr(len, 16); 8819 jccb(Assembler::notZero, copy_32_loop); 8820 8821 // compress next vector of 8 chars (if any) 8822 bind(copy_16); 8823 // len = 0 8824 testl(result, 0x00000008); // check if there's a block of 8 chars to compress 8825 jccb(Assembler::zero, copy_tail_sse); 8826 8827 pxor(tmp3Reg, tmp3Reg); 8828 8829 movdqu(tmp2Reg, Address(src, 0)); 8830 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 8831 jccb(Assembler::notZero, reset_for_copy_tail); 8832 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 8833 movq(Address(dst, 0), tmp2Reg); 8834 addptr(src, 16); 8835 addptr(dst, 8); 8836 jmpb(copy_tail_sse); 8837 8838 bind(reset_for_copy_tail); 8839 movl(tmp5, result); 8840 andl(tmp5, 0x0000000f); 8841 lea(src, Address(src, tmp5, Address::times_2)); 8842 lea(dst, Address(dst, tmp5, Address::times_1)); 8843 subptr(len, tmp5); 8844 jmpb(copy_chars_loop); 8845 8846 bind(copy_tail_sse); 8847 movl(len, result); 8848 andl(len, 0x00000007); // tail count (in chars) 8849 } 8850 // compress 1 char per iter 8851 bind(copy_tail); 8852 testl(len, len); 8853 jccb(Assembler::zero, done); 8854 lea(src, Address(src, len, Address::times_2)); 8855 lea(dst, Address(dst, len, Address::times_1)); 8856 negptr(len); 8857 8858 bind(copy_chars_loop); 8859 load_unsigned_short(tmp5, Address(src, len, Address::times_2)); 8860 testl(tmp5, 0xff00); // check if Unicode char 8861 jccb(Assembler::notZero, reset_sp); 8862 movb(Address(dst, len, Address::times_1), tmp5); // ASCII char; compress to 1 byte 8863 increment(len); 8864 jccb(Assembler::notZero, copy_chars_loop); 8865 8866 // add len then return (len will be zero if compress succeeded, otherwise negative) 8867 bind(reset_sp); 8868 addl(result, len); 8869 8870 bind(done); 8871 } 8872 8873 // Inflate byte[] array to char[]. 8874 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 8875 // @IntrinsicCandidate 8876 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 8877 // for (int i = 0; i < len; i++) { 8878 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 8879 // } 8880 // } 8881 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 8882 XMMRegister tmp1, Register tmp2, KRegister mask) { 8883 Label copy_chars_loop, done, below_threshold, avx3_threshold; 8884 // rsi: src 8885 // rdi: dst 8886 // rdx: len 8887 // rcx: tmp2 8888 8889 // rsi holds start addr of source byte[] to be inflated 8890 // rdi holds start addr of destination char[] 8891 // rdx holds length 8892 assert_different_registers(src, dst, len, tmp2); 8893 movl(tmp2, len); 8894 if ((UseAVX > 2) && // AVX512 8895 VM_Version::supports_avx512vlbw() && 8896 VM_Version::supports_bmi2()) { 8897 8898 Label copy_32_loop, copy_tail; 8899 Register tmp3_aliased = len; 8900 8901 // if length of the string is less than 16, handle it in an old fashioned way 8902 testl(len, -16); 8903 jcc(Assembler::zero, below_threshold); 8904 8905 testl(len, -1 * AVX3Threshold); 8906 jcc(Assembler::zero, avx3_threshold); 8907 8908 // In order to use only one arithmetic operation for the main loop we use 8909 // this pre-calculation 8910 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 8911 andl(len, -32); // vector count 8912 jccb(Assembler::zero, copy_tail); 8913 8914 lea(src, Address(src, len, Address::times_1)); 8915 lea(dst, Address(dst, len, Address::times_2)); 8916 negptr(len); 8917 8918 8919 // inflate 32 chars per iter 8920 bind(copy_32_loop); 8921 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 8922 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 8923 addptr(len, 32); 8924 jcc(Assembler::notZero, copy_32_loop); 8925 8926 bind(copy_tail); 8927 // bail out when there is nothing to be done 8928 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 8929 jcc(Assembler::zero, done); 8930 8931 // ~(~0 << length), where length is the # of remaining elements to process 8932 movl(tmp3_aliased, -1); 8933 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 8934 notl(tmp3_aliased); 8935 kmovdl(mask, tmp3_aliased); 8936 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 8937 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 8938 8939 jmp(done); 8940 bind(avx3_threshold); 8941 } 8942 if (UseSSE42Intrinsics) { 8943 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 8944 8945 if (UseAVX > 1) { 8946 andl(tmp2, (16 - 1)); 8947 andl(len, -16); 8948 jccb(Assembler::zero, copy_new_tail); 8949 } else { 8950 andl(tmp2, 0x00000007); // tail count (in chars) 8951 andl(len, 0xfffffff8); // vector count (in chars) 8952 jccb(Assembler::zero, copy_tail); 8953 } 8954 8955 // vectored inflation 8956 lea(src, Address(src, len, Address::times_1)); 8957 lea(dst, Address(dst, len, Address::times_2)); 8958 negptr(len); 8959 8960 if (UseAVX > 1) { 8961 bind(copy_16_loop); 8962 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 8963 vmovdqu(Address(dst, len, Address::times_2), tmp1); 8964 addptr(len, 16); 8965 jcc(Assembler::notZero, copy_16_loop); 8966 8967 bind(below_threshold); 8968 bind(copy_new_tail); 8969 movl(len, tmp2); 8970 andl(tmp2, 0x00000007); 8971 andl(len, 0xFFFFFFF8); 8972 jccb(Assembler::zero, copy_tail); 8973 8974 pmovzxbw(tmp1, Address(src, 0)); 8975 movdqu(Address(dst, 0), tmp1); 8976 addptr(src, 8); 8977 addptr(dst, 2 * 8); 8978 8979 jmp(copy_tail, true); 8980 } 8981 8982 // inflate 8 chars per iter 8983 bind(copy_8_loop); 8984 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 8985 movdqu(Address(dst, len, Address::times_2), tmp1); 8986 addptr(len, 8); 8987 jcc(Assembler::notZero, copy_8_loop); 8988 8989 bind(copy_tail); 8990 movl(len, tmp2); 8991 8992 cmpl(len, 4); 8993 jccb(Assembler::less, copy_bytes); 8994 8995 movdl(tmp1, Address(src, 0)); // load 4 byte chars 8996 pmovzxbw(tmp1, tmp1); 8997 movq(Address(dst, 0), tmp1); 8998 subptr(len, 4); 8999 addptr(src, 4); 9000 addptr(dst, 8); 9001 9002 bind(copy_bytes); 9003 } else { 9004 bind(below_threshold); 9005 } 9006 9007 testl(len, len); 9008 jccb(Assembler::zero, done); 9009 lea(src, Address(src, len, Address::times_1)); 9010 lea(dst, Address(dst, len, Address::times_2)); 9011 negptr(len); 9012 9013 // inflate 1 char per iter 9014 bind(copy_chars_loop); 9015 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 9016 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 9017 increment(len); 9018 jcc(Assembler::notZero, copy_chars_loop); 9019 9020 bind(done); 9021 } 9022 9023 9024 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 9025 switch(type) { 9026 case T_BYTE: 9027 case T_BOOLEAN: 9028 evmovdqub(dst, kmask, src, merge, vector_len); 9029 break; 9030 case T_CHAR: 9031 case T_SHORT: 9032 evmovdquw(dst, kmask, src, merge, vector_len); 9033 break; 9034 case T_INT: 9035 case T_FLOAT: 9036 evmovdqul(dst, kmask, src, merge, vector_len); 9037 break; 9038 case T_LONG: 9039 case T_DOUBLE: 9040 evmovdquq(dst, kmask, src, merge, vector_len); 9041 break; 9042 default: 9043 fatal("Unexpected type argument %s", type2name(type)); 9044 break; 9045 } 9046 } 9047 9048 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 9049 switch(type) { 9050 case T_BYTE: 9051 case T_BOOLEAN: 9052 evmovdqub(dst, kmask, src, merge, vector_len); 9053 break; 9054 case T_CHAR: 9055 case T_SHORT: 9056 evmovdquw(dst, kmask, src, merge, vector_len); 9057 break; 9058 case T_INT: 9059 case T_FLOAT: 9060 evmovdqul(dst, kmask, src, merge, vector_len); 9061 break; 9062 case T_LONG: 9063 case T_DOUBLE: 9064 evmovdquq(dst, kmask, src, merge, vector_len); 9065 break; 9066 default: 9067 fatal("Unexpected type argument %s", type2name(type)); 9068 break; 9069 } 9070 } 9071 9072 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 9073 switch(masklen) { 9074 case 2: 9075 knotbl(dst, src); 9076 movl(rtmp, 3); 9077 kmovbl(ktmp, rtmp); 9078 kandbl(dst, ktmp, dst); 9079 break; 9080 case 4: 9081 knotbl(dst, src); 9082 movl(rtmp, 15); 9083 kmovbl(ktmp, rtmp); 9084 kandbl(dst, ktmp, dst); 9085 break; 9086 case 8: 9087 knotbl(dst, src); 9088 break; 9089 case 16: 9090 knotwl(dst, src); 9091 break; 9092 case 32: 9093 knotdl(dst, src); 9094 break; 9095 case 64: 9096 knotql(dst, src); 9097 break; 9098 default: 9099 fatal("Unexpected vector length %d", masklen); 9100 break; 9101 } 9102 } 9103 9104 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9105 switch(type) { 9106 case T_BOOLEAN: 9107 case T_BYTE: 9108 kandbl(dst, src1, src2); 9109 break; 9110 case T_CHAR: 9111 case T_SHORT: 9112 kandwl(dst, src1, src2); 9113 break; 9114 case T_INT: 9115 case T_FLOAT: 9116 kanddl(dst, src1, src2); 9117 break; 9118 case T_LONG: 9119 case T_DOUBLE: 9120 kandql(dst, src1, src2); 9121 break; 9122 default: 9123 fatal("Unexpected type argument %s", type2name(type)); 9124 break; 9125 } 9126 } 9127 9128 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9129 switch(type) { 9130 case T_BOOLEAN: 9131 case T_BYTE: 9132 korbl(dst, src1, src2); 9133 break; 9134 case T_CHAR: 9135 case T_SHORT: 9136 korwl(dst, src1, src2); 9137 break; 9138 case T_INT: 9139 case T_FLOAT: 9140 kordl(dst, src1, src2); 9141 break; 9142 case T_LONG: 9143 case T_DOUBLE: 9144 korql(dst, src1, src2); 9145 break; 9146 default: 9147 fatal("Unexpected type argument %s", type2name(type)); 9148 break; 9149 } 9150 } 9151 9152 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9153 switch(type) { 9154 case T_BOOLEAN: 9155 case T_BYTE: 9156 kxorbl(dst, src1, src2); 9157 break; 9158 case T_CHAR: 9159 case T_SHORT: 9160 kxorwl(dst, src1, src2); 9161 break; 9162 case T_INT: 9163 case T_FLOAT: 9164 kxordl(dst, src1, src2); 9165 break; 9166 case T_LONG: 9167 case T_DOUBLE: 9168 kxorql(dst, src1, src2); 9169 break; 9170 default: 9171 fatal("Unexpected type argument %s", type2name(type)); 9172 break; 9173 } 9174 } 9175 9176 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9177 switch(type) { 9178 case T_BOOLEAN: 9179 case T_BYTE: 9180 evpermb(dst, mask, nds, src, merge, vector_len); break; 9181 case T_CHAR: 9182 case T_SHORT: 9183 evpermw(dst, mask, nds, src, merge, vector_len); break; 9184 case T_INT: 9185 case T_FLOAT: 9186 evpermd(dst, mask, nds, src, merge, vector_len); break; 9187 case T_LONG: 9188 case T_DOUBLE: 9189 evpermq(dst, mask, nds, src, merge, vector_len); break; 9190 default: 9191 fatal("Unexpected type argument %s", type2name(type)); break; 9192 } 9193 } 9194 9195 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9196 switch(type) { 9197 case T_BOOLEAN: 9198 case T_BYTE: 9199 evpermb(dst, mask, nds, src, merge, vector_len); break; 9200 case T_CHAR: 9201 case T_SHORT: 9202 evpermw(dst, mask, nds, src, merge, vector_len); break; 9203 case T_INT: 9204 case T_FLOAT: 9205 evpermd(dst, mask, nds, src, merge, vector_len); break; 9206 case T_LONG: 9207 case T_DOUBLE: 9208 evpermq(dst, mask, nds, src, merge, vector_len); break; 9209 default: 9210 fatal("Unexpected type argument %s", type2name(type)); break; 9211 } 9212 } 9213 9214 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9215 switch(type) { 9216 case T_BYTE: 9217 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9218 case T_SHORT: 9219 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9220 case T_INT: 9221 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9222 case T_LONG: 9223 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9224 default: 9225 fatal("Unexpected type argument %s", type2name(type)); break; 9226 } 9227 } 9228 9229 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9230 switch(type) { 9231 case T_BYTE: 9232 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9233 case T_SHORT: 9234 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9235 case T_INT: 9236 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9237 case T_LONG: 9238 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9239 default: 9240 fatal("Unexpected type argument %s", type2name(type)); break; 9241 } 9242 } 9243 9244 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9245 switch(type) { 9246 case T_BYTE: 9247 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9248 case T_SHORT: 9249 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9250 case T_INT: 9251 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9252 case T_LONG: 9253 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9254 default: 9255 fatal("Unexpected type argument %s", type2name(type)); break; 9256 } 9257 } 9258 9259 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9260 switch(type) { 9261 case T_BYTE: 9262 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9263 case T_SHORT: 9264 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9265 case T_INT: 9266 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9267 case T_LONG: 9268 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9269 default: 9270 fatal("Unexpected type argument %s", type2name(type)); break; 9271 } 9272 } 9273 9274 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9275 switch(type) { 9276 case T_INT: 9277 evpxord(dst, mask, nds, src, merge, vector_len); break; 9278 case T_LONG: 9279 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9280 default: 9281 fatal("Unexpected type argument %s", type2name(type)); break; 9282 } 9283 } 9284 9285 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9286 switch(type) { 9287 case T_INT: 9288 evpxord(dst, mask, nds, src, merge, vector_len); break; 9289 case T_LONG: 9290 evpxorq(dst, mask, nds, src, merge, vector_len); break; 9291 default: 9292 fatal("Unexpected type argument %s", type2name(type)); break; 9293 } 9294 } 9295 9296 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9297 switch(type) { 9298 case T_INT: 9299 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9300 case T_LONG: 9301 evporq(dst, mask, nds, src, merge, vector_len); break; 9302 default: 9303 fatal("Unexpected type argument %s", type2name(type)); break; 9304 } 9305 } 9306 9307 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9308 switch(type) { 9309 case T_INT: 9310 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 9311 case T_LONG: 9312 evporq(dst, mask, nds, src, merge, vector_len); break; 9313 default: 9314 fatal("Unexpected type argument %s", type2name(type)); break; 9315 } 9316 } 9317 9318 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9319 switch(type) { 9320 case T_INT: 9321 evpandd(dst, mask, nds, src, merge, vector_len); break; 9322 case T_LONG: 9323 evpandq(dst, mask, nds, src, merge, vector_len); break; 9324 default: 9325 fatal("Unexpected type argument %s", type2name(type)); break; 9326 } 9327 } 9328 9329 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9330 switch(type) { 9331 case T_INT: 9332 evpandd(dst, mask, nds, src, merge, vector_len); break; 9333 case T_LONG: 9334 evpandq(dst, mask, nds, src, merge, vector_len); break; 9335 default: 9336 fatal("Unexpected type argument %s", type2name(type)); break; 9337 } 9338 } 9339 9340 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 9341 switch(masklen) { 9342 case 8: 9343 kortestbl(src1, src2); 9344 break; 9345 case 16: 9346 kortestwl(src1, src2); 9347 break; 9348 case 32: 9349 kortestdl(src1, src2); 9350 break; 9351 case 64: 9352 kortestql(src1, src2); 9353 break; 9354 default: 9355 fatal("Unexpected mask length %d", masklen); 9356 break; 9357 } 9358 } 9359 9360 9361 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 9362 switch(masklen) { 9363 case 8: 9364 ktestbl(src1, src2); 9365 break; 9366 case 16: 9367 ktestwl(src1, src2); 9368 break; 9369 case 32: 9370 ktestdl(src1, src2); 9371 break; 9372 case 64: 9373 ktestql(src1, src2); 9374 break; 9375 default: 9376 fatal("Unexpected mask length %d", masklen); 9377 break; 9378 } 9379 } 9380 9381 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9382 switch(type) { 9383 case T_INT: 9384 evprold(dst, mask, src, shift, merge, vlen_enc); break; 9385 case T_LONG: 9386 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 9387 default: 9388 fatal("Unexpected type argument %s", type2name(type)); break; 9389 break; 9390 } 9391 } 9392 9393 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 9394 switch(type) { 9395 case T_INT: 9396 evprord(dst, mask, src, shift, merge, vlen_enc); break; 9397 case T_LONG: 9398 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 9399 default: 9400 fatal("Unexpected type argument %s", type2name(type)); break; 9401 } 9402 } 9403 9404 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9405 switch(type) { 9406 case T_INT: 9407 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 9408 case T_LONG: 9409 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 9410 default: 9411 fatal("Unexpected type argument %s", type2name(type)); break; 9412 } 9413 } 9414 9415 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9416 switch(type) { 9417 case T_INT: 9418 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 9419 case T_LONG: 9420 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 9421 default: 9422 fatal("Unexpected type argument %s", type2name(type)); break; 9423 } 9424 } 9425 9426 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9427 assert(rscratch != noreg || always_reachable(src), "missing"); 9428 9429 if (reachable(src)) { 9430 evpandq(dst, nds, as_Address(src), vector_len); 9431 } else { 9432 lea(rscratch, src); 9433 evpandq(dst, nds, Address(rscratch, 0), vector_len); 9434 } 9435 } 9436 9437 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 9438 assert(rscratch != noreg || always_reachable(src), "missing"); 9439 9440 if (reachable(src)) { 9441 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 9442 } else { 9443 lea(rscratch, src); 9444 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 9445 } 9446 } 9447 9448 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9449 assert(rscratch != noreg || always_reachable(src), "missing"); 9450 9451 if (reachable(src)) { 9452 evporq(dst, nds, as_Address(src), vector_len); 9453 } else { 9454 lea(rscratch, src); 9455 evporq(dst, nds, Address(rscratch, 0), vector_len); 9456 } 9457 } 9458 9459 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9460 assert(rscratch != noreg || always_reachable(src), "missing"); 9461 9462 if (reachable(src)) { 9463 vpshufb(dst, nds, as_Address(src), vector_len); 9464 } else { 9465 lea(rscratch, src); 9466 vpshufb(dst, nds, Address(rscratch, 0), vector_len); 9467 } 9468 } 9469 9470 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9471 assert(rscratch != noreg || always_reachable(src), "missing"); 9472 9473 if (reachable(src)) { 9474 Assembler::vpor(dst, nds, as_Address(src), vector_len); 9475 } else { 9476 lea(rscratch, src); 9477 Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len); 9478 } 9479 } 9480 9481 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 9482 assert(rscratch != noreg || always_reachable(src3), "missing"); 9483 9484 if (reachable(src3)) { 9485 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 9486 } else { 9487 lea(rscratch, src3); 9488 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 9489 } 9490 } 9491 9492 #if COMPILER2_OR_JVMCI 9493 9494 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 9495 Register length, Register temp, int vec_enc) { 9496 // Computing mask for predicated vector store. 9497 movptr(temp, -1); 9498 bzhiq(temp, temp, length); 9499 kmov(mask, temp); 9500 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 9501 } 9502 9503 // Set memory operation for length "less than" 64 bytes. 9504 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 9505 XMMRegister xmm, KRegister mask, Register length, 9506 Register temp, bool use64byteVector) { 9507 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9508 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9509 if (!use64byteVector) { 9510 fill32(dst, disp, xmm); 9511 subptr(length, 32 >> shift); 9512 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 9513 } else { 9514 assert(MaxVectorSize == 64, "vector length != 64"); 9515 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 9516 } 9517 } 9518 9519 9520 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 9521 XMMRegister xmm, KRegister mask, Register length, 9522 Register temp) { 9523 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9524 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9525 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 9526 } 9527 9528 9529 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 9530 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9531 vmovdqu(dst, xmm); 9532 } 9533 9534 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 9535 fill32(Address(dst, disp), xmm); 9536 } 9537 9538 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 9539 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9540 if (!use64byteVector) { 9541 fill32(dst, xmm); 9542 fill32(dst.plus_disp(32), xmm); 9543 } else { 9544 evmovdquq(dst, xmm, Assembler::AVX_512bit); 9545 } 9546 } 9547 9548 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 9549 fill64(Address(dst, disp), xmm, use64byteVector); 9550 } 9551 9552 #ifdef _LP64 9553 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 9554 Register count, Register rtmp, XMMRegister xtmp) { 9555 Label L_exit; 9556 Label L_fill_start; 9557 Label L_fill_64_bytes; 9558 Label L_fill_96_bytes; 9559 Label L_fill_128_bytes; 9560 Label L_fill_128_bytes_loop; 9561 Label L_fill_128_loop_header; 9562 Label L_fill_128_bytes_loop_header; 9563 Label L_fill_128_bytes_loop_pre_header; 9564 Label L_fill_zmm_sequence; 9565 9566 int shift = -1; 9567 int avx3threshold = VM_Version::avx3_threshold(); 9568 switch(type) { 9569 case T_BYTE: shift = 0; 9570 break; 9571 case T_SHORT: shift = 1; 9572 break; 9573 case T_INT: shift = 2; 9574 break; 9575 /* Uncomment when LONG fill stubs are supported. 9576 case T_LONG: shift = 3; 9577 break; 9578 */ 9579 default: 9580 fatal("Unhandled type: %s\n", type2name(type)); 9581 } 9582 9583 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 9584 9585 if (MaxVectorSize == 64) { 9586 cmpq(count, avx3threshold >> shift); 9587 jcc(Assembler::greater, L_fill_zmm_sequence); 9588 } 9589 9590 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 9591 9592 bind(L_fill_start); 9593 9594 cmpq(count, 32 >> shift); 9595 jccb(Assembler::greater, L_fill_64_bytes); 9596 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 9597 jmp(L_exit); 9598 9599 bind(L_fill_64_bytes); 9600 cmpq(count, 64 >> shift); 9601 jccb(Assembler::greater, L_fill_96_bytes); 9602 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 9603 jmp(L_exit); 9604 9605 bind(L_fill_96_bytes); 9606 cmpq(count, 96 >> shift); 9607 jccb(Assembler::greater, L_fill_128_bytes); 9608 fill64(to, 0, xtmp); 9609 subq(count, 64 >> shift); 9610 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 9611 jmp(L_exit); 9612 9613 bind(L_fill_128_bytes); 9614 cmpq(count, 128 >> shift); 9615 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 9616 fill64(to, 0, xtmp); 9617 fill32(to, 64, xtmp); 9618 subq(count, 96 >> shift); 9619 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 9620 jmp(L_exit); 9621 9622 bind(L_fill_128_bytes_loop_pre_header); 9623 { 9624 mov(rtmp, to); 9625 andq(rtmp, 31); 9626 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 9627 negq(rtmp); 9628 addq(rtmp, 32); 9629 mov64(r8, -1L); 9630 bzhiq(r8, r8, rtmp); 9631 kmovql(k2, r8); 9632 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 9633 addq(to, rtmp); 9634 shrq(rtmp, shift); 9635 subq(count, rtmp); 9636 } 9637 9638 cmpq(count, 128 >> shift); 9639 jcc(Assembler::less, L_fill_start); 9640 9641 bind(L_fill_128_bytes_loop_header); 9642 subq(count, 128 >> shift); 9643 9644 align32(); 9645 bind(L_fill_128_bytes_loop); 9646 fill64(to, 0, xtmp); 9647 fill64(to, 64, xtmp); 9648 addq(to, 128); 9649 subq(count, 128 >> shift); 9650 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 9651 9652 addq(count, 128 >> shift); 9653 jcc(Assembler::zero, L_exit); 9654 jmp(L_fill_start); 9655 } 9656 9657 if (MaxVectorSize == 64) { 9658 // Sequence using 64 byte ZMM register. 9659 Label L_fill_128_bytes_zmm; 9660 Label L_fill_192_bytes_zmm; 9661 Label L_fill_192_bytes_loop_zmm; 9662 Label L_fill_192_bytes_loop_header_zmm; 9663 Label L_fill_192_bytes_loop_pre_header_zmm; 9664 Label L_fill_start_zmm_sequence; 9665 9666 bind(L_fill_zmm_sequence); 9667 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 9668 9669 bind(L_fill_start_zmm_sequence); 9670 cmpq(count, 64 >> shift); 9671 jccb(Assembler::greater, L_fill_128_bytes_zmm); 9672 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 9673 jmp(L_exit); 9674 9675 bind(L_fill_128_bytes_zmm); 9676 cmpq(count, 128 >> shift); 9677 jccb(Assembler::greater, L_fill_192_bytes_zmm); 9678 fill64(to, 0, xtmp, true); 9679 subq(count, 64 >> shift); 9680 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 9681 jmp(L_exit); 9682 9683 bind(L_fill_192_bytes_zmm); 9684 cmpq(count, 192 >> shift); 9685 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 9686 fill64(to, 0, xtmp, true); 9687 fill64(to, 64, xtmp, true); 9688 subq(count, 128 >> shift); 9689 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 9690 jmp(L_exit); 9691 9692 bind(L_fill_192_bytes_loop_pre_header_zmm); 9693 { 9694 movq(rtmp, to); 9695 andq(rtmp, 63); 9696 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 9697 negq(rtmp); 9698 addq(rtmp, 64); 9699 mov64(r8, -1L); 9700 bzhiq(r8, r8, rtmp); 9701 kmovql(k2, r8); 9702 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 9703 addq(to, rtmp); 9704 shrq(rtmp, shift); 9705 subq(count, rtmp); 9706 } 9707 9708 cmpq(count, 192 >> shift); 9709 jcc(Assembler::less, L_fill_start_zmm_sequence); 9710 9711 bind(L_fill_192_bytes_loop_header_zmm); 9712 subq(count, 192 >> shift); 9713 9714 align32(); 9715 bind(L_fill_192_bytes_loop_zmm); 9716 fill64(to, 0, xtmp, true); 9717 fill64(to, 64, xtmp, true); 9718 fill64(to, 128, xtmp, true); 9719 addq(to, 192); 9720 subq(count, 192 >> shift); 9721 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 9722 9723 addq(count, 192 >> shift); 9724 jcc(Assembler::zero, L_exit); 9725 jmp(L_fill_start_zmm_sequence); 9726 } 9727 bind(L_exit); 9728 } 9729 #endif 9730 #endif //COMPILER2_OR_JVMCI 9731 9732 9733 #ifdef _LP64 9734 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 9735 Label done; 9736 cvttss2sil(dst, src); 9737 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9738 cmpl(dst, 0x80000000); // float_sign_flip 9739 jccb(Assembler::notEqual, done); 9740 subptr(rsp, 8); 9741 movflt(Address(rsp, 0), src); 9742 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 9743 pop(dst); 9744 bind(done); 9745 } 9746 9747 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 9748 Label done; 9749 cvttsd2sil(dst, src); 9750 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9751 cmpl(dst, 0x80000000); // float_sign_flip 9752 jccb(Assembler::notEqual, done); 9753 subptr(rsp, 8); 9754 movdbl(Address(rsp, 0), src); 9755 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 9756 pop(dst); 9757 bind(done); 9758 } 9759 9760 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 9761 Label done; 9762 cvttss2siq(dst, src); 9763 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9764 jccb(Assembler::notEqual, done); 9765 subptr(rsp, 8); 9766 movflt(Address(rsp, 0), src); 9767 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 9768 pop(dst); 9769 bind(done); 9770 } 9771 9772 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9773 // Following code is line by line assembly translation rounding algorithm. 9774 // Please refer to java.lang.Math.round(float) algorithm for details. 9775 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 9776 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 9777 const int32_t FloatConsts_EXP_BIAS = 127; 9778 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 9779 const int32_t MINUS_32 = 0xFFFFFFE0; 9780 Label L_special_case, L_block1, L_exit; 9781 movl(rtmp, FloatConsts_EXP_BIT_MASK); 9782 movdl(dst, src); 9783 andl(dst, rtmp); 9784 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 9785 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 9786 subl(rtmp, dst); 9787 movl(rcx, rtmp); 9788 movl(dst, MINUS_32); 9789 testl(rtmp, dst); 9790 jccb(Assembler::notEqual, L_special_case); 9791 movdl(dst, src); 9792 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 9793 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 9794 movdl(rtmp, src); 9795 testl(rtmp, rtmp); 9796 jccb(Assembler::greaterEqual, L_block1); 9797 negl(dst); 9798 bind(L_block1); 9799 sarl(dst); 9800 addl(dst, 0x1); 9801 sarl(dst, 0x1); 9802 jmp(L_exit); 9803 bind(L_special_case); 9804 convert_f2i(dst, src); 9805 bind(L_exit); 9806 } 9807 9808 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9809 // Following code is line by line assembly translation rounding algorithm. 9810 // Please refer to java.lang.Math.round(double) algorithm for details. 9811 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 9812 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 9813 const int64_t DoubleConsts_EXP_BIAS = 1023; 9814 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 9815 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 9816 Label L_special_case, L_block1, L_exit; 9817 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 9818 movq(dst, src); 9819 andq(dst, rtmp); 9820 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 9821 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 9822 subq(rtmp, dst); 9823 movq(rcx, rtmp); 9824 mov64(dst, MINUS_64); 9825 testq(rtmp, dst); 9826 jccb(Assembler::notEqual, L_special_case); 9827 movq(dst, src); 9828 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 9829 andq(dst, rtmp); 9830 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 9831 orq(dst, rtmp); 9832 movq(rtmp, src); 9833 testq(rtmp, rtmp); 9834 jccb(Assembler::greaterEqual, L_block1); 9835 negq(dst); 9836 bind(L_block1); 9837 sarq(dst); 9838 addq(dst, 0x1); 9839 sarq(dst, 0x1); 9840 jmp(L_exit); 9841 bind(L_special_case); 9842 convert_d2l(dst, src); 9843 bind(L_exit); 9844 } 9845 9846 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 9847 Label done; 9848 cvttsd2siq(dst, src); 9849 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9850 jccb(Assembler::notEqual, done); 9851 subptr(rsp, 8); 9852 movdbl(Address(rsp, 0), src); 9853 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 9854 pop(dst); 9855 bind(done); 9856 } 9857 9858 void MacroAssembler::cache_wb(Address line) 9859 { 9860 // 64 bit cpus always support clflush 9861 assert(VM_Version::supports_clflush(), "clflush should be available"); 9862 bool optimized = VM_Version::supports_clflushopt(); 9863 bool no_evict = VM_Version::supports_clwb(); 9864 9865 // prefer clwb (writeback without evict) otherwise 9866 // prefer clflushopt (potentially parallel writeback with evict) 9867 // otherwise fallback on clflush (serial writeback with evict) 9868 9869 if (optimized) { 9870 if (no_evict) { 9871 clwb(line); 9872 } else { 9873 clflushopt(line); 9874 } 9875 } else { 9876 // no need for fence when using CLFLUSH 9877 clflush(line); 9878 } 9879 } 9880 9881 void MacroAssembler::cache_wbsync(bool is_pre) 9882 { 9883 assert(VM_Version::supports_clflush(), "clflush should be available"); 9884 bool optimized = VM_Version::supports_clflushopt(); 9885 bool no_evict = VM_Version::supports_clwb(); 9886 9887 // pick the correct implementation 9888 9889 if (!is_pre && (optimized || no_evict)) { 9890 // need an sfence for post flush when using clflushopt or clwb 9891 // otherwise no no need for any synchroniaztion 9892 9893 sfence(); 9894 } 9895 } 9896 9897 #endif // _LP64 9898 9899 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 9900 switch (cond) { 9901 // Note some conditions are synonyms for others 9902 case Assembler::zero: return Assembler::notZero; 9903 case Assembler::notZero: return Assembler::zero; 9904 case Assembler::less: return Assembler::greaterEqual; 9905 case Assembler::lessEqual: return Assembler::greater; 9906 case Assembler::greater: return Assembler::lessEqual; 9907 case Assembler::greaterEqual: return Assembler::less; 9908 case Assembler::below: return Assembler::aboveEqual; 9909 case Assembler::belowEqual: return Assembler::above; 9910 case Assembler::above: return Assembler::belowEqual; 9911 case Assembler::aboveEqual: return Assembler::below; 9912 case Assembler::overflow: return Assembler::noOverflow; 9913 case Assembler::noOverflow: return Assembler::overflow; 9914 case Assembler::negative: return Assembler::positive; 9915 case Assembler::positive: return Assembler::negative; 9916 case Assembler::parity: return Assembler::noParity; 9917 case Assembler::noParity: return Assembler::parity; 9918 } 9919 ShouldNotReachHere(); return Assembler::overflow; 9920 } 9921 9922 SkipIfEqual::SkipIfEqual( 9923 MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) { 9924 _masm = masm; 9925 _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch); 9926 _masm->jcc(Assembler::equal, _label); 9927 } 9928 9929 SkipIfEqual::~SkipIfEqual() { 9930 _masm->bind(_label); 9931 } 9932 9933 // 32-bit Windows has its own fast-path implementation 9934 // of get_thread 9935 #if !defined(WIN32) || defined(_LP64) 9936 9937 // This is simply a call to Thread::current() 9938 void MacroAssembler::get_thread(Register thread) { 9939 if (thread != rax) { 9940 push(rax); 9941 } 9942 LP64_ONLY(push(rdi);) 9943 LP64_ONLY(push(rsi);) 9944 push(rdx); 9945 push(rcx); 9946 #ifdef _LP64 9947 push(r8); 9948 push(r9); 9949 push(r10); 9950 push(r11); 9951 #endif 9952 9953 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 9954 9955 #ifdef _LP64 9956 pop(r11); 9957 pop(r10); 9958 pop(r9); 9959 pop(r8); 9960 #endif 9961 pop(rcx); 9962 pop(rdx); 9963 LP64_ONLY(pop(rsi);) 9964 LP64_ONLY(pop(rdi);) 9965 if (thread != rax) { 9966 mov(thread, rax); 9967 pop(rax); 9968 } 9969 } 9970 9971 9972 #endif // !WIN32 || _LP64 9973 9974 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 9975 Label L_stack_ok; 9976 if (bias == 0) { 9977 testptr(sp, 2 * wordSize - 1); 9978 } else { 9979 // lea(tmp, Address(rsp, bias); 9980 mov(tmp, sp); 9981 addptr(tmp, bias); 9982 testptr(tmp, 2 * wordSize - 1); 9983 } 9984 jcc(Assembler::equal, L_stack_ok); 9985 block_comment(msg); 9986 stop(msg); 9987 bind(L_stack_ok); 9988 } 9989 9990 // Implements lightweight-locking. 9991 // 9992 // obj: the object to be locked 9993 // reg_rax: rax 9994 // thread: the thread which attempts to lock obj 9995 // tmp: a temporary register 9996 // 9997 // x86_32 Note: basic_lock and thread may alias each other due to limited register 9998 // availiability. 9999 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 10000 assert(reg_rax == rax, ""); 10001 assert_different_registers(obj, reg_rax, thread, tmp); 10002 10003 Label push; 10004 const Register top = tmp; 10005 10006 // Preload the markWord. It is important that this is the first 10007 // instruction emitted as it is part of C1's null check semantics. 10008 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 10009 10010 movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0); 10011 10012 #ifndef _LP64 10013 if (thread == basic_lock) { 10014 get_thread(thread); 10015 } 10016 #endif // !_LP64 10017 // Load top. 10018 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10019 10020 // Check if the lock-stack is full. 10021 cmpl(top, LockStack::end_offset()); 10022 jcc(Assembler::greaterEqual, slow); 10023 10024 // Check for recursion. 10025 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 10026 jcc(Assembler::equal, push); 10027 10028 // Check header for monitor (0b10). 10029 testptr(reg_rax, markWord::monitor_value); 10030 jcc(Assembler::notZero, slow); 10031 10032 // Try to lock. Transition lock bits 0b01 => 0b00 10033 movptr(tmp, reg_rax); 10034 andptr(tmp, ~(int32_t)markWord::unlocked_value); 10035 orptr(reg_rax, markWord::unlocked_value); 10036 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10037 jcc(Assembler::notEqual, slow); 10038 10039 // Restore top, CAS clobbers register. 10040 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10041 10042 bind(push); 10043 // After successful lock, push object on lock-stack. 10044 movptr(Address(thread, top), obj); 10045 incrementl(top, oopSize); 10046 movl(Address(thread, JavaThread::lock_stack_top_offset()), top); 10047 } 10048 10049 // Implements lightweight-unlocking. 10050 // 10051 // obj: the object to be unlocked 10052 // reg_rax: rax 10053 // thread: the thread 10054 // tmp: a temporary register 10055 // 10056 // x86_32 Note: reg_rax and thread may alias each other due to limited register 10057 // availiability. 10058 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 10059 assert(reg_rax == rax, ""); 10060 assert_different_registers(obj, reg_rax, tmp); 10061 LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);) 10062 10063 Label unlocked, push_and_slow; 10064 const Register top = tmp; 10065 10066 // Check if obj is top of lock-stack. 10067 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10068 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 10069 jcc(Assembler::notEqual, slow); 10070 10071 // Pop lock-stack. 10072 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);) 10073 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10074 10075 // Check if recursive. 10076 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize)); 10077 jcc(Assembler::equal, unlocked); 10078 10079 // Not recursive. Check header for monitor (0b10). 10080 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 10081 testptr(reg_rax, markWord::monitor_value); 10082 jcc(Assembler::notZero, push_and_slow); 10083 10084 #ifdef ASSERT 10085 // Check header not unlocked (0b01). 10086 Label not_unlocked; 10087 testptr(reg_rax, markWord::unlocked_value); 10088 jcc(Assembler::zero, not_unlocked); 10089 stop("lightweight_unlock already unlocked"); 10090 bind(not_unlocked); 10091 #endif 10092 10093 // Try to unlock. Transition lock bits 0b00 => 0b01 10094 movptr(tmp, reg_rax); 10095 orptr(tmp, markWord::unlocked_value); 10096 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10097 jcc(Assembler::equal, unlocked); 10098 10099 bind(push_and_slow); 10100 // Restore lock-stack and handle the unlock in runtime. 10101 if (thread == reg_rax) { 10102 // On x86_32 we may lose the thread. 10103 get_thread(thread); 10104 } 10105 #ifdef ASSERT 10106 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10107 movptr(Address(thread, top), obj); 10108 #endif 10109 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10110 jmp(slow); 10111 10112 bind(unlocked); 10113 }