1 /* 2 * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/macroAssembler.hpp" 28 #include "classfile/vmIntrinsics.hpp" 29 #include "compiler/oopMap.hpp" 30 #include "gc/shared/barrierSet.hpp" 31 #include "gc/shared/barrierSetAssembler.hpp" 32 #include "gc/shared/barrierSetNMethod.hpp" 33 #include "gc/shared/gc_globals.hpp" 34 #include "memory/universe.hpp" 35 #include "prims/jvmtiExport.hpp" 36 #include "runtime/arguments.hpp" 37 #include "runtime/javaThread.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "runtime/stubRoutines.hpp" 40 #include "utilities/macros.hpp" 41 #include "vmreg_x86.inline.hpp" 42 #include "stubGenerator_x86_64.hpp" 43 #ifdef COMPILER2 44 #include "opto/runtime.hpp" 45 #include "opto/c2_globals.hpp" 46 #endif 47 #if INCLUDE_JVMCI 48 #include "jvmci/jvmci_globals.hpp" 49 #endif 50 #if INCLUDE_JFR 51 #include "jfr/support/jfrIntrinsics.hpp" 52 #endif 53 54 // For a more detailed description of the stub routine structure 55 // see the comment in stubRoutines.hpp 56 57 #define __ _masm-> 58 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) 59 60 #ifdef PRODUCT 61 #define BLOCK_COMMENT(str) /* nothing */ 62 #else 63 #define BLOCK_COMMENT(str) __ block_comment(str) 64 #endif // PRODUCT 65 66 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 67 68 // 69 // Linux Arguments: 70 // c_rarg0: call wrapper address address 71 // c_rarg1: result address 72 // c_rarg2: result type BasicType 73 // c_rarg3: method Method* 74 // c_rarg4: (interpreter) entry point address 75 // c_rarg5: parameters intptr_t* 76 // 16(rbp): parameter size (in words) int 77 // 24(rbp): thread Thread* 78 // 79 // [ return_from_Java ] <--- rsp 80 // [ argument word n ] 81 // ... 82 // -12 [ argument word 1 ] 83 // -11 [ saved r15 ] <--- rsp_after_call 84 // -10 [ saved r14 ] 85 // -9 [ saved r13 ] 86 // -8 [ saved r12 ] 87 // -7 [ saved rbx ] 88 // -6 [ call wrapper ] 89 // -5 [ result ] 90 // -4 [ result type ] 91 // -3 [ method ] 92 // -2 [ entry point ] 93 // -1 [ parameters ] 94 // 0 [ saved rbp ] <--- rbp 95 // 1 [ return address ] 96 // 2 [ parameter size ] 97 // 3 [ thread ] 98 // 99 // Windows Arguments: 100 // c_rarg0: call wrapper address address 101 // c_rarg1: result address 102 // c_rarg2: result type BasicType 103 // c_rarg3: method Method* 104 // 48(rbp): (interpreter) entry point address 105 // 56(rbp): parameters intptr_t* 106 // 64(rbp): parameter size (in words) int 107 // 72(rbp): thread Thread* 108 // 109 // [ return_from_Java ] <--- rsp 110 // [ argument word n ] 111 // ... 112 // -60 [ argument word 1 ] 113 // -59 [ saved xmm31 ] <--- rsp after_call 114 // [ saved xmm16-xmm30 ] (EVEX enabled, else the space is blank) 115 // -27 [ saved xmm15 ] 116 // [ saved xmm7-xmm14 ] 117 // -9 [ saved xmm6 ] (each xmm register takes 2 slots) 118 // -7 [ saved r15 ] 119 // -6 [ saved r14 ] 120 // -5 [ saved r13 ] 121 // -4 [ saved r12 ] 122 // -3 [ saved rdi ] 123 // -2 [ saved rsi ] 124 // -1 [ saved rbx ] 125 // 0 [ saved rbp ] <--- rbp 126 // 1 [ return address ] 127 // 2 [ call wrapper ] 128 // 3 [ result ] 129 // 4 [ result type ] 130 // 5 [ method ] 131 // 6 [ entry point ] 132 // 7 [ parameters ] 133 // 8 [ parameter size ] 134 // 9 [ thread ] 135 // 136 // Windows reserves the callers stack space for arguments 1-4. 137 // We spill c_rarg0-c_rarg3 to this space. 138 139 // Call stub stack layout word offsets from rbp 140 #ifdef _WIN64 141 enum call_stub_layout { 142 xmm_save_first = 6, // save from xmm6 143 xmm_save_last = 31, // to xmm31 144 xmm_save_base = -9, 145 rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27 146 r15_off = -7, 147 r14_off = -6, 148 r13_off = -5, 149 r12_off = -4, 150 rdi_off = -3, 151 rsi_off = -2, 152 rbx_off = -1, 153 rbp_off = 0, 154 retaddr_off = 1, 155 call_wrapper_off = 2, 156 result_off = 3, 157 result_type_off = 4, 158 method_off = 5, 159 entry_point_off = 6, 160 parameters_off = 7, 161 parameter_size_off = 8, 162 thread_off = 9 163 }; 164 165 static Address xmm_save(int reg) { 166 assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range"); 167 return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize); 168 } 169 #else // !_WIN64 170 enum call_stub_layout { 171 rsp_after_call_off = -12, 172 mxcsr_off = rsp_after_call_off, 173 r15_off = -11, 174 r14_off = -10, 175 r13_off = -9, 176 r12_off = -8, 177 rbx_off = -7, 178 call_wrapper_off = -6, 179 result_off = -5, 180 result_type_off = -4, 181 method_off = -3, 182 entry_point_off = -2, 183 parameters_off = -1, 184 rbp_off = 0, 185 retaddr_off = 1, 186 parameter_size_off = 2, 187 thread_off = 3 188 }; 189 #endif // _WIN64 190 191 address StubGenerator::generate_call_stub(address& return_address) { 192 193 assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && 194 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, 195 "adjust this code"); 196 StubCodeMark mark(this, "StubRoutines", "call_stub"); 197 address start = __ pc(); 198 199 // same as in generate_catch_exception()! 200 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 201 202 const Address call_wrapper (rbp, call_wrapper_off * wordSize); 203 const Address result (rbp, result_off * wordSize); 204 const Address result_type (rbp, result_type_off * wordSize); 205 const Address method (rbp, method_off * wordSize); 206 const Address entry_point (rbp, entry_point_off * wordSize); 207 const Address parameters (rbp, parameters_off * wordSize); 208 const Address parameter_size(rbp, parameter_size_off * wordSize); 209 210 // same as in generate_catch_exception()! 211 const Address thread (rbp, thread_off * wordSize); 212 213 const Address r15_save(rbp, r15_off * wordSize); 214 const Address r14_save(rbp, r14_off * wordSize); 215 const Address r13_save(rbp, r13_off * wordSize); 216 const Address r12_save(rbp, r12_off * wordSize); 217 const Address rbx_save(rbp, rbx_off * wordSize); 218 219 // stub code 220 __ enter(); 221 __ subptr(rsp, -rsp_after_call_off * wordSize); 222 223 // save register parameters 224 #ifndef _WIN64 225 __ movptr(parameters, c_rarg5); // parameters 226 __ movptr(entry_point, c_rarg4); // entry_point 227 #endif 228 229 __ movptr(method, c_rarg3); // method 230 __ movl(result_type, c_rarg2); // result type 231 __ movptr(result, c_rarg1); // result 232 __ movptr(call_wrapper, c_rarg0); // call wrapper 233 234 // save regs belonging to calling function 235 __ movptr(rbx_save, rbx); 236 __ movptr(r12_save, r12); 237 __ movptr(r13_save, r13); 238 __ movptr(r14_save, r14); 239 __ movptr(r15_save, r15); 240 241 #ifdef _WIN64 242 int last_reg = 15; 243 if (UseAVX > 2) { 244 last_reg = 31; 245 } 246 if (VM_Version::supports_evex()) { 247 for (int i = xmm_save_first; i <= last_reg; i++) { 248 __ vextractf32x4(xmm_save(i), as_XMMRegister(i), 0); 249 } 250 } else { 251 for (int i = xmm_save_first; i <= last_reg; i++) { 252 __ movdqu(xmm_save(i), as_XMMRegister(i)); 253 } 254 } 255 256 const Address rdi_save(rbp, rdi_off * wordSize); 257 const Address rsi_save(rbp, rsi_off * wordSize); 258 259 __ movptr(rsi_save, rsi); 260 __ movptr(rdi_save, rdi); 261 #else 262 const Address mxcsr_save(rbp, mxcsr_off * wordSize); 263 { 264 Label skip_ldmx; 265 __ stmxcsr(mxcsr_save); 266 __ movl(rax, mxcsr_save); 267 __ andl(rax, 0xFFC0); // Mask out any pending exceptions (only check control and mask bits) 268 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 269 __ cmp32(rax, mxcsr_std, rscratch1); 270 __ jcc(Assembler::equal, skip_ldmx); 271 __ ldmxcsr(mxcsr_std, rscratch1); 272 __ bind(skip_ldmx); 273 } 274 #endif 275 276 // Load up thread register 277 __ movptr(r15_thread, thread); 278 __ reinit_heapbase(); 279 280 #ifdef ASSERT 281 // make sure we have no pending exceptions 282 { 283 Label L; 284 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 285 __ jcc(Assembler::equal, L); 286 __ stop("StubRoutines::call_stub: entered with pending exception"); 287 __ bind(L); 288 } 289 #endif 290 291 // pass parameters if any 292 BLOCK_COMMENT("pass parameters if any"); 293 Label parameters_done; 294 __ movl(c_rarg3, parameter_size); 295 __ testl(c_rarg3, c_rarg3); 296 __ jcc(Assembler::zero, parameters_done); 297 298 Label loop; 299 __ movptr(c_rarg2, parameters); // parameter pointer 300 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 301 __ BIND(loop); 302 __ movptr(rax, Address(c_rarg2, 0));// get parameter 303 __ addptr(c_rarg2, wordSize); // advance to next parameter 304 __ decrementl(c_rarg1); // decrement counter 305 __ push(rax); // pass parameter 306 __ jcc(Assembler::notZero, loop); 307 308 // call Java function 309 __ BIND(parameters_done); 310 __ movptr(rbx, method); // get Method* 311 __ movptr(c_rarg1, entry_point); // get entry_point 312 __ mov(r13, rsp); // set sender sp 313 BLOCK_COMMENT("call Java function"); 314 __ call(c_rarg1); 315 316 BLOCK_COMMENT("call_stub_return_address:"); 317 return_address = __ pc(); 318 319 // store result depending on type (everything that is not 320 // T_OBJECT, T_PRIMITIVE_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) 321 __ movptr(r13, result); 322 Label is_long, is_float, is_double, check_prim, exit; 323 __ movl(rbx, result_type); 324 __ cmpl(rbx, T_OBJECT); 325 __ jcc(Assembler::equal, check_prim); 326 __ cmpl(rbx, T_PRIMITIVE_OBJECT); 327 __ jcc(Assembler::equal, check_prim); 328 __ cmpl(rbx, T_LONG); 329 __ jcc(Assembler::equal, is_long); 330 __ cmpl(rbx, T_FLOAT); 331 __ jcc(Assembler::equal, is_float); 332 __ cmpl(rbx, T_DOUBLE); 333 __ jcc(Assembler::equal, is_double); 334 335 // handle T_INT case 336 __ movl(Address(r13, 0), rax); 337 338 __ BIND(exit); 339 340 // pop parameters 341 __ lea(rsp, rsp_after_call); 342 343 #ifdef ASSERT 344 // verify that threads correspond 345 { 346 Label L1, L2, L3; 347 __ cmpptr(r15_thread, thread); 348 __ jcc(Assembler::equal, L1); 349 __ stop("StubRoutines::call_stub: r15_thread is corrupted"); 350 __ bind(L1); 351 __ get_thread(rbx); 352 __ cmpptr(r15_thread, thread); 353 __ jcc(Assembler::equal, L2); 354 __ stop("StubRoutines::call_stub: r15_thread is modified by call"); 355 __ bind(L2); 356 __ cmpptr(r15_thread, rbx); 357 __ jcc(Assembler::equal, L3); 358 __ stop("StubRoutines::call_stub: threads must correspond"); 359 __ bind(L3); 360 } 361 #endif 362 363 __ pop_cont_fastpath(); 364 365 // restore regs belonging to calling function 366 #ifdef _WIN64 367 // emit the restores for xmm regs 368 if (VM_Version::supports_evex()) { 369 for (int i = xmm_save_first; i <= last_reg; i++) { 370 __ vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0); 371 } 372 } else { 373 for (int i = xmm_save_first; i <= last_reg; i++) { 374 __ movdqu(as_XMMRegister(i), xmm_save(i)); 375 } 376 } 377 #endif 378 __ movptr(r15, r15_save); 379 __ movptr(r14, r14_save); 380 __ movptr(r13, r13_save); 381 __ movptr(r12, r12_save); 382 __ movptr(rbx, rbx_save); 383 384 #ifdef _WIN64 385 __ movptr(rdi, rdi_save); 386 __ movptr(rsi, rsi_save); 387 #else 388 __ ldmxcsr(mxcsr_save); 389 #endif 390 391 // restore rsp 392 __ addptr(rsp, -rsp_after_call_off * wordSize); 393 394 // return 395 __ vzeroupper(); 396 __ pop(rbp); 397 __ ret(0); 398 399 // handle return types different from T_INT 400 __ BIND(check_prim); 401 if (InlineTypeReturnedAsFields) { 402 // Check for scalarized return value 403 __ testptr(rax, 1); 404 __ jcc(Assembler::zero, is_long); 405 // Load pack handler address 406 __ andptr(rax, -2); 407 __ movptr(rax, Address(rax, InstanceKlass::adr_inlineklass_fixed_block_offset())); 408 __ movptr(rbx, Address(rax, InlineKlass::pack_handler_jobject_offset())); 409 // Call pack handler to initialize the buffer 410 __ call(rbx); 411 __ jmp(exit); 412 } 413 __ BIND(is_long); 414 __ movq(Address(r13, 0), rax); 415 __ jmp(exit); 416 417 __ BIND(is_float); 418 __ movflt(Address(r13, 0), xmm0); 419 __ jmp(exit); 420 421 __ BIND(is_double); 422 __ movdbl(Address(r13, 0), xmm0); 423 __ jmp(exit); 424 425 return start; 426 } 427 428 // Return point for a Java call if there's an exception thrown in 429 // Java code. The exception is caught and transformed into a 430 // pending exception stored in JavaThread that can be tested from 431 // within the VM. 432 // 433 // Note: Usually the parameters are removed by the callee. In case 434 // of an exception crossing an activation frame boundary, that is 435 // not the case if the callee is compiled code => need to setup the 436 // rsp. 437 // 438 // rax: exception oop 439 440 address StubGenerator::generate_catch_exception() { 441 StubCodeMark mark(this, "StubRoutines", "catch_exception"); 442 address start = __ pc(); 443 444 // same as in generate_call_stub(): 445 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 446 const Address thread (rbp, thread_off * wordSize); 447 448 #ifdef ASSERT 449 // verify that threads correspond 450 { 451 Label L1, L2, L3; 452 __ cmpptr(r15_thread, thread); 453 __ jcc(Assembler::equal, L1); 454 __ stop("StubRoutines::catch_exception: r15_thread is corrupted"); 455 __ bind(L1); 456 __ get_thread(rbx); 457 __ cmpptr(r15_thread, thread); 458 __ jcc(Assembler::equal, L2); 459 __ stop("StubRoutines::catch_exception: r15_thread is modified by call"); 460 __ bind(L2); 461 __ cmpptr(r15_thread, rbx); 462 __ jcc(Assembler::equal, L3); 463 __ stop("StubRoutines::catch_exception: threads must correspond"); 464 __ bind(L3); 465 } 466 #endif 467 468 // set pending exception 469 __ verify_oop(rax); 470 471 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); 472 __ lea(rscratch1, ExternalAddress((address)__FILE__)); 473 __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); 474 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); 475 476 // complete return to VM 477 assert(StubRoutines::_call_stub_return_address != nullptr, 478 "_call_stub_return_address must have been generated before"); 479 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); 480 481 return start; 482 } 483 484 // Continuation point for runtime calls returning with a pending 485 // exception. The pending exception check happened in the runtime 486 // or native call stub. The pending exception in Thread is 487 // converted into a Java-level exception. 488 // 489 // Contract with Java-level exception handlers: 490 // rax: exception 491 // rdx: throwing pc 492 // 493 // NOTE: At entry of this stub, exception-pc must be on stack !! 494 495 address StubGenerator::generate_forward_exception() { 496 StubCodeMark mark(this, "StubRoutines", "forward exception"); 497 address start = __ pc(); 498 499 // Upon entry, the sp points to the return address returning into 500 // Java (interpreted or compiled) code; i.e., the return address 501 // becomes the throwing pc. 502 // 503 // Arguments pushed before the runtime call are still on the stack 504 // but the exception handler will reset the stack pointer -> 505 // ignore them. A potential result in registers can be ignored as 506 // well. 507 508 #ifdef ASSERT 509 // make sure this code is only executed if there is a pending exception 510 { 511 Label L; 512 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 513 __ jcc(Assembler::notEqual, L); 514 __ stop("StubRoutines::forward exception: no pending exception (1)"); 515 __ bind(L); 516 } 517 #endif 518 519 // compute exception handler into rbx 520 __ movptr(c_rarg0, Address(rsp, 0)); 521 BLOCK_COMMENT("call exception_handler_for_return_address"); 522 __ call_VM_leaf(CAST_FROM_FN_PTR(address, 523 SharedRuntime::exception_handler_for_return_address), 524 r15_thread, c_rarg0); 525 __ mov(rbx, rax); 526 527 // setup rax & rdx, remove return address & clear pending exception 528 __ pop(rdx); 529 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 530 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 531 532 #ifdef ASSERT 533 // make sure exception is set 534 { 535 Label L; 536 __ testptr(rax, rax); 537 __ jcc(Assembler::notEqual, L); 538 __ stop("StubRoutines::forward exception: no pending exception (2)"); 539 __ bind(L); 540 } 541 #endif 542 543 // continue at exception handler (return address removed) 544 // rax: exception 545 // rbx: exception handler 546 // rdx: throwing pc 547 __ verify_oop(rax); 548 __ jmp(rbx); 549 550 return start; 551 } 552 553 // Support for intptr_t OrderAccess::fence() 554 // 555 // Arguments : 556 // 557 // Result: 558 address StubGenerator::generate_orderaccess_fence() { 559 StubCodeMark mark(this, "StubRoutines", "orderaccess_fence"); 560 address start = __ pc(); 561 562 __ membar(Assembler::StoreLoad); 563 __ ret(0); 564 565 return start; 566 } 567 568 569 // Support for intptr_t get_previous_sp() 570 // 571 // This routine is used to find the previous stack pointer for the 572 // caller. 573 address StubGenerator::generate_get_previous_sp() { 574 StubCodeMark mark(this, "StubRoutines", "get_previous_sp"); 575 address start = __ pc(); 576 577 __ movptr(rax, rsp); 578 __ addptr(rax, 8); // return address is at the top of the stack. 579 __ ret(0); 580 581 return start; 582 } 583 584 //---------------------------------------------------------------------------------------------------- 585 // Support for void verify_mxcsr() 586 // 587 // This routine is used with -Xcheck:jni to verify that native 588 // JNI code does not return to Java code without restoring the 589 // MXCSR register to our expected state. 590 591 address StubGenerator::generate_verify_mxcsr() { 592 StubCodeMark mark(this, "StubRoutines", "verify_mxcsr"); 593 address start = __ pc(); 594 595 const Address mxcsr_save(rsp, 0); 596 597 if (CheckJNICalls) { 598 Label ok_ret; 599 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 600 __ push(rax); 601 __ subptr(rsp, wordSize); // allocate a temp location 602 __ stmxcsr(mxcsr_save); 603 __ movl(rax, mxcsr_save); 604 __ andl(rax, 0xFFC0); // Mask out any pending exceptions (only check control and mask bits) 605 __ cmp32(rax, mxcsr_std, rscratch1); 606 __ jcc(Assembler::equal, ok_ret); 607 608 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall"); 609 610 __ ldmxcsr(mxcsr_std, rscratch1); 611 612 __ bind(ok_ret); 613 __ addptr(rsp, wordSize); 614 __ pop(rax); 615 } 616 617 __ ret(0); 618 619 return start; 620 } 621 622 address StubGenerator::generate_f2i_fixup() { 623 StubCodeMark mark(this, "StubRoutines", "f2i_fixup"); 624 Address inout(rsp, 5 * wordSize); // return address + 4 saves 625 626 address start = __ pc(); 627 628 Label L; 629 630 __ push(rax); 631 __ push(c_rarg3); 632 __ push(c_rarg2); 633 __ push(c_rarg1); 634 635 __ movl(rax, 0x7f800000); 636 __ xorl(c_rarg3, c_rarg3); 637 __ movl(c_rarg2, inout); 638 __ movl(c_rarg1, c_rarg2); 639 __ andl(c_rarg1, 0x7fffffff); 640 __ cmpl(rax, c_rarg1); // NaN? -> 0 641 __ jcc(Assembler::negative, L); 642 __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint 643 __ movl(c_rarg3, 0x80000000); 644 __ movl(rax, 0x7fffffff); 645 __ cmovl(Assembler::positive, c_rarg3, rax); 646 647 __ bind(L); 648 __ movptr(inout, c_rarg3); 649 650 __ pop(c_rarg1); 651 __ pop(c_rarg2); 652 __ pop(c_rarg3); 653 __ pop(rax); 654 655 __ ret(0); 656 657 return start; 658 } 659 660 address StubGenerator::generate_f2l_fixup() { 661 StubCodeMark mark(this, "StubRoutines", "f2l_fixup"); 662 Address inout(rsp, 5 * wordSize); // return address + 4 saves 663 address start = __ pc(); 664 665 Label L; 666 667 __ push(rax); 668 __ push(c_rarg3); 669 __ push(c_rarg2); 670 __ push(c_rarg1); 671 672 __ movl(rax, 0x7f800000); 673 __ xorl(c_rarg3, c_rarg3); 674 __ movl(c_rarg2, inout); 675 __ movl(c_rarg1, c_rarg2); 676 __ andl(c_rarg1, 0x7fffffff); 677 __ cmpl(rax, c_rarg1); // NaN? -> 0 678 __ jcc(Assembler::negative, L); 679 __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong 680 __ mov64(c_rarg3, 0x8000000000000000); 681 __ mov64(rax, 0x7fffffffffffffff); 682 __ cmov(Assembler::positive, c_rarg3, rax); 683 684 __ bind(L); 685 __ movptr(inout, c_rarg3); 686 687 __ pop(c_rarg1); 688 __ pop(c_rarg2); 689 __ pop(c_rarg3); 690 __ pop(rax); 691 692 __ ret(0); 693 694 return start; 695 } 696 697 address StubGenerator::generate_d2i_fixup() { 698 StubCodeMark mark(this, "StubRoutines", "d2i_fixup"); 699 Address inout(rsp, 6 * wordSize); // return address + 5 saves 700 701 address start = __ pc(); 702 703 Label L; 704 705 __ push(rax); 706 __ push(c_rarg3); 707 __ push(c_rarg2); 708 __ push(c_rarg1); 709 __ push(c_rarg0); 710 711 __ movl(rax, 0x7ff00000); 712 __ movq(c_rarg2, inout); 713 __ movl(c_rarg3, c_rarg2); 714 __ mov(c_rarg1, c_rarg2); 715 __ mov(c_rarg0, c_rarg2); 716 __ negl(c_rarg3); 717 __ shrptr(c_rarg1, 0x20); 718 __ orl(c_rarg3, c_rarg2); 719 __ andl(c_rarg1, 0x7fffffff); 720 __ xorl(c_rarg2, c_rarg2); 721 __ shrl(c_rarg3, 0x1f); 722 __ orl(c_rarg1, c_rarg3); 723 __ cmpl(rax, c_rarg1); 724 __ jcc(Assembler::negative, L); // NaN -> 0 725 __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint 726 __ movl(c_rarg2, 0x80000000); 727 __ movl(rax, 0x7fffffff); 728 __ cmov(Assembler::positive, c_rarg2, rax); 729 730 __ bind(L); 731 __ movptr(inout, c_rarg2); 732 733 __ pop(c_rarg0); 734 __ pop(c_rarg1); 735 __ pop(c_rarg2); 736 __ pop(c_rarg3); 737 __ pop(rax); 738 739 __ ret(0); 740 741 return start; 742 } 743 744 address StubGenerator::generate_d2l_fixup() { 745 StubCodeMark mark(this, "StubRoutines", "d2l_fixup"); 746 Address inout(rsp, 6 * wordSize); // return address + 5 saves 747 748 address start = __ pc(); 749 750 Label L; 751 752 __ push(rax); 753 __ push(c_rarg3); 754 __ push(c_rarg2); 755 __ push(c_rarg1); 756 __ push(c_rarg0); 757 758 __ movl(rax, 0x7ff00000); 759 __ movq(c_rarg2, inout); 760 __ movl(c_rarg3, c_rarg2); 761 __ mov(c_rarg1, c_rarg2); 762 __ mov(c_rarg0, c_rarg2); 763 __ negl(c_rarg3); 764 __ shrptr(c_rarg1, 0x20); 765 __ orl(c_rarg3, c_rarg2); 766 __ andl(c_rarg1, 0x7fffffff); 767 __ xorl(c_rarg2, c_rarg2); 768 __ shrl(c_rarg3, 0x1f); 769 __ orl(c_rarg1, c_rarg3); 770 __ cmpl(rax, c_rarg1); 771 __ jcc(Assembler::negative, L); // NaN -> 0 772 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong 773 __ mov64(c_rarg2, 0x8000000000000000); 774 __ mov64(rax, 0x7fffffffffffffff); 775 __ cmovq(Assembler::positive, c_rarg2, rax); 776 777 __ bind(L); 778 __ movq(inout, c_rarg2); 779 780 __ pop(c_rarg0); 781 __ pop(c_rarg1); 782 __ pop(c_rarg2); 783 __ pop(c_rarg3); 784 __ pop(rax); 785 786 __ ret(0); 787 788 return start; 789 } 790 791 address StubGenerator::generate_count_leading_zeros_lut(const char *stub_name) { 792 __ align64(); 793 StubCodeMark mark(this, "StubRoutines", stub_name); 794 address start = __ pc(); 795 796 __ emit_data64(0x0101010102020304, relocInfo::none); 797 __ emit_data64(0x0000000000000000, relocInfo::none); 798 __ emit_data64(0x0101010102020304, relocInfo::none); 799 __ emit_data64(0x0000000000000000, relocInfo::none); 800 __ emit_data64(0x0101010102020304, relocInfo::none); 801 __ emit_data64(0x0000000000000000, relocInfo::none); 802 __ emit_data64(0x0101010102020304, relocInfo::none); 803 __ emit_data64(0x0000000000000000, relocInfo::none); 804 805 return start; 806 } 807 808 address StubGenerator::generate_popcount_avx_lut(const char *stub_name) { 809 __ align64(); 810 StubCodeMark mark(this, "StubRoutines", stub_name); 811 address start = __ pc(); 812 813 __ emit_data64(0x0302020102010100, relocInfo::none); 814 __ emit_data64(0x0403030203020201, relocInfo::none); 815 __ emit_data64(0x0302020102010100, relocInfo::none); 816 __ emit_data64(0x0403030203020201, relocInfo::none); 817 __ emit_data64(0x0302020102010100, relocInfo::none); 818 __ emit_data64(0x0403030203020201, relocInfo::none); 819 __ emit_data64(0x0302020102010100, relocInfo::none); 820 __ emit_data64(0x0403030203020201, relocInfo::none); 821 822 return start; 823 } 824 825 address StubGenerator::generate_iota_indices(const char *stub_name) { 826 __ align(CodeEntryAlignment); 827 StubCodeMark mark(this, "StubRoutines", stub_name); 828 address start = __ pc(); 829 // B 830 __ emit_data64(0x0706050403020100, relocInfo::none); 831 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); 832 __ emit_data64(0x1716151413121110, relocInfo::none); 833 __ emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none); 834 __ emit_data64(0x2726252423222120, relocInfo::none); 835 __ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none); 836 __ emit_data64(0x3736353433323130, relocInfo::none); 837 __ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none); 838 // W 839 __ emit_data64(0x0003000200010000, relocInfo::none); 840 __ emit_data64(0x0007000600050004, relocInfo::none); 841 __ emit_data64(0x000B000A00090008, relocInfo::none); 842 __ emit_data64(0x000F000E000D000C, relocInfo::none); 843 __ emit_data64(0x0013001200110010, relocInfo::none); 844 __ emit_data64(0x0017001600150014, relocInfo::none); 845 __ emit_data64(0x001B001A00190018, relocInfo::none); 846 __ emit_data64(0x001F001E001D001C, relocInfo::none); 847 // D 848 __ emit_data64(0x0000000100000000, relocInfo::none); 849 __ emit_data64(0x0000000300000002, relocInfo::none); 850 __ emit_data64(0x0000000500000004, relocInfo::none); 851 __ emit_data64(0x0000000700000006, relocInfo::none); 852 __ emit_data64(0x0000000900000008, relocInfo::none); 853 __ emit_data64(0x0000000B0000000A, relocInfo::none); 854 __ emit_data64(0x0000000D0000000C, relocInfo::none); 855 __ emit_data64(0x0000000F0000000E, relocInfo::none); 856 // Q 857 __ emit_data64(0x0000000000000000, relocInfo::none); 858 __ emit_data64(0x0000000000000001, relocInfo::none); 859 __ emit_data64(0x0000000000000002, relocInfo::none); 860 __ emit_data64(0x0000000000000003, relocInfo::none); 861 __ emit_data64(0x0000000000000004, relocInfo::none); 862 __ emit_data64(0x0000000000000005, relocInfo::none); 863 __ emit_data64(0x0000000000000006, relocInfo::none); 864 __ emit_data64(0x0000000000000007, relocInfo::none); 865 // D - FP 866 __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f 867 __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f 868 __ emit_data64(0x40A0000040800000, relocInfo::none); // 4.0f, 5.0f 869 __ emit_data64(0x40E0000040C00000, relocInfo::none); // 6.0f, 7.0f 870 __ emit_data64(0x4110000041000000, relocInfo::none); // 8.0f, 9.0f 871 __ emit_data64(0x4130000041200000, relocInfo::none); // 10.0f, 11.0f 872 __ emit_data64(0x4150000041400000, relocInfo::none); // 12.0f, 13.0f 873 __ emit_data64(0x4170000041600000, relocInfo::none); // 14.0f, 15.0f 874 // Q - FP 875 __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d 876 __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d 877 __ emit_data64(0x4000000000000000, relocInfo::none); // 2.0d 878 __ emit_data64(0x4008000000000000, relocInfo::none); // 3.0d 879 __ emit_data64(0x4010000000000000, relocInfo::none); // 4.0d 880 __ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d 881 __ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d 882 __ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d 883 return start; 884 } 885 886 address StubGenerator::generate_vector_reverse_bit_lut(const char *stub_name) { 887 __ align(CodeEntryAlignment); 888 StubCodeMark mark(this, "StubRoutines", stub_name); 889 address start = __ pc(); 890 891 __ emit_data64(0x0E060A020C040800, relocInfo::none); 892 __ emit_data64(0x0F070B030D050901, relocInfo::none); 893 __ emit_data64(0x0E060A020C040800, relocInfo::none); 894 __ emit_data64(0x0F070B030D050901, relocInfo::none); 895 __ emit_data64(0x0E060A020C040800, relocInfo::none); 896 __ emit_data64(0x0F070B030D050901, relocInfo::none); 897 __ emit_data64(0x0E060A020C040800, relocInfo::none); 898 __ emit_data64(0x0F070B030D050901, relocInfo::none); 899 900 return start; 901 } 902 903 address StubGenerator::generate_vector_reverse_byte_perm_mask_long(const char *stub_name) { 904 __ align(CodeEntryAlignment); 905 StubCodeMark mark(this, "StubRoutines", stub_name); 906 address start = __ pc(); 907 908 __ emit_data64(0x0001020304050607, relocInfo::none); 909 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 910 __ emit_data64(0x0001020304050607, relocInfo::none); 911 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 912 __ emit_data64(0x0001020304050607, relocInfo::none); 913 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 914 __ emit_data64(0x0001020304050607, relocInfo::none); 915 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 916 917 return start; 918 } 919 920 address StubGenerator::generate_vector_reverse_byte_perm_mask_int(const char *stub_name) { 921 __ align(CodeEntryAlignment); 922 StubCodeMark mark(this, "StubRoutines", stub_name); 923 address start = __ pc(); 924 925 __ emit_data64(0x0405060700010203, relocInfo::none); 926 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 927 __ emit_data64(0x0405060700010203, relocInfo::none); 928 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 929 __ emit_data64(0x0405060700010203, relocInfo::none); 930 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 931 __ emit_data64(0x0405060700010203, relocInfo::none); 932 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 933 934 return start; 935 } 936 937 address StubGenerator::generate_vector_reverse_byte_perm_mask_short(const char *stub_name) { 938 __ align(CodeEntryAlignment); 939 StubCodeMark mark(this, "StubRoutines", stub_name); 940 address start = __ pc(); 941 942 __ emit_data64(0x0607040502030001, relocInfo::none); 943 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 944 __ emit_data64(0x0607040502030001, relocInfo::none); 945 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 946 __ emit_data64(0x0607040502030001, relocInfo::none); 947 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 948 __ emit_data64(0x0607040502030001, relocInfo::none); 949 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 950 951 return start; 952 } 953 954 address StubGenerator::generate_vector_byte_shuffle_mask(const char *stub_name) { 955 __ align(CodeEntryAlignment); 956 StubCodeMark mark(this, "StubRoutines", stub_name); 957 address start = __ pc(); 958 959 __ emit_data64(0x7070707070707070, relocInfo::none); 960 __ emit_data64(0x7070707070707070, relocInfo::none); 961 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 962 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 963 964 return start; 965 } 966 967 address StubGenerator::generate_fp_mask(const char *stub_name, int64_t mask) { 968 __ align(CodeEntryAlignment); 969 StubCodeMark mark(this, "StubRoutines", stub_name); 970 address start = __ pc(); 971 972 __ emit_data64( mask, relocInfo::none ); 973 __ emit_data64( mask, relocInfo::none ); 974 975 return start; 976 } 977 978 address StubGenerator::generate_vector_mask(const char *stub_name, int64_t mask) { 979 __ align(CodeEntryAlignment); 980 StubCodeMark mark(this, "StubRoutines", stub_name); 981 address start = __ pc(); 982 983 __ emit_data64(mask, relocInfo::none); 984 __ emit_data64(mask, relocInfo::none); 985 __ emit_data64(mask, relocInfo::none); 986 __ emit_data64(mask, relocInfo::none); 987 __ emit_data64(mask, relocInfo::none); 988 __ emit_data64(mask, relocInfo::none); 989 __ emit_data64(mask, relocInfo::none); 990 __ emit_data64(mask, relocInfo::none); 991 992 return start; 993 } 994 995 address StubGenerator::generate_vector_byte_perm_mask(const char *stub_name) { 996 __ align(CodeEntryAlignment); 997 StubCodeMark mark(this, "StubRoutines", stub_name); 998 address start = __ pc(); 999 1000 __ emit_data64(0x0000000000000001, relocInfo::none); 1001 __ emit_data64(0x0000000000000003, relocInfo::none); 1002 __ emit_data64(0x0000000000000005, relocInfo::none); 1003 __ emit_data64(0x0000000000000007, relocInfo::none); 1004 __ emit_data64(0x0000000000000000, relocInfo::none); 1005 __ emit_data64(0x0000000000000002, relocInfo::none); 1006 __ emit_data64(0x0000000000000004, relocInfo::none); 1007 __ emit_data64(0x0000000000000006, relocInfo::none); 1008 1009 return start; 1010 } 1011 1012 address StubGenerator::generate_vector_fp_mask(const char *stub_name, int64_t mask) { 1013 __ align(CodeEntryAlignment); 1014 StubCodeMark mark(this, "StubRoutines", stub_name); 1015 address start = __ pc(); 1016 1017 __ emit_data64(mask, relocInfo::none); 1018 __ emit_data64(mask, relocInfo::none); 1019 __ emit_data64(mask, relocInfo::none); 1020 __ emit_data64(mask, relocInfo::none); 1021 __ emit_data64(mask, relocInfo::none); 1022 __ emit_data64(mask, relocInfo::none); 1023 __ emit_data64(mask, relocInfo::none); 1024 __ emit_data64(mask, relocInfo::none); 1025 1026 return start; 1027 } 1028 1029 address StubGenerator::generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len, 1030 int32_t val0, int32_t val1, int32_t val2, int32_t val3, 1031 int32_t val4, int32_t val5, int32_t val6, int32_t val7, 1032 int32_t val8, int32_t val9, int32_t val10, int32_t val11, 1033 int32_t val12, int32_t val13, int32_t val14, int32_t val15) { 1034 __ align(CodeEntryAlignment); 1035 StubCodeMark mark(this, "StubRoutines", stub_name); 1036 address start = __ pc(); 1037 1038 assert(len != Assembler::AVX_NoVec, "vector len must be specified"); 1039 __ emit_data(val0, relocInfo::none, 0); 1040 __ emit_data(val1, relocInfo::none, 0); 1041 __ emit_data(val2, relocInfo::none, 0); 1042 __ emit_data(val3, relocInfo::none, 0); 1043 if (len >= Assembler::AVX_256bit) { 1044 __ emit_data(val4, relocInfo::none, 0); 1045 __ emit_data(val5, relocInfo::none, 0); 1046 __ emit_data(val6, relocInfo::none, 0); 1047 __ emit_data(val7, relocInfo::none, 0); 1048 if (len >= Assembler::AVX_512bit) { 1049 __ emit_data(val8, relocInfo::none, 0); 1050 __ emit_data(val9, relocInfo::none, 0); 1051 __ emit_data(val10, relocInfo::none, 0); 1052 __ emit_data(val11, relocInfo::none, 0); 1053 __ emit_data(val12, relocInfo::none, 0); 1054 __ emit_data(val13, relocInfo::none, 0); 1055 __ emit_data(val14, relocInfo::none, 0); 1056 __ emit_data(val15, relocInfo::none, 0); 1057 } 1058 } 1059 return start; 1060 } 1061 1062 // Non-destructive plausibility checks for oops 1063 // 1064 // Arguments: 1065 // all args on stack! 1066 // 1067 // Stack after saving c_rarg3: 1068 // [tos + 0]: saved c_rarg3 1069 // [tos + 1]: saved c_rarg2 1070 // [tos + 2]: saved r12 (several TemplateTable methods use it) 1071 // [tos + 3]: saved flags 1072 // [tos + 4]: return address 1073 // * [tos + 5]: error message (char*) 1074 // * [tos + 6]: object to verify (oop) 1075 // * [tos + 7]: saved rax - saved by caller and bashed 1076 // * [tos + 8]: saved r10 (rscratch1) - saved by caller 1077 // * = popped on exit 1078 address StubGenerator::generate_verify_oop() { 1079 StubCodeMark mark(this, "StubRoutines", "verify_oop"); 1080 address start = __ pc(); 1081 1082 Label exit, error; 1083 1084 __ pushf(); 1085 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()), rscratch1); 1086 1087 __ push(r12); 1088 1089 // save c_rarg2 and c_rarg3 1090 __ push(c_rarg2); 1091 __ push(c_rarg3); 1092 1093 enum { 1094 // After previous pushes. 1095 oop_to_verify = 6 * wordSize, 1096 saved_rax = 7 * wordSize, 1097 saved_r10 = 8 * wordSize, 1098 1099 // Before the call to MacroAssembler::debug(), see below. 1100 return_addr = 16 * wordSize, 1101 error_msg = 17 * wordSize 1102 }; 1103 1104 // get object 1105 __ movptr(rax, Address(rsp, oop_to_verify)); 1106 1107 // make sure object is 'reasonable' 1108 __ testptr(rax, rax); 1109 __ jcc(Assembler::zero, exit); // if obj is null it is OK 1110 1111 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); 1112 bs_asm->check_oop(_masm, rax, c_rarg2, c_rarg3, error); 1113 1114 // return if everything seems ok 1115 __ bind(exit); 1116 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1117 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1118 __ pop(c_rarg3); // restore c_rarg3 1119 __ pop(c_rarg2); // restore c_rarg2 1120 __ pop(r12); // restore r12 1121 __ popf(); // restore flags 1122 __ ret(4 * wordSize); // pop caller saved stuff 1123 1124 // handle errors 1125 __ bind(error); 1126 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1127 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1128 __ pop(c_rarg3); // get saved c_rarg3 back 1129 __ pop(c_rarg2); // get saved c_rarg2 back 1130 __ pop(r12); // get saved r12 back 1131 __ popf(); // get saved flags off stack -- 1132 // will be ignored 1133 1134 __ pusha(); // push registers 1135 // (rip is already 1136 // already pushed) 1137 // debug(char* msg, int64_t pc, int64_t regs[]) 1138 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and 1139 // pushed all the registers, so now the stack looks like: 1140 // [tos + 0] 16 saved registers 1141 // [tos + 16] return address 1142 // * [tos + 17] error message (char*) 1143 // * [tos + 18] object to verify (oop) 1144 // * [tos + 19] saved rax - saved by caller and bashed 1145 // * [tos + 20] saved r10 (rscratch1) - saved by caller 1146 // * = popped on exit 1147 1148 __ movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message 1149 __ movptr(c_rarg1, Address(rsp, return_addr)); // pass return address 1150 __ movq(c_rarg2, rsp); // pass address of regs on stack 1151 __ mov(r12, rsp); // remember rsp 1152 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 1153 __ andptr(rsp, -16); // align stack as required by ABI 1154 BLOCK_COMMENT("call MacroAssembler::debug"); 1155 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 1156 __ hlt(); 1157 1158 return start; 1159 } 1160 1161 1162 // Shuffle first three arg regs on Windows into Linux/Solaris locations. 1163 // 1164 // Outputs: 1165 // rdi - rcx 1166 // rsi - rdx 1167 // rdx - r8 1168 // rcx - r9 1169 // 1170 // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter 1171 // are non-volatile. r9 and r10 should not be used by the caller. 1172 // 1173 void StubGenerator::setup_arg_regs(int nargs) { 1174 const Register saved_rdi = r9; 1175 const Register saved_rsi = r10; 1176 assert(nargs == 3 || nargs == 4, "else fix"); 1177 #ifdef _WIN64 1178 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1179 "unexpected argument registers"); 1180 if (nargs == 4) { 1181 __ mov(rax, r9); // r9 is also saved_rdi 1182 } 1183 __ movptr(saved_rdi, rdi); 1184 __ movptr(saved_rsi, rsi); 1185 __ mov(rdi, rcx); // c_rarg0 1186 __ mov(rsi, rdx); // c_rarg1 1187 __ mov(rdx, r8); // c_rarg2 1188 if (nargs == 4) { 1189 __ mov(rcx, rax); // c_rarg3 (via rax) 1190 } 1191 #else 1192 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1193 "unexpected argument registers"); 1194 #endif 1195 DEBUG_ONLY(_regs_in_thread = false;) 1196 } 1197 1198 1199 void StubGenerator::restore_arg_regs() { 1200 assert(!_regs_in_thread, "wrong call to restore_arg_regs"); 1201 const Register saved_rdi = r9; 1202 const Register saved_rsi = r10; 1203 #ifdef _WIN64 1204 __ movptr(rdi, saved_rdi); 1205 __ movptr(rsi, saved_rsi); 1206 #endif 1207 } 1208 1209 1210 // This is used in places where r10 is a scratch register, and can 1211 // be adapted if r9 is needed also. 1212 void StubGenerator::setup_arg_regs_using_thread(int nargs) { 1213 const Register saved_r15 = r9; 1214 assert(nargs == 3 || nargs == 4, "else fix"); 1215 #ifdef _WIN64 1216 if (nargs == 4) { 1217 __ mov(rax, r9); // r9 is also saved_r15 1218 } 1219 __ mov(saved_r15, r15); // r15 is callee saved and needs to be restored 1220 __ get_thread(r15_thread); 1221 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1222 "unexpected argument registers"); 1223 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi); 1224 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi); 1225 1226 __ mov(rdi, rcx); // c_rarg0 1227 __ mov(rsi, rdx); // c_rarg1 1228 __ mov(rdx, r8); // c_rarg2 1229 if (nargs == 4) { 1230 __ mov(rcx, rax); // c_rarg3 (via rax) 1231 } 1232 #else 1233 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1234 "unexpected argument registers"); 1235 #endif 1236 DEBUG_ONLY(_regs_in_thread = true;) 1237 } 1238 1239 1240 void StubGenerator::restore_arg_regs_using_thread() { 1241 assert(_regs_in_thread, "wrong call to restore_arg_regs"); 1242 const Register saved_r15 = r9; 1243 #ifdef _WIN64 1244 __ get_thread(r15_thread); 1245 __ movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset()))); 1246 __ movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset()))); 1247 __ mov(r15, saved_r15); // r15 is callee saved and needs to be restored 1248 #endif 1249 } 1250 1251 1252 void StubGenerator::setup_argument_regs(BasicType type) { 1253 if (type == T_BYTE || type == T_SHORT) { 1254 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1255 // r9 and r10 may be used to save non-volatile registers 1256 } else { 1257 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx 1258 // r9 is used to save r15_thread 1259 } 1260 } 1261 1262 1263 void StubGenerator::restore_argument_regs(BasicType type) { 1264 if (type == T_BYTE || type == T_SHORT) { 1265 restore_arg_regs(); 1266 } else { 1267 restore_arg_regs_using_thread(); 1268 } 1269 } 1270 1271 address StubGenerator::generate_data_cache_writeback() { 1272 const Register src = c_rarg0; // source address 1273 1274 __ align(CodeEntryAlignment); 1275 1276 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback"); 1277 1278 address start = __ pc(); 1279 1280 __ enter(); 1281 __ cache_wb(Address(src, 0)); 1282 __ leave(); 1283 __ ret(0); 1284 1285 return start; 1286 } 1287 1288 address StubGenerator::generate_data_cache_writeback_sync() { 1289 const Register is_pre = c_rarg0; // pre or post sync 1290 1291 __ align(CodeEntryAlignment); 1292 1293 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync"); 1294 1295 // pre wbsync is a no-op 1296 // post wbsync translates to an sfence 1297 1298 Label skip; 1299 address start = __ pc(); 1300 1301 __ enter(); 1302 __ cmpl(is_pre, 0); 1303 __ jcc(Assembler::notEqual, skip); 1304 __ cache_wbsync(false); 1305 __ bind(skip); 1306 __ leave(); 1307 __ ret(0); 1308 1309 return start; 1310 } 1311 1312 // ofs and limit are use for multi-block byte array. 1313 // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs) 1314 address StubGenerator::generate_md5_implCompress(bool multi_block, const char *name) { 1315 __ align(CodeEntryAlignment); 1316 StubCodeMark mark(this, "StubRoutines", name); 1317 address start = __ pc(); 1318 1319 const Register buf_param = r15; 1320 const Address state_param(rsp, 0 * wordSize); 1321 const Address ofs_param (rsp, 1 * wordSize ); 1322 const Address limit_param(rsp, 1 * wordSize + 4); 1323 1324 __ enter(); 1325 __ push(rbx); 1326 __ push(rdi); 1327 __ push(rsi); 1328 __ push(r15); 1329 __ subptr(rsp, 2 * wordSize); 1330 1331 __ movptr(buf_param, c_rarg0); 1332 __ movptr(state_param, c_rarg1); 1333 if (multi_block) { 1334 __ movl(ofs_param, c_rarg2); 1335 __ movl(limit_param, c_rarg3); 1336 } 1337 __ fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block); 1338 1339 __ addptr(rsp, 2 * wordSize); 1340 __ pop(r15); 1341 __ pop(rsi); 1342 __ pop(rdi); 1343 __ pop(rbx); 1344 __ leave(); 1345 __ ret(0); 1346 1347 return start; 1348 } 1349 1350 address StubGenerator::generate_upper_word_mask() { 1351 __ align64(); 1352 StubCodeMark mark(this, "StubRoutines", "upper_word_mask"); 1353 address start = __ pc(); 1354 1355 __ emit_data64(0x0000000000000000, relocInfo::none); 1356 __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); 1357 1358 return start; 1359 } 1360 1361 address StubGenerator::generate_shuffle_byte_flip_mask() { 1362 __ align64(); 1363 StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask"); 1364 address start = __ pc(); 1365 1366 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1367 __ emit_data64(0x0001020304050607, relocInfo::none); 1368 1369 return start; 1370 } 1371 1372 // ofs and limit are use for multi-block byte array. 1373 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1374 address StubGenerator::generate_sha1_implCompress(bool multi_block, const char *name) { 1375 __ align(CodeEntryAlignment); 1376 StubCodeMark mark(this, "StubRoutines", name); 1377 address start = __ pc(); 1378 1379 Register buf = c_rarg0; 1380 Register state = c_rarg1; 1381 Register ofs = c_rarg2; 1382 Register limit = c_rarg3; 1383 1384 const XMMRegister abcd = xmm0; 1385 const XMMRegister e0 = xmm1; 1386 const XMMRegister e1 = xmm2; 1387 const XMMRegister msg0 = xmm3; 1388 1389 const XMMRegister msg1 = xmm4; 1390 const XMMRegister msg2 = xmm5; 1391 const XMMRegister msg3 = xmm6; 1392 const XMMRegister shuf_mask = xmm7; 1393 1394 __ enter(); 1395 1396 __ subptr(rsp, 4 * wordSize); 1397 1398 __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask, 1399 buf, state, ofs, limit, rsp, multi_block); 1400 1401 __ addptr(rsp, 4 * wordSize); 1402 1403 __ leave(); 1404 __ ret(0); 1405 1406 return start; 1407 } 1408 1409 address StubGenerator::generate_pshuffle_byte_flip_mask() { 1410 __ align64(); 1411 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask"); 1412 address start = __ pc(); 1413 1414 __ emit_data64(0x0405060700010203, relocInfo::none); 1415 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1416 1417 if (VM_Version::supports_avx2()) { 1418 __ emit_data64(0x0405060700010203, relocInfo::none); // second copy 1419 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1420 // _SHUF_00BA 1421 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1422 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1423 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1424 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1425 // _SHUF_DC00 1426 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1427 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1428 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1429 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1430 } 1431 1432 return start; 1433 } 1434 1435 //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 1436 address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() { 1437 __ align32(); 1438 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512"); 1439 address start = __ pc(); 1440 1441 if (VM_Version::supports_avx2()) { 1442 __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK 1443 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1444 __ emit_data64(0x1011121314151617, relocInfo::none); 1445 __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); 1446 __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO 1447 __ emit_data64(0x0000000000000000, relocInfo::none); 1448 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1449 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1450 } 1451 1452 return start; 1453 } 1454 1455 // ofs and limit are use for multi-block byte array. 1456 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1457 address StubGenerator::generate_sha256_implCompress(bool multi_block, const char *name) { 1458 assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); 1459 __ align(CodeEntryAlignment); 1460 StubCodeMark mark(this, "StubRoutines", name); 1461 address start = __ pc(); 1462 1463 Register buf = c_rarg0; 1464 Register state = c_rarg1; 1465 Register ofs = c_rarg2; 1466 Register limit = c_rarg3; 1467 1468 const XMMRegister msg = xmm0; 1469 const XMMRegister state0 = xmm1; 1470 const XMMRegister state1 = xmm2; 1471 const XMMRegister msgtmp0 = xmm3; 1472 1473 const XMMRegister msgtmp1 = xmm4; 1474 const XMMRegister msgtmp2 = xmm5; 1475 const XMMRegister msgtmp3 = xmm6; 1476 const XMMRegister msgtmp4 = xmm7; 1477 1478 const XMMRegister shuf_mask = xmm8; 1479 1480 __ enter(); 1481 1482 __ subptr(rsp, 4 * wordSize); 1483 1484 if (VM_Version::supports_sha()) { 1485 __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1486 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1487 } else if (VM_Version::supports_avx2()) { 1488 __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1489 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1490 } 1491 __ addptr(rsp, 4 * wordSize); 1492 __ vzeroupper(); 1493 __ leave(); 1494 __ ret(0); 1495 1496 return start; 1497 } 1498 1499 address StubGenerator::generate_sha512_implCompress(bool multi_block, const char *name) { 1500 assert(VM_Version::supports_avx2(), ""); 1501 assert(VM_Version::supports_bmi2(), ""); 1502 __ align(CodeEntryAlignment); 1503 StubCodeMark mark(this, "StubRoutines", name); 1504 address start = __ pc(); 1505 1506 Register buf = c_rarg0; 1507 Register state = c_rarg1; 1508 Register ofs = c_rarg2; 1509 Register limit = c_rarg3; 1510 1511 const XMMRegister msg = xmm0; 1512 const XMMRegister state0 = xmm1; 1513 const XMMRegister state1 = xmm2; 1514 const XMMRegister msgtmp0 = xmm3; 1515 const XMMRegister msgtmp1 = xmm4; 1516 const XMMRegister msgtmp2 = xmm5; 1517 const XMMRegister msgtmp3 = xmm6; 1518 const XMMRegister msgtmp4 = xmm7; 1519 1520 const XMMRegister shuf_mask = xmm8; 1521 1522 __ enter(); 1523 1524 __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1525 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1526 1527 __ vzeroupper(); 1528 __ leave(); 1529 __ ret(0); 1530 1531 return start; 1532 } 1533 1534 address StubGenerator::base64_shuffle_addr() { 1535 __ align64(); 1536 StubCodeMark mark(this, "StubRoutines", "shuffle_base64"); 1537 address start = __ pc(); 1538 1539 assert(((unsigned long long)start & 0x3f) == 0, 1540 "Alignment problem (0x%08llx)", (unsigned long long)start); 1541 __ emit_data64(0x0405030401020001, relocInfo::none); 1542 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1543 __ emit_data64(0x10110f100d0e0c0d, relocInfo::none); 1544 __ emit_data64(0x1617151613141213, relocInfo::none); 1545 __ emit_data64(0x1c1d1b1c191a1819, relocInfo::none); 1546 __ emit_data64(0x222321221f201e1f, relocInfo::none); 1547 __ emit_data64(0x2829272825262425, relocInfo::none); 1548 __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); 1549 1550 return start; 1551 } 1552 1553 address StubGenerator::base64_avx2_shuffle_addr() { 1554 __ align32(); 1555 StubCodeMark mark(this, "StubRoutines", "avx2_shuffle_base64"); 1556 address start = __ pc(); 1557 1558 __ emit_data64(0x0809070805060405, relocInfo::none); 1559 __ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none); 1560 __ emit_data64(0x0405030401020001, relocInfo::none); 1561 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1562 1563 return start; 1564 } 1565 1566 address StubGenerator::base64_avx2_input_mask_addr() { 1567 __ align32(); 1568 StubCodeMark mark(this, "StubRoutines", "avx2_input_mask_base64"); 1569 address start = __ pc(); 1570 1571 __ emit_data64(0x8000000000000000, relocInfo::none); 1572 __ emit_data64(0x8000000080000000, relocInfo::none); 1573 __ emit_data64(0x8000000080000000, relocInfo::none); 1574 __ emit_data64(0x8000000080000000, relocInfo::none); 1575 1576 return start; 1577 } 1578 1579 address StubGenerator::base64_avx2_lut_addr() { 1580 __ align32(); 1581 StubCodeMark mark(this, "StubRoutines", "avx2_lut_base64"); 1582 address start = __ pc(); 1583 1584 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1585 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1586 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1587 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1588 1589 // URL LUT 1590 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1591 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1592 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1593 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1594 1595 return start; 1596 } 1597 1598 address StubGenerator::base64_encoding_table_addr() { 1599 __ align64(); 1600 StubCodeMark mark(this, "StubRoutines", "encoding_table_base64"); 1601 address start = __ pc(); 1602 1603 assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); 1604 __ emit_data64(0x4847464544434241, relocInfo::none); 1605 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1606 __ emit_data64(0x5857565554535251, relocInfo::none); 1607 __ emit_data64(0x6665646362615a59, relocInfo::none); 1608 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1609 __ emit_data64(0x767574737271706f, relocInfo::none); 1610 __ emit_data64(0x333231307a797877, relocInfo::none); 1611 __ emit_data64(0x2f2b393837363534, relocInfo::none); 1612 1613 // URL table 1614 __ emit_data64(0x4847464544434241, relocInfo::none); 1615 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1616 __ emit_data64(0x5857565554535251, relocInfo::none); 1617 __ emit_data64(0x6665646362615a59, relocInfo::none); 1618 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1619 __ emit_data64(0x767574737271706f, relocInfo::none); 1620 __ emit_data64(0x333231307a797877, relocInfo::none); 1621 __ emit_data64(0x5f2d393837363534, relocInfo::none); 1622 1623 return start; 1624 } 1625 1626 // Code for generating Base64 encoding. 1627 // Intrinsic function prototype in Base64.java: 1628 // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, 1629 // boolean isURL) { 1630 address StubGenerator::generate_base64_encodeBlock() 1631 { 1632 __ align(CodeEntryAlignment); 1633 StubCodeMark mark(this, "StubRoutines", "implEncode"); 1634 address start = __ pc(); 1635 1636 __ enter(); 1637 1638 // Save callee-saved registers before using them 1639 __ push(r12); 1640 __ push(r13); 1641 __ push(r14); 1642 __ push(r15); 1643 1644 // arguments 1645 const Register source = c_rarg0; // Source Array 1646 const Register start_offset = c_rarg1; // start offset 1647 const Register end_offset = c_rarg2; // end offset 1648 const Register dest = c_rarg3; // destination array 1649 1650 #ifndef _WIN64 1651 const Register dp = c_rarg4; // Position for writing to dest array 1652 const Register isURL = c_rarg5; // Base64 or URL character set 1653 #else 1654 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 1655 const Address isURL_mem(rbp, 7 * wordSize); 1656 const Register isURL = r10; // pick the volatile windows register 1657 const Register dp = r12; 1658 __ movl(dp, dp_mem); 1659 __ movl(isURL, isURL_mem); 1660 #endif 1661 1662 const Register length = r14; 1663 const Register encode_table = r13; 1664 Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop; 1665 1666 // calculate length from offsets 1667 __ movl(length, end_offset); 1668 __ subl(length, start_offset); 1669 __ jcc(Assembler::lessEqual, L_exit); 1670 1671 // Code for 512-bit VBMI encoding. Encodes 48 input bytes into 64 1672 // output bytes. We read 64 input bytes and ignore the last 16, so be 1673 // sure not to read past the end of the input buffer. 1674 if (VM_Version::supports_avx512_vbmi()) { 1675 __ cmpl(length, 64); // Do not overrun input buffer. 1676 __ jcc(Assembler::below, L_not512); 1677 1678 __ shll(isURL, 6); // index into decode table based on isURL 1679 __ lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 1680 __ addptr(encode_table, isURL); 1681 __ shrl(isURL, 6); // restore isURL 1682 1683 __ mov64(rax, 0x3036242a1016040aull); // Shifts 1684 __ evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15); 1685 __ evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit); 1686 __ evpbroadcastq(xmm1, rax, Assembler::AVX_512bit); 1687 1688 __ align32(); 1689 __ BIND(L_vbmiLoop); 1690 1691 __ vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit); 1692 __ subl(length, 48); 1693 1694 // Put the input bytes into the proper lanes for writing, then 1695 // encode them. 1696 __ evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit); 1697 __ vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit); 1698 1699 // Write to destination 1700 __ evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit); 1701 1702 __ addptr(dest, 64); 1703 __ addptr(source, 48); 1704 __ cmpl(length, 64); 1705 __ jcc(Assembler::aboveEqual, L_vbmiLoop); 1706 1707 __ vzeroupper(); 1708 } 1709 1710 __ BIND(L_not512); 1711 if (VM_Version::supports_avx2()) { 1712 /* 1713 ** This AVX2 encoder is based off the paper at: 1714 ** https://dl.acm.org/doi/10.1145/3132709 1715 ** 1716 ** We use AVX2 SIMD instructions to encode 24 bytes into 32 1717 ** output bytes. 1718 ** 1719 */ 1720 // Lengths under 32 bytes are done with scalar routine 1721 __ cmpl(length, 31); 1722 __ jcc(Assembler::belowEqual, L_process3); 1723 1724 // Set up supporting constant table data 1725 __ vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax); 1726 // 6-bit mask for 2nd and 4th (and multiples) 6-bit values 1727 __ movl(rax, 0x0fc0fc00); 1728 __ movdl(xmm8, rax); 1729 __ vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax); 1730 __ vpbroadcastd(xmm8, xmm8, Assembler::AVX_256bit); 1731 1732 // Multiplication constant for "shifting" right by 6 and 10 1733 // bits 1734 __ movl(rax, 0x04000040); 1735 1736 __ subl(length, 24); 1737 __ movdl(xmm7, rax); 1738 __ vpbroadcastd(xmm7, xmm7, Assembler::AVX_256bit); 1739 1740 // For the first load, we mask off reading of the first 4 1741 // bytes into the register. This is so we can get 4 3-byte 1742 // chunks into each lane of the register, avoiding having to 1743 // handle end conditions. We then shuffle these bytes into a 1744 // specific order so that manipulation is easier. 1745 // 1746 // The initial read loads the XMM register like this: 1747 // 1748 // Lower 128-bit lane: 1749 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1750 // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1 1751 // | C2 | D0 | D1 | D2 | 1752 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1753 // 1754 // Upper 128-bit lane: 1755 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1756 // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2 1757 // | XX | XX | XX | XX | 1758 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1759 // 1760 // Where A0 is the first input byte, B0 is the fourth, etc. 1761 // The alphabetical significance denotes the 3 bytes to be 1762 // consumed and encoded into 4 bytes. 1763 // 1764 // We then shuffle the register so each 32-bit word contains 1765 // the sequence: 1766 // A1 A0 A2 A1, B1, B0, B2, B1, etc. 1767 // Each of these byte sequences are then manipulated into 4 1768 // 6-bit values ready for encoding. 1769 // 1770 // If we focus on one set of 3-byte chunks, changing the 1771 // nomenclature such that A0 => a, A1 => b, and A2 => c, we 1772 // shuffle such that each 24-bit chunk contains: 1773 // 1774 // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6 1775 // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0 1776 // Explain this step. 1777 // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4 1778 // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2 1779 // 1780 // W first and off all but bits 4-9 and 16-21 (c5..c0 and 1781 // a5..a0) and shift them using a vector multiplication 1782 // operation (vpmulhuw) which effectively shifts c right by 6 1783 // bits and a right by 10 bits. We similarly mask bits 10-15 1784 // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4 1785 // bits respectively. This is done using vpmullw. We end up 1786 // with 4 6-bit values, thus splitting the 3 input bytes, 1787 // ready for encoding: 1788 // 0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0 1789 // 1790 // For translation, we recognize that there are 5 distinct 1791 // ranges of legal Base64 characters as below: 1792 // 1793 // +-------------+-------------+------------+ 1794 // | 6-bit value | ASCII range | offset | 1795 // +-------------+-------------+------------+ 1796 // | 0..25 | A..Z | 65 | 1797 // | 26..51 | a..z | 71 | 1798 // | 52..61 | 0..9 | -4 | 1799 // | 62 | + or - | -19 or -17 | 1800 // | 63 | / or _ | -16 or 32 | 1801 // +-------------+-------------+------------+ 1802 // 1803 // We note that vpshufb does a parallel lookup in a 1804 // destination register using the lower 4 bits of bytes from a 1805 // source register. If we use a saturated subtraction and 1806 // subtract 51 from each 6-bit value, bytes from [0,51] 1807 // saturate to 0, and [52,63] map to a range of [1,12]. We 1808 // distinguish the [0,25] and [26,51] ranges by assigning a 1809 // value of 13 for all 6-bit values less than 26. We end up 1810 // with: 1811 // 1812 // +-------------+-------------+------------+ 1813 // | 6-bit value | Reduced | offset | 1814 // +-------------+-------------+------------+ 1815 // | 0..25 | 13 | 65 | 1816 // | 26..51 | 0 | 71 | 1817 // | 52..61 | 0..9 | -4 | 1818 // | 62 | 11 | -19 or -17 | 1819 // | 63 | 12 | -16 or 32 | 1820 // +-------------+-------------+------------+ 1821 // 1822 // We then use a final vpshufb to add the appropriate offset, 1823 // translating the bytes. 1824 // 1825 // Load input bytes - only 28 bytes. Mask the first load to 1826 // not load into the full register. 1827 __ vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit); 1828 1829 // Move 3-byte chunks of input (12 bytes) into 16 bytes, 1830 // ordering by: 1831 // 1, 0, 2, 1; 4, 3, 5, 4; etc. This groups 6-bit chunks 1832 // for easy masking 1833 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 1834 1835 __ addl(start_offset, 24); 1836 1837 // Load masking register for first and third (and multiples) 1838 // 6-bit values. 1839 __ movl(rax, 0x003f03f0); 1840 __ movdl(xmm6, rax); 1841 __ vpbroadcastd(xmm6, xmm6, Assembler::AVX_256bit); 1842 // Multiplication constant for "shifting" left by 4 and 8 bits 1843 __ movl(rax, 0x01000010); 1844 __ movdl(xmm5, rax); 1845 __ vpbroadcastd(xmm5, xmm5, Assembler::AVX_256bit); 1846 1847 // Isolate 6-bit chunks of interest 1848 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 1849 1850 // Load constants for encoding 1851 __ movl(rax, 0x19191919); 1852 __ movdl(xmm3, rax); 1853 __ vpbroadcastd(xmm3, xmm3, Assembler::AVX_256bit); 1854 __ movl(rax, 0x33333333); 1855 __ movdl(xmm4, rax); 1856 __ vpbroadcastd(xmm4, xmm4, Assembler::AVX_256bit); 1857 1858 // Shift output bytes 0 and 2 into proper lanes 1859 __ vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit); 1860 1861 // Mask and shift output bytes 1 and 3 into proper lanes and 1862 // combine 1863 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 1864 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 1865 __ vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 1866 1867 // Find out which are 0..25. This indicates which input 1868 // values fall in the range of 'A'-'Z', which require an 1869 // additional offset (see comments above) 1870 __ vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit); 1871 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 1872 __ vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit); 1873 1874 // Load the proper lookup table 1875 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr())); 1876 __ movl(r15, isURL); 1877 __ shll(r15, 5); 1878 __ vmovdqu(xmm2, Address(r11, r15)); 1879 1880 // Shuffle the offsets based on the range calculation done 1881 // above. This allows us to add the correct offset to the 1882 // 6-bit value corresponding to the range documented above. 1883 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 1884 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 1885 1886 // Store the encoded bytes 1887 __ vmovdqu(Address(dest, dp), xmm0); 1888 __ addl(dp, 32); 1889 1890 __ cmpl(length, 31); 1891 __ jcc(Assembler::belowEqual, L_process3); 1892 1893 __ align32(); 1894 __ BIND(L_32byteLoop); 1895 1896 // Get next 32 bytes 1897 __ vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4)); 1898 1899 __ subl(length, 24); 1900 __ addl(start_offset, 24); 1901 1902 // This logic is identical to the above, with only constant 1903 // register loads removed. Shuffle the input, mask off 6-bit 1904 // chunks, shift them into place, then add the offset to 1905 // encode. 1906 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 1907 1908 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 1909 __ vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit); 1910 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 1911 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 1912 __ vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 1913 __ vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit); 1914 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 1915 __ vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit); 1916 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 1917 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 1918 1919 // Store the encoded bytes 1920 __ vmovdqu(Address(dest, dp), xmm0); 1921 __ addl(dp, 32); 1922 1923 __ cmpl(length, 31); 1924 __ jcc(Assembler::above, L_32byteLoop); 1925 1926 __ BIND(L_process3); 1927 __ vzeroupper(); 1928 } else { 1929 __ BIND(L_process3); 1930 } 1931 1932 __ cmpl(length, 3); 1933 __ jcc(Assembler::below, L_exit); 1934 1935 // Load the encoding table based on isURL 1936 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 1937 __ movl(r15, isURL); 1938 __ shll(r15, 6); 1939 __ addptr(r11, r15); 1940 1941 __ BIND(L_processdata); 1942 1943 // Load 3 bytes 1944 __ load_unsigned_byte(r15, Address(source, start_offset)); 1945 __ load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1)); 1946 __ load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2)); 1947 1948 // Build a 32-bit word with bytes 1, 2, 0, 1 1949 __ movl(rax, r10); 1950 __ shll(r10, 24); 1951 __ orl(rax, r10); 1952 1953 __ subl(length, 3); 1954 1955 __ shll(r15, 8); 1956 __ shll(r13, 16); 1957 __ orl(rax, r15); 1958 1959 __ addl(start_offset, 3); 1960 1961 __ orl(rax, r13); 1962 // At this point, rax contains | byte1 | byte2 | byte0 | byte1 1963 // r13 has byte2 << 16 - need low-order 6 bits to translate. 1964 // This translated byte is the fourth output byte. 1965 __ shrl(r13, 16); 1966 __ andl(r13, 0x3f); 1967 1968 // The high-order 6 bits of r15 (byte0) is translated. 1969 // The translated byte is the first output byte. 1970 __ shrl(r15, 10); 1971 1972 __ load_unsigned_byte(r13, Address(r11, r13)); 1973 __ load_unsigned_byte(r15, Address(r11, r15)); 1974 1975 __ movb(Address(dest, dp, Address::times_1, 3), r13); 1976 1977 // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0. 1978 // This translated byte is the second output byte. 1979 __ shrl(rax, 4); 1980 __ movl(r10, rax); 1981 __ andl(rax, 0x3f); 1982 1983 __ movb(Address(dest, dp, Address::times_1, 0), r15); 1984 1985 __ load_unsigned_byte(rax, Address(r11, rax)); 1986 1987 // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2. 1988 // This translated byte is the third output byte. 1989 __ shrl(r10, 18); 1990 __ andl(r10, 0x3f); 1991 1992 __ load_unsigned_byte(r10, Address(r11, r10)); 1993 1994 __ movb(Address(dest, dp, Address::times_1, 1), rax); 1995 __ movb(Address(dest, dp, Address::times_1, 2), r10); 1996 1997 __ addl(dp, 4); 1998 __ cmpl(length, 3); 1999 __ jcc(Assembler::aboveEqual, L_processdata); 2000 2001 __ BIND(L_exit); 2002 __ pop(r15); 2003 __ pop(r14); 2004 __ pop(r13); 2005 __ pop(r12); 2006 __ leave(); 2007 __ ret(0); 2008 2009 return start; 2010 } 2011 2012 // base64 AVX512vbmi tables 2013 address StubGenerator::base64_vbmi_lookup_lo_addr() { 2014 __ align64(); 2015 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64"); 2016 address start = __ pc(); 2017 2018 assert(((unsigned long long)start & 0x3f) == 0, 2019 "Alignment problem (0x%08llx)", (unsigned long long)start); 2020 __ emit_data64(0x8080808080808080, relocInfo::none); 2021 __ emit_data64(0x8080808080808080, relocInfo::none); 2022 __ emit_data64(0x8080808080808080, relocInfo::none); 2023 __ emit_data64(0x8080808080808080, relocInfo::none); 2024 __ emit_data64(0x8080808080808080, relocInfo::none); 2025 __ emit_data64(0x3f8080803e808080, relocInfo::none); 2026 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2027 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2028 2029 return start; 2030 } 2031 2032 address StubGenerator::base64_vbmi_lookup_hi_addr() { 2033 __ align64(); 2034 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64"); 2035 address start = __ pc(); 2036 2037 assert(((unsigned long long)start & 0x3f) == 0, 2038 "Alignment problem (0x%08llx)", (unsigned long long)start); 2039 __ emit_data64(0x0605040302010080, relocInfo::none); 2040 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2041 __ emit_data64(0x161514131211100f, relocInfo::none); 2042 __ emit_data64(0x8080808080191817, relocInfo::none); 2043 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2044 __ emit_data64(0x2827262524232221, relocInfo::none); 2045 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2046 __ emit_data64(0x8080808080333231, relocInfo::none); 2047 2048 return start; 2049 } 2050 address StubGenerator::base64_vbmi_lookup_lo_url_addr() { 2051 __ align64(); 2052 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64url"); 2053 address start = __ pc(); 2054 2055 assert(((unsigned long long)start & 0x3f) == 0, 2056 "Alignment problem (0x%08llx)", (unsigned long long)start); 2057 __ emit_data64(0x8080808080808080, relocInfo::none); 2058 __ emit_data64(0x8080808080808080, relocInfo::none); 2059 __ emit_data64(0x8080808080808080, relocInfo::none); 2060 __ emit_data64(0x8080808080808080, relocInfo::none); 2061 __ emit_data64(0x8080808080808080, relocInfo::none); 2062 __ emit_data64(0x80803e8080808080, relocInfo::none); 2063 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2064 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2065 2066 return start; 2067 } 2068 2069 address StubGenerator::base64_vbmi_lookup_hi_url_addr() { 2070 __ align64(); 2071 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64url"); 2072 address start = __ pc(); 2073 2074 assert(((unsigned long long)start & 0x3f) == 0, 2075 "Alignment problem (0x%08llx)", (unsigned long long)start); 2076 __ emit_data64(0x0605040302010080, relocInfo::none); 2077 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2078 __ emit_data64(0x161514131211100f, relocInfo::none); 2079 __ emit_data64(0x3f80808080191817, relocInfo::none); 2080 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2081 __ emit_data64(0x2827262524232221, relocInfo::none); 2082 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2083 __ emit_data64(0x8080808080333231, relocInfo::none); 2084 2085 return start; 2086 } 2087 2088 address StubGenerator::base64_vbmi_pack_vec_addr() { 2089 __ align64(); 2090 StubCodeMark mark(this, "StubRoutines", "pack_vec_base64"); 2091 address start = __ pc(); 2092 2093 assert(((unsigned long long)start & 0x3f) == 0, 2094 "Alignment problem (0x%08llx)", (unsigned long long)start); 2095 __ emit_data64(0x090a040506000102, relocInfo::none); 2096 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2097 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2098 __ emit_data64(0x292a242526202122, relocInfo::none); 2099 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2100 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2101 __ emit_data64(0x0000000000000000, relocInfo::none); 2102 __ emit_data64(0x0000000000000000, relocInfo::none); 2103 2104 return start; 2105 } 2106 2107 address StubGenerator::base64_vbmi_join_0_1_addr() { 2108 __ align64(); 2109 StubCodeMark mark(this, "StubRoutines", "join_0_1_base64"); 2110 address start = __ pc(); 2111 2112 assert(((unsigned long long)start & 0x3f) == 0, 2113 "Alignment problem (0x%08llx)", (unsigned long long)start); 2114 __ emit_data64(0x090a040506000102, relocInfo::none); 2115 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2116 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2117 __ emit_data64(0x292a242526202122, relocInfo::none); 2118 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2119 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2120 __ emit_data64(0x494a444546404142, relocInfo::none); 2121 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2122 2123 return start; 2124 } 2125 2126 address StubGenerator::base64_vbmi_join_1_2_addr() { 2127 __ align64(); 2128 StubCodeMark mark(this, "StubRoutines", "join_1_2_base64"); 2129 address start = __ pc(); 2130 2131 assert(((unsigned long long)start & 0x3f) == 0, 2132 "Alignment problem (0x%08llx)", (unsigned long long)start); 2133 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2134 __ emit_data64(0x292a242526202122, relocInfo::none); 2135 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2136 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2137 __ emit_data64(0x494a444546404142, relocInfo::none); 2138 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2139 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2140 __ emit_data64(0x696a646566606162, relocInfo::none); 2141 2142 return start; 2143 } 2144 2145 address StubGenerator::base64_vbmi_join_2_3_addr() { 2146 __ align64(); 2147 StubCodeMark mark(this, "StubRoutines", "join_2_3_base64"); 2148 address start = __ pc(); 2149 2150 assert(((unsigned long long)start & 0x3f) == 0, 2151 "Alignment problem (0x%08llx)", (unsigned long long)start); 2152 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2153 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2154 __ emit_data64(0x494a444546404142, relocInfo::none); 2155 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2156 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2157 __ emit_data64(0x696a646566606162, relocInfo::none); 2158 __ emit_data64(0x767071726c6d6e68, relocInfo::none); 2159 __ emit_data64(0x7c7d7e78797a7475, relocInfo::none); 2160 2161 return start; 2162 } 2163 2164 address StubGenerator::base64_AVX2_decode_tables_addr() { 2165 __ align64(); 2166 StubCodeMark mark(this, "StubRoutines", "AVX2_tables_base64"); 2167 address start = __ pc(); 2168 2169 assert(((unsigned long long)start & 0x3f) == 0, 2170 "Alignment problem (0x%08llx)", (unsigned long long)start); 2171 __ emit_data(0x2f2f2f2f, relocInfo::none, 0); 2172 __ emit_data(0x5f5f5f5f, relocInfo::none, 0); // for URL 2173 2174 __ emit_data(0xffffffff, relocInfo::none, 0); 2175 __ emit_data(0xfcfcfcfc, relocInfo::none, 0); // for URL 2176 2177 // Permute table 2178 __ emit_data64(0x0000000100000000, relocInfo::none); 2179 __ emit_data64(0x0000000400000002, relocInfo::none); 2180 __ emit_data64(0x0000000600000005, relocInfo::none); 2181 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2182 2183 // Shuffle table 2184 __ emit_data64(0x090a040506000102, relocInfo::none); 2185 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2186 __ emit_data64(0x090a040506000102, relocInfo::none); 2187 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2188 2189 // merge table 2190 __ emit_data(0x01400140, relocInfo::none, 0); 2191 2192 // merge multiplier 2193 __ emit_data(0x00011000, relocInfo::none, 0); 2194 2195 return start; 2196 } 2197 2198 address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { 2199 __ align64(); 2200 StubCodeMark mark(this, "StubRoutines", "AVX2_tables_URL_base64"); 2201 address start = __ pc(); 2202 2203 assert(((unsigned long long)start & 0x3f) == 0, 2204 "Alignment problem (0x%08llx)", (unsigned long long)start); 2205 // lut_lo 2206 __ emit_data64(0x1111111111111115, relocInfo::none); 2207 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2208 __ emit_data64(0x1111111111111115, relocInfo::none); 2209 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2210 2211 // lut_roll 2212 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2213 __ emit_data64(0x0000000000000000, relocInfo::none); 2214 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2215 __ emit_data64(0x0000000000000000, relocInfo::none); 2216 2217 // lut_lo URL 2218 __ emit_data64(0x1111111111111115, relocInfo::none); 2219 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2220 __ emit_data64(0x1111111111111115, relocInfo::none); 2221 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2222 2223 // lut_roll URL 2224 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2225 __ emit_data64(0x0000000000000000, relocInfo::none); 2226 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2227 __ emit_data64(0x0000000000000000, relocInfo::none); 2228 2229 // lut_hi 2230 __ emit_data64(0x0804080402011010, relocInfo::none); 2231 __ emit_data64(0x1010101010101010, relocInfo::none); 2232 __ emit_data64(0x0804080402011010, relocInfo::none); 2233 __ emit_data64(0x1010101010101010, relocInfo::none); 2234 2235 return start; 2236 } 2237 2238 address StubGenerator::base64_decoding_table_addr() { 2239 StubCodeMark mark(this, "StubRoutines", "decoding_table_base64"); 2240 address start = __ pc(); 2241 2242 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2243 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2244 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2245 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2246 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2247 __ emit_data64(0x3fffffff3effffff, relocInfo::none); 2248 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2249 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2250 __ emit_data64(0x06050403020100ff, relocInfo::none); 2251 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2252 __ emit_data64(0x161514131211100f, relocInfo::none); 2253 __ emit_data64(0xffffffffff191817, relocInfo::none); 2254 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2255 __ emit_data64(0x2827262524232221, relocInfo::none); 2256 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2257 __ emit_data64(0xffffffffff333231, relocInfo::none); 2258 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2259 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2260 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2261 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2262 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2263 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2264 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2265 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2266 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2267 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2268 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2269 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2270 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2271 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2272 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2273 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2274 2275 // URL table 2276 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2277 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2278 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2279 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2280 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2281 __ emit_data64(0xffff3effffffffff, relocInfo::none); 2282 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2283 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2284 __ emit_data64(0x06050403020100ff, relocInfo::none); 2285 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2286 __ emit_data64(0x161514131211100f, relocInfo::none); 2287 __ emit_data64(0x3fffffffff191817, relocInfo::none); 2288 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2289 __ emit_data64(0x2827262524232221, relocInfo::none); 2290 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2291 __ emit_data64(0xffffffffff333231, relocInfo::none); 2292 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2293 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2294 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2295 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2296 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2297 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2298 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2299 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2300 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2301 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2302 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2303 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2304 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2305 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2306 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2307 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2308 2309 return start; 2310 } 2311 2312 2313 // Code for generating Base64 decoding. 2314 // 2315 // Based on the article (and associated code) from https://arxiv.org/abs/1910.05109. 2316 // 2317 // Intrinsic function prototype in Base64.java: 2318 // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { 2319 address StubGenerator::generate_base64_decodeBlock() { 2320 __ align(CodeEntryAlignment); 2321 StubCodeMark mark(this, "StubRoutines", "implDecode"); 2322 address start = __ pc(); 2323 2324 __ enter(); 2325 2326 // Save callee-saved registers before using them 2327 __ push(r12); 2328 __ push(r13); 2329 __ push(r14); 2330 __ push(r15); 2331 __ push(rbx); 2332 2333 // arguments 2334 const Register source = c_rarg0; // Source Array 2335 const Register start_offset = c_rarg1; // start offset 2336 const Register end_offset = c_rarg2; // end offset 2337 const Register dest = c_rarg3; // destination array 2338 const Register isMIME = rbx; 2339 2340 #ifndef _WIN64 2341 const Register dp = c_rarg4; // Position for writing to dest array 2342 const Register isURL = c_rarg5;// Base64 or URL character set 2343 __ movl(isMIME, Address(rbp, 2 * wordSize)); 2344 #else 2345 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 2346 const Address isURL_mem(rbp, 7 * wordSize); 2347 const Register isURL = r10; // pick the volatile windows register 2348 const Register dp = r12; 2349 __ movl(dp, dp_mem); 2350 __ movl(isURL, isURL_mem); 2351 __ movl(isMIME, Address(rbp, 8 * wordSize)); 2352 #endif 2353 2354 const XMMRegister lookup_lo = xmm5; 2355 const XMMRegister lookup_hi = xmm6; 2356 const XMMRegister errorvec = xmm7; 2357 const XMMRegister pack16_op = xmm9; 2358 const XMMRegister pack32_op = xmm8; 2359 const XMMRegister input0 = xmm3; 2360 const XMMRegister input1 = xmm20; 2361 const XMMRegister input2 = xmm21; 2362 const XMMRegister input3 = xmm19; 2363 const XMMRegister join01 = xmm12; 2364 const XMMRegister join12 = xmm11; 2365 const XMMRegister join23 = xmm10; 2366 const XMMRegister translated0 = xmm2; 2367 const XMMRegister translated1 = xmm1; 2368 const XMMRegister translated2 = xmm0; 2369 const XMMRegister translated3 = xmm4; 2370 2371 const XMMRegister merged0 = xmm2; 2372 const XMMRegister merged1 = xmm1; 2373 const XMMRegister merged2 = xmm0; 2374 const XMMRegister merged3 = xmm4; 2375 const XMMRegister merge_ab_bc0 = xmm2; 2376 const XMMRegister merge_ab_bc1 = xmm1; 2377 const XMMRegister merge_ab_bc2 = xmm0; 2378 const XMMRegister merge_ab_bc3 = xmm4; 2379 2380 const XMMRegister pack24bits = xmm4; 2381 2382 const Register length = r14; 2383 const Register output_size = r13; 2384 const Register output_mask = r15; 2385 const KRegister input_mask = k1; 2386 2387 const XMMRegister input_initial_valid_b64 = xmm0; 2388 const XMMRegister tmp = xmm10; 2389 const XMMRegister mask = xmm0; 2390 const XMMRegister invalid_b64 = xmm1; 2391 2392 Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL; 2393 Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce; 2394 Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero, L_lastChunk; 2395 2396 // calculate length from offsets 2397 __ movl(length, end_offset); 2398 __ subl(length, start_offset); 2399 __ push(dest); // Save for return value calc 2400 2401 // If AVX512 VBMI not supported, just compile non-AVX code 2402 if(VM_Version::supports_avx512_vbmi() && 2403 VM_Version::supports_avx512bw()) { 2404 __ cmpl(length, 31); // 32-bytes is break-even for AVX-512 2405 __ jcc(Assembler::lessEqual, L_lastChunk); 2406 2407 __ cmpl(isMIME, 0); 2408 __ jcc(Assembler::notEqual, L_lastChunk); 2409 2410 // Load lookup tables based on isURL 2411 __ cmpl(isURL, 0); 2412 __ jcc(Assembler::notZero, L_loadURL); 2413 2414 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13); 2415 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13); 2416 2417 __ BIND(L_continue); 2418 2419 __ movl(r15, 0x01400140); 2420 __ evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit); 2421 2422 __ movl(r15, 0x00011000); 2423 __ evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit); 2424 2425 __ cmpl(length, 0xff); 2426 __ jcc(Assembler::lessEqual, L_process64); 2427 2428 // load masks required for decoding data 2429 __ BIND(L_processdata); 2430 __ evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13); 2431 __ evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13); 2432 __ evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13); 2433 2434 __ align32(); 2435 __ BIND(L_process256); 2436 // Grab input data 2437 __ evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit); 2438 __ evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit); 2439 __ evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit); 2440 __ evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit); 2441 2442 // Copy the low part of the lookup table into the destination of the permutation 2443 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2444 __ evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit); 2445 __ evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit); 2446 __ evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit); 2447 2448 // Translate the base64 input into "decoded" bytes 2449 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2450 __ evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit); 2451 __ evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit); 2452 __ evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit); 2453 2454 // OR all of the translations together to check for errors (high-order bit of byte set) 2455 __ vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit); 2456 2457 __ vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit); 2458 __ vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit); 2459 __ vpor(errorvec, input3, input0, Assembler::AVX_512bit); 2460 2461 // Check if there was an error - if so, try 64-byte chunks 2462 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2463 __ kortestql(k3, k3); 2464 __ jcc(Assembler::notZero, L_process64); 2465 2466 // The merging and shuffling happens here 2467 // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa] 2468 // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd] 2469 // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40 2470 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2471 __ vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit); 2472 __ vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit); 2473 __ vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit); 2474 2475 // Now do the same with packed 16-bit values. 2476 // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb] 2477 // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12 2478 // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd] 2479 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2480 __ vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit); 2481 __ vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit); 2482 __ vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit); 2483 2484 // The join vectors specify which byte from which vector goes into the outputs 2485 // One of every 4 bytes in the extended vector is zero, so we pack them into their 2486 // final positions in the register for storing (256 bytes in, 192 bytes out) 2487 __ evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit); 2488 __ evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit); 2489 __ evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit); 2490 2491 // Store result 2492 __ evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit); 2493 __ evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit); 2494 __ evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit); 2495 2496 __ addptr(source, 0x100); 2497 __ addptr(dest, 0xc0); 2498 __ subl(length, 0x100); 2499 __ cmpl(length, 64 * 4); 2500 __ jcc(Assembler::greaterEqual, L_process256); 2501 2502 // At this point, we've decoded 64 * 4 * n bytes. 2503 // The remaining length will be <= 64 * 4 - 1. 2504 // UNLESS there was an error decoding the first 256-byte chunk. In this 2505 // case, the length will be arbitrarily long. 2506 // 2507 // Note that this will be the path for MIME-encoded strings. 2508 2509 __ BIND(L_process64); 2510 2511 __ evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13); 2512 2513 __ cmpl(length, 63); 2514 __ jcc(Assembler::lessEqual, L_finalBit); 2515 2516 __ mov64(rax, 0x0000ffffffffffff); 2517 __ kmovql(k2, rax); 2518 2519 __ align32(); 2520 __ BIND(L_process64Loop); 2521 2522 // Handle first 64-byte block 2523 2524 __ evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit); 2525 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2526 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2527 2528 __ vpor(errorvec, translated0, input0, Assembler::AVX_512bit); 2529 2530 // Check for error and bomb out before updating dest 2531 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2532 __ kortestql(k3, k3); 2533 __ jcc(Assembler::notZero, L_exit); 2534 2535 // Pack output register, selecting correct byte ordering 2536 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2537 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2538 __ vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit); 2539 2540 __ evmovdqub(Address(dest, dp), k2, merged0, true, Assembler::AVX_512bit); 2541 2542 __ subl(length, 64); 2543 __ addptr(source, 64); 2544 __ addptr(dest, 48); 2545 2546 __ cmpl(length, 64); 2547 __ jcc(Assembler::greaterEqual, L_process64Loop); 2548 2549 __ cmpl(length, 0); 2550 __ jcc(Assembler::lessEqual, L_exit); 2551 2552 __ BIND(L_finalBit); 2553 // Now have 1 to 63 bytes left to decode 2554 2555 // I was going to let Java take care of the final fragment 2556 // however it will repeatedly call this routine for every 4 bytes 2557 // of input data, so handle the rest here. 2558 __ movq(rax, -1); 2559 __ bzhiq(rax, rax, length); // Input mask in rax 2560 2561 __ movl(output_size, length); 2562 __ shrl(output_size, 2); // Find (len / 4) * 3 (output length) 2563 __ lea(output_size, Address(output_size, output_size, Address::times_2, 0)); 2564 // output_size in r13 2565 2566 // Strip pad characters, if any, and adjust length and mask 2567 __ cmpb(Address(source, length, Address::times_1, -1), '='); 2568 __ jcc(Assembler::equal, L_padding); 2569 2570 __ BIND(L_donePadding); 2571 2572 // Output size is (64 - output_size), output mask is (all 1s >> output_size). 2573 __ kmovql(input_mask, rax); 2574 __ movq(output_mask, -1); 2575 __ bzhiq(output_mask, output_mask, output_size); 2576 2577 // Load initial input with all valid base64 characters. Will be used 2578 // in merging source bytes to avoid masking when determining if an error occurred. 2579 __ movl(rax, 0x61616161); 2580 __ evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit); 2581 2582 // A register containing all invalid base64 decoded values 2583 __ movl(rax, 0x80808080); 2584 __ evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit); 2585 2586 // input_mask is in k1 2587 // output_size is in r13 2588 // output_mask is in r15 2589 // zmm0 - free 2590 // zmm1 - 0x00011000 2591 // zmm2 - 0x01400140 2592 // zmm3 - errorvec 2593 // zmm4 - pack vector 2594 // zmm5 - lookup_lo 2595 // zmm6 - lookup_hi 2596 // zmm7 - errorvec 2597 // zmm8 - 0x61616161 2598 // zmm9 - 0x80808080 2599 2600 // Load only the bytes from source, merging into our "fully-valid" register 2601 __ evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit); 2602 2603 // Decode all bytes within our merged input 2604 __ evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit); 2605 __ evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit); 2606 __ evporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit); 2607 2608 // Check for error. Compare (decoded | initial) to all invalid. 2609 // If any bytes have their high-order bit set, then we have an error. 2610 __ evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit); 2611 __ kortestql(k2, k2); 2612 2613 // If we have an error, use the brute force loop to decode what we can (4-byte chunks). 2614 __ jcc(Assembler::notZero, L_bruteForce); 2615 2616 // Shuffle output bytes 2617 __ vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit); 2618 __ vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit); 2619 2620 __ vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit); 2621 __ kmovql(k1, output_mask); 2622 __ evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit); 2623 2624 __ addptr(dest, output_size); 2625 2626 __ BIND(L_exit); 2627 __ vzeroupper(); 2628 __ pop(rax); // Get original dest value 2629 __ subptr(dest, rax); // Number of bytes converted 2630 __ movptr(rax, dest); 2631 __ pop(rbx); 2632 __ pop(r15); 2633 __ pop(r14); 2634 __ pop(r13); 2635 __ pop(r12); 2636 __ leave(); 2637 __ ret(0); 2638 2639 __ BIND(L_loadURL); 2640 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13); 2641 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13); 2642 __ jmp(L_continue); 2643 2644 __ BIND(L_padding); 2645 __ decrementq(output_size, 1); 2646 __ shrq(rax, 1); 2647 2648 __ cmpb(Address(source, length, Address::times_1, -2), '='); 2649 __ jcc(Assembler::notEqual, L_donePadding); 2650 2651 __ decrementq(output_size, 1); 2652 __ shrq(rax, 1); 2653 __ jmp(L_donePadding); 2654 2655 __ align32(); 2656 __ BIND(L_bruteForce); 2657 } // End of if(avx512_vbmi) 2658 2659 if (VM_Version::supports_avx2()) { 2660 Label L_tailProc, L_topLoop, L_enterLoop; 2661 2662 __ cmpl(isMIME, 0); 2663 __ jcc(Assembler::notEqual, L_lastChunk); 2664 2665 // Check for buffer too small (for algorithm) 2666 __ subl(length, 0x2c); 2667 __ jcc(Assembler::less, L_tailProc); 2668 2669 __ shll(isURL, 2); 2670 2671 // Algorithm adapted from https://arxiv.org/abs/1704.00605, "Faster Base64 2672 // Encoding and Decoding using AVX2 Instructions". URL modifications added. 2673 2674 // Set up constants 2675 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_tables_addr())); 2676 __ vpbroadcastd(xmm4, Address(r13, isURL, Address::times_1), Assembler::AVX_256bit); // 2F or 5F 2677 __ vpbroadcastd(xmm10, Address(r13, isURL, Address::times_1, 0x08), Assembler::AVX_256bit); // -1 or -4 2678 __ vmovdqu(xmm12, Address(r13, 0x10)); // permute 2679 __ vmovdqu(xmm13, Address(r13, 0x30)); // shuffle 2680 __ vpbroadcastd(xmm7, Address(r13, 0x50), Assembler::AVX_256bit); // merge 2681 __ vpbroadcastd(xmm6, Address(r13, 0x54), Assembler::AVX_256bit); // merge mult 2682 2683 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_LUT_tables_addr())); 2684 __ shll(isURL, 4); 2685 __ vmovdqu(xmm11, Address(r13, isURL, Address::times_1, 0x00)); // lut_lo 2686 __ vmovdqu(xmm8, Address(r13, isURL, Address::times_1, 0x20)); // lut_roll 2687 __ shrl(isURL, 6); // restore isURL 2688 __ vmovdqu(xmm9, Address(r13, 0x80)); // lut_hi 2689 __ jmp(L_enterLoop); 2690 2691 __ align32(); 2692 __ bind(L_topLoop); 2693 // Add in the offset value (roll) to get 6-bit out values 2694 __ vpaddb(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 2695 // Merge and permute the output bits into appropriate output byte lanes 2696 __ vpmaddubsw(xmm0, xmm0, xmm7, Assembler::AVX_256bit); 2697 __ vpmaddwd(xmm0, xmm0, xmm6, Assembler::AVX_256bit); 2698 __ vpshufb(xmm0, xmm0, xmm13, Assembler::AVX_256bit); 2699 __ vpermd(xmm0, xmm12, xmm0, Assembler::AVX_256bit); 2700 // Store the output bytes 2701 __ vmovdqu(Address(dest, dp, Address::times_1, 0), xmm0); 2702 __ addptr(source, 0x20); 2703 __ addptr(dest, 0x18); 2704 __ subl(length, 0x20); 2705 __ jcc(Assembler::less, L_tailProc); 2706 2707 __ bind(L_enterLoop); 2708 2709 // Load in encoded string (32 bytes) 2710 __ vmovdqu(xmm2, Address(source, start_offset, Address::times_1, 0x0)); 2711 // Extract the high nibble for indexing into the lut tables. High 4 bits are don't care. 2712 __ vpsrld(xmm1, xmm2, 0x4, Assembler::AVX_256bit); 2713 __ vpand(xmm1, xmm4, xmm1, Assembler::AVX_256bit); 2714 // Extract the low nibble. 5F/2F will isolate the low-order 4 bits. High 4 bits are don't care. 2715 __ vpand(xmm3, xmm2, xmm4, Assembler::AVX_256bit); 2716 // Check for special-case (0x2F or 0x5F (URL)) 2717 __ vpcmpeqb(xmm0, xmm4, xmm2, Assembler::AVX_256bit); 2718 // Get the bitset based on the low nibble. vpshufb uses low-order 4 bits only. 2719 __ vpshufb(xmm3, xmm11, xmm3, Assembler::AVX_256bit); 2720 // Get the bit value of the high nibble 2721 __ vpshufb(xmm5, xmm9, xmm1, Assembler::AVX_256bit); 2722 // Make sure 2F / 5F shows as valid 2723 __ vpandn(xmm3, xmm0, xmm3, Assembler::AVX_256bit); 2724 // Make adjustment for roll index. For non-URL, this is a no-op, 2725 // for URL, this adjusts by -4. This is to properly index the 2726 // roll value for 2F / 5F. 2727 __ vpand(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 2728 // If the and of the two is non-zero, we have an invalid input character 2729 __ vptest(xmm3, xmm5); 2730 // Extract the "roll" value - value to add to the input to get 6-bit out value 2731 __ vpaddb(xmm0, xmm0, xmm1, Assembler::AVX_256bit); // Handle 2F / 5F 2732 __ vpshufb(xmm0, xmm8, xmm0, Assembler::AVX_256bit); 2733 __ jcc(Assembler::equal, L_topLoop); // Fall through on error 2734 2735 __ bind(L_tailProc); 2736 2737 __ addl(length, 0x2c); 2738 2739 __ vzeroupper(); 2740 } 2741 2742 // Use non-AVX code to decode 4-byte chunks into 3 bytes of output 2743 2744 // Register state (Linux): 2745 // r12-15 - saved on stack 2746 // rdi - src 2747 // rsi - sp 2748 // rdx - sl 2749 // rcx - dst 2750 // r8 - dp 2751 // r9 - isURL 2752 2753 // Register state (Windows): 2754 // r12-15 - saved on stack 2755 // rcx - src 2756 // rdx - sp 2757 // r8 - sl 2758 // r9 - dst 2759 // r12 - dp 2760 // r10 - isURL 2761 2762 // Registers (common): 2763 // length (r14) - bytes in src 2764 2765 const Register decode_table = r11; 2766 const Register out_byte_count = rbx; 2767 const Register byte1 = r13; 2768 const Register byte2 = r15; 2769 const Register byte3 = WIN64_ONLY(r8) NOT_WIN64(rdx); 2770 const Register byte4 = WIN64_ONLY(r10) NOT_WIN64(r9); 2771 2772 __ bind(L_lastChunk); 2773 2774 __ shrl(length, 2); // Multiple of 4 bytes only - length is # 4-byte chunks 2775 __ cmpl(length, 0); 2776 __ jcc(Assembler::lessEqual, L_exit_no_vzero); 2777 2778 __ shll(isURL, 8); // index into decode table based on isURL 2779 __ lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr())); 2780 __ addptr(decode_table, isURL); 2781 2782 __ jmp(L_bottomLoop); 2783 2784 __ align32(); 2785 __ BIND(L_forceLoop); 2786 __ shll(byte1, 18); 2787 __ shll(byte2, 12); 2788 __ shll(byte3, 6); 2789 __ orl(byte1, byte2); 2790 __ orl(byte1, byte3); 2791 __ orl(byte1, byte4); 2792 2793 __ addptr(source, 4); 2794 2795 __ movb(Address(dest, dp, Address::times_1, 2), byte1); 2796 __ shrl(byte1, 8); 2797 __ movb(Address(dest, dp, Address::times_1, 1), byte1); 2798 __ shrl(byte1, 8); 2799 __ movb(Address(dest, dp, Address::times_1, 0), byte1); 2800 2801 __ addptr(dest, 3); 2802 __ decrementl(length, 1); 2803 __ jcc(Assembler::zero, L_exit_no_vzero); 2804 2805 __ BIND(L_bottomLoop); 2806 __ load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00)); 2807 __ load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01)); 2808 __ load_signed_byte(byte1, Address(decode_table, byte1)); 2809 __ load_signed_byte(byte2, Address(decode_table, byte2)); 2810 __ load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02)); 2811 __ load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03)); 2812 __ load_signed_byte(byte3, Address(decode_table, byte3)); 2813 __ load_signed_byte(byte4, Address(decode_table, byte4)); 2814 2815 __ mov(rax, byte1); 2816 __ orl(rax, byte2); 2817 __ orl(rax, byte3); 2818 __ orl(rax, byte4); 2819 __ jcc(Assembler::positive, L_forceLoop); 2820 2821 __ BIND(L_exit_no_vzero); 2822 __ pop(rax); // Get original dest value 2823 __ subptr(dest, rax); // Number of bytes converted 2824 __ movptr(rax, dest); 2825 __ pop(rbx); 2826 __ pop(r15); 2827 __ pop(r14); 2828 __ pop(r13); 2829 __ pop(r12); 2830 __ leave(); 2831 __ ret(0); 2832 2833 return start; 2834 } 2835 2836 2837 /** 2838 * Arguments: 2839 * 2840 * Inputs: 2841 * c_rarg0 - int crc 2842 * c_rarg1 - byte* buf 2843 * c_rarg2 - int length 2844 * 2845 * Output: 2846 * rax - int crc result 2847 */ 2848 address StubGenerator::generate_updateBytesCRC32() { 2849 assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions"); 2850 2851 __ align(CodeEntryAlignment); 2852 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32"); 2853 2854 address start = __ pc(); 2855 2856 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 2857 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 2858 // rscratch1: r10 2859 const Register crc = c_rarg0; // crc 2860 const Register buf = c_rarg1; // source java byte array address 2861 const Register len = c_rarg2; // length 2862 const Register table = c_rarg3; // crc_table address (reuse register) 2863 const Register tmp1 = r11; 2864 const Register tmp2 = r10; 2865 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax); 2866 2867 BLOCK_COMMENT("Entry:"); 2868 __ enter(); // required for proper stackwalking of RuntimeStub frame 2869 2870 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 2871 VM_Version::supports_avx512bw() && 2872 VM_Version::supports_avx512vl()) { 2873 // The constants used in the CRC32 algorithm requires the 1's compliment of the initial crc value. 2874 // However, the constant table for CRC32-C assumes the original crc value. Account for this 2875 // difference before calling and after returning. 2876 __ lea(table, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr())); 2877 __ notl(crc); 2878 __ kernel_crc32_avx512(crc, buf, len, table, tmp1, tmp2); 2879 __ notl(crc); 2880 } else { 2881 __ kernel_crc32(crc, buf, len, table, tmp1); 2882 } 2883 2884 __ movl(rax, crc); 2885 __ vzeroupper(); 2886 __ leave(); // required for proper stackwalking of RuntimeStub frame 2887 __ ret(0); 2888 2889 return start; 2890 } 2891 2892 /** 2893 * Arguments: 2894 * 2895 * Inputs: 2896 * c_rarg0 - int crc 2897 * c_rarg1 - byte* buf 2898 * c_rarg2 - long length 2899 * c_rarg3 - table_start - optional (present only when doing a library_call, 2900 * not used by x86 algorithm) 2901 * 2902 * Output: 2903 * rax - int crc result 2904 */ 2905 address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { 2906 assert(UseCRC32CIntrinsics, "need SSE4_2"); 2907 __ align(CodeEntryAlignment); 2908 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C"); 2909 address start = __ pc(); 2910 2911 //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs 2912 //Windows RCX RDX R8 R9 none none XMM0..XMM3 2913 //Lin / Sol RDI RSI RDX RCX R8 R9 XMM0..XMM7 2914 const Register crc = c_rarg0; // crc 2915 const Register buf = c_rarg1; // source java byte array address 2916 const Register len = c_rarg2; // length 2917 const Register a = rax; 2918 const Register j = r9; 2919 const Register k = r10; 2920 const Register l = r11; 2921 #ifdef _WIN64 2922 const Register y = rdi; 2923 const Register z = rsi; 2924 #else 2925 const Register y = rcx; 2926 const Register z = r8; 2927 #endif 2928 assert_different_registers(crc, buf, len, a, j, k, l, y, z); 2929 2930 BLOCK_COMMENT("Entry:"); 2931 __ enter(); // required for proper stackwalking of RuntimeStub frame 2932 Label L_continue; 2933 2934 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 2935 VM_Version::supports_avx512bw() && 2936 VM_Version::supports_avx512vl()) { 2937 Label L_doSmall; 2938 2939 __ cmpl(len, 384); 2940 __ jcc(Assembler::lessEqual, L_doSmall); 2941 2942 __ lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr())); 2943 __ kernel_crc32_avx512(crc, buf, len, j, l, k); 2944 2945 __ jmp(L_continue); 2946 2947 __ bind(L_doSmall); 2948 } 2949 #ifdef _WIN64 2950 __ push(y); 2951 __ push(z); 2952 #endif 2953 __ crc32c_ipl_alg2_alt2(crc, buf, len, 2954 a, j, k, 2955 l, y, z, 2956 c_farg0, c_farg1, c_farg2, 2957 is_pclmulqdq_supported); 2958 #ifdef _WIN64 2959 __ pop(z); 2960 __ pop(y); 2961 #endif 2962 2963 __ bind(L_continue); 2964 __ movl(rax, crc); 2965 __ vzeroupper(); 2966 __ leave(); // required for proper stackwalking of RuntimeStub frame 2967 __ ret(0); 2968 2969 return start; 2970 } 2971 2972 2973 /** 2974 * Arguments: 2975 * 2976 * Input: 2977 * c_rarg0 - x address 2978 * c_rarg1 - x length 2979 * c_rarg2 - y address 2980 * c_rarg3 - y length 2981 * not Win64 2982 * c_rarg4 - z address 2983 * c_rarg5 - z length 2984 * Win64 2985 * rsp+40 - z address 2986 * rsp+48 - z length 2987 */ 2988 address StubGenerator::generate_multiplyToLen() { 2989 __ align(CodeEntryAlignment); 2990 StubCodeMark mark(this, "StubRoutines", "multiplyToLen"); 2991 address start = __ pc(); 2992 2993 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 2994 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 2995 const Register x = rdi; 2996 const Register xlen = rax; 2997 const Register y = rsi; 2998 const Register ylen = rcx; 2999 const Register z = r8; 3000 const Register zlen = r11; 3001 3002 // Next registers will be saved on stack in multiply_to_len(). 3003 const Register tmp1 = r12; 3004 const Register tmp2 = r13; 3005 const Register tmp3 = r14; 3006 const Register tmp4 = r15; 3007 const Register tmp5 = rbx; 3008 3009 BLOCK_COMMENT("Entry:"); 3010 __ enter(); // required for proper stackwalking of RuntimeStub frame 3011 3012 #ifndef _WIN64 3013 __ movptr(zlen, r9); // Save r9 in r11 - zlen 3014 #endif 3015 setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx 3016 // ylen => rcx, z => r8, zlen => r11 3017 // r9 and r10 may be used to save non-volatile registers 3018 #ifdef _WIN64 3019 // last 2 arguments (#4, #5) are on stack on Win64 3020 __ movptr(z, Address(rsp, 6 * wordSize)); 3021 __ movptr(zlen, Address(rsp, 7 * wordSize)); 3022 #endif 3023 3024 __ movptr(xlen, rsi); 3025 __ movptr(y, rdx); 3026 __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5); 3027 3028 restore_arg_regs(); 3029 3030 __ leave(); // required for proper stackwalking of RuntimeStub frame 3031 __ ret(0); 3032 3033 return start; 3034 } 3035 3036 /** 3037 * Arguments: 3038 * 3039 * Input: 3040 * c_rarg0 - obja address 3041 * c_rarg1 - objb address 3042 * c_rarg3 - length length 3043 * c_rarg4 - scale log2_array_indxscale 3044 * 3045 * Output: 3046 * rax - int >= mismatched index, < 0 bitwise complement of tail 3047 */ 3048 address StubGenerator::generate_vectorizedMismatch() { 3049 __ align(CodeEntryAlignment); 3050 StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch"); 3051 address start = __ pc(); 3052 3053 BLOCK_COMMENT("Entry:"); 3054 __ enter(); 3055 3056 #ifdef _WIN64 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3057 const Register scale = c_rarg0; //rcx, will exchange with r9 3058 const Register objb = c_rarg1; //rdx 3059 const Register length = c_rarg2; //r8 3060 const Register obja = c_rarg3; //r9 3061 __ xchgq(obja, scale); //now obja and scale contains the correct contents 3062 3063 const Register tmp1 = r10; 3064 const Register tmp2 = r11; 3065 #endif 3066 #ifndef _WIN64 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3067 const Register obja = c_rarg0; //U:rdi 3068 const Register objb = c_rarg1; //U:rsi 3069 const Register length = c_rarg2; //U:rdx 3070 const Register scale = c_rarg3; //U:rcx 3071 const Register tmp1 = r8; 3072 const Register tmp2 = r9; 3073 #endif 3074 const Register result = rax; //return value 3075 const XMMRegister vec0 = xmm0; 3076 const XMMRegister vec1 = xmm1; 3077 const XMMRegister vec2 = xmm2; 3078 3079 __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2); 3080 3081 __ vzeroupper(); 3082 __ leave(); 3083 __ ret(0); 3084 3085 return start; 3086 } 3087 3088 /** 3089 * Arguments: 3090 * 3091 // Input: 3092 // c_rarg0 - x address 3093 // c_rarg1 - x length 3094 // c_rarg2 - z address 3095 // c_rarg3 - z length 3096 * 3097 */ 3098 address StubGenerator::generate_squareToLen() { 3099 3100 __ align(CodeEntryAlignment); 3101 StubCodeMark mark(this, "StubRoutines", "squareToLen"); 3102 address start = __ pc(); 3103 3104 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3105 // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...) 3106 const Register x = rdi; 3107 const Register len = rsi; 3108 const Register z = r8; 3109 const Register zlen = rcx; 3110 3111 const Register tmp1 = r12; 3112 const Register tmp2 = r13; 3113 const Register tmp3 = r14; 3114 const Register tmp4 = r15; 3115 const Register tmp5 = rbx; 3116 3117 BLOCK_COMMENT("Entry:"); 3118 __ enter(); // required for proper stackwalking of RuntimeStub frame 3119 3120 setup_arg_regs(4); // x => rdi, len => rsi, z => rdx 3121 // zlen => rcx 3122 // r9 and r10 may be used to save non-volatile registers 3123 __ movptr(r8, rdx); 3124 __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3125 3126 restore_arg_regs(); 3127 3128 __ leave(); // required for proper stackwalking of RuntimeStub frame 3129 __ ret(0); 3130 3131 return start; 3132 } 3133 3134 address StubGenerator::generate_method_entry_barrier() { 3135 __ align(CodeEntryAlignment); 3136 StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); 3137 address start = __ pc(); 3138 3139 Label deoptimize_label; 3140 3141 __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing 3142 3143 BLOCK_COMMENT("Entry:"); 3144 __ enter(); // save rbp 3145 3146 // save c_rarg0, because we want to use that value. 3147 // We could do without it but then we depend on the number of slots used by pusha 3148 __ push(c_rarg0); 3149 3150 __ lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address 3151 3152 __ pusha(); 3153 3154 // The method may have floats as arguments, and we must spill them before calling 3155 // the VM runtime. 3156 assert(Argument::n_float_register_parameters_j == 8, "Assumption"); 3157 const int xmm_size = wordSize * 2; 3158 const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j; 3159 __ subptr(rsp, xmm_spill_size); 3160 __ movdqu(Address(rsp, xmm_size * 7), xmm7); 3161 __ movdqu(Address(rsp, xmm_size * 6), xmm6); 3162 __ movdqu(Address(rsp, xmm_size * 5), xmm5); 3163 __ movdqu(Address(rsp, xmm_size * 4), xmm4); 3164 __ movdqu(Address(rsp, xmm_size * 3), xmm3); 3165 __ movdqu(Address(rsp, xmm_size * 2), xmm2); 3166 __ movdqu(Address(rsp, xmm_size * 1), xmm1); 3167 __ movdqu(Address(rsp, xmm_size * 0), xmm0); 3168 3169 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), 1); 3170 3171 __ movdqu(xmm0, Address(rsp, xmm_size * 0)); 3172 __ movdqu(xmm1, Address(rsp, xmm_size * 1)); 3173 __ movdqu(xmm2, Address(rsp, xmm_size * 2)); 3174 __ movdqu(xmm3, Address(rsp, xmm_size * 3)); 3175 __ movdqu(xmm4, Address(rsp, xmm_size * 4)); 3176 __ movdqu(xmm5, Address(rsp, xmm_size * 5)); 3177 __ movdqu(xmm6, Address(rsp, xmm_size * 6)); 3178 __ movdqu(xmm7, Address(rsp, xmm_size * 7)); 3179 __ addptr(rsp, xmm_spill_size); 3180 3181 __ cmpl(rax, 1); // 1 means deoptimize 3182 __ jcc(Assembler::equal, deoptimize_label); 3183 3184 __ popa(); 3185 __ pop(c_rarg0); 3186 3187 __ leave(); 3188 3189 __ addptr(rsp, 1 * wordSize); // cookie 3190 __ ret(0); 3191 3192 3193 __ BIND(deoptimize_label); 3194 3195 __ popa(); 3196 __ pop(c_rarg0); 3197 3198 __ leave(); 3199 3200 // this can be taken out, but is good for verification purposes. getting a SIGSEGV 3201 // here while still having a correct stack is valuable 3202 __ testptr(rsp, Address(rsp, 0)); 3203 3204 __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier 3205 __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point 3206 3207 return start; 3208 } 3209 3210 /** 3211 * Arguments: 3212 * 3213 * Input: 3214 * c_rarg0 - out address 3215 * c_rarg1 - in address 3216 * c_rarg2 - offset 3217 * c_rarg3 - len 3218 * not Win64 3219 * c_rarg4 - k 3220 * Win64 3221 * rsp+40 - k 3222 */ 3223 address StubGenerator::generate_mulAdd() { 3224 __ align(CodeEntryAlignment); 3225 StubCodeMark mark(this, "StubRoutines", "mulAdd"); 3226 address start = __ pc(); 3227 3228 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3229 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3230 const Register out = rdi; 3231 const Register in = rsi; 3232 const Register offset = r11; 3233 const Register len = rcx; 3234 const Register k = r8; 3235 3236 // Next registers will be saved on stack in mul_add(). 3237 const Register tmp1 = r12; 3238 const Register tmp2 = r13; 3239 const Register tmp3 = r14; 3240 const Register tmp4 = r15; 3241 const Register tmp5 = rbx; 3242 3243 BLOCK_COMMENT("Entry:"); 3244 __ enter(); // required for proper stackwalking of RuntimeStub frame 3245 3246 setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx 3247 // len => rcx, k => r8 3248 // r9 and r10 may be used to save non-volatile registers 3249 #ifdef _WIN64 3250 // last argument is on stack on Win64 3251 __ movl(k, Address(rsp, 6 * wordSize)); 3252 #endif 3253 __ movptr(r11, rdx); // move offset in rdx to offset(r11) 3254 __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3255 3256 restore_arg_regs(); 3257 3258 __ leave(); // required for proper stackwalking of RuntimeStub frame 3259 __ ret(0); 3260 3261 return start; 3262 } 3263 3264 address StubGenerator::generate_bigIntegerRightShift() { 3265 __ align(CodeEntryAlignment); 3266 StubCodeMark mark(this, "StubRoutines", "bigIntegerRightShiftWorker"); 3267 address start = __ pc(); 3268 3269 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3270 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3271 const Register newArr = rdi; 3272 const Register oldArr = rsi; 3273 const Register newIdx = rdx; 3274 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3275 const Register totalNumIter = r8; 3276 3277 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3278 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3279 const Register tmp1 = r11; // Caller save. 3280 const Register tmp2 = rax; // Caller save. 3281 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3282 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3283 const Register tmp5 = r14; // Callee save. 3284 const Register tmp6 = r15; 3285 3286 const XMMRegister x0 = xmm0; 3287 const XMMRegister x1 = xmm1; 3288 const XMMRegister x2 = xmm2; 3289 3290 BLOCK_COMMENT("Entry:"); 3291 __ enter(); // required for proper stackwalking of RuntimeStub frame 3292 3293 #ifdef _WIN64 3294 setup_arg_regs(4); 3295 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3296 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3297 // Save callee save registers. 3298 __ push(tmp3); 3299 __ push(tmp4); 3300 #endif 3301 __ push(tmp5); 3302 3303 // Rename temps used throughout the code. 3304 const Register idx = tmp1; 3305 const Register nIdx = tmp2; 3306 3307 __ xorl(idx, idx); 3308 3309 // Start right shift from end of the array. 3310 // For example, if #iteration = 4 and newIdx = 1 3311 // then dest[4] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3312 // if #iteration = 4 and newIdx = 0 3313 // then dest[3] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3314 __ movl(idx, totalNumIter); 3315 __ movl(nIdx, idx); 3316 __ addl(nIdx, newIdx); 3317 3318 // If vectorization is enabled, check if the number of iterations is at least 64 3319 // If not, then go to ShifTwo processing 2 iterations 3320 if (VM_Version::supports_avx512_vbmi2()) { 3321 __ cmpptr(totalNumIter, (AVX3Threshold/64)); 3322 __ jcc(Assembler::less, ShiftTwo); 3323 3324 if (AVX3Threshold < 16 * 64) { 3325 __ cmpl(totalNumIter, 16); 3326 __ jcc(Assembler::less, ShiftTwo); 3327 } 3328 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3329 __ subl(idx, 16); 3330 __ subl(nIdx, 16); 3331 __ BIND(Shift512Loop); 3332 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 4), Assembler::AVX_512bit); 3333 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3334 __ vpshrdvd(x2, x1, x0, Assembler::AVX_512bit); 3335 __ evmovdqul(Address(newArr, nIdx, Address::times_4), x2, Assembler::AVX_512bit); 3336 __ subl(nIdx, 16); 3337 __ subl(idx, 16); 3338 __ jcc(Assembler::greaterEqual, Shift512Loop); 3339 __ addl(idx, 16); 3340 __ addl(nIdx, 16); 3341 } 3342 __ BIND(ShiftTwo); 3343 __ cmpl(idx, 2); 3344 __ jcc(Assembler::less, ShiftOne); 3345 __ subl(idx, 2); 3346 __ subl(nIdx, 2); 3347 __ BIND(ShiftTwoLoop); 3348 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 8)); 3349 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3350 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3351 __ shrdl(tmp5, tmp4); 3352 __ shrdl(tmp4, tmp3); 3353 __ movl(Address(newArr, nIdx, Address::times_4, 4), tmp5); 3354 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3355 __ subl(nIdx, 2); 3356 __ subl(idx, 2); 3357 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3358 __ addl(idx, 2); 3359 __ addl(nIdx, 2); 3360 3361 // Do the last iteration 3362 __ BIND(ShiftOne); 3363 __ cmpl(idx, 1); 3364 __ jcc(Assembler::less, Exit); 3365 __ subl(idx, 1); 3366 __ subl(nIdx, 1); 3367 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3368 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3369 __ shrdl(tmp4, tmp3); 3370 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3371 __ BIND(Exit); 3372 __ vzeroupper(); 3373 // Restore callee save registers. 3374 __ pop(tmp5); 3375 #ifdef _WIN64 3376 __ pop(tmp4); 3377 __ pop(tmp3); 3378 restore_arg_regs(); 3379 #endif 3380 __ leave(); // required for proper stackwalking of RuntimeStub frame 3381 __ ret(0); 3382 3383 return start; 3384 } 3385 3386 /** 3387 * Arguments: 3388 * 3389 * Input: 3390 * c_rarg0 - newArr address 3391 * c_rarg1 - oldArr address 3392 * c_rarg2 - newIdx 3393 * c_rarg3 - shiftCount 3394 * not Win64 3395 * c_rarg4 - numIter 3396 * Win64 3397 * rsp40 - numIter 3398 */ 3399 address StubGenerator::generate_bigIntegerLeftShift() { 3400 __ align(CodeEntryAlignment); 3401 StubCodeMark mark(this, "StubRoutines", "bigIntegerLeftShiftWorker"); 3402 address start = __ pc(); 3403 3404 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3405 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3406 const Register newArr = rdi; 3407 const Register oldArr = rsi; 3408 const Register newIdx = rdx; 3409 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3410 const Register totalNumIter = r8; 3411 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3412 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3413 const Register tmp1 = r11; // Caller save. 3414 const Register tmp2 = rax; // Caller save. 3415 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3416 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3417 const Register tmp5 = r14; // Callee save. 3418 3419 const XMMRegister x0 = xmm0; 3420 const XMMRegister x1 = xmm1; 3421 const XMMRegister x2 = xmm2; 3422 BLOCK_COMMENT("Entry:"); 3423 __ enter(); // required for proper stackwalking of RuntimeStub frame 3424 3425 #ifdef _WIN64 3426 setup_arg_regs(4); 3427 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3428 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3429 // Save callee save registers. 3430 __ push(tmp3); 3431 __ push(tmp4); 3432 #endif 3433 __ push(tmp5); 3434 3435 // Rename temps used throughout the code 3436 const Register idx = tmp1; 3437 const Register numIterTmp = tmp2; 3438 3439 // Start idx from zero. 3440 __ xorl(idx, idx); 3441 // Compute interior pointer for new array. We do this so that we can use same index for both old and new arrays. 3442 __ lea(newArr, Address(newArr, newIdx, Address::times_4)); 3443 __ movl(numIterTmp, totalNumIter); 3444 3445 // If vectorization is enabled, check if the number of iterations is at least 64 3446 // If not, then go to ShiftTwo shifting two numbers at a time 3447 if (VM_Version::supports_avx512_vbmi2()) { 3448 __ cmpl(totalNumIter, (AVX3Threshold/64)); 3449 __ jcc(Assembler::less, ShiftTwo); 3450 3451 if (AVX3Threshold < 16 * 64) { 3452 __ cmpl(totalNumIter, 16); 3453 __ jcc(Assembler::less, ShiftTwo); 3454 } 3455 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3456 __ subl(numIterTmp, 16); 3457 __ BIND(Shift512Loop); 3458 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3459 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 0x4), Assembler::AVX_512bit); 3460 __ vpshldvd(x1, x2, x0, Assembler::AVX_512bit); 3461 __ evmovdqul(Address(newArr, idx, Address::times_4), x1, Assembler::AVX_512bit); 3462 __ addl(idx, 16); 3463 __ subl(numIterTmp, 16); 3464 __ jcc(Assembler::greaterEqual, Shift512Loop); 3465 __ addl(numIterTmp, 16); 3466 } 3467 __ BIND(ShiftTwo); 3468 __ cmpl(totalNumIter, 1); 3469 __ jcc(Assembler::less, Exit); 3470 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3471 __ subl(numIterTmp, 2); 3472 __ jcc(Assembler::less, ShiftOne); 3473 3474 __ BIND(ShiftTwoLoop); 3475 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3476 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 0x8)); 3477 __ shldl(tmp3, tmp4); 3478 __ shldl(tmp4, tmp5); 3479 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3480 __ movl(Address(newArr, idx, Address::times_4, 0x4), tmp4); 3481 __ movl(tmp3, tmp5); 3482 __ addl(idx, 2); 3483 __ subl(numIterTmp, 2); 3484 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3485 3486 // Do the last iteration 3487 __ BIND(ShiftOne); 3488 __ addl(numIterTmp, 2); 3489 __ cmpl(numIterTmp, 1); 3490 __ jcc(Assembler::less, Exit); 3491 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3492 __ shldl(tmp3, tmp4); 3493 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3494 3495 __ BIND(Exit); 3496 __ vzeroupper(); 3497 // Restore callee save registers. 3498 __ pop(tmp5); 3499 #ifdef _WIN64 3500 __ pop(tmp4); 3501 __ pop(tmp3); 3502 restore_arg_regs(); 3503 #endif 3504 __ leave(); // required for proper stackwalking of RuntimeStub frame 3505 __ ret(0); 3506 3507 return start; 3508 } 3509 3510 void StubGenerator::generate_libm_stubs() { 3511 if (UseLibmIntrinsic && InlineIntrinsics) { 3512 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) { 3513 StubRoutines::_dsin = generate_libmSin(); // from stubGenerator_x86_64_sin.cpp 3514 } 3515 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { 3516 StubRoutines::_dcos = generate_libmCos(); // from stubGenerator_x86_64_cos.cpp 3517 } 3518 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) { 3519 StubRoutines::_dtan = generate_libmTan(); // from stubGenerator_x86_64_tan.cpp 3520 } 3521 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) { 3522 StubRoutines::_dexp = generate_libmExp(); // from stubGenerator_x86_64_exp.cpp 3523 } 3524 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) { 3525 StubRoutines::_dpow = generate_libmPow(); // from stubGenerator_x86_64_pow.cpp 3526 } 3527 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) { 3528 StubRoutines::_dlog = generate_libmLog(); // from stubGenerator_x86_64_log.cpp 3529 } 3530 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) { 3531 StubRoutines::_dlog10 = generate_libmLog10(); // from stubGenerator_x86_64_log.cpp 3532 } 3533 } 3534 } 3535 3536 /** 3537 * Arguments: 3538 * 3539 * Input: 3540 * c_rarg0 - float16 jshort 3541 * 3542 * Output: 3543 * xmm0 - float 3544 */ 3545 address StubGenerator::generate_float16ToFloat() { 3546 StubCodeMark mark(this, "StubRoutines", "float16ToFloat"); 3547 3548 address start = __ pc(); 3549 3550 BLOCK_COMMENT("Entry:"); 3551 // No need for RuntimeStub frame since it is called only during JIT compilation 3552 3553 // Load value into xmm0 and convert 3554 __ flt16_to_flt(xmm0, c_rarg0); 3555 3556 __ ret(0); 3557 3558 return start; 3559 } 3560 3561 /** 3562 * Arguments: 3563 * 3564 * Input: 3565 * xmm0 - float 3566 * 3567 * Output: 3568 * rax - float16 jshort 3569 */ 3570 address StubGenerator::generate_floatToFloat16() { 3571 StubCodeMark mark(this, "StubRoutines", "floatToFloat16"); 3572 3573 address start = __ pc(); 3574 3575 BLOCK_COMMENT("Entry:"); 3576 // No need for RuntimeStub frame since it is called only during JIT compilation 3577 3578 // Convert and put result into rax 3579 __ flt_to_flt16(rax, xmm0, xmm1); 3580 3581 __ ret(0); 3582 3583 return start; 3584 } 3585 3586 address StubGenerator::generate_cont_thaw(const char* label, Continuation::thaw_kind kind) { 3587 if (!Continuations::enabled()) return nullptr; 3588 3589 bool return_barrier = Continuation::is_thaw_return_barrier(kind); 3590 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); 3591 3592 StubCodeMark mark(this, "StubRoutines", label); 3593 address start = __ pc(); 3594 3595 // TODO: Handle Valhalla return types. May require generating different return barriers. 3596 3597 if (!return_barrier) { 3598 // Pop return address. If we don't do this, we get a drift, 3599 // where the bottom-most frozen frame continuously grows. 3600 __ pop(c_rarg3); 3601 } else { 3602 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3603 } 3604 3605 #ifdef ASSERT 3606 { 3607 Label L_good_sp; 3608 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3609 __ jcc(Assembler::equal, L_good_sp); 3610 __ stop("Incorrect rsp at thaw entry"); 3611 __ BIND(L_good_sp); 3612 } 3613 #endif // ASSERT 3614 3615 if (return_barrier) { 3616 // Preserve possible return value from a method returning to the return barrier. 3617 __ push(rax); 3618 __ push_d(xmm0); 3619 } 3620 3621 __ movptr(c_rarg0, r15_thread); 3622 __ movptr(c_rarg1, (return_barrier ? 1 : 0)); 3623 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2); 3624 __ movptr(rbx, rax); 3625 3626 if (return_barrier) { 3627 // Restore return value from a method returning to the return barrier. 3628 // No safepoint in the call to thaw, so even an oop return value should be OK. 3629 __ pop_d(xmm0); 3630 __ pop(rax); 3631 } 3632 3633 #ifdef ASSERT 3634 { 3635 Label L_good_sp; 3636 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3637 __ jcc(Assembler::equal, L_good_sp); 3638 __ stop("Incorrect rsp after prepare thaw"); 3639 __ BIND(L_good_sp); 3640 } 3641 #endif // ASSERT 3642 3643 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames 3644 Label L_thaw_success; 3645 __ testptr(rbx, rbx); 3646 __ jccb(Assembler::notZero, L_thaw_success); 3647 __ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry())); 3648 __ bind(L_thaw_success); 3649 3650 // Make room for the thawed frames and align the stack. 3651 __ subptr(rsp, rbx); 3652 __ andptr(rsp, -StackAlignmentInBytes); 3653 3654 if (return_barrier) { 3655 // Preserve possible return value from a method returning to the return barrier. (Again.) 3656 __ push(rax); 3657 __ push_d(xmm0); 3658 } 3659 3660 // If we want, we can templatize thaw by kind, and have three different entries. 3661 __ movptr(c_rarg0, r15_thread); 3662 __ movptr(c_rarg1, kind); 3663 __ call_VM_leaf(Continuation::thaw_entry(), 2); 3664 __ movptr(rbx, rax); 3665 3666 if (return_barrier) { 3667 // Restore return value from a method returning to the return barrier. (Again.) 3668 // No safepoint in the call to thaw, so even an oop return value should be OK. 3669 __ pop_d(xmm0); 3670 __ pop(rax); 3671 } else { 3672 // Return 0 (success) from doYield. 3673 __ xorptr(rax, rax); 3674 } 3675 3676 // After thawing, rbx is the SP of the yielding frame. 3677 // Move there, and then to saved RBP slot. 3678 __ movptr(rsp, rbx); 3679 __ subptr(rsp, 2*wordSize); 3680 3681 if (return_barrier_exception) { 3682 __ movptr(c_rarg0, r15_thread); 3683 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address 3684 3685 // rax still holds the original exception oop, save it before the call 3686 __ push(rax); 3687 3688 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2); 3689 __ movptr(rbx, rax); 3690 3691 // Continue at exception handler: 3692 // rax: exception oop 3693 // rbx: exception handler 3694 // rdx: exception pc 3695 __ pop(rax); 3696 __ verify_oop(rax); 3697 __ pop(rbp); // pop out RBP here too 3698 __ pop(rdx); 3699 __ jmp(rbx); 3700 } else { 3701 // We are "returning" into the topmost thawed frame; see Thaw::push_return_frame 3702 __ pop(rbp); 3703 __ ret(0); 3704 } 3705 3706 return start; 3707 } 3708 3709 address StubGenerator::generate_cont_thaw() { 3710 return generate_cont_thaw("Cont thaw", Continuation::thaw_top); 3711 } 3712 3713 // TODO: will probably need multiple return barriers depending on return type 3714 3715 address StubGenerator::generate_cont_returnBarrier() { 3716 return generate_cont_thaw("Cont thaw return barrier", Continuation::thaw_return_barrier); 3717 } 3718 3719 address StubGenerator::generate_cont_returnBarrier_exception() { 3720 return generate_cont_thaw("Cont thaw return barrier exception", Continuation::thaw_return_barrier_exception); 3721 } 3722 3723 #if INCLUDE_JFR 3724 3725 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. 3726 // It returns a jobject handle to the event writer. 3727 // The handle is dereferenced and the return value is the event writer oop. 3728 RuntimeStub* StubGenerator::generate_jfr_write_checkpoint() { 3729 enum layout { 3730 rbp_off, 3731 rbpH_off, 3732 return_off, 3733 return_off2, 3734 framesize // inclusive of return address 3735 }; 3736 3737 CodeBuffer code("jfr_write_checkpoint", 1024, 64); 3738 MacroAssembler* _masm = new MacroAssembler(&code); 3739 address start = __ pc(); 3740 3741 __ enter(); 3742 address the_pc = __ pc(); 3743 3744 int frame_complete = the_pc - start; 3745 3746 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1); 3747 __ movptr(c_rarg0, r15_thread); 3748 __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1); 3749 __ reset_last_Java_frame(true); 3750 3751 // rax is jobject handle result, unpack and process it through a barrier. 3752 __ resolve_global_jobject(rax, r15_thread, c_rarg0); 3753 3754 __ leave(); 3755 __ ret(0); 3756 3757 OopMapSet* oop_maps = new OopMapSet(); 3758 OopMap* map = new OopMap(framesize, 1); 3759 oop_maps->add_gc_map(frame_complete, map); 3760 3761 RuntimeStub* stub = 3762 RuntimeStub::new_runtime_stub(code.name(), 3763 &code, 3764 frame_complete, 3765 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 3766 oop_maps, 3767 false); 3768 return stub; 3769 } 3770 3771 #endif // INCLUDE_JFR 3772 3773 // Continuation point for throwing of implicit exceptions that are 3774 // not handled in the current activation. Fabricates an exception 3775 // oop and initiates normal exception dispatching in this 3776 // frame. Since we need to preserve callee-saved values (currently 3777 // only for C2, but done for C1 as well) we need a callee-saved oop 3778 // map and therefore have to make these stubs into RuntimeStubs 3779 // rather than BufferBlobs. If the compiler needs all registers to 3780 // be preserved between the fault point and the exception handler 3781 // then it must assume responsibility for that in 3782 // AbstractCompiler::continuation_for_implicit_null_exception or 3783 // continuation_for_implicit_division_by_zero_exception. All other 3784 // implicit exceptions (e.g., NullPointerException or 3785 // AbstractMethodError on entry) are either at call sites or 3786 // otherwise assume that stack unwinding will be initiated, so 3787 // caller saved registers were assumed volatile in the compiler. 3788 address StubGenerator::generate_throw_exception(const char* name, 3789 address runtime_entry, 3790 Register arg1, 3791 Register arg2) { 3792 // Information about frame layout at time of blocking runtime call. 3793 // Note that we only have to preserve callee-saved registers since 3794 // the compilers are responsible for supplying a continuation point 3795 // if they expect all registers to be preserved. 3796 enum layout { 3797 rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt, 3798 rbp_off2, 3799 return_off, 3800 return_off2, 3801 framesize // inclusive of return address 3802 }; 3803 3804 int insts_size = 512; 3805 int locs_size = 64; 3806 3807 CodeBuffer code(name, insts_size, locs_size); 3808 OopMapSet* oop_maps = new OopMapSet(); 3809 MacroAssembler* _masm = new MacroAssembler(&code); 3810 3811 address start = __ pc(); 3812 3813 // This is an inlined and slightly modified version of call_VM 3814 // which has the ability to fetch the return PC out of 3815 // thread-local storage and also sets up last_Java_sp slightly 3816 // differently than the real call_VM 3817 3818 __ enter(); // required for proper stackwalking of RuntimeStub frame 3819 3820 assert(is_even(framesize/2), "sp not 16-byte aligned"); 3821 3822 // return address and rbp are already in place 3823 __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog 3824 3825 int frame_complete = __ pc() - start; 3826 3827 // Set up last_Java_sp and last_Java_fp 3828 address the_pc = __ pc(); 3829 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1); 3830 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack 3831 3832 // Call runtime 3833 if (arg1 != noreg) { 3834 assert(arg2 != c_rarg1, "clobbered"); 3835 __ movptr(c_rarg1, arg1); 3836 } 3837 if (arg2 != noreg) { 3838 __ movptr(c_rarg2, arg2); 3839 } 3840 __ movptr(c_rarg0, r15_thread); 3841 BLOCK_COMMENT("call runtime_entry"); 3842 __ call(RuntimeAddress(runtime_entry)); 3843 3844 // Generate oop map 3845 OopMap* map = new OopMap(framesize, 0); 3846 3847 oop_maps->add_gc_map(the_pc - start, map); 3848 3849 __ reset_last_Java_frame(true); 3850 3851 __ leave(); // required for proper stackwalking of RuntimeStub frame 3852 3853 // check for pending exceptions 3854 #ifdef ASSERT 3855 Label L; 3856 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 3857 __ jcc(Assembler::notEqual, L); 3858 __ should_not_reach_here(); 3859 __ bind(L); 3860 #endif // ASSERT 3861 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 3862 3863 3864 // codeBlob framesize is in words (not VMRegImpl::slot_size) 3865 RuntimeStub* stub = 3866 RuntimeStub::new_runtime_stub(name, 3867 &code, 3868 frame_complete, 3869 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 3870 oop_maps, false); 3871 return stub->entry_point(); 3872 } 3873 3874 void StubGenerator::create_control_words() { 3875 // Round to nearest, 64-bit mode, exceptions masked 3876 StubRoutines::x86::_mxcsr_std = 0x1F80; 3877 } 3878 3879 // Initialization 3880 void StubGenerator::generate_initial_stubs() { 3881 // Generates all stubs and initializes the entry points 3882 3883 // This platform-specific settings are needed by generate_call_stub() 3884 create_control_words(); 3885 3886 // Initialize table for unsafe copy memeory check. 3887 if (UnsafeCopyMemory::_table == nullptr) { 3888 UnsafeCopyMemory::create_table(16); 3889 } 3890 3891 // entry points that exist in all platforms Note: This is code 3892 // that could be shared among different platforms - however the 3893 // benefit seems to be smaller than the disadvantage of having a 3894 // much more complicated generator structure. See also comment in 3895 // stubRoutines.hpp. 3896 3897 StubRoutines::_forward_exception_entry = generate_forward_exception(); 3898 3899 // Generate these first because they are called from other stubs 3900 if (InlineTypeReturnedAsFields) { 3901 StubRoutines::_load_inline_type_fields_in_regs = 3902 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), 3903 "load_inline_type_fields_in_regs", false); 3904 StubRoutines::_store_inline_type_fields_to_buf = 3905 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), 3906 "store_inline_type_fields_to_buf", true); 3907 } 3908 3909 StubRoutines::_call_stub_entry = 3910 generate_call_stub(StubRoutines::_call_stub_return_address); 3911 3912 // is referenced by megamorphic call 3913 StubRoutines::_catch_exception_entry = generate_catch_exception(); 3914 3915 // atomic calls 3916 StubRoutines::_fence_entry = generate_orderaccess_fence(); 3917 3918 // platform dependent 3919 StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp(); 3920 3921 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr(); 3922 3923 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); 3924 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); 3925 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup(); 3926 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup(); 3927 3928 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); 3929 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); 3930 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); 3931 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); 3932 3933 // Build this early so it's available for the interpreter. 3934 StubRoutines::_throw_StackOverflowError_entry = 3935 generate_throw_exception("StackOverflowError throw_exception", 3936 CAST_FROM_FN_PTR(address, 3937 SharedRuntime:: 3938 throw_StackOverflowError)); 3939 StubRoutines::_throw_delayed_StackOverflowError_entry = 3940 generate_throw_exception("delayed StackOverflowError throw_exception", 3941 CAST_FROM_FN_PTR(address, 3942 SharedRuntime:: 3943 throw_delayed_StackOverflowError)); 3944 if (UseCRC32Intrinsics) { 3945 // set table address before stub generation which use it 3946 StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table; 3947 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32(); 3948 } 3949 3950 if (UseCRC32CIntrinsics) { 3951 bool supports_clmul = VM_Version::supports_clmul(); 3952 StubRoutines::x86::generate_CRC32C_table(supports_clmul); 3953 StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table; 3954 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul); 3955 } 3956 3957 if (VM_Version::supports_float16()) { 3958 // For results consistency both intrinsics should be enabled. 3959 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here. 3960 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) && 3961 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) { 3962 StubRoutines::_hf2f = generate_float16ToFloat(); 3963 StubRoutines::_f2hf = generate_floatToFloat16(); 3964 } 3965 } 3966 3967 generate_libm_stubs(); 3968 } 3969 3970 // Call here from the interpreter or compiled code to either load 3971 // multiple returned values from the inline type instance being 3972 // returned to registers or to store returned values to a newly 3973 // allocated inline type instance. 3974 // Register is a class, but it would be assigned numerical value. 3975 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull. 3976 PRAGMA_DIAG_PUSH 3977 PRAGMA_NONNULL_IGNORED 3978 address StubGenerator::generate_return_value_stub(address destination, const char* name, bool has_res) { 3979 // We need to save all registers the calling convention may use so 3980 // the runtime calls read or update those registers. This needs to 3981 // be in sync with SharedRuntime::java_return_convention(). 3982 enum layout { 3983 pad_off = frame::arg_reg_save_area_bytes/BytesPerInt, pad_off_2, 3984 rax_off, rax_off_2, 3985 j_rarg5_off, j_rarg5_2, 3986 j_rarg4_off, j_rarg4_2, 3987 j_rarg3_off, j_rarg3_2, 3988 j_rarg2_off, j_rarg2_2, 3989 j_rarg1_off, j_rarg1_2, 3990 j_rarg0_off, j_rarg0_2, 3991 j_farg0_off, j_farg0_2, 3992 j_farg1_off, j_farg1_2, 3993 j_farg2_off, j_farg2_2, 3994 j_farg3_off, j_farg3_2, 3995 j_farg4_off, j_farg4_2, 3996 j_farg5_off, j_farg5_2, 3997 j_farg6_off, j_farg6_2, 3998 j_farg7_off, j_farg7_2, 3999 rbp_off, rbp_off_2, 4000 return_off, return_off_2, 4001 4002 framesize 4003 }; 4004 4005 CodeBuffer buffer(name, 1000, 512); 4006 MacroAssembler* _masm = new MacroAssembler(&buffer); 4007 4008 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16); 4009 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned"); 4010 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt; 4011 int frame_size_in_words = frame_size_in_bytes / wordSize; 4012 4013 OopMapSet *oop_maps = new OopMapSet(); 4014 OopMap* map = new OopMap(frame_size_in_slots, 0); 4015 4016 map->set_callee_saved(VMRegImpl::stack2reg(rax_off), rax->as_VMReg()); 4017 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg()); 4018 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg()); 4019 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg()); 4020 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg()); 4021 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg()); 4022 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg()); 4023 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg()); 4024 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg()); 4025 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg()); 4026 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg()); 4027 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg()); 4028 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg()); 4029 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg()); 4030 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg()); 4031 4032 int start = __ offset(); 4033 4034 __ subptr(rsp, frame_size_in_bytes - 8 /* return address*/); 4035 4036 __ movptr(Address(rsp, rbp_off * BytesPerInt), rbp); 4037 __ movdbl(Address(rsp, j_farg7_off * BytesPerInt), j_farg7); 4038 __ movdbl(Address(rsp, j_farg6_off * BytesPerInt), j_farg6); 4039 __ movdbl(Address(rsp, j_farg5_off * BytesPerInt), j_farg5); 4040 __ movdbl(Address(rsp, j_farg4_off * BytesPerInt), j_farg4); 4041 __ movdbl(Address(rsp, j_farg3_off * BytesPerInt), j_farg3); 4042 __ movdbl(Address(rsp, j_farg2_off * BytesPerInt), j_farg2); 4043 __ movdbl(Address(rsp, j_farg1_off * BytesPerInt), j_farg1); 4044 __ movdbl(Address(rsp, j_farg0_off * BytesPerInt), j_farg0); 4045 4046 __ movptr(Address(rsp, j_rarg0_off * BytesPerInt), j_rarg0); 4047 __ movptr(Address(rsp, j_rarg1_off * BytesPerInt), j_rarg1); 4048 __ movptr(Address(rsp, j_rarg2_off * BytesPerInt), j_rarg2); 4049 __ movptr(Address(rsp, j_rarg3_off * BytesPerInt), j_rarg3); 4050 __ movptr(Address(rsp, j_rarg4_off * BytesPerInt), j_rarg4); 4051 __ movptr(Address(rsp, j_rarg5_off * BytesPerInt), j_rarg5); 4052 __ movptr(Address(rsp, rax_off * BytesPerInt), rax); 4053 4054 int frame_complete = __ offset(); 4055 4056 __ set_last_Java_frame(noreg, noreg, NULL, rscratch1); 4057 4058 __ mov(c_rarg0, r15_thread); 4059 __ mov(c_rarg1, rax); 4060 4061 __ call(RuntimeAddress(destination)); 4062 4063 // Set an oopmap for the call site. 4064 4065 oop_maps->add_gc_map( __ offset() - start, map); 4066 4067 // clear last_Java_sp 4068 __ reset_last_Java_frame(false); 4069 4070 __ movptr(rbp, Address(rsp, rbp_off * BytesPerInt)); 4071 __ movdbl(j_farg7, Address(rsp, j_farg7_off * BytesPerInt)); 4072 __ movdbl(j_farg6, Address(rsp, j_farg6_off * BytesPerInt)); 4073 __ movdbl(j_farg5, Address(rsp, j_farg5_off * BytesPerInt)); 4074 __ movdbl(j_farg4, Address(rsp, j_farg4_off * BytesPerInt)); 4075 __ movdbl(j_farg3, Address(rsp, j_farg3_off * BytesPerInt)); 4076 __ movdbl(j_farg2, Address(rsp, j_farg2_off * BytesPerInt)); 4077 __ movdbl(j_farg1, Address(rsp, j_farg1_off * BytesPerInt)); 4078 __ movdbl(j_farg0, Address(rsp, j_farg0_off * BytesPerInt)); 4079 4080 __ movptr(j_rarg0, Address(rsp, j_rarg0_off * BytesPerInt)); 4081 __ movptr(j_rarg1, Address(rsp, j_rarg1_off * BytesPerInt)); 4082 __ movptr(j_rarg2, Address(rsp, j_rarg2_off * BytesPerInt)); 4083 __ movptr(j_rarg3, Address(rsp, j_rarg3_off * BytesPerInt)); 4084 __ movptr(j_rarg4, Address(rsp, j_rarg4_off * BytesPerInt)); 4085 __ movptr(j_rarg5, Address(rsp, j_rarg5_off * BytesPerInt)); 4086 __ movptr(rax, Address(rsp, rax_off * BytesPerInt)); 4087 4088 __ addptr(rsp, frame_size_in_bytes-8); 4089 4090 // check for pending exceptions 4091 Label pending; 4092 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD); 4093 __ jcc(Assembler::notEqual, pending); 4094 4095 if (has_res) { 4096 __ get_vm_result(rax, r15_thread); 4097 } 4098 4099 __ ret(0); 4100 4101 __ bind(pending); 4102 4103 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 4104 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 4105 4106 // ------------- 4107 // make sure all code is generated 4108 _masm->flush(); 4109 4110 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false); 4111 return stub->entry_point(); 4112 } 4113 4114 void StubGenerator::generate_continuation_stubs() { 4115 // Continuation stubs: 4116 StubRoutines::_cont_thaw = generate_cont_thaw(); 4117 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); 4118 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); 4119 4120 JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) 4121 JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) 4122 } 4123 4124 void StubGenerator::generate_final_stubs() { 4125 // Generates the rest of stubs and initializes the entry points 4126 4127 // These entry points require SharedInfo::stack0 to be set up in 4128 // non-core builds and need to be relocatable, so they each 4129 // fabricate a RuntimeStub internally. 4130 StubRoutines::_throw_AbstractMethodError_entry = 4131 generate_throw_exception("AbstractMethodError throw_exception", 4132 CAST_FROM_FN_PTR(address, 4133 SharedRuntime:: 4134 throw_AbstractMethodError)); 4135 4136 StubRoutines::_throw_IncompatibleClassChangeError_entry = 4137 generate_throw_exception("IncompatibleClassChangeError throw_exception", 4138 CAST_FROM_FN_PTR(address, 4139 SharedRuntime:: 4140 throw_IncompatibleClassChangeError)); 4141 4142 StubRoutines::_throw_NullPointerException_at_call_entry = 4143 generate_throw_exception("NullPointerException at call throw_exception", 4144 CAST_FROM_FN_PTR(address, 4145 SharedRuntime:: 4146 throw_NullPointerException_at_call)); 4147 4148 // support for verify_oop (must happen after universe_init) 4149 if (VerifyOops) { 4150 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); 4151 } 4152 4153 // data cache line writeback 4154 StubRoutines::_data_cache_writeback = generate_data_cache_writeback(); 4155 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync(); 4156 4157 // arraycopy stubs used by compilers 4158 generate_arraycopy_stubs(); 4159 4160 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 4161 if (bs_nm != nullptr) { 4162 StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier(); 4163 } 4164 4165 if (UseVectorizedMismatchIntrinsic) { 4166 StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch(); 4167 } 4168 } 4169 4170 void StubGenerator::generate_compiler_stubs() { 4171 #if COMPILER2_OR_JVMCI 4172 4173 // Entry points that are C2 compiler specific. 4174 4175 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF); 4176 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000); 4177 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF); 4178 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000); 4179 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF); 4180 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x0000000100000001); 4181 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff); 4182 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask"); 4183 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff); 4184 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff); 4185 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit, 4186 0xFFFFFFFF, 0, 0, 0); 4187 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit, 4188 0xFFFFFFFF, 0xFFFFFFFF, 0, 0); 4189 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100); 4190 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask"); 4191 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100); 4192 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000); 4193 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000); 4194 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices"); 4195 StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut("count_leading_zeros_lut"); 4196 StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut("reverse_bit_lut"); 4197 StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long("perm_mask_long"); 4198 StubRoutines::x86::_vector_reverse_byte_perm_mask_int = generate_vector_reverse_byte_perm_mask_int("perm_mask_int"); 4199 StubRoutines::x86::_vector_reverse_byte_perm_mask_short = generate_vector_reverse_byte_perm_mask_short("perm_mask_short"); 4200 4201 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) { 4202 // lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight. 4203 StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut("popcount_lut"); 4204 } 4205 4206 generate_aes_stubs(); 4207 4208 generate_ghash_stubs(); 4209 4210 generate_chacha_stubs(); 4211 4212 if (UseAdler32Intrinsics) { 4213 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32(); 4214 } 4215 4216 if (UsePoly1305Intrinsics) { 4217 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks(); 4218 } 4219 4220 if (UseMD5Intrinsics) { 4221 StubRoutines::_md5_implCompress = generate_md5_implCompress(false, "md5_implCompress"); 4222 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(true, "md5_implCompressMB"); 4223 } 4224 4225 if (UseSHA1Intrinsics) { 4226 StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask(); 4227 StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask(); 4228 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress"); 4229 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB"); 4230 } 4231 4232 if (UseSHA256Intrinsics) { 4233 StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; 4234 char* dst = (char*)StubRoutines::x86::_k256_W; 4235 char* src = (char*)StubRoutines::x86::_k256; 4236 for (int ii = 0; ii < 16; ++ii) { 4237 memcpy(dst + 32 * ii, src + 16 * ii, 16); 4238 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); 4239 } 4240 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; 4241 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); 4242 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress"); 4243 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB"); 4244 } 4245 4246 if (UseSHA512Intrinsics) { 4247 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; 4248 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); 4249 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress"); 4250 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB"); 4251 } 4252 4253 if (UseBASE64Intrinsics) { 4254 if(VM_Version::supports_avx2()) { 4255 StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr(); 4256 StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr(); 4257 StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr(); 4258 StubRoutines::x86::_avx2_decode_tables_base64 = base64_AVX2_decode_tables_addr(); 4259 StubRoutines::x86::_avx2_decode_lut_tables_base64 = base64_AVX2_decode_LUT_tables_addr(); 4260 } 4261 StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr(); 4262 if (VM_Version::supports_avx512_vbmi()) { 4263 StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr(); 4264 StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr(); 4265 StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr(); 4266 StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr(); 4267 StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr(); 4268 StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr(); 4269 StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr(); 4270 StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr(); 4271 StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr(); 4272 } 4273 StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr(); 4274 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock(); 4275 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); 4276 } 4277 4278 #ifdef COMPILER2 4279 if (UseMultiplyToLenIntrinsic) { 4280 StubRoutines::_multiplyToLen = generate_multiplyToLen(); 4281 } 4282 if (UseSquareToLenIntrinsic) { 4283 StubRoutines::_squareToLen = generate_squareToLen(); 4284 } 4285 if (UseMulAddIntrinsic) { 4286 StubRoutines::_mulAdd = generate_mulAdd(); 4287 } 4288 if (VM_Version::supports_avx512_vbmi2()) { 4289 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); 4290 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); 4291 } 4292 if (UseMontgomeryMultiplyIntrinsic) { 4293 StubRoutines::_montgomeryMultiply 4294 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply); 4295 } 4296 if (UseMontgomerySquareIntrinsic) { 4297 StubRoutines::_montgomerySquare 4298 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square); 4299 } 4300 4301 // Get svml stub routine addresses 4302 void *libjsvml = nullptr; 4303 char ebuf[1024]; 4304 char dll_name[JVM_MAXPATHLEN]; 4305 if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "jsvml")) { 4306 libjsvml = os::dll_load(dll_name, ebuf, sizeof ebuf); 4307 } 4308 if (libjsvml != nullptr) { 4309 // SVML method naming convention 4310 // All the methods are named as __jsvml_op<T><N>_ha_<VV> 4311 // Where: 4312 // ha stands for high accuracy 4313 // <T> is optional to indicate float/double 4314 // Set to f for vector float operation 4315 // Omitted for vector double operation 4316 // <N> is the number of elements in the vector 4317 // 1, 2, 4, 8, 16 4318 // e.g. 128 bit float vector has 4 float elements 4319 // <VV> indicates the avx/sse level: 4320 // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 4321 // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns 4322 // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns 4323 4324 log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "jsvml" JNI_LIB_SUFFIX, p2i(libjsvml)); 4325 if (UseAVX > 2) { 4326 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { 4327 int vop = VectorSupport::VECTOR_OP_SVML_START + op; 4328 if ((!VM_Version::supports_avx512dq()) && 4329 (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { 4330 continue; 4331 } 4332 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::svmlname[op]); 4333 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); 4334 4335 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::svmlname[op]); 4336 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); 4337 } 4338 } 4339 const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex"); 4340 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { 4341 int vop = VectorSupport::VECTOR_OP_SVML_START + op; 4342 if (vop == VectorSupport::VECTOR_OP_POW) { 4343 continue; 4344 } 4345 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4346 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); 4347 4348 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4349 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); 4350 4351 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4352 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); 4353 4354 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4355 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); 4356 4357 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4358 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); 4359 4360 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4361 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); 4362 } 4363 } 4364 #endif // COMPILER2 4365 #endif // COMPILER2_OR_JVMCI 4366 } 4367 4368 StubGenerator::StubGenerator(CodeBuffer* code, StubsKind kind) : StubCodeGenerator(code) { 4369 DEBUG_ONLY( _regs_in_thread = false; ) 4370 switch(kind) { 4371 case Initial_stubs: 4372 generate_initial_stubs(); 4373 break; 4374 case Continuation_stubs: 4375 generate_continuation_stubs(); 4376 break; 4377 case Compiler_stubs: 4378 generate_compiler_stubs(); 4379 break; 4380 case Final_stubs: 4381 generate_final_stubs(); 4382 break; 4383 default: 4384 fatal("unexpected stubs kind: %d", kind); 4385 break; 4386 }; 4387 } 4388 4389 void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind) { 4390 StubGenerator g(code, kind); 4391 } 4392 4393 #undef __