1 /* 2 * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/macroAssembler.hpp" 28 #include "classfile/vmIntrinsics.hpp" 29 #include "compiler/oopMap.hpp" 30 #include "gc/shared/barrierSet.hpp" 31 #include "gc/shared/barrierSetAssembler.hpp" 32 #include "gc/shared/barrierSetNMethod.hpp" 33 #include "gc/shared/gc_globals.hpp" 34 #include "memory/universe.hpp" 35 #include "prims/jvmtiExport.hpp" 36 #include "prims/upcallLinker.hpp" 37 #include "runtime/arguments.hpp" 38 #include "runtime/javaThread.hpp" 39 #include "runtime/sharedRuntime.hpp" 40 #include "runtime/stubRoutines.hpp" 41 #include "utilities/macros.hpp" 42 #include "vmreg_x86.inline.hpp" 43 #include "stubGenerator_x86_64.hpp" 44 #ifdef COMPILER2 45 #include "opto/runtime.hpp" 46 #include "opto/c2_globals.hpp" 47 #endif 48 #if INCLUDE_JVMCI 49 #include "jvmci/jvmci_globals.hpp" 50 #endif 51 52 // For a more detailed description of the stub routine structure 53 // see the comment in stubRoutines.hpp 54 55 #define __ _masm-> 56 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) 57 58 #ifdef PRODUCT 59 #define BLOCK_COMMENT(str) /* nothing */ 60 #else 61 #define BLOCK_COMMENT(str) __ block_comment(str) 62 #endif // PRODUCT 63 64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 65 66 // 67 // Linux Arguments: 68 // c_rarg0: call wrapper address address 69 // c_rarg1: result address 70 // c_rarg2: result type BasicType 71 // c_rarg3: method Method* 72 // c_rarg4: (interpreter) entry point address 73 // c_rarg5: parameters intptr_t* 74 // 16(rbp): parameter size (in words) int 75 // 24(rbp): thread Thread* 76 // 77 // [ return_from_Java ] <--- rsp 78 // [ argument word n ] 79 // ... 80 // -12 [ argument word 1 ] 81 // -11 [ saved r15 ] <--- rsp_after_call 82 // -10 [ saved r14 ] 83 // -9 [ saved r13 ] 84 // -8 [ saved r12 ] 85 // -7 [ saved rbx ] 86 // -6 [ call wrapper ] 87 // -5 [ result ] 88 // -4 [ result type ] 89 // -3 [ method ] 90 // -2 [ entry point ] 91 // -1 [ parameters ] 92 // 0 [ saved rbp ] <--- rbp 93 // 1 [ return address ] 94 // 2 [ parameter size ] 95 // 3 [ thread ] 96 // 97 // Windows Arguments: 98 // c_rarg0: call wrapper address address 99 // c_rarg1: result address 100 // c_rarg2: result type BasicType 101 // c_rarg3: method Method* 102 // 48(rbp): (interpreter) entry point address 103 // 56(rbp): parameters intptr_t* 104 // 64(rbp): parameter size (in words) int 105 // 72(rbp): thread Thread* 106 // 107 // [ return_from_Java ] <--- rsp 108 // [ argument word n ] 109 // ... 110 // -28 [ argument word 1 ] 111 // -27 [ saved xmm15 ] <--- rsp after_call 112 // [ saved xmm7-xmm14 ] 113 // -9 [ saved xmm6 ] (each xmm register takes 2 slots) 114 // -7 [ saved r15 ] 115 // -6 [ saved r14 ] 116 // -5 [ saved r13 ] 117 // -4 [ saved r12 ] 118 // -3 [ saved rdi ] 119 // -2 [ saved rsi ] 120 // -1 [ saved rbx ] 121 // 0 [ saved rbp ] <--- rbp 122 // 1 [ return address ] 123 // 2 [ call wrapper ] 124 // 3 [ result ] 125 // 4 [ result type ] 126 // 5 [ method ] 127 // 6 [ entry point ] 128 // 7 [ parameters ] 129 // 8 [ parameter size ] 130 // 9 [ thread ] 131 // 132 // Windows reserves the callers stack space for arguments 1-4. 133 // We spill c_rarg0-c_rarg3 to this space. 134 135 // Call stub stack layout word offsets from rbp 136 #ifdef _WIN64 137 enum call_stub_layout { 138 xmm_save_first = 6, // save from xmm6 139 xmm_save_last = 15, // to xmm15 140 xmm_save_base = -9, 141 rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27 142 r15_off = -7, 143 r14_off = -6, 144 r13_off = -5, 145 r12_off = -4, 146 rdi_off = -3, 147 rsi_off = -2, 148 rbx_off = -1, 149 rbp_off = 0, 150 retaddr_off = 1, 151 call_wrapper_off = 2, 152 result_off = 3, 153 result_type_off = 4, 154 method_off = 5, 155 entry_point_off = 6, 156 parameters_off = 7, 157 parameter_size_off = 8, 158 thread_off = 9 159 }; 160 161 static Address xmm_save(int reg) { 162 assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range"); 163 return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize); 164 } 165 #else // !_WIN64 166 enum call_stub_layout { 167 rsp_after_call_off = -12, 168 mxcsr_off = rsp_after_call_off, 169 r15_off = -11, 170 r14_off = -10, 171 r13_off = -9, 172 r12_off = -8, 173 rbx_off = -7, 174 call_wrapper_off = -6, 175 result_off = -5, 176 result_type_off = -4, 177 method_off = -3, 178 entry_point_off = -2, 179 parameters_off = -1, 180 rbp_off = 0, 181 retaddr_off = 1, 182 parameter_size_off = 2, 183 thread_off = 3 184 }; 185 #endif // _WIN64 186 187 address StubGenerator::generate_call_stub(address& return_address) { 188 189 assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && 190 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, 191 "adjust this code"); 192 StubCodeMark mark(this, "StubRoutines", "call_stub"); 193 address start = __ pc(); 194 195 // same as in generate_catch_exception()! 196 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 197 198 const Address call_wrapper (rbp, call_wrapper_off * wordSize); 199 const Address result (rbp, result_off * wordSize); 200 const Address result_type (rbp, result_type_off * wordSize); 201 const Address method (rbp, method_off * wordSize); 202 const Address entry_point (rbp, entry_point_off * wordSize); 203 const Address parameters (rbp, parameters_off * wordSize); 204 const Address parameter_size(rbp, parameter_size_off * wordSize); 205 206 // same as in generate_catch_exception()! 207 const Address thread (rbp, thread_off * wordSize); 208 209 const Address r15_save(rbp, r15_off * wordSize); 210 const Address r14_save(rbp, r14_off * wordSize); 211 const Address r13_save(rbp, r13_off * wordSize); 212 const Address r12_save(rbp, r12_off * wordSize); 213 const Address rbx_save(rbp, rbx_off * wordSize); 214 215 // stub code 216 __ enter(); 217 __ subptr(rsp, -rsp_after_call_off * wordSize); 218 219 // save register parameters 220 #ifndef _WIN64 221 __ movptr(parameters, c_rarg5); // parameters 222 __ movptr(entry_point, c_rarg4); // entry_point 223 #endif 224 225 __ movptr(method, c_rarg3); // method 226 __ movl(result_type, c_rarg2); // result type 227 __ movptr(result, c_rarg1); // result 228 __ movptr(call_wrapper, c_rarg0); // call wrapper 229 230 // save regs belonging to calling function 231 __ movptr(rbx_save, rbx); 232 __ movptr(r12_save, r12); 233 __ movptr(r13_save, r13); 234 __ movptr(r14_save, r14); 235 __ movptr(r15_save, r15); 236 237 #ifdef _WIN64 238 int last_reg = 15; 239 for (int i = xmm_save_first; i <= last_reg; i++) { 240 __ movdqu(xmm_save(i), as_XMMRegister(i)); 241 } 242 243 const Address rdi_save(rbp, rdi_off * wordSize); 244 const Address rsi_save(rbp, rsi_off * wordSize); 245 246 __ movptr(rsi_save, rsi); 247 __ movptr(rdi_save, rdi); 248 #else 249 const Address mxcsr_save(rbp, mxcsr_off * wordSize); 250 { 251 Label skip_ldmx; 252 __ stmxcsr(mxcsr_save); 253 __ movl(rax, mxcsr_save); 254 __ andl(rax, 0xFFC0); // Mask out any pending exceptions (only check control and mask bits) 255 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 256 __ cmp32(rax, mxcsr_std, rscratch1); 257 __ jcc(Assembler::equal, skip_ldmx); 258 __ ldmxcsr(mxcsr_std, rscratch1); 259 __ bind(skip_ldmx); 260 } 261 #endif 262 263 // Load up thread register 264 __ movptr(r15_thread, thread); 265 __ reinit_heapbase(); 266 267 #ifdef ASSERT 268 // make sure we have no pending exceptions 269 { 270 Label L; 271 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 272 __ jcc(Assembler::equal, L); 273 __ stop("StubRoutines::call_stub: entered with pending exception"); 274 __ bind(L); 275 } 276 #endif 277 278 // pass parameters if any 279 BLOCK_COMMENT("pass parameters if any"); 280 Label parameters_done; 281 __ movl(c_rarg3, parameter_size); 282 __ testl(c_rarg3, c_rarg3); 283 __ jcc(Assembler::zero, parameters_done); 284 285 Label loop; 286 __ movptr(c_rarg2, parameters); // parameter pointer 287 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 288 __ BIND(loop); 289 __ movptr(rax, Address(c_rarg2, 0));// get parameter 290 __ addptr(c_rarg2, wordSize); // advance to next parameter 291 __ decrementl(c_rarg1); // decrement counter 292 __ push(rax); // pass parameter 293 __ jcc(Assembler::notZero, loop); 294 295 // call Java function 296 __ BIND(parameters_done); 297 __ movptr(rbx, method); // get Method* 298 __ movptr(c_rarg1, entry_point); // get entry_point 299 __ mov(r13, rsp); // set sender sp 300 BLOCK_COMMENT("call Java function"); 301 __ call(c_rarg1); 302 303 BLOCK_COMMENT("call_stub_return_address:"); 304 return_address = __ pc(); 305 306 // store result depending on type (everything that is not 307 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) 308 __ movptr(r13, result); 309 Label is_long, is_float, is_double, check_prim, exit; 310 __ movl(rbx, result_type); 311 __ cmpl(rbx, T_OBJECT); 312 __ jcc(Assembler::equal, check_prim); 313 __ cmpl(rbx, T_LONG); 314 __ jcc(Assembler::equal, is_long); 315 __ cmpl(rbx, T_FLOAT); 316 __ jcc(Assembler::equal, is_float); 317 __ cmpl(rbx, T_DOUBLE); 318 __ jcc(Assembler::equal, is_double); 319 #ifdef ASSERT 320 // make sure the type is INT 321 { 322 Label L; 323 __ cmpl(rbx, T_INT); 324 __ jcc(Assembler::equal, L); 325 __ stop("StubRoutines::call_stub: unexpected result type"); 326 __ bind(L); 327 } 328 #endif 329 330 // handle T_INT case 331 __ movl(Address(r13, 0), rax); 332 333 __ BIND(exit); 334 335 // pop parameters 336 __ lea(rsp, rsp_after_call); 337 338 #ifdef ASSERT 339 // verify that threads correspond 340 { 341 Label L1, L2, L3; 342 __ cmpptr(r15_thread, thread); 343 __ jcc(Assembler::equal, L1); 344 __ stop("StubRoutines::call_stub: r15_thread is corrupted"); 345 __ bind(L1); 346 __ get_thread(rbx); 347 __ cmpptr(r15_thread, thread); 348 __ jcc(Assembler::equal, L2); 349 __ stop("StubRoutines::call_stub: r15_thread is modified by call"); 350 __ bind(L2); 351 __ cmpptr(r15_thread, rbx); 352 __ jcc(Assembler::equal, L3); 353 __ stop("StubRoutines::call_stub: threads must correspond"); 354 __ bind(L3); 355 } 356 #endif 357 358 __ pop_cont_fastpath(); 359 360 // restore regs belonging to calling function 361 #ifdef _WIN64 362 // emit the restores for xmm regs 363 for (int i = xmm_save_first; i <= last_reg; i++) { 364 __ movdqu(as_XMMRegister(i), xmm_save(i)); 365 } 366 #endif 367 __ movptr(r15, r15_save); 368 __ movptr(r14, r14_save); 369 __ movptr(r13, r13_save); 370 __ movptr(r12, r12_save); 371 __ movptr(rbx, rbx_save); 372 373 #ifdef _WIN64 374 __ movptr(rdi, rdi_save); 375 __ movptr(rsi, rsi_save); 376 #else 377 __ ldmxcsr(mxcsr_save); 378 #endif 379 380 // restore rsp 381 __ addptr(rsp, -rsp_after_call_off * wordSize); 382 383 // return 384 __ vzeroupper(); 385 __ pop(rbp); 386 __ ret(0); 387 388 // handle return types different from T_INT 389 __ BIND(check_prim); 390 if (InlineTypeReturnedAsFields) { 391 // Check for scalarized return value 392 __ testptr(rax, 1); 393 __ jcc(Assembler::zero, is_long); 394 // Load pack handler address 395 __ andptr(rax, -2); 396 __ movptr(rax, Address(rax, InstanceKlass::adr_inlineklass_fixed_block_offset())); 397 __ movptr(rbx, Address(rax, InlineKlass::pack_handler_jobject_offset())); 398 // Call pack handler to initialize the buffer 399 __ call(rbx); 400 __ jmp(exit); 401 } 402 __ BIND(is_long); 403 __ movq(Address(r13, 0), rax); 404 __ jmp(exit); 405 406 __ BIND(is_float); 407 __ movflt(Address(r13, 0), xmm0); 408 __ jmp(exit); 409 410 __ BIND(is_double); 411 __ movdbl(Address(r13, 0), xmm0); 412 __ jmp(exit); 413 414 return start; 415 } 416 417 // Return point for a Java call if there's an exception thrown in 418 // Java code. The exception is caught and transformed into a 419 // pending exception stored in JavaThread that can be tested from 420 // within the VM. 421 // 422 // Note: Usually the parameters are removed by the callee. In case 423 // of an exception crossing an activation frame boundary, that is 424 // not the case if the callee is compiled code => need to setup the 425 // rsp. 426 // 427 // rax: exception oop 428 429 address StubGenerator::generate_catch_exception() { 430 StubCodeMark mark(this, "StubRoutines", "catch_exception"); 431 address start = __ pc(); 432 433 // same as in generate_call_stub(): 434 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 435 const Address thread (rbp, thread_off * wordSize); 436 437 #ifdef ASSERT 438 // verify that threads correspond 439 { 440 Label L1, L2, L3; 441 __ cmpptr(r15_thread, thread); 442 __ jcc(Assembler::equal, L1); 443 __ stop("StubRoutines::catch_exception: r15_thread is corrupted"); 444 __ bind(L1); 445 __ get_thread(rbx); 446 __ cmpptr(r15_thread, thread); 447 __ jcc(Assembler::equal, L2); 448 __ stop("StubRoutines::catch_exception: r15_thread is modified by call"); 449 __ bind(L2); 450 __ cmpptr(r15_thread, rbx); 451 __ jcc(Assembler::equal, L3); 452 __ stop("StubRoutines::catch_exception: threads must correspond"); 453 __ bind(L3); 454 } 455 #endif 456 457 // set pending exception 458 __ verify_oop(rax); 459 460 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); 461 __ lea(rscratch1, ExternalAddress((address)__FILE__)); 462 __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); 463 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); 464 465 // complete return to VM 466 assert(StubRoutines::_call_stub_return_address != nullptr, 467 "_call_stub_return_address must have been generated before"); 468 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); 469 470 return start; 471 } 472 473 // Continuation point for runtime calls returning with a pending 474 // exception. The pending exception check happened in the runtime 475 // or native call stub. The pending exception in Thread is 476 // converted into a Java-level exception. 477 // 478 // Contract with Java-level exception handlers: 479 // rax: exception 480 // rdx: throwing pc 481 // 482 // NOTE: At entry of this stub, exception-pc must be on stack !! 483 484 address StubGenerator::generate_forward_exception() { 485 StubCodeMark mark(this, "StubRoutines", "forward exception"); 486 address start = __ pc(); 487 488 // Upon entry, the sp points to the return address returning into 489 // Java (interpreted or compiled) code; i.e., the return address 490 // becomes the throwing pc. 491 // 492 // Arguments pushed before the runtime call are still on the stack 493 // but the exception handler will reset the stack pointer -> 494 // ignore them. A potential result in registers can be ignored as 495 // well. 496 497 #ifdef ASSERT 498 // make sure this code is only executed if there is a pending exception 499 { 500 Label L; 501 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 502 __ jcc(Assembler::notEqual, L); 503 __ stop("StubRoutines::forward exception: no pending exception (1)"); 504 __ bind(L); 505 } 506 #endif 507 508 // compute exception handler into rbx 509 __ movptr(c_rarg0, Address(rsp, 0)); 510 BLOCK_COMMENT("call exception_handler_for_return_address"); 511 __ call_VM_leaf(CAST_FROM_FN_PTR(address, 512 SharedRuntime::exception_handler_for_return_address), 513 r15_thread, c_rarg0); 514 __ mov(rbx, rax); 515 516 // setup rax & rdx, remove return address & clear pending exception 517 __ pop(rdx); 518 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 519 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 520 521 #ifdef ASSERT 522 // make sure exception is set 523 { 524 Label L; 525 __ testptr(rax, rax); 526 __ jcc(Assembler::notEqual, L); 527 __ stop("StubRoutines::forward exception: no pending exception (2)"); 528 __ bind(L); 529 } 530 #endif 531 532 // continue at exception handler (return address removed) 533 // rax: exception 534 // rbx: exception handler 535 // rdx: throwing pc 536 __ verify_oop(rax); 537 __ jmp(rbx); 538 539 return start; 540 } 541 542 // Support for intptr_t OrderAccess::fence() 543 // 544 // Arguments : 545 // 546 // Result: 547 address StubGenerator::generate_orderaccess_fence() { 548 StubCodeMark mark(this, "StubRoutines", "orderaccess_fence"); 549 address start = __ pc(); 550 551 __ membar(Assembler::StoreLoad); 552 __ ret(0); 553 554 return start; 555 } 556 557 558 // Support for intptr_t get_previous_sp() 559 // 560 // This routine is used to find the previous stack pointer for the 561 // caller. 562 address StubGenerator::generate_get_previous_sp() { 563 StubCodeMark mark(this, "StubRoutines", "get_previous_sp"); 564 address start = __ pc(); 565 566 __ movptr(rax, rsp); 567 __ addptr(rax, 8); // return address is at the top of the stack. 568 __ ret(0); 569 570 return start; 571 } 572 573 //---------------------------------------------------------------------------------------------------- 574 // Support for void verify_mxcsr() 575 // 576 // This routine is used with -Xcheck:jni to verify that native 577 // JNI code does not return to Java code without restoring the 578 // MXCSR register to our expected state. 579 580 address StubGenerator::generate_verify_mxcsr() { 581 StubCodeMark mark(this, "StubRoutines", "verify_mxcsr"); 582 address start = __ pc(); 583 584 const Address mxcsr_save(rsp, 0); 585 586 if (CheckJNICalls) { 587 Label ok_ret; 588 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 589 __ push(rax); 590 __ subptr(rsp, wordSize); // allocate a temp location 591 __ stmxcsr(mxcsr_save); 592 __ movl(rax, mxcsr_save); 593 __ andl(rax, 0xFFC0); // Mask out any pending exceptions (only check control and mask bits) 594 __ cmp32(rax, mxcsr_std, rscratch1); 595 __ jcc(Assembler::equal, ok_ret); 596 597 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall"); 598 599 __ ldmxcsr(mxcsr_std, rscratch1); 600 601 __ bind(ok_ret); 602 __ addptr(rsp, wordSize); 603 __ pop(rax); 604 } 605 606 __ ret(0); 607 608 return start; 609 } 610 611 address StubGenerator::generate_f2i_fixup() { 612 StubCodeMark mark(this, "StubRoutines", "f2i_fixup"); 613 Address inout(rsp, 5 * wordSize); // return address + 4 saves 614 615 address start = __ pc(); 616 617 Label L; 618 619 __ push(rax); 620 __ push(c_rarg3); 621 __ push(c_rarg2); 622 __ push(c_rarg1); 623 624 __ movl(rax, 0x7f800000); 625 __ xorl(c_rarg3, c_rarg3); 626 __ movl(c_rarg2, inout); 627 __ movl(c_rarg1, c_rarg2); 628 __ andl(c_rarg1, 0x7fffffff); 629 __ cmpl(rax, c_rarg1); // NaN? -> 0 630 __ jcc(Assembler::negative, L); 631 __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint 632 __ movl(c_rarg3, 0x80000000); 633 __ movl(rax, 0x7fffffff); 634 __ cmovl(Assembler::positive, c_rarg3, rax); 635 636 __ bind(L); 637 __ movptr(inout, c_rarg3); 638 639 __ pop(c_rarg1); 640 __ pop(c_rarg2); 641 __ pop(c_rarg3); 642 __ pop(rax); 643 644 __ ret(0); 645 646 return start; 647 } 648 649 address StubGenerator::generate_f2l_fixup() { 650 StubCodeMark mark(this, "StubRoutines", "f2l_fixup"); 651 Address inout(rsp, 5 * wordSize); // return address + 4 saves 652 address start = __ pc(); 653 654 Label L; 655 656 __ push(rax); 657 __ push(c_rarg3); 658 __ push(c_rarg2); 659 __ push(c_rarg1); 660 661 __ movl(rax, 0x7f800000); 662 __ xorl(c_rarg3, c_rarg3); 663 __ movl(c_rarg2, inout); 664 __ movl(c_rarg1, c_rarg2); 665 __ andl(c_rarg1, 0x7fffffff); 666 __ cmpl(rax, c_rarg1); // NaN? -> 0 667 __ jcc(Assembler::negative, L); 668 __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong 669 __ mov64(c_rarg3, 0x8000000000000000); 670 __ mov64(rax, 0x7fffffffffffffff); 671 __ cmov(Assembler::positive, c_rarg3, rax); 672 673 __ bind(L); 674 __ movptr(inout, c_rarg3); 675 676 __ pop(c_rarg1); 677 __ pop(c_rarg2); 678 __ pop(c_rarg3); 679 __ pop(rax); 680 681 __ ret(0); 682 683 return start; 684 } 685 686 address StubGenerator::generate_d2i_fixup() { 687 StubCodeMark mark(this, "StubRoutines", "d2i_fixup"); 688 Address inout(rsp, 6 * wordSize); // return address + 5 saves 689 690 address start = __ pc(); 691 692 Label L; 693 694 __ push(rax); 695 __ push(c_rarg3); 696 __ push(c_rarg2); 697 __ push(c_rarg1); 698 __ push(c_rarg0); 699 700 __ movl(rax, 0x7ff00000); 701 __ movq(c_rarg2, inout); 702 __ movl(c_rarg3, c_rarg2); 703 __ mov(c_rarg1, c_rarg2); 704 __ mov(c_rarg0, c_rarg2); 705 __ negl(c_rarg3); 706 __ shrptr(c_rarg1, 0x20); 707 __ orl(c_rarg3, c_rarg2); 708 __ andl(c_rarg1, 0x7fffffff); 709 __ xorl(c_rarg2, c_rarg2); 710 __ shrl(c_rarg3, 0x1f); 711 __ orl(c_rarg1, c_rarg3); 712 __ cmpl(rax, c_rarg1); 713 __ jcc(Assembler::negative, L); // NaN -> 0 714 __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint 715 __ movl(c_rarg2, 0x80000000); 716 __ movl(rax, 0x7fffffff); 717 __ cmov(Assembler::positive, c_rarg2, rax); 718 719 __ bind(L); 720 __ movptr(inout, c_rarg2); 721 722 __ pop(c_rarg0); 723 __ pop(c_rarg1); 724 __ pop(c_rarg2); 725 __ pop(c_rarg3); 726 __ pop(rax); 727 728 __ ret(0); 729 730 return start; 731 } 732 733 address StubGenerator::generate_d2l_fixup() { 734 StubCodeMark mark(this, "StubRoutines", "d2l_fixup"); 735 Address inout(rsp, 6 * wordSize); // return address + 5 saves 736 737 address start = __ pc(); 738 739 Label L; 740 741 __ push(rax); 742 __ push(c_rarg3); 743 __ push(c_rarg2); 744 __ push(c_rarg1); 745 __ push(c_rarg0); 746 747 __ movl(rax, 0x7ff00000); 748 __ movq(c_rarg2, inout); 749 __ movl(c_rarg3, c_rarg2); 750 __ mov(c_rarg1, c_rarg2); 751 __ mov(c_rarg0, c_rarg2); 752 __ negl(c_rarg3); 753 __ shrptr(c_rarg1, 0x20); 754 __ orl(c_rarg3, c_rarg2); 755 __ andl(c_rarg1, 0x7fffffff); 756 __ xorl(c_rarg2, c_rarg2); 757 __ shrl(c_rarg3, 0x1f); 758 __ orl(c_rarg1, c_rarg3); 759 __ cmpl(rax, c_rarg1); 760 __ jcc(Assembler::negative, L); // NaN -> 0 761 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong 762 __ mov64(c_rarg2, 0x8000000000000000); 763 __ mov64(rax, 0x7fffffffffffffff); 764 __ cmovq(Assembler::positive, c_rarg2, rax); 765 766 __ bind(L); 767 __ movq(inout, c_rarg2); 768 769 __ pop(c_rarg0); 770 __ pop(c_rarg1); 771 __ pop(c_rarg2); 772 __ pop(c_rarg3); 773 __ pop(rax); 774 775 __ ret(0); 776 777 return start; 778 } 779 780 address StubGenerator::generate_count_leading_zeros_lut(const char *stub_name) { 781 __ align64(); 782 StubCodeMark mark(this, "StubRoutines", stub_name); 783 address start = __ pc(); 784 785 __ emit_data64(0x0101010102020304, relocInfo::none); 786 __ emit_data64(0x0000000000000000, relocInfo::none); 787 __ emit_data64(0x0101010102020304, relocInfo::none); 788 __ emit_data64(0x0000000000000000, relocInfo::none); 789 __ emit_data64(0x0101010102020304, relocInfo::none); 790 __ emit_data64(0x0000000000000000, relocInfo::none); 791 __ emit_data64(0x0101010102020304, relocInfo::none); 792 __ emit_data64(0x0000000000000000, relocInfo::none); 793 794 return start; 795 } 796 797 address StubGenerator::generate_popcount_avx_lut(const char *stub_name) { 798 __ align64(); 799 StubCodeMark mark(this, "StubRoutines", stub_name); 800 address start = __ pc(); 801 802 __ emit_data64(0x0302020102010100, relocInfo::none); 803 __ emit_data64(0x0403030203020201, relocInfo::none); 804 __ emit_data64(0x0302020102010100, relocInfo::none); 805 __ emit_data64(0x0403030203020201, relocInfo::none); 806 __ emit_data64(0x0302020102010100, relocInfo::none); 807 __ emit_data64(0x0403030203020201, relocInfo::none); 808 __ emit_data64(0x0302020102010100, relocInfo::none); 809 __ emit_data64(0x0403030203020201, relocInfo::none); 810 811 return start; 812 } 813 814 address StubGenerator::generate_iota_indices(const char *stub_name) { 815 __ align(CodeEntryAlignment); 816 StubCodeMark mark(this, "StubRoutines", stub_name); 817 address start = __ pc(); 818 // B 819 __ emit_data64(0x0706050403020100, relocInfo::none); 820 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); 821 __ emit_data64(0x1716151413121110, relocInfo::none); 822 __ emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none); 823 __ emit_data64(0x2726252423222120, relocInfo::none); 824 __ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none); 825 __ emit_data64(0x3736353433323130, relocInfo::none); 826 __ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none); 827 // W 828 __ emit_data64(0x0003000200010000, relocInfo::none); 829 __ emit_data64(0x0007000600050004, relocInfo::none); 830 __ emit_data64(0x000B000A00090008, relocInfo::none); 831 __ emit_data64(0x000F000E000D000C, relocInfo::none); 832 __ emit_data64(0x0013001200110010, relocInfo::none); 833 __ emit_data64(0x0017001600150014, relocInfo::none); 834 __ emit_data64(0x001B001A00190018, relocInfo::none); 835 __ emit_data64(0x001F001E001D001C, relocInfo::none); 836 // D 837 __ emit_data64(0x0000000100000000, relocInfo::none); 838 __ emit_data64(0x0000000300000002, relocInfo::none); 839 __ emit_data64(0x0000000500000004, relocInfo::none); 840 __ emit_data64(0x0000000700000006, relocInfo::none); 841 __ emit_data64(0x0000000900000008, relocInfo::none); 842 __ emit_data64(0x0000000B0000000A, relocInfo::none); 843 __ emit_data64(0x0000000D0000000C, relocInfo::none); 844 __ emit_data64(0x0000000F0000000E, relocInfo::none); 845 // Q 846 __ emit_data64(0x0000000000000000, relocInfo::none); 847 __ emit_data64(0x0000000000000001, relocInfo::none); 848 __ emit_data64(0x0000000000000002, relocInfo::none); 849 __ emit_data64(0x0000000000000003, relocInfo::none); 850 __ emit_data64(0x0000000000000004, relocInfo::none); 851 __ emit_data64(0x0000000000000005, relocInfo::none); 852 __ emit_data64(0x0000000000000006, relocInfo::none); 853 __ emit_data64(0x0000000000000007, relocInfo::none); 854 // D - FP 855 __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f 856 __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f 857 __ emit_data64(0x40A0000040800000, relocInfo::none); // 4.0f, 5.0f 858 __ emit_data64(0x40E0000040C00000, relocInfo::none); // 6.0f, 7.0f 859 __ emit_data64(0x4110000041000000, relocInfo::none); // 8.0f, 9.0f 860 __ emit_data64(0x4130000041200000, relocInfo::none); // 10.0f, 11.0f 861 __ emit_data64(0x4150000041400000, relocInfo::none); // 12.0f, 13.0f 862 __ emit_data64(0x4170000041600000, relocInfo::none); // 14.0f, 15.0f 863 // Q - FP 864 __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d 865 __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d 866 __ emit_data64(0x4000000000000000, relocInfo::none); // 2.0d 867 __ emit_data64(0x4008000000000000, relocInfo::none); // 3.0d 868 __ emit_data64(0x4010000000000000, relocInfo::none); // 4.0d 869 __ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d 870 __ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d 871 __ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d 872 return start; 873 } 874 875 address StubGenerator::generate_vector_reverse_bit_lut(const char *stub_name) { 876 __ align(CodeEntryAlignment); 877 StubCodeMark mark(this, "StubRoutines", stub_name); 878 address start = __ pc(); 879 880 __ emit_data64(0x0E060A020C040800, relocInfo::none); 881 __ emit_data64(0x0F070B030D050901, relocInfo::none); 882 __ emit_data64(0x0E060A020C040800, relocInfo::none); 883 __ emit_data64(0x0F070B030D050901, relocInfo::none); 884 __ emit_data64(0x0E060A020C040800, relocInfo::none); 885 __ emit_data64(0x0F070B030D050901, relocInfo::none); 886 __ emit_data64(0x0E060A020C040800, relocInfo::none); 887 __ emit_data64(0x0F070B030D050901, relocInfo::none); 888 889 return start; 890 } 891 892 address StubGenerator::generate_vector_reverse_byte_perm_mask_long(const char *stub_name) { 893 __ align(CodeEntryAlignment); 894 StubCodeMark mark(this, "StubRoutines", stub_name); 895 address start = __ pc(); 896 897 __ emit_data64(0x0001020304050607, relocInfo::none); 898 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 899 __ emit_data64(0x0001020304050607, relocInfo::none); 900 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 901 __ emit_data64(0x0001020304050607, relocInfo::none); 902 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 903 __ emit_data64(0x0001020304050607, relocInfo::none); 904 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 905 906 return start; 907 } 908 909 address StubGenerator::generate_vector_reverse_byte_perm_mask_int(const char *stub_name) { 910 __ align(CodeEntryAlignment); 911 StubCodeMark mark(this, "StubRoutines", stub_name); 912 address start = __ pc(); 913 914 __ emit_data64(0x0405060700010203, relocInfo::none); 915 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 916 __ emit_data64(0x0405060700010203, relocInfo::none); 917 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 918 __ emit_data64(0x0405060700010203, relocInfo::none); 919 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 920 __ emit_data64(0x0405060700010203, relocInfo::none); 921 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 922 923 return start; 924 } 925 926 address StubGenerator::generate_vector_reverse_byte_perm_mask_short(const char *stub_name) { 927 __ align(CodeEntryAlignment); 928 StubCodeMark mark(this, "StubRoutines", stub_name); 929 address start = __ pc(); 930 931 __ emit_data64(0x0607040502030001, relocInfo::none); 932 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 933 __ emit_data64(0x0607040502030001, relocInfo::none); 934 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 935 __ emit_data64(0x0607040502030001, relocInfo::none); 936 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 937 __ emit_data64(0x0607040502030001, relocInfo::none); 938 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 939 940 return start; 941 } 942 943 address StubGenerator::generate_vector_byte_shuffle_mask(const char *stub_name) { 944 __ align(CodeEntryAlignment); 945 StubCodeMark mark(this, "StubRoutines", stub_name); 946 address start = __ pc(); 947 948 __ emit_data64(0x7070707070707070, relocInfo::none); 949 __ emit_data64(0x7070707070707070, relocInfo::none); 950 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 951 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 952 953 return start; 954 } 955 956 address StubGenerator::generate_fp_mask(const char *stub_name, int64_t mask) { 957 __ align(CodeEntryAlignment); 958 StubCodeMark mark(this, "StubRoutines", stub_name); 959 address start = __ pc(); 960 961 __ emit_data64( mask, relocInfo::none ); 962 __ emit_data64( mask, relocInfo::none ); 963 964 return start; 965 } 966 967 address StubGenerator::generate_compress_perm_table(const char *stub_name, int32_t esize) { 968 __ align(CodeEntryAlignment); 969 StubCodeMark mark(this, "StubRoutines", stub_name); 970 address start = __ pc(); 971 if (esize == 32) { 972 // Loop to generate 256 x 8 int compression permute index table. A row is 973 // accessed using 8 bit index computed using vector mask. An entry in 974 // a row holds either a valid permute index corresponding to set bit position 975 // or a -1 (default) value. 976 for (int mask = 0; mask < 256; mask++) { 977 int ctr = 0; 978 for (int j = 0; j < 8; j++) { 979 if (mask & (1 << j)) { 980 __ emit_data(j, relocInfo::none); 981 ctr++; 982 } 983 } 984 for (; ctr < 8; ctr++) { 985 __ emit_data(-1, relocInfo::none); 986 } 987 } 988 } else { 989 assert(esize == 64, ""); 990 // Loop to generate 16 x 4 long compression permute index table. A row is 991 // accessed using 4 bit index computed using vector mask. An entry in 992 // a row holds either a valid permute index pair for a quadword corresponding 993 // to set bit position or a -1 (default) value. 994 for (int mask = 0; mask < 16; mask++) { 995 int ctr = 0; 996 for (int j = 0; j < 4; j++) { 997 if (mask & (1 << j)) { 998 __ emit_data(2 * j, relocInfo::none); 999 __ emit_data(2 * j + 1, relocInfo::none); 1000 ctr++; 1001 } 1002 } 1003 for (; ctr < 4; ctr++) { 1004 __ emit_data64(-1L, relocInfo::none); 1005 } 1006 } 1007 } 1008 return start; 1009 } 1010 1011 address StubGenerator::generate_expand_perm_table(const char *stub_name, int32_t esize) { 1012 __ align(CodeEntryAlignment); 1013 StubCodeMark mark(this, "StubRoutines", stub_name); 1014 address start = __ pc(); 1015 if (esize == 32) { 1016 // Loop to generate 256 x 8 int expand permute index table. A row is accessed 1017 // using 8 bit index computed using vector mask. An entry in a row holds either 1018 // a valid permute index (starting from least significant lane) placed at poisition 1019 // corresponding to set bit position or a -1 (default) value. 1020 for (int mask = 0; mask < 256; mask++) { 1021 int ctr = 0; 1022 for (int j = 0; j < 8; j++) { 1023 if (mask & (1 << j)) { 1024 __ emit_data(ctr++, relocInfo::none); 1025 } else { 1026 __ emit_data(-1, relocInfo::none); 1027 } 1028 } 1029 } 1030 } else { 1031 assert(esize == 64, ""); 1032 // Loop to generate 16 x 4 long expand permute index table. A row is accessed 1033 // using 4 bit index computed using vector mask. An entry in a row holds either 1034 // a valid doubleword permute index pair representing a quadword index (starting 1035 // from least significant lane) placed at poisition corresponding to set bit 1036 // position or a -1 (default) value. 1037 for (int mask = 0; mask < 16; mask++) { 1038 int ctr = 0; 1039 for (int j = 0; j < 4; j++) { 1040 if (mask & (1 << j)) { 1041 __ emit_data(2 * ctr, relocInfo::none); 1042 __ emit_data(2 * ctr + 1, relocInfo::none); 1043 ctr++; 1044 } else { 1045 __ emit_data64(-1L, relocInfo::none); 1046 } 1047 } 1048 } 1049 } 1050 return start; 1051 } 1052 1053 address StubGenerator::generate_vector_mask(const char *stub_name, int64_t mask) { 1054 __ align(CodeEntryAlignment); 1055 StubCodeMark mark(this, "StubRoutines", stub_name); 1056 address start = __ pc(); 1057 1058 __ emit_data64(mask, relocInfo::none); 1059 __ emit_data64(mask, relocInfo::none); 1060 __ emit_data64(mask, relocInfo::none); 1061 __ emit_data64(mask, relocInfo::none); 1062 __ emit_data64(mask, relocInfo::none); 1063 __ emit_data64(mask, relocInfo::none); 1064 __ emit_data64(mask, relocInfo::none); 1065 __ emit_data64(mask, relocInfo::none); 1066 1067 return start; 1068 } 1069 1070 address StubGenerator::generate_vector_byte_perm_mask(const char *stub_name) { 1071 __ align(CodeEntryAlignment); 1072 StubCodeMark mark(this, "StubRoutines", stub_name); 1073 address start = __ pc(); 1074 1075 __ emit_data64(0x0000000000000001, relocInfo::none); 1076 __ emit_data64(0x0000000000000003, relocInfo::none); 1077 __ emit_data64(0x0000000000000005, relocInfo::none); 1078 __ emit_data64(0x0000000000000007, relocInfo::none); 1079 __ emit_data64(0x0000000000000000, relocInfo::none); 1080 __ emit_data64(0x0000000000000002, relocInfo::none); 1081 __ emit_data64(0x0000000000000004, relocInfo::none); 1082 __ emit_data64(0x0000000000000006, relocInfo::none); 1083 1084 return start; 1085 } 1086 1087 address StubGenerator::generate_vector_fp_mask(const char *stub_name, int64_t mask) { 1088 __ align(CodeEntryAlignment); 1089 StubCodeMark mark(this, "StubRoutines", stub_name); 1090 address start = __ pc(); 1091 1092 __ emit_data64(mask, relocInfo::none); 1093 __ emit_data64(mask, relocInfo::none); 1094 __ emit_data64(mask, relocInfo::none); 1095 __ emit_data64(mask, relocInfo::none); 1096 __ emit_data64(mask, relocInfo::none); 1097 __ emit_data64(mask, relocInfo::none); 1098 __ emit_data64(mask, relocInfo::none); 1099 __ emit_data64(mask, relocInfo::none); 1100 1101 return start; 1102 } 1103 1104 address StubGenerator::generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len, 1105 int32_t val0, int32_t val1, int32_t val2, int32_t val3, 1106 int32_t val4, int32_t val5, int32_t val6, int32_t val7, 1107 int32_t val8, int32_t val9, int32_t val10, int32_t val11, 1108 int32_t val12, int32_t val13, int32_t val14, int32_t val15) { 1109 __ align(CodeEntryAlignment); 1110 StubCodeMark mark(this, "StubRoutines", stub_name); 1111 address start = __ pc(); 1112 1113 assert(len != Assembler::AVX_NoVec, "vector len must be specified"); 1114 __ emit_data(val0, relocInfo::none, 0); 1115 __ emit_data(val1, relocInfo::none, 0); 1116 __ emit_data(val2, relocInfo::none, 0); 1117 __ emit_data(val3, relocInfo::none, 0); 1118 if (len >= Assembler::AVX_256bit) { 1119 __ emit_data(val4, relocInfo::none, 0); 1120 __ emit_data(val5, relocInfo::none, 0); 1121 __ emit_data(val6, relocInfo::none, 0); 1122 __ emit_data(val7, relocInfo::none, 0); 1123 if (len >= Assembler::AVX_512bit) { 1124 __ emit_data(val8, relocInfo::none, 0); 1125 __ emit_data(val9, relocInfo::none, 0); 1126 __ emit_data(val10, relocInfo::none, 0); 1127 __ emit_data(val11, relocInfo::none, 0); 1128 __ emit_data(val12, relocInfo::none, 0); 1129 __ emit_data(val13, relocInfo::none, 0); 1130 __ emit_data(val14, relocInfo::none, 0); 1131 __ emit_data(val15, relocInfo::none, 0); 1132 } 1133 } 1134 return start; 1135 } 1136 1137 // Non-destructive plausibility checks for oops 1138 // 1139 // Arguments: 1140 // all args on stack! 1141 // 1142 // Stack after saving c_rarg3: 1143 // [tos + 0]: saved c_rarg3 1144 // [tos + 1]: saved c_rarg2 1145 // [tos + 2]: saved r12 (several TemplateTable methods use it) 1146 // [tos + 3]: saved flags 1147 // [tos + 4]: return address 1148 // * [tos + 5]: error message (char*) 1149 // * [tos + 6]: object to verify (oop) 1150 // * [tos + 7]: saved rax - saved by caller and bashed 1151 // * [tos + 8]: saved r10 (rscratch1) - saved by caller 1152 // * = popped on exit 1153 address StubGenerator::generate_verify_oop() { 1154 StubCodeMark mark(this, "StubRoutines", "verify_oop"); 1155 address start = __ pc(); 1156 1157 Label exit, error; 1158 1159 __ pushf(); 1160 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()), rscratch1); 1161 1162 __ push(r12); 1163 1164 // save c_rarg2 and c_rarg3 1165 __ push(c_rarg2); 1166 __ push(c_rarg3); 1167 1168 enum { 1169 // After previous pushes. 1170 oop_to_verify = 6 * wordSize, 1171 saved_rax = 7 * wordSize, 1172 saved_r10 = 8 * wordSize, 1173 1174 // Before the call to MacroAssembler::debug(), see below. 1175 return_addr = 16 * wordSize, 1176 error_msg = 17 * wordSize 1177 }; 1178 1179 // get object 1180 __ movptr(rax, Address(rsp, oop_to_verify)); 1181 1182 // make sure object is 'reasonable' 1183 __ testptr(rax, rax); 1184 __ jcc(Assembler::zero, exit); // if obj is null it is OK 1185 1186 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); 1187 bs_asm->check_oop(_masm, rax, c_rarg2, c_rarg3, error); 1188 1189 // return if everything seems ok 1190 __ bind(exit); 1191 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1192 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1193 __ pop(c_rarg3); // restore c_rarg3 1194 __ pop(c_rarg2); // restore c_rarg2 1195 __ pop(r12); // restore r12 1196 __ popf(); // restore flags 1197 __ ret(4 * wordSize); // pop caller saved stuff 1198 1199 // handle errors 1200 __ bind(error); 1201 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1202 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1203 __ pop(c_rarg3); // get saved c_rarg3 back 1204 __ pop(c_rarg2); // get saved c_rarg2 back 1205 __ pop(r12); // get saved r12 back 1206 __ popf(); // get saved flags off stack -- 1207 // will be ignored 1208 1209 __ pusha(); // push registers 1210 // (rip is already 1211 // already pushed) 1212 // debug(char* msg, int64_t pc, int64_t regs[]) 1213 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and 1214 // pushed all the registers, so now the stack looks like: 1215 // [tos + 0] 16 saved registers 1216 // [tos + 16] return address 1217 // * [tos + 17] error message (char*) 1218 // * [tos + 18] object to verify (oop) 1219 // * [tos + 19] saved rax - saved by caller and bashed 1220 // * [tos + 20] saved r10 (rscratch1) - saved by caller 1221 // * = popped on exit 1222 1223 __ movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message 1224 __ movptr(c_rarg1, Address(rsp, return_addr)); // pass return address 1225 __ movq(c_rarg2, rsp); // pass address of regs on stack 1226 __ mov(r12, rsp); // remember rsp 1227 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 1228 __ andptr(rsp, -16); // align stack as required by ABI 1229 BLOCK_COMMENT("call MacroAssembler::debug"); 1230 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 1231 __ hlt(); 1232 1233 return start; 1234 } 1235 1236 1237 // Shuffle first three arg regs on Windows into Linux/Solaris locations. 1238 // 1239 // Outputs: 1240 // rdi - rcx 1241 // rsi - rdx 1242 // rdx - r8 1243 // rcx - r9 1244 // 1245 // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter 1246 // are non-volatile. r9 and r10 should not be used by the caller. 1247 // 1248 void StubGenerator::setup_arg_regs(int nargs) { 1249 const Register saved_rdi = r9; 1250 const Register saved_rsi = r10; 1251 assert(nargs == 3 || nargs == 4, "else fix"); 1252 #ifdef _WIN64 1253 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1254 "unexpected argument registers"); 1255 if (nargs == 4) { 1256 __ mov(rax, r9); // r9 is also saved_rdi 1257 } 1258 __ movptr(saved_rdi, rdi); 1259 __ movptr(saved_rsi, rsi); 1260 __ mov(rdi, rcx); // c_rarg0 1261 __ mov(rsi, rdx); // c_rarg1 1262 __ mov(rdx, r8); // c_rarg2 1263 if (nargs == 4) { 1264 __ mov(rcx, rax); // c_rarg3 (via rax) 1265 } 1266 #else 1267 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1268 "unexpected argument registers"); 1269 #endif 1270 DEBUG_ONLY(_regs_in_thread = false;) 1271 } 1272 1273 1274 void StubGenerator::restore_arg_regs() { 1275 assert(!_regs_in_thread, "wrong call to restore_arg_regs"); 1276 const Register saved_rdi = r9; 1277 const Register saved_rsi = r10; 1278 #ifdef _WIN64 1279 __ movptr(rdi, saved_rdi); 1280 __ movptr(rsi, saved_rsi); 1281 #endif 1282 } 1283 1284 1285 // This is used in places where r10 is a scratch register, and can 1286 // be adapted if r9 is needed also. 1287 void StubGenerator::setup_arg_regs_using_thread(int nargs) { 1288 const Register saved_r15 = r9; 1289 assert(nargs == 3 || nargs == 4, "else fix"); 1290 #ifdef _WIN64 1291 if (nargs == 4) { 1292 __ mov(rax, r9); // r9 is also saved_r15 1293 } 1294 __ mov(saved_r15, r15); // r15 is callee saved and needs to be restored 1295 __ get_thread(r15_thread); 1296 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1297 "unexpected argument registers"); 1298 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi); 1299 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi); 1300 1301 __ mov(rdi, rcx); // c_rarg0 1302 __ mov(rsi, rdx); // c_rarg1 1303 __ mov(rdx, r8); // c_rarg2 1304 if (nargs == 4) { 1305 __ mov(rcx, rax); // c_rarg3 (via rax) 1306 } 1307 #else 1308 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1309 "unexpected argument registers"); 1310 #endif 1311 DEBUG_ONLY(_regs_in_thread = true;) 1312 } 1313 1314 1315 void StubGenerator::restore_arg_regs_using_thread() { 1316 assert(_regs_in_thread, "wrong call to restore_arg_regs"); 1317 const Register saved_r15 = r9; 1318 #ifdef _WIN64 1319 __ get_thread(r15_thread); 1320 __ movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset()))); 1321 __ movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset()))); 1322 __ mov(r15, saved_r15); // r15 is callee saved and needs to be restored 1323 #endif 1324 } 1325 1326 1327 void StubGenerator::setup_argument_regs(BasicType type) { 1328 if (type == T_BYTE || type == T_SHORT) { 1329 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1330 // r9 and r10 may be used to save non-volatile registers 1331 } else { 1332 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx 1333 // r9 is used to save r15_thread 1334 } 1335 } 1336 1337 1338 void StubGenerator::restore_argument_regs(BasicType type) { 1339 if (type == T_BYTE || type == T_SHORT) { 1340 restore_arg_regs(); 1341 } else { 1342 restore_arg_regs_using_thread(); 1343 } 1344 } 1345 1346 address StubGenerator::generate_data_cache_writeback() { 1347 const Register src = c_rarg0; // source address 1348 1349 __ align(CodeEntryAlignment); 1350 1351 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback"); 1352 1353 address start = __ pc(); 1354 1355 __ enter(); 1356 __ cache_wb(Address(src, 0)); 1357 __ leave(); 1358 __ ret(0); 1359 1360 return start; 1361 } 1362 1363 address StubGenerator::generate_data_cache_writeback_sync() { 1364 const Register is_pre = c_rarg0; // pre or post sync 1365 1366 __ align(CodeEntryAlignment); 1367 1368 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync"); 1369 1370 // pre wbsync is a no-op 1371 // post wbsync translates to an sfence 1372 1373 Label skip; 1374 address start = __ pc(); 1375 1376 __ enter(); 1377 __ cmpl(is_pre, 0); 1378 __ jcc(Assembler::notEqual, skip); 1379 __ cache_wbsync(false); 1380 __ bind(skip); 1381 __ leave(); 1382 __ ret(0); 1383 1384 return start; 1385 } 1386 1387 // ofs and limit are use for multi-block byte array. 1388 // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs) 1389 address StubGenerator::generate_md5_implCompress(bool multi_block, const char *name) { 1390 __ align(CodeEntryAlignment); 1391 StubCodeMark mark(this, "StubRoutines", name); 1392 address start = __ pc(); 1393 1394 const Register buf_param = r15; 1395 const Address state_param(rsp, 0 * wordSize); 1396 const Address ofs_param (rsp, 1 * wordSize ); 1397 const Address limit_param(rsp, 1 * wordSize + 4); 1398 1399 __ enter(); 1400 __ push(rbx); 1401 __ push(rdi); 1402 __ push(rsi); 1403 __ push(r15); 1404 __ subptr(rsp, 2 * wordSize); 1405 1406 __ movptr(buf_param, c_rarg0); 1407 __ movptr(state_param, c_rarg1); 1408 if (multi_block) { 1409 __ movl(ofs_param, c_rarg2); 1410 __ movl(limit_param, c_rarg3); 1411 } 1412 __ fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block); 1413 1414 __ addptr(rsp, 2 * wordSize); 1415 __ pop(r15); 1416 __ pop(rsi); 1417 __ pop(rdi); 1418 __ pop(rbx); 1419 __ leave(); 1420 __ ret(0); 1421 1422 return start; 1423 } 1424 1425 address StubGenerator::generate_upper_word_mask() { 1426 __ align64(); 1427 StubCodeMark mark(this, "StubRoutines", "upper_word_mask"); 1428 address start = __ pc(); 1429 1430 __ emit_data64(0x0000000000000000, relocInfo::none); 1431 __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); 1432 1433 return start; 1434 } 1435 1436 address StubGenerator::generate_shuffle_byte_flip_mask() { 1437 __ align64(); 1438 StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask"); 1439 address start = __ pc(); 1440 1441 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1442 __ emit_data64(0x0001020304050607, relocInfo::none); 1443 1444 return start; 1445 } 1446 1447 // ofs and limit are use for multi-block byte array. 1448 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1449 address StubGenerator::generate_sha1_implCompress(bool multi_block, const char *name) { 1450 __ align(CodeEntryAlignment); 1451 StubCodeMark mark(this, "StubRoutines", name); 1452 address start = __ pc(); 1453 1454 Register buf = c_rarg0; 1455 Register state = c_rarg1; 1456 Register ofs = c_rarg2; 1457 Register limit = c_rarg3; 1458 1459 const XMMRegister abcd = xmm0; 1460 const XMMRegister e0 = xmm1; 1461 const XMMRegister e1 = xmm2; 1462 const XMMRegister msg0 = xmm3; 1463 1464 const XMMRegister msg1 = xmm4; 1465 const XMMRegister msg2 = xmm5; 1466 const XMMRegister msg3 = xmm6; 1467 const XMMRegister shuf_mask = xmm7; 1468 1469 __ enter(); 1470 1471 __ subptr(rsp, 4 * wordSize); 1472 1473 __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask, 1474 buf, state, ofs, limit, rsp, multi_block); 1475 1476 __ addptr(rsp, 4 * wordSize); 1477 1478 __ leave(); 1479 __ ret(0); 1480 1481 return start; 1482 } 1483 1484 address StubGenerator::generate_pshuffle_byte_flip_mask() { 1485 __ align64(); 1486 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask"); 1487 address start = __ pc(); 1488 1489 __ emit_data64(0x0405060700010203, relocInfo::none); 1490 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1491 1492 if (VM_Version::supports_avx2()) { 1493 __ emit_data64(0x0405060700010203, relocInfo::none); // second copy 1494 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1495 // _SHUF_00BA 1496 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1497 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1498 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1499 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1500 // _SHUF_DC00 1501 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1502 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1503 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1504 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1505 } 1506 1507 return start; 1508 } 1509 1510 //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 1511 address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() { 1512 __ align32(); 1513 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512"); 1514 address start = __ pc(); 1515 1516 if (VM_Version::supports_avx2()) { 1517 __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK 1518 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1519 __ emit_data64(0x1011121314151617, relocInfo::none); 1520 __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); 1521 __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO 1522 __ emit_data64(0x0000000000000000, relocInfo::none); 1523 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1524 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1525 } 1526 1527 return start; 1528 } 1529 1530 // ofs and limit are use for multi-block byte array. 1531 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1532 address StubGenerator::generate_sha256_implCompress(bool multi_block, const char *name) { 1533 assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); 1534 __ align(CodeEntryAlignment); 1535 StubCodeMark mark(this, "StubRoutines", name); 1536 address start = __ pc(); 1537 1538 Register buf = c_rarg0; 1539 Register state = c_rarg1; 1540 Register ofs = c_rarg2; 1541 Register limit = c_rarg3; 1542 1543 const XMMRegister msg = xmm0; 1544 const XMMRegister state0 = xmm1; 1545 const XMMRegister state1 = xmm2; 1546 const XMMRegister msgtmp0 = xmm3; 1547 1548 const XMMRegister msgtmp1 = xmm4; 1549 const XMMRegister msgtmp2 = xmm5; 1550 const XMMRegister msgtmp3 = xmm6; 1551 const XMMRegister msgtmp4 = xmm7; 1552 1553 const XMMRegister shuf_mask = xmm8; 1554 1555 __ enter(); 1556 1557 __ subptr(rsp, 4 * wordSize); 1558 1559 if (VM_Version::supports_sha()) { 1560 __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1561 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1562 } else if (VM_Version::supports_avx2()) { 1563 __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1564 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1565 } 1566 __ addptr(rsp, 4 * wordSize); 1567 __ vzeroupper(); 1568 __ leave(); 1569 __ ret(0); 1570 1571 return start; 1572 } 1573 1574 address StubGenerator::generate_sha512_implCompress(bool multi_block, const char *name) { 1575 assert(VM_Version::supports_avx2(), ""); 1576 assert(VM_Version::supports_bmi2(), ""); 1577 __ align(CodeEntryAlignment); 1578 StubCodeMark mark(this, "StubRoutines", name); 1579 address start = __ pc(); 1580 1581 Register buf = c_rarg0; 1582 Register state = c_rarg1; 1583 Register ofs = c_rarg2; 1584 Register limit = c_rarg3; 1585 1586 const XMMRegister msg = xmm0; 1587 const XMMRegister state0 = xmm1; 1588 const XMMRegister state1 = xmm2; 1589 const XMMRegister msgtmp0 = xmm3; 1590 const XMMRegister msgtmp1 = xmm4; 1591 const XMMRegister msgtmp2 = xmm5; 1592 const XMMRegister msgtmp3 = xmm6; 1593 const XMMRegister msgtmp4 = xmm7; 1594 1595 const XMMRegister shuf_mask = xmm8; 1596 1597 __ enter(); 1598 1599 __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1600 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1601 1602 __ vzeroupper(); 1603 __ leave(); 1604 __ ret(0); 1605 1606 return start; 1607 } 1608 1609 address StubGenerator::base64_shuffle_addr() { 1610 __ align64(); 1611 StubCodeMark mark(this, "StubRoutines", "shuffle_base64"); 1612 address start = __ pc(); 1613 1614 assert(((unsigned long long)start & 0x3f) == 0, 1615 "Alignment problem (0x%08llx)", (unsigned long long)start); 1616 __ emit_data64(0x0405030401020001, relocInfo::none); 1617 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1618 __ emit_data64(0x10110f100d0e0c0d, relocInfo::none); 1619 __ emit_data64(0x1617151613141213, relocInfo::none); 1620 __ emit_data64(0x1c1d1b1c191a1819, relocInfo::none); 1621 __ emit_data64(0x222321221f201e1f, relocInfo::none); 1622 __ emit_data64(0x2829272825262425, relocInfo::none); 1623 __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); 1624 1625 return start; 1626 } 1627 1628 address StubGenerator::base64_avx2_shuffle_addr() { 1629 __ align32(); 1630 StubCodeMark mark(this, "StubRoutines", "avx2_shuffle_base64"); 1631 address start = __ pc(); 1632 1633 __ emit_data64(0x0809070805060405, relocInfo::none); 1634 __ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none); 1635 __ emit_data64(0x0405030401020001, relocInfo::none); 1636 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1637 1638 return start; 1639 } 1640 1641 address StubGenerator::base64_avx2_input_mask_addr() { 1642 __ align32(); 1643 StubCodeMark mark(this, "StubRoutines", "avx2_input_mask_base64"); 1644 address start = __ pc(); 1645 1646 __ emit_data64(0x8000000000000000, relocInfo::none); 1647 __ emit_data64(0x8000000080000000, relocInfo::none); 1648 __ emit_data64(0x8000000080000000, relocInfo::none); 1649 __ emit_data64(0x8000000080000000, relocInfo::none); 1650 1651 return start; 1652 } 1653 1654 address StubGenerator::base64_avx2_lut_addr() { 1655 __ align32(); 1656 StubCodeMark mark(this, "StubRoutines", "avx2_lut_base64"); 1657 address start = __ pc(); 1658 1659 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1660 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1661 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1662 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1663 1664 // URL LUT 1665 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1666 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1667 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1668 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1669 1670 return start; 1671 } 1672 1673 address StubGenerator::base64_encoding_table_addr() { 1674 __ align64(); 1675 StubCodeMark mark(this, "StubRoutines", "encoding_table_base64"); 1676 address start = __ pc(); 1677 1678 assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); 1679 __ emit_data64(0x4847464544434241, relocInfo::none); 1680 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1681 __ emit_data64(0x5857565554535251, relocInfo::none); 1682 __ emit_data64(0x6665646362615a59, relocInfo::none); 1683 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1684 __ emit_data64(0x767574737271706f, relocInfo::none); 1685 __ emit_data64(0x333231307a797877, relocInfo::none); 1686 __ emit_data64(0x2f2b393837363534, relocInfo::none); 1687 1688 // URL table 1689 __ emit_data64(0x4847464544434241, relocInfo::none); 1690 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1691 __ emit_data64(0x5857565554535251, relocInfo::none); 1692 __ emit_data64(0x6665646362615a59, relocInfo::none); 1693 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1694 __ emit_data64(0x767574737271706f, relocInfo::none); 1695 __ emit_data64(0x333231307a797877, relocInfo::none); 1696 __ emit_data64(0x5f2d393837363534, relocInfo::none); 1697 1698 return start; 1699 } 1700 1701 // Code for generating Base64 encoding. 1702 // Intrinsic function prototype in Base64.java: 1703 // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, 1704 // boolean isURL) { 1705 address StubGenerator::generate_base64_encodeBlock() 1706 { 1707 __ align(CodeEntryAlignment); 1708 StubCodeMark mark(this, "StubRoutines", "implEncode"); 1709 address start = __ pc(); 1710 1711 __ enter(); 1712 1713 // Save callee-saved registers before using them 1714 __ push(r12); 1715 __ push(r13); 1716 __ push(r14); 1717 __ push(r15); 1718 1719 // arguments 1720 const Register source = c_rarg0; // Source Array 1721 const Register start_offset = c_rarg1; // start offset 1722 const Register end_offset = c_rarg2; // end offset 1723 const Register dest = c_rarg3; // destination array 1724 1725 #ifndef _WIN64 1726 const Register dp = c_rarg4; // Position for writing to dest array 1727 const Register isURL = c_rarg5; // Base64 or URL character set 1728 #else 1729 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 1730 const Address isURL_mem(rbp, 7 * wordSize); 1731 const Register isURL = r10; // pick the volatile windows register 1732 const Register dp = r12; 1733 __ movl(dp, dp_mem); 1734 __ movl(isURL, isURL_mem); 1735 #endif 1736 1737 const Register length = r14; 1738 const Register encode_table = r13; 1739 Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop; 1740 1741 // calculate length from offsets 1742 __ movl(length, end_offset); 1743 __ subl(length, start_offset); 1744 __ jcc(Assembler::lessEqual, L_exit); 1745 1746 // Code for 512-bit VBMI encoding. Encodes 48 input bytes into 64 1747 // output bytes. We read 64 input bytes and ignore the last 16, so be 1748 // sure not to read past the end of the input buffer. 1749 if (VM_Version::supports_avx512_vbmi()) { 1750 __ cmpl(length, 64); // Do not overrun input buffer. 1751 __ jcc(Assembler::below, L_not512); 1752 1753 __ shll(isURL, 6); // index into decode table based on isURL 1754 __ lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 1755 __ addptr(encode_table, isURL); 1756 __ shrl(isURL, 6); // restore isURL 1757 1758 __ mov64(rax, 0x3036242a1016040aull); // Shifts 1759 __ evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15); 1760 __ evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit); 1761 __ evpbroadcastq(xmm1, rax, Assembler::AVX_512bit); 1762 1763 __ align32(); 1764 __ BIND(L_vbmiLoop); 1765 1766 __ vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit); 1767 __ subl(length, 48); 1768 1769 // Put the input bytes into the proper lanes for writing, then 1770 // encode them. 1771 __ evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit); 1772 __ vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit); 1773 1774 // Write to destination 1775 __ evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit); 1776 1777 __ addptr(dest, 64); 1778 __ addptr(source, 48); 1779 __ cmpl(length, 64); 1780 __ jcc(Assembler::aboveEqual, L_vbmiLoop); 1781 1782 __ vzeroupper(); 1783 } 1784 1785 __ BIND(L_not512); 1786 if (VM_Version::supports_avx2()) { 1787 /* 1788 ** This AVX2 encoder is based off the paper at: 1789 ** https://dl.acm.org/doi/10.1145/3132709 1790 ** 1791 ** We use AVX2 SIMD instructions to encode 24 bytes into 32 1792 ** output bytes. 1793 ** 1794 */ 1795 // Lengths under 32 bytes are done with scalar routine 1796 __ cmpl(length, 31); 1797 __ jcc(Assembler::belowEqual, L_process3); 1798 1799 // Set up supporting constant table data 1800 __ vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax); 1801 // 6-bit mask for 2nd and 4th (and multiples) 6-bit values 1802 __ movl(rax, 0x0fc0fc00); 1803 __ movdl(xmm8, rax); 1804 __ vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax); 1805 __ vpbroadcastd(xmm8, xmm8, Assembler::AVX_256bit); 1806 1807 // Multiplication constant for "shifting" right by 6 and 10 1808 // bits 1809 __ movl(rax, 0x04000040); 1810 1811 __ subl(length, 24); 1812 __ movdl(xmm7, rax); 1813 __ vpbroadcastd(xmm7, xmm7, Assembler::AVX_256bit); 1814 1815 // For the first load, we mask off reading of the first 4 1816 // bytes into the register. This is so we can get 4 3-byte 1817 // chunks into each lane of the register, avoiding having to 1818 // handle end conditions. We then shuffle these bytes into a 1819 // specific order so that manipulation is easier. 1820 // 1821 // The initial read loads the XMM register like this: 1822 // 1823 // Lower 128-bit lane: 1824 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1825 // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1 1826 // | C2 | D0 | D1 | D2 | 1827 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1828 // 1829 // Upper 128-bit lane: 1830 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1831 // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2 1832 // | XX | XX | XX | XX | 1833 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1834 // 1835 // Where A0 is the first input byte, B0 is the fourth, etc. 1836 // The alphabetical significance denotes the 3 bytes to be 1837 // consumed and encoded into 4 bytes. 1838 // 1839 // We then shuffle the register so each 32-bit word contains 1840 // the sequence: 1841 // A1 A0 A2 A1, B1, B0, B2, B1, etc. 1842 // Each of these byte sequences are then manipulated into 4 1843 // 6-bit values ready for encoding. 1844 // 1845 // If we focus on one set of 3-byte chunks, changing the 1846 // nomenclature such that A0 => a, A1 => b, and A2 => c, we 1847 // shuffle such that each 24-bit chunk contains: 1848 // 1849 // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6 1850 // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0 1851 // Explain this step. 1852 // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4 1853 // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2 1854 // 1855 // W first and off all but bits 4-9 and 16-21 (c5..c0 and 1856 // a5..a0) and shift them using a vector multiplication 1857 // operation (vpmulhuw) which effectively shifts c right by 6 1858 // bits and a right by 10 bits. We similarly mask bits 10-15 1859 // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4 1860 // bits respectively. This is done using vpmullw. We end up 1861 // with 4 6-bit values, thus splitting the 3 input bytes, 1862 // ready for encoding: 1863 // 0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0 1864 // 1865 // For translation, we recognize that there are 5 distinct 1866 // ranges of legal Base64 characters as below: 1867 // 1868 // +-------------+-------------+------------+ 1869 // | 6-bit value | ASCII range | offset | 1870 // +-------------+-------------+------------+ 1871 // | 0..25 | A..Z | 65 | 1872 // | 26..51 | a..z | 71 | 1873 // | 52..61 | 0..9 | -4 | 1874 // | 62 | + or - | -19 or -17 | 1875 // | 63 | / or _ | -16 or 32 | 1876 // +-------------+-------------+------------+ 1877 // 1878 // We note that vpshufb does a parallel lookup in a 1879 // destination register using the lower 4 bits of bytes from a 1880 // source register. If we use a saturated subtraction and 1881 // subtract 51 from each 6-bit value, bytes from [0,51] 1882 // saturate to 0, and [52,63] map to a range of [1,12]. We 1883 // distinguish the [0,25] and [26,51] ranges by assigning a 1884 // value of 13 for all 6-bit values less than 26. We end up 1885 // with: 1886 // 1887 // +-------------+-------------+------------+ 1888 // | 6-bit value | Reduced | offset | 1889 // +-------------+-------------+------------+ 1890 // | 0..25 | 13 | 65 | 1891 // | 26..51 | 0 | 71 | 1892 // | 52..61 | 0..9 | -4 | 1893 // | 62 | 11 | -19 or -17 | 1894 // | 63 | 12 | -16 or 32 | 1895 // +-------------+-------------+------------+ 1896 // 1897 // We then use a final vpshufb to add the appropriate offset, 1898 // translating the bytes. 1899 // 1900 // Load input bytes - only 28 bytes. Mask the first load to 1901 // not load into the full register. 1902 __ vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit); 1903 1904 // Move 3-byte chunks of input (12 bytes) into 16 bytes, 1905 // ordering by: 1906 // 1, 0, 2, 1; 4, 3, 5, 4; etc. This groups 6-bit chunks 1907 // for easy masking 1908 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 1909 1910 __ addl(start_offset, 24); 1911 1912 // Load masking register for first and third (and multiples) 1913 // 6-bit values. 1914 __ movl(rax, 0x003f03f0); 1915 __ movdl(xmm6, rax); 1916 __ vpbroadcastd(xmm6, xmm6, Assembler::AVX_256bit); 1917 // Multiplication constant for "shifting" left by 4 and 8 bits 1918 __ movl(rax, 0x01000010); 1919 __ movdl(xmm5, rax); 1920 __ vpbroadcastd(xmm5, xmm5, Assembler::AVX_256bit); 1921 1922 // Isolate 6-bit chunks of interest 1923 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 1924 1925 // Load constants for encoding 1926 __ movl(rax, 0x19191919); 1927 __ movdl(xmm3, rax); 1928 __ vpbroadcastd(xmm3, xmm3, Assembler::AVX_256bit); 1929 __ movl(rax, 0x33333333); 1930 __ movdl(xmm4, rax); 1931 __ vpbroadcastd(xmm4, xmm4, Assembler::AVX_256bit); 1932 1933 // Shift output bytes 0 and 2 into proper lanes 1934 __ vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit); 1935 1936 // Mask and shift output bytes 1 and 3 into proper lanes and 1937 // combine 1938 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 1939 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 1940 __ vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 1941 1942 // Find out which are 0..25. This indicates which input 1943 // values fall in the range of 'A'-'Z', which require an 1944 // additional offset (see comments above) 1945 __ vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit); 1946 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 1947 __ vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit); 1948 1949 // Load the proper lookup table 1950 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr())); 1951 __ movl(r15, isURL); 1952 __ shll(r15, 5); 1953 __ vmovdqu(xmm2, Address(r11, r15)); 1954 1955 // Shuffle the offsets based on the range calculation done 1956 // above. This allows us to add the correct offset to the 1957 // 6-bit value corresponding to the range documented above. 1958 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 1959 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 1960 1961 // Store the encoded bytes 1962 __ vmovdqu(Address(dest, dp), xmm0); 1963 __ addl(dp, 32); 1964 1965 __ cmpl(length, 31); 1966 __ jcc(Assembler::belowEqual, L_process3); 1967 1968 __ align32(); 1969 __ BIND(L_32byteLoop); 1970 1971 // Get next 32 bytes 1972 __ vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4)); 1973 1974 __ subl(length, 24); 1975 __ addl(start_offset, 24); 1976 1977 // This logic is identical to the above, with only constant 1978 // register loads removed. Shuffle the input, mask off 6-bit 1979 // chunks, shift them into place, then add the offset to 1980 // encode. 1981 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 1982 1983 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 1984 __ vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit); 1985 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 1986 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 1987 __ vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 1988 __ vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit); 1989 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 1990 __ vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit); 1991 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 1992 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 1993 1994 // Store the encoded bytes 1995 __ vmovdqu(Address(dest, dp), xmm0); 1996 __ addl(dp, 32); 1997 1998 __ cmpl(length, 31); 1999 __ jcc(Assembler::above, L_32byteLoop); 2000 2001 __ BIND(L_process3); 2002 __ vzeroupper(); 2003 } else { 2004 __ BIND(L_process3); 2005 } 2006 2007 __ cmpl(length, 3); 2008 __ jcc(Assembler::below, L_exit); 2009 2010 // Load the encoding table based on isURL 2011 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 2012 __ movl(r15, isURL); 2013 __ shll(r15, 6); 2014 __ addptr(r11, r15); 2015 2016 __ BIND(L_processdata); 2017 2018 // Load 3 bytes 2019 __ load_unsigned_byte(r15, Address(source, start_offset)); 2020 __ load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1)); 2021 __ load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2)); 2022 2023 // Build a 32-bit word with bytes 1, 2, 0, 1 2024 __ movl(rax, r10); 2025 __ shll(r10, 24); 2026 __ orl(rax, r10); 2027 2028 __ subl(length, 3); 2029 2030 __ shll(r15, 8); 2031 __ shll(r13, 16); 2032 __ orl(rax, r15); 2033 2034 __ addl(start_offset, 3); 2035 2036 __ orl(rax, r13); 2037 // At this point, rax contains | byte1 | byte2 | byte0 | byte1 2038 // r13 has byte2 << 16 - need low-order 6 bits to translate. 2039 // This translated byte is the fourth output byte. 2040 __ shrl(r13, 16); 2041 __ andl(r13, 0x3f); 2042 2043 // The high-order 6 bits of r15 (byte0) is translated. 2044 // The translated byte is the first output byte. 2045 __ shrl(r15, 10); 2046 2047 __ load_unsigned_byte(r13, Address(r11, r13)); 2048 __ load_unsigned_byte(r15, Address(r11, r15)); 2049 2050 __ movb(Address(dest, dp, Address::times_1, 3), r13); 2051 2052 // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0. 2053 // This translated byte is the second output byte. 2054 __ shrl(rax, 4); 2055 __ movl(r10, rax); 2056 __ andl(rax, 0x3f); 2057 2058 __ movb(Address(dest, dp, Address::times_1, 0), r15); 2059 2060 __ load_unsigned_byte(rax, Address(r11, rax)); 2061 2062 // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2. 2063 // This translated byte is the third output byte. 2064 __ shrl(r10, 18); 2065 __ andl(r10, 0x3f); 2066 2067 __ load_unsigned_byte(r10, Address(r11, r10)); 2068 2069 __ movb(Address(dest, dp, Address::times_1, 1), rax); 2070 __ movb(Address(dest, dp, Address::times_1, 2), r10); 2071 2072 __ addl(dp, 4); 2073 __ cmpl(length, 3); 2074 __ jcc(Assembler::aboveEqual, L_processdata); 2075 2076 __ BIND(L_exit); 2077 __ pop(r15); 2078 __ pop(r14); 2079 __ pop(r13); 2080 __ pop(r12); 2081 __ leave(); 2082 __ ret(0); 2083 2084 return start; 2085 } 2086 2087 // base64 AVX512vbmi tables 2088 address StubGenerator::base64_vbmi_lookup_lo_addr() { 2089 __ align64(); 2090 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64"); 2091 address start = __ pc(); 2092 2093 assert(((unsigned long long)start & 0x3f) == 0, 2094 "Alignment problem (0x%08llx)", (unsigned long long)start); 2095 __ emit_data64(0x8080808080808080, relocInfo::none); 2096 __ emit_data64(0x8080808080808080, relocInfo::none); 2097 __ emit_data64(0x8080808080808080, relocInfo::none); 2098 __ emit_data64(0x8080808080808080, relocInfo::none); 2099 __ emit_data64(0x8080808080808080, relocInfo::none); 2100 __ emit_data64(0x3f8080803e808080, relocInfo::none); 2101 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2102 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2103 2104 return start; 2105 } 2106 2107 address StubGenerator::base64_vbmi_lookup_hi_addr() { 2108 __ align64(); 2109 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64"); 2110 address start = __ pc(); 2111 2112 assert(((unsigned long long)start & 0x3f) == 0, 2113 "Alignment problem (0x%08llx)", (unsigned long long)start); 2114 __ emit_data64(0x0605040302010080, relocInfo::none); 2115 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2116 __ emit_data64(0x161514131211100f, relocInfo::none); 2117 __ emit_data64(0x8080808080191817, relocInfo::none); 2118 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2119 __ emit_data64(0x2827262524232221, relocInfo::none); 2120 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2121 __ emit_data64(0x8080808080333231, relocInfo::none); 2122 2123 return start; 2124 } 2125 address StubGenerator::base64_vbmi_lookup_lo_url_addr() { 2126 __ align64(); 2127 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64url"); 2128 address start = __ pc(); 2129 2130 assert(((unsigned long long)start & 0x3f) == 0, 2131 "Alignment problem (0x%08llx)", (unsigned long long)start); 2132 __ emit_data64(0x8080808080808080, relocInfo::none); 2133 __ emit_data64(0x8080808080808080, relocInfo::none); 2134 __ emit_data64(0x8080808080808080, relocInfo::none); 2135 __ emit_data64(0x8080808080808080, relocInfo::none); 2136 __ emit_data64(0x8080808080808080, relocInfo::none); 2137 __ emit_data64(0x80803e8080808080, relocInfo::none); 2138 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2139 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2140 2141 return start; 2142 } 2143 2144 address StubGenerator::base64_vbmi_lookup_hi_url_addr() { 2145 __ align64(); 2146 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64url"); 2147 address start = __ pc(); 2148 2149 assert(((unsigned long long)start & 0x3f) == 0, 2150 "Alignment problem (0x%08llx)", (unsigned long long)start); 2151 __ emit_data64(0x0605040302010080, relocInfo::none); 2152 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2153 __ emit_data64(0x161514131211100f, relocInfo::none); 2154 __ emit_data64(0x3f80808080191817, relocInfo::none); 2155 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2156 __ emit_data64(0x2827262524232221, relocInfo::none); 2157 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2158 __ emit_data64(0x8080808080333231, relocInfo::none); 2159 2160 return start; 2161 } 2162 2163 address StubGenerator::base64_vbmi_pack_vec_addr() { 2164 __ align64(); 2165 StubCodeMark mark(this, "StubRoutines", "pack_vec_base64"); 2166 address start = __ pc(); 2167 2168 assert(((unsigned long long)start & 0x3f) == 0, 2169 "Alignment problem (0x%08llx)", (unsigned long long)start); 2170 __ emit_data64(0x090a040506000102, relocInfo::none); 2171 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2172 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2173 __ emit_data64(0x292a242526202122, relocInfo::none); 2174 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2175 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2176 __ emit_data64(0x0000000000000000, relocInfo::none); 2177 __ emit_data64(0x0000000000000000, relocInfo::none); 2178 2179 return start; 2180 } 2181 2182 address StubGenerator::base64_vbmi_join_0_1_addr() { 2183 __ align64(); 2184 StubCodeMark mark(this, "StubRoutines", "join_0_1_base64"); 2185 address start = __ pc(); 2186 2187 assert(((unsigned long long)start & 0x3f) == 0, 2188 "Alignment problem (0x%08llx)", (unsigned long long)start); 2189 __ emit_data64(0x090a040506000102, relocInfo::none); 2190 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2191 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2192 __ emit_data64(0x292a242526202122, relocInfo::none); 2193 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2194 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2195 __ emit_data64(0x494a444546404142, relocInfo::none); 2196 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2197 2198 return start; 2199 } 2200 2201 address StubGenerator::base64_vbmi_join_1_2_addr() { 2202 __ align64(); 2203 StubCodeMark mark(this, "StubRoutines", "join_1_2_base64"); 2204 address start = __ pc(); 2205 2206 assert(((unsigned long long)start & 0x3f) == 0, 2207 "Alignment problem (0x%08llx)", (unsigned long long)start); 2208 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2209 __ emit_data64(0x292a242526202122, relocInfo::none); 2210 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2211 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2212 __ emit_data64(0x494a444546404142, relocInfo::none); 2213 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2214 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2215 __ emit_data64(0x696a646566606162, relocInfo::none); 2216 2217 return start; 2218 } 2219 2220 address StubGenerator::base64_vbmi_join_2_3_addr() { 2221 __ align64(); 2222 StubCodeMark mark(this, "StubRoutines", "join_2_3_base64"); 2223 address start = __ pc(); 2224 2225 assert(((unsigned long long)start & 0x3f) == 0, 2226 "Alignment problem (0x%08llx)", (unsigned long long)start); 2227 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2228 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2229 __ emit_data64(0x494a444546404142, relocInfo::none); 2230 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2231 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2232 __ emit_data64(0x696a646566606162, relocInfo::none); 2233 __ emit_data64(0x767071726c6d6e68, relocInfo::none); 2234 __ emit_data64(0x7c7d7e78797a7475, relocInfo::none); 2235 2236 return start; 2237 } 2238 2239 address StubGenerator::base64_AVX2_decode_tables_addr() { 2240 __ align64(); 2241 StubCodeMark mark(this, "StubRoutines", "AVX2_tables_base64"); 2242 address start = __ pc(); 2243 2244 assert(((unsigned long long)start & 0x3f) == 0, 2245 "Alignment problem (0x%08llx)", (unsigned long long)start); 2246 __ emit_data(0x2f2f2f2f, relocInfo::none, 0); 2247 __ emit_data(0x5f5f5f5f, relocInfo::none, 0); // for URL 2248 2249 __ emit_data(0xffffffff, relocInfo::none, 0); 2250 __ emit_data(0xfcfcfcfc, relocInfo::none, 0); // for URL 2251 2252 // Permute table 2253 __ emit_data64(0x0000000100000000, relocInfo::none); 2254 __ emit_data64(0x0000000400000002, relocInfo::none); 2255 __ emit_data64(0x0000000600000005, relocInfo::none); 2256 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2257 2258 // Shuffle table 2259 __ emit_data64(0x090a040506000102, relocInfo::none); 2260 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2261 __ emit_data64(0x090a040506000102, relocInfo::none); 2262 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2263 2264 // merge table 2265 __ emit_data(0x01400140, relocInfo::none, 0); 2266 2267 // merge multiplier 2268 __ emit_data(0x00011000, relocInfo::none, 0); 2269 2270 return start; 2271 } 2272 2273 address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { 2274 __ align64(); 2275 StubCodeMark mark(this, "StubRoutines", "AVX2_tables_URL_base64"); 2276 address start = __ pc(); 2277 2278 assert(((unsigned long long)start & 0x3f) == 0, 2279 "Alignment problem (0x%08llx)", (unsigned long long)start); 2280 // lut_lo 2281 __ emit_data64(0x1111111111111115, relocInfo::none); 2282 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2283 __ emit_data64(0x1111111111111115, relocInfo::none); 2284 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2285 2286 // lut_roll 2287 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2288 __ emit_data64(0x0000000000000000, relocInfo::none); 2289 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2290 __ emit_data64(0x0000000000000000, relocInfo::none); 2291 2292 // lut_lo URL 2293 __ emit_data64(0x1111111111111115, relocInfo::none); 2294 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2295 __ emit_data64(0x1111111111111115, relocInfo::none); 2296 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2297 2298 // lut_roll URL 2299 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2300 __ emit_data64(0x0000000000000000, relocInfo::none); 2301 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2302 __ emit_data64(0x0000000000000000, relocInfo::none); 2303 2304 // lut_hi 2305 __ emit_data64(0x0804080402011010, relocInfo::none); 2306 __ emit_data64(0x1010101010101010, relocInfo::none); 2307 __ emit_data64(0x0804080402011010, relocInfo::none); 2308 __ emit_data64(0x1010101010101010, relocInfo::none); 2309 2310 return start; 2311 } 2312 2313 address StubGenerator::base64_decoding_table_addr() { 2314 StubCodeMark mark(this, "StubRoutines", "decoding_table_base64"); 2315 address start = __ pc(); 2316 2317 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2318 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2319 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2320 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2321 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2322 __ emit_data64(0x3fffffff3effffff, relocInfo::none); 2323 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2324 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2325 __ emit_data64(0x06050403020100ff, relocInfo::none); 2326 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2327 __ emit_data64(0x161514131211100f, relocInfo::none); 2328 __ emit_data64(0xffffffffff191817, relocInfo::none); 2329 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2330 __ emit_data64(0x2827262524232221, relocInfo::none); 2331 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2332 __ emit_data64(0xffffffffff333231, relocInfo::none); 2333 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2334 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2335 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2336 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2337 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2338 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2339 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2340 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2341 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2342 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2343 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2344 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2345 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2346 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2347 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2348 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2349 2350 // URL table 2351 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2352 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2353 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2354 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2355 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2356 __ emit_data64(0xffff3effffffffff, relocInfo::none); 2357 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2358 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2359 __ emit_data64(0x06050403020100ff, relocInfo::none); 2360 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2361 __ emit_data64(0x161514131211100f, relocInfo::none); 2362 __ emit_data64(0x3fffffffff191817, relocInfo::none); 2363 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2364 __ emit_data64(0x2827262524232221, relocInfo::none); 2365 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2366 __ emit_data64(0xffffffffff333231, relocInfo::none); 2367 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2368 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2369 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2370 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2371 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2372 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2373 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2374 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2375 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2376 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2377 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2378 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2379 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2380 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2381 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2382 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2383 2384 return start; 2385 } 2386 2387 2388 // Code for generating Base64 decoding. 2389 // 2390 // Based on the article (and associated code) from https://arxiv.org/abs/1910.05109. 2391 // 2392 // Intrinsic function prototype in Base64.java: 2393 // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { 2394 address StubGenerator::generate_base64_decodeBlock() { 2395 __ align(CodeEntryAlignment); 2396 StubCodeMark mark(this, "StubRoutines", "implDecode"); 2397 address start = __ pc(); 2398 2399 __ enter(); 2400 2401 // Save callee-saved registers before using them 2402 __ push(r12); 2403 __ push(r13); 2404 __ push(r14); 2405 __ push(r15); 2406 __ push(rbx); 2407 2408 // arguments 2409 const Register source = c_rarg0; // Source Array 2410 const Register start_offset = c_rarg1; // start offset 2411 const Register end_offset = c_rarg2; // end offset 2412 const Register dest = c_rarg3; // destination array 2413 const Register isMIME = rbx; 2414 2415 #ifndef _WIN64 2416 const Register dp = c_rarg4; // Position for writing to dest array 2417 const Register isURL = c_rarg5;// Base64 or URL character set 2418 __ movl(isMIME, Address(rbp, 2 * wordSize)); 2419 #else 2420 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 2421 const Address isURL_mem(rbp, 7 * wordSize); 2422 const Register isURL = r10; // pick the volatile windows register 2423 const Register dp = r12; 2424 __ movl(dp, dp_mem); 2425 __ movl(isURL, isURL_mem); 2426 __ movl(isMIME, Address(rbp, 8 * wordSize)); 2427 #endif 2428 2429 const XMMRegister lookup_lo = xmm5; 2430 const XMMRegister lookup_hi = xmm6; 2431 const XMMRegister errorvec = xmm7; 2432 const XMMRegister pack16_op = xmm9; 2433 const XMMRegister pack32_op = xmm8; 2434 const XMMRegister input0 = xmm3; 2435 const XMMRegister input1 = xmm20; 2436 const XMMRegister input2 = xmm21; 2437 const XMMRegister input3 = xmm19; 2438 const XMMRegister join01 = xmm12; 2439 const XMMRegister join12 = xmm11; 2440 const XMMRegister join23 = xmm10; 2441 const XMMRegister translated0 = xmm2; 2442 const XMMRegister translated1 = xmm1; 2443 const XMMRegister translated2 = xmm0; 2444 const XMMRegister translated3 = xmm4; 2445 2446 const XMMRegister merged0 = xmm2; 2447 const XMMRegister merged1 = xmm1; 2448 const XMMRegister merged2 = xmm0; 2449 const XMMRegister merged3 = xmm4; 2450 const XMMRegister merge_ab_bc0 = xmm2; 2451 const XMMRegister merge_ab_bc1 = xmm1; 2452 const XMMRegister merge_ab_bc2 = xmm0; 2453 const XMMRegister merge_ab_bc3 = xmm4; 2454 2455 const XMMRegister pack24bits = xmm4; 2456 2457 const Register length = r14; 2458 const Register output_size = r13; 2459 const Register output_mask = r15; 2460 const KRegister input_mask = k1; 2461 2462 const XMMRegister input_initial_valid_b64 = xmm0; 2463 const XMMRegister tmp = xmm10; 2464 const XMMRegister mask = xmm0; 2465 const XMMRegister invalid_b64 = xmm1; 2466 2467 Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL; 2468 Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce; 2469 Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero, L_lastChunk; 2470 2471 // calculate length from offsets 2472 __ movl(length, end_offset); 2473 __ subl(length, start_offset); 2474 __ push(dest); // Save for return value calc 2475 2476 // If AVX512 VBMI not supported, just compile non-AVX code 2477 if(VM_Version::supports_avx512_vbmi() && 2478 VM_Version::supports_avx512bw()) { 2479 __ cmpl(length, 31); // 32-bytes is break-even for AVX-512 2480 __ jcc(Assembler::lessEqual, L_lastChunk); 2481 2482 __ cmpl(isMIME, 0); 2483 __ jcc(Assembler::notEqual, L_lastChunk); 2484 2485 // Load lookup tables based on isURL 2486 __ cmpl(isURL, 0); 2487 __ jcc(Assembler::notZero, L_loadURL); 2488 2489 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13); 2490 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13); 2491 2492 __ BIND(L_continue); 2493 2494 __ movl(r15, 0x01400140); 2495 __ evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit); 2496 2497 __ movl(r15, 0x00011000); 2498 __ evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit); 2499 2500 __ cmpl(length, 0xff); 2501 __ jcc(Assembler::lessEqual, L_process64); 2502 2503 // load masks required for decoding data 2504 __ BIND(L_processdata); 2505 __ evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13); 2506 __ evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13); 2507 __ evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13); 2508 2509 __ align32(); 2510 __ BIND(L_process256); 2511 // Grab input data 2512 __ evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit); 2513 __ evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit); 2514 __ evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit); 2515 __ evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit); 2516 2517 // Copy the low part of the lookup table into the destination of the permutation 2518 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2519 __ evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit); 2520 __ evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit); 2521 __ evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit); 2522 2523 // Translate the base64 input into "decoded" bytes 2524 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2525 __ evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit); 2526 __ evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit); 2527 __ evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit); 2528 2529 // OR all of the translations together to check for errors (high-order bit of byte set) 2530 __ vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit); 2531 2532 __ vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit); 2533 __ vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit); 2534 __ vpor(errorvec, input3, input0, Assembler::AVX_512bit); 2535 2536 // Check if there was an error - if so, try 64-byte chunks 2537 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2538 __ kortestql(k3, k3); 2539 __ jcc(Assembler::notZero, L_process64); 2540 2541 // The merging and shuffling happens here 2542 // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa] 2543 // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd] 2544 // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40 2545 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2546 __ vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit); 2547 __ vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit); 2548 __ vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit); 2549 2550 // Now do the same with packed 16-bit values. 2551 // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb] 2552 // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12 2553 // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd] 2554 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2555 __ vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit); 2556 __ vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit); 2557 __ vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit); 2558 2559 // The join vectors specify which byte from which vector goes into the outputs 2560 // One of every 4 bytes in the extended vector is zero, so we pack them into their 2561 // final positions in the register for storing (256 bytes in, 192 bytes out) 2562 __ evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit); 2563 __ evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit); 2564 __ evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit); 2565 2566 // Store result 2567 __ evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit); 2568 __ evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit); 2569 __ evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit); 2570 2571 __ addptr(source, 0x100); 2572 __ addptr(dest, 0xc0); 2573 __ subl(length, 0x100); 2574 __ cmpl(length, 64 * 4); 2575 __ jcc(Assembler::greaterEqual, L_process256); 2576 2577 // At this point, we've decoded 64 * 4 * n bytes. 2578 // The remaining length will be <= 64 * 4 - 1. 2579 // UNLESS there was an error decoding the first 256-byte chunk. In this 2580 // case, the length will be arbitrarily long. 2581 // 2582 // Note that this will be the path for MIME-encoded strings. 2583 2584 __ BIND(L_process64); 2585 2586 __ evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13); 2587 2588 __ cmpl(length, 63); 2589 __ jcc(Assembler::lessEqual, L_finalBit); 2590 2591 __ mov64(rax, 0x0000ffffffffffff); 2592 __ kmovql(k2, rax); 2593 2594 __ align32(); 2595 __ BIND(L_process64Loop); 2596 2597 // Handle first 64-byte block 2598 2599 __ evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit); 2600 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2601 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2602 2603 __ vpor(errorvec, translated0, input0, Assembler::AVX_512bit); 2604 2605 // Check for error and bomb out before updating dest 2606 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2607 __ kortestql(k3, k3); 2608 __ jcc(Assembler::notZero, L_exit); 2609 2610 // Pack output register, selecting correct byte ordering 2611 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2612 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2613 __ vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit); 2614 2615 __ evmovdqub(Address(dest, dp), k2, merged0, true, Assembler::AVX_512bit); 2616 2617 __ subl(length, 64); 2618 __ addptr(source, 64); 2619 __ addptr(dest, 48); 2620 2621 __ cmpl(length, 64); 2622 __ jcc(Assembler::greaterEqual, L_process64Loop); 2623 2624 __ cmpl(length, 0); 2625 __ jcc(Assembler::lessEqual, L_exit); 2626 2627 __ BIND(L_finalBit); 2628 // Now have 1 to 63 bytes left to decode 2629 2630 // I was going to let Java take care of the final fragment 2631 // however it will repeatedly call this routine for every 4 bytes 2632 // of input data, so handle the rest here. 2633 __ movq(rax, -1); 2634 __ bzhiq(rax, rax, length); // Input mask in rax 2635 2636 __ movl(output_size, length); 2637 __ shrl(output_size, 2); // Find (len / 4) * 3 (output length) 2638 __ lea(output_size, Address(output_size, output_size, Address::times_2, 0)); 2639 // output_size in r13 2640 2641 // Strip pad characters, if any, and adjust length and mask 2642 __ addq(length, start_offset); 2643 __ cmpb(Address(source, length, Address::times_1, -1), '='); 2644 __ jcc(Assembler::equal, L_padding); 2645 2646 __ BIND(L_donePadding); 2647 __ subq(length, start_offset); 2648 2649 // Output size is (64 - output_size), output mask is (all 1s >> output_size). 2650 __ kmovql(input_mask, rax); 2651 __ movq(output_mask, -1); 2652 __ bzhiq(output_mask, output_mask, output_size); 2653 2654 // Load initial input with all valid base64 characters. Will be used 2655 // in merging source bytes to avoid masking when determining if an error occurred. 2656 __ movl(rax, 0x61616161); 2657 __ evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit); 2658 2659 // A register containing all invalid base64 decoded values 2660 __ movl(rax, 0x80808080); 2661 __ evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit); 2662 2663 // input_mask is in k1 2664 // output_size is in r13 2665 // output_mask is in r15 2666 // zmm0 - free 2667 // zmm1 - 0x00011000 2668 // zmm2 - 0x01400140 2669 // zmm3 - errorvec 2670 // zmm4 - pack vector 2671 // zmm5 - lookup_lo 2672 // zmm6 - lookup_hi 2673 // zmm7 - errorvec 2674 // zmm8 - 0x61616161 2675 // zmm9 - 0x80808080 2676 2677 // Load only the bytes from source, merging into our "fully-valid" register 2678 __ evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit); 2679 2680 // Decode all bytes within our merged input 2681 __ evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit); 2682 __ evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit); 2683 __ evporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit); 2684 2685 // Check for error. Compare (decoded | initial) to all invalid. 2686 // If any bytes have their high-order bit set, then we have an error. 2687 __ evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit); 2688 __ kortestql(k2, k2); 2689 2690 // If we have an error, use the brute force loop to decode what we can (4-byte chunks). 2691 __ jcc(Assembler::notZero, L_bruteForce); 2692 2693 // Shuffle output bytes 2694 __ vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit); 2695 __ vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit); 2696 2697 __ vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit); 2698 __ kmovql(k1, output_mask); 2699 __ evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit); 2700 2701 __ addptr(dest, output_size); 2702 2703 __ BIND(L_exit); 2704 __ vzeroupper(); 2705 __ pop(rax); // Get original dest value 2706 __ subptr(dest, rax); // Number of bytes converted 2707 __ movptr(rax, dest); 2708 __ pop(rbx); 2709 __ pop(r15); 2710 __ pop(r14); 2711 __ pop(r13); 2712 __ pop(r12); 2713 __ leave(); 2714 __ ret(0); 2715 2716 __ BIND(L_loadURL); 2717 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13); 2718 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13); 2719 __ jmp(L_continue); 2720 2721 __ BIND(L_padding); 2722 __ decrementq(output_size, 1); 2723 __ shrq(rax, 1); 2724 2725 __ cmpb(Address(source, length, Address::times_1, -2), '='); 2726 __ jcc(Assembler::notEqual, L_donePadding); 2727 2728 __ decrementq(output_size, 1); 2729 __ shrq(rax, 1); 2730 __ jmp(L_donePadding); 2731 2732 __ align32(); 2733 __ BIND(L_bruteForce); 2734 } // End of if(avx512_vbmi) 2735 2736 if (VM_Version::supports_avx2()) { 2737 Label L_tailProc, L_topLoop, L_enterLoop; 2738 2739 __ cmpl(isMIME, 0); 2740 __ jcc(Assembler::notEqual, L_lastChunk); 2741 2742 // Check for buffer too small (for algorithm) 2743 __ subl(length, 0x2c); 2744 __ jcc(Assembler::less, L_tailProc); 2745 2746 __ shll(isURL, 2); 2747 2748 // Algorithm adapted from https://arxiv.org/abs/1704.00605, "Faster Base64 2749 // Encoding and Decoding using AVX2 Instructions". URL modifications added. 2750 2751 // Set up constants 2752 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_tables_addr())); 2753 __ vpbroadcastd(xmm4, Address(r13, isURL, Address::times_1), Assembler::AVX_256bit); // 2F or 5F 2754 __ vpbroadcastd(xmm10, Address(r13, isURL, Address::times_1, 0x08), Assembler::AVX_256bit); // -1 or -4 2755 __ vmovdqu(xmm12, Address(r13, 0x10)); // permute 2756 __ vmovdqu(xmm13, Address(r13, 0x30)); // shuffle 2757 __ vpbroadcastd(xmm7, Address(r13, 0x50), Assembler::AVX_256bit); // merge 2758 __ vpbroadcastd(xmm6, Address(r13, 0x54), Assembler::AVX_256bit); // merge mult 2759 2760 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_LUT_tables_addr())); 2761 __ shll(isURL, 4); 2762 __ vmovdqu(xmm11, Address(r13, isURL, Address::times_1, 0x00)); // lut_lo 2763 __ vmovdqu(xmm8, Address(r13, isURL, Address::times_1, 0x20)); // lut_roll 2764 __ shrl(isURL, 6); // restore isURL 2765 __ vmovdqu(xmm9, Address(r13, 0x80)); // lut_hi 2766 __ jmp(L_enterLoop); 2767 2768 __ align32(); 2769 __ bind(L_topLoop); 2770 // Add in the offset value (roll) to get 6-bit out values 2771 __ vpaddb(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 2772 // Merge and permute the output bits into appropriate output byte lanes 2773 __ vpmaddubsw(xmm0, xmm0, xmm7, Assembler::AVX_256bit); 2774 __ vpmaddwd(xmm0, xmm0, xmm6, Assembler::AVX_256bit); 2775 __ vpshufb(xmm0, xmm0, xmm13, Assembler::AVX_256bit); 2776 __ vpermd(xmm0, xmm12, xmm0, Assembler::AVX_256bit); 2777 // Store the output bytes 2778 __ vmovdqu(Address(dest, dp, Address::times_1, 0), xmm0); 2779 __ addptr(source, 0x20); 2780 __ addptr(dest, 0x18); 2781 __ subl(length, 0x20); 2782 __ jcc(Assembler::less, L_tailProc); 2783 2784 __ bind(L_enterLoop); 2785 2786 // Load in encoded string (32 bytes) 2787 __ vmovdqu(xmm2, Address(source, start_offset, Address::times_1, 0x0)); 2788 // Extract the high nibble for indexing into the lut tables. High 4 bits are don't care. 2789 __ vpsrld(xmm1, xmm2, 0x4, Assembler::AVX_256bit); 2790 __ vpand(xmm1, xmm4, xmm1, Assembler::AVX_256bit); 2791 // Extract the low nibble. 5F/2F will isolate the low-order 4 bits. High 4 bits are don't care. 2792 __ vpand(xmm3, xmm2, xmm4, Assembler::AVX_256bit); 2793 // Check for special-case (0x2F or 0x5F (URL)) 2794 __ vpcmpeqb(xmm0, xmm4, xmm2, Assembler::AVX_256bit); 2795 // Get the bitset based on the low nibble. vpshufb uses low-order 4 bits only. 2796 __ vpshufb(xmm3, xmm11, xmm3, Assembler::AVX_256bit); 2797 // Get the bit value of the high nibble 2798 __ vpshufb(xmm5, xmm9, xmm1, Assembler::AVX_256bit); 2799 // Make sure 2F / 5F shows as valid 2800 __ vpandn(xmm3, xmm0, xmm3, Assembler::AVX_256bit); 2801 // Make adjustment for roll index. For non-URL, this is a no-op, 2802 // for URL, this adjusts by -4. This is to properly index the 2803 // roll value for 2F / 5F. 2804 __ vpand(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 2805 // If the and of the two is non-zero, we have an invalid input character 2806 __ vptest(xmm3, xmm5); 2807 // Extract the "roll" value - value to add to the input to get 6-bit out value 2808 __ vpaddb(xmm0, xmm0, xmm1, Assembler::AVX_256bit); // Handle 2F / 5F 2809 __ vpshufb(xmm0, xmm8, xmm0, Assembler::AVX_256bit); 2810 __ jcc(Assembler::equal, L_topLoop); // Fall through on error 2811 2812 __ bind(L_tailProc); 2813 2814 __ addl(length, 0x2c); 2815 2816 __ vzeroupper(); 2817 } 2818 2819 // Use non-AVX code to decode 4-byte chunks into 3 bytes of output 2820 2821 // Register state (Linux): 2822 // r12-15 - saved on stack 2823 // rdi - src 2824 // rsi - sp 2825 // rdx - sl 2826 // rcx - dst 2827 // r8 - dp 2828 // r9 - isURL 2829 2830 // Register state (Windows): 2831 // r12-15 - saved on stack 2832 // rcx - src 2833 // rdx - sp 2834 // r8 - sl 2835 // r9 - dst 2836 // r12 - dp 2837 // r10 - isURL 2838 2839 // Registers (common): 2840 // length (r14) - bytes in src 2841 2842 const Register decode_table = r11; 2843 const Register out_byte_count = rbx; 2844 const Register byte1 = r13; 2845 const Register byte2 = r15; 2846 const Register byte3 = WIN64_ONLY(r8) NOT_WIN64(rdx); 2847 const Register byte4 = WIN64_ONLY(r10) NOT_WIN64(r9); 2848 2849 __ bind(L_lastChunk); 2850 2851 __ shrl(length, 2); // Multiple of 4 bytes only - length is # 4-byte chunks 2852 __ cmpl(length, 0); 2853 __ jcc(Assembler::lessEqual, L_exit_no_vzero); 2854 2855 __ shll(isURL, 8); // index into decode table based on isURL 2856 __ lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr())); 2857 __ addptr(decode_table, isURL); 2858 2859 __ jmp(L_bottomLoop); 2860 2861 __ align32(); 2862 __ BIND(L_forceLoop); 2863 __ shll(byte1, 18); 2864 __ shll(byte2, 12); 2865 __ shll(byte3, 6); 2866 __ orl(byte1, byte2); 2867 __ orl(byte1, byte3); 2868 __ orl(byte1, byte4); 2869 2870 __ addptr(source, 4); 2871 2872 __ movb(Address(dest, dp, Address::times_1, 2), byte1); 2873 __ shrl(byte1, 8); 2874 __ movb(Address(dest, dp, Address::times_1, 1), byte1); 2875 __ shrl(byte1, 8); 2876 __ movb(Address(dest, dp, Address::times_1, 0), byte1); 2877 2878 __ addptr(dest, 3); 2879 __ decrementl(length, 1); 2880 __ jcc(Assembler::zero, L_exit_no_vzero); 2881 2882 __ BIND(L_bottomLoop); 2883 __ load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00)); 2884 __ load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01)); 2885 __ load_signed_byte(byte1, Address(decode_table, byte1)); 2886 __ load_signed_byte(byte2, Address(decode_table, byte2)); 2887 __ load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02)); 2888 __ load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03)); 2889 __ load_signed_byte(byte3, Address(decode_table, byte3)); 2890 __ load_signed_byte(byte4, Address(decode_table, byte4)); 2891 2892 __ mov(rax, byte1); 2893 __ orl(rax, byte2); 2894 __ orl(rax, byte3); 2895 __ orl(rax, byte4); 2896 __ jcc(Assembler::positive, L_forceLoop); 2897 2898 __ BIND(L_exit_no_vzero); 2899 __ pop(rax); // Get original dest value 2900 __ subptr(dest, rax); // Number of bytes converted 2901 __ movptr(rax, dest); 2902 __ pop(rbx); 2903 __ pop(r15); 2904 __ pop(r14); 2905 __ pop(r13); 2906 __ pop(r12); 2907 __ leave(); 2908 __ ret(0); 2909 2910 return start; 2911 } 2912 2913 2914 /** 2915 * Arguments: 2916 * 2917 * Inputs: 2918 * c_rarg0 - int crc 2919 * c_rarg1 - byte* buf 2920 * c_rarg2 - int length 2921 * 2922 * Output: 2923 * rax - int crc result 2924 */ 2925 address StubGenerator::generate_updateBytesCRC32() { 2926 assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions"); 2927 2928 __ align(CodeEntryAlignment); 2929 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32"); 2930 2931 address start = __ pc(); 2932 2933 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 2934 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 2935 // rscratch1: r10 2936 const Register crc = c_rarg0; // crc 2937 const Register buf = c_rarg1; // source java byte array address 2938 const Register len = c_rarg2; // length 2939 const Register table = c_rarg3; // crc_table address (reuse register) 2940 const Register tmp1 = r11; 2941 const Register tmp2 = r10; 2942 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax); 2943 2944 BLOCK_COMMENT("Entry:"); 2945 __ enter(); // required for proper stackwalking of RuntimeStub frame 2946 2947 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 2948 VM_Version::supports_avx512bw() && 2949 VM_Version::supports_avx512vl()) { 2950 // The constants used in the CRC32 algorithm requires the 1's compliment of the initial crc value. 2951 // However, the constant table for CRC32-C assumes the original crc value. Account for this 2952 // difference before calling and after returning. 2953 __ lea(table, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr())); 2954 __ notl(crc); 2955 __ kernel_crc32_avx512(crc, buf, len, table, tmp1, tmp2); 2956 __ notl(crc); 2957 } else { 2958 __ kernel_crc32(crc, buf, len, table, tmp1); 2959 } 2960 2961 __ movl(rax, crc); 2962 __ vzeroupper(); 2963 __ leave(); // required for proper stackwalking of RuntimeStub frame 2964 __ ret(0); 2965 2966 return start; 2967 } 2968 2969 /** 2970 * Arguments: 2971 * 2972 * Inputs: 2973 * c_rarg0 - int crc 2974 * c_rarg1 - byte* buf 2975 * c_rarg2 - long length 2976 * c_rarg3 - table_start - optional (present only when doing a library_call, 2977 * not used by x86 algorithm) 2978 * 2979 * Output: 2980 * rax - int crc result 2981 */ 2982 address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { 2983 assert(UseCRC32CIntrinsics, "need SSE4_2"); 2984 __ align(CodeEntryAlignment); 2985 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C"); 2986 address start = __ pc(); 2987 2988 //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs 2989 //Windows RCX RDX R8 R9 none none XMM0..XMM3 2990 //Lin / Sol RDI RSI RDX RCX R8 R9 XMM0..XMM7 2991 const Register crc = c_rarg0; // crc 2992 const Register buf = c_rarg1; // source java byte array address 2993 const Register len = c_rarg2; // length 2994 const Register a = rax; 2995 const Register j = r9; 2996 const Register k = r10; 2997 const Register l = r11; 2998 #ifdef _WIN64 2999 const Register y = rdi; 3000 const Register z = rsi; 3001 #else 3002 const Register y = rcx; 3003 const Register z = r8; 3004 #endif 3005 assert_different_registers(crc, buf, len, a, j, k, l, y, z); 3006 3007 BLOCK_COMMENT("Entry:"); 3008 __ enter(); // required for proper stackwalking of RuntimeStub frame 3009 Label L_continue; 3010 3011 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 3012 VM_Version::supports_avx512bw() && 3013 VM_Version::supports_avx512vl()) { 3014 Label L_doSmall; 3015 3016 __ cmpl(len, 384); 3017 __ jcc(Assembler::lessEqual, L_doSmall); 3018 3019 __ lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr())); 3020 __ kernel_crc32_avx512(crc, buf, len, j, l, k); 3021 3022 __ jmp(L_continue); 3023 3024 __ bind(L_doSmall); 3025 } 3026 #ifdef _WIN64 3027 __ push(y); 3028 __ push(z); 3029 #endif 3030 __ crc32c_ipl_alg2_alt2(crc, buf, len, 3031 a, j, k, 3032 l, y, z, 3033 c_farg0, c_farg1, c_farg2, 3034 is_pclmulqdq_supported); 3035 #ifdef _WIN64 3036 __ pop(z); 3037 __ pop(y); 3038 #endif 3039 3040 __ bind(L_continue); 3041 __ movl(rax, crc); 3042 __ vzeroupper(); 3043 __ leave(); // required for proper stackwalking of RuntimeStub frame 3044 __ ret(0); 3045 3046 return start; 3047 } 3048 3049 3050 /** 3051 * Arguments: 3052 * 3053 * Input: 3054 * c_rarg0 - x address 3055 * c_rarg1 - x length 3056 * c_rarg2 - y address 3057 * c_rarg3 - y length 3058 * not Win64 3059 * c_rarg4 - z address 3060 * Win64 3061 * rsp+40 - z address 3062 */ 3063 address StubGenerator::generate_multiplyToLen() { 3064 __ align(CodeEntryAlignment); 3065 StubCodeMark mark(this, "StubRoutines", "multiplyToLen"); 3066 address start = __ pc(); 3067 3068 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3069 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3070 const Register x = rdi; 3071 const Register xlen = rax; 3072 const Register y = rsi; 3073 const Register ylen = rcx; 3074 const Register z = r8; 3075 3076 // Next registers will be saved on stack in multiply_to_len(). 3077 const Register tmp0 = r11; 3078 const Register tmp1 = r12; 3079 const Register tmp2 = r13; 3080 const Register tmp3 = r14; 3081 const Register tmp4 = r15; 3082 const Register tmp5 = rbx; 3083 3084 BLOCK_COMMENT("Entry:"); 3085 __ enter(); // required for proper stackwalking of RuntimeStub frame 3086 3087 setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx 3088 // ylen => rcx, z => r8 3089 // r9 and r10 may be used to save non-volatile registers 3090 #ifdef _WIN64 3091 // last argument (#4) is on stack on Win64 3092 __ movptr(z, Address(rsp, 6 * wordSize)); 3093 #endif 3094 3095 __ movptr(xlen, rsi); 3096 __ movptr(y, rdx); 3097 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 3098 3099 restore_arg_regs(); 3100 3101 __ leave(); // required for proper stackwalking of RuntimeStub frame 3102 __ ret(0); 3103 3104 return start; 3105 } 3106 3107 /** 3108 * Arguments: 3109 * 3110 * Input: 3111 * c_rarg0 - obja address 3112 * c_rarg1 - objb address 3113 * c_rarg3 - length length 3114 * c_rarg4 - scale log2_array_indxscale 3115 * 3116 * Output: 3117 * rax - int >= mismatched index, < 0 bitwise complement of tail 3118 */ 3119 address StubGenerator::generate_vectorizedMismatch() { 3120 __ align(CodeEntryAlignment); 3121 StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch"); 3122 address start = __ pc(); 3123 3124 BLOCK_COMMENT("Entry:"); 3125 __ enter(); 3126 3127 #ifdef _WIN64 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3128 const Register scale = c_rarg0; //rcx, will exchange with r9 3129 const Register objb = c_rarg1; //rdx 3130 const Register length = c_rarg2; //r8 3131 const Register obja = c_rarg3; //r9 3132 __ xchgq(obja, scale); //now obja and scale contains the correct contents 3133 3134 const Register tmp1 = r10; 3135 const Register tmp2 = r11; 3136 #endif 3137 #ifndef _WIN64 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3138 const Register obja = c_rarg0; //U:rdi 3139 const Register objb = c_rarg1; //U:rsi 3140 const Register length = c_rarg2; //U:rdx 3141 const Register scale = c_rarg3; //U:rcx 3142 const Register tmp1 = r8; 3143 const Register tmp2 = r9; 3144 #endif 3145 const Register result = rax; //return value 3146 const XMMRegister vec0 = xmm0; 3147 const XMMRegister vec1 = xmm1; 3148 const XMMRegister vec2 = xmm2; 3149 3150 __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2); 3151 3152 __ vzeroupper(); 3153 __ leave(); 3154 __ ret(0); 3155 3156 return start; 3157 } 3158 3159 /** 3160 * Arguments: 3161 * 3162 // Input: 3163 // c_rarg0 - x address 3164 // c_rarg1 - x length 3165 // c_rarg2 - z address 3166 // c_rarg3 - z length 3167 * 3168 */ 3169 address StubGenerator::generate_squareToLen() { 3170 3171 __ align(CodeEntryAlignment); 3172 StubCodeMark mark(this, "StubRoutines", "squareToLen"); 3173 address start = __ pc(); 3174 3175 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3176 // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...) 3177 const Register x = rdi; 3178 const Register len = rsi; 3179 const Register z = r8; 3180 const Register zlen = rcx; 3181 3182 const Register tmp1 = r12; 3183 const Register tmp2 = r13; 3184 const Register tmp3 = r14; 3185 const Register tmp4 = r15; 3186 const Register tmp5 = rbx; 3187 3188 BLOCK_COMMENT("Entry:"); 3189 __ enter(); // required for proper stackwalking of RuntimeStub frame 3190 3191 setup_arg_regs(4); // x => rdi, len => rsi, z => rdx 3192 // zlen => rcx 3193 // r9 and r10 may be used to save non-volatile registers 3194 __ movptr(r8, rdx); 3195 __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3196 3197 restore_arg_regs(); 3198 3199 __ leave(); // required for proper stackwalking of RuntimeStub frame 3200 __ ret(0); 3201 3202 return start; 3203 } 3204 3205 address StubGenerator::generate_method_entry_barrier() { 3206 __ align(CodeEntryAlignment); 3207 StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); 3208 address start = __ pc(); 3209 3210 Label deoptimize_label; 3211 3212 __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing 3213 3214 BLOCK_COMMENT("Entry:"); 3215 __ enter(); // save rbp 3216 3217 // save c_rarg0, because we want to use that value. 3218 // We could do without it but then we depend on the number of slots used by pusha 3219 __ push(c_rarg0); 3220 3221 __ lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address 3222 3223 __ pusha(); 3224 3225 // The method may have floats as arguments, and we must spill them before calling 3226 // the VM runtime. 3227 assert(Argument::n_float_register_parameters_j == 8, "Assumption"); 3228 const int xmm_size = wordSize * 2; 3229 const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j; 3230 __ subptr(rsp, xmm_spill_size); 3231 __ movdqu(Address(rsp, xmm_size * 7), xmm7); 3232 __ movdqu(Address(rsp, xmm_size * 6), xmm6); 3233 __ movdqu(Address(rsp, xmm_size * 5), xmm5); 3234 __ movdqu(Address(rsp, xmm_size * 4), xmm4); 3235 __ movdqu(Address(rsp, xmm_size * 3), xmm3); 3236 __ movdqu(Address(rsp, xmm_size * 2), xmm2); 3237 __ movdqu(Address(rsp, xmm_size * 1), xmm1); 3238 __ movdqu(Address(rsp, xmm_size * 0), xmm0); 3239 3240 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), 1); 3241 3242 __ movdqu(xmm0, Address(rsp, xmm_size * 0)); 3243 __ movdqu(xmm1, Address(rsp, xmm_size * 1)); 3244 __ movdqu(xmm2, Address(rsp, xmm_size * 2)); 3245 __ movdqu(xmm3, Address(rsp, xmm_size * 3)); 3246 __ movdqu(xmm4, Address(rsp, xmm_size * 4)); 3247 __ movdqu(xmm5, Address(rsp, xmm_size * 5)); 3248 __ movdqu(xmm6, Address(rsp, xmm_size * 6)); 3249 __ movdqu(xmm7, Address(rsp, xmm_size * 7)); 3250 __ addptr(rsp, xmm_spill_size); 3251 3252 __ cmpl(rax, 1); // 1 means deoptimize 3253 __ jcc(Assembler::equal, deoptimize_label); 3254 3255 __ popa(); 3256 __ pop(c_rarg0); 3257 3258 __ leave(); 3259 3260 __ addptr(rsp, 1 * wordSize); // cookie 3261 __ ret(0); 3262 3263 3264 __ BIND(deoptimize_label); 3265 3266 __ popa(); 3267 __ pop(c_rarg0); 3268 3269 __ leave(); 3270 3271 // this can be taken out, but is good for verification purposes. getting a SIGSEGV 3272 // here while still having a correct stack is valuable 3273 __ testptr(rsp, Address(rsp, 0)); 3274 3275 __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier 3276 __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point 3277 3278 return start; 3279 } 3280 3281 /** 3282 * Arguments: 3283 * 3284 * Input: 3285 * c_rarg0 - out address 3286 * c_rarg1 - in address 3287 * c_rarg2 - offset 3288 * c_rarg3 - len 3289 * not Win64 3290 * c_rarg4 - k 3291 * Win64 3292 * rsp+40 - k 3293 */ 3294 address StubGenerator::generate_mulAdd() { 3295 __ align(CodeEntryAlignment); 3296 StubCodeMark mark(this, "StubRoutines", "mulAdd"); 3297 address start = __ pc(); 3298 3299 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3300 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3301 const Register out = rdi; 3302 const Register in = rsi; 3303 const Register offset = r11; 3304 const Register len = rcx; 3305 const Register k = r8; 3306 3307 // Next registers will be saved on stack in mul_add(). 3308 const Register tmp1 = r12; 3309 const Register tmp2 = r13; 3310 const Register tmp3 = r14; 3311 const Register tmp4 = r15; 3312 const Register tmp5 = rbx; 3313 3314 BLOCK_COMMENT("Entry:"); 3315 __ enter(); // required for proper stackwalking of RuntimeStub frame 3316 3317 setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx 3318 // len => rcx, k => r8 3319 // r9 and r10 may be used to save non-volatile registers 3320 #ifdef _WIN64 3321 // last argument is on stack on Win64 3322 __ movl(k, Address(rsp, 6 * wordSize)); 3323 #endif 3324 __ movptr(r11, rdx); // move offset in rdx to offset(r11) 3325 __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3326 3327 restore_arg_regs(); 3328 3329 __ leave(); // required for proper stackwalking of RuntimeStub frame 3330 __ ret(0); 3331 3332 return start; 3333 } 3334 3335 address StubGenerator::generate_bigIntegerRightShift() { 3336 __ align(CodeEntryAlignment); 3337 StubCodeMark mark(this, "StubRoutines", "bigIntegerRightShiftWorker"); 3338 address start = __ pc(); 3339 3340 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3341 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3342 const Register newArr = rdi; 3343 const Register oldArr = rsi; 3344 const Register newIdx = rdx; 3345 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3346 const Register totalNumIter = r8; 3347 3348 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3349 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3350 const Register tmp1 = r11; // Caller save. 3351 const Register tmp2 = rax; // Caller save. 3352 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3353 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3354 const Register tmp5 = r14; // Callee save. 3355 const Register tmp6 = r15; 3356 3357 const XMMRegister x0 = xmm0; 3358 const XMMRegister x1 = xmm1; 3359 const XMMRegister x2 = xmm2; 3360 3361 BLOCK_COMMENT("Entry:"); 3362 __ enter(); // required for proper stackwalking of RuntimeStub frame 3363 3364 #ifdef _WIN64 3365 setup_arg_regs(4); 3366 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3367 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3368 // Save callee save registers. 3369 __ push(tmp3); 3370 __ push(tmp4); 3371 #endif 3372 __ push(tmp5); 3373 3374 // Rename temps used throughout the code. 3375 const Register idx = tmp1; 3376 const Register nIdx = tmp2; 3377 3378 __ xorl(idx, idx); 3379 3380 // Start right shift from end of the array. 3381 // For example, if #iteration = 4 and newIdx = 1 3382 // then dest[4] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3383 // if #iteration = 4 and newIdx = 0 3384 // then dest[3] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3385 __ movl(idx, totalNumIter); 3386 __ movl(nIdx, idx); 3387 __ addl(nIdx, newIdx); 3388 3389 // If vectorization is enabled, check if the number of iterations is at least 64 3390 // If not, then go to ShifTwo processing 2 iterations 3391 if (VM_Version::supports_avx512_vbmi2()) { 3392 __ cmpptr(totalNumIter, (AVX3Threshold/64)); 3393 __ jcc(Assembler::less, ShiftTwo); 3394 3395 if (AVX3Threshold < 16 * 64) { 3396 __ cmpl(totalNumIter, 16); 3397 __ jcc(Assembler::less, ShiftTwo); 3398 } 3399 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3400 __ subl(idx, 16); 3401 __ subl(nIdx, 16); 3402 __ BIND(Shift512Loop); 3403 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 4), Assembler::AVX_512bit); 3404 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3405 __ vpshrdvd(x2, x1, x0, Assembler::AVX_512bit); 3406 __ evmovdqul(Address(newArr, nIdx, Address::times_4), x2, Assembler::AVX_512bit); 3407 __ subl(nIdx, 16); 3408 __ subl(idx, 16); 3409 __ jcc(Assembler::greaterEqual, Shift512Loop); 3410 __ addl(idx, 16); 3411 __ addl(nIdx, 16); 3412 } 3413 __ BIND(ShiftTwo); 3414 __ cmpl(idx, 2); 3415 __ jcc(Assembler::less, ShiftOne); 3416 __ subl(idx, 2); 3417 __ subl(nIdx, 2); 3418 __ BIND(ShiftTwoLoop); 3419 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 8)); 3420 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3421 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3422 __ shrdl(tmp5, tmp4); 3423 __ shrdl(tmp4, tmp3); 3424 __ movl(Address(newArr, nIdx, Address::times_4, 4), tmp5); 3425 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3426 __ subl(nIdx, 2); 3427 __ subl(idx, 2); 3428 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3429 __ addl(idx, 2); 3430 __ addl(nIdx, 2); 3431 3432 // Do the last iteration 3433 __ BIND(ShiftOne); 3434 __ cmpl(idx, 1); 3435 __ jcc(Assembler::less, Exit); 3436 __ subl(idx, 1); 3437 __ subl(nIdx, 1); 3438 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3439 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3440 __ shrdl(tmp4, tmp3); 3441 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3442 __ BIND(Exit); 3443 __ vzeroupper(); 3444 // Restore callee save registers. 3445 __ pop(tmp5); 3446 #ifdef _WIN64 3447 __ pop(tmp4); 3448 __ pop(tmp3); 3449 restore_arg_regs(); 3450 #endif 3451 __ leave(); // required for proper stackwalking of RuntimeStub frame 3452 __ ret(0); 3453 3454 return start; 3455 } 3456 3457 /** 3458 * Arguments: 3459 * 3460 * Input: 3461 * c_rarg0 - newArr address 3462 * c_rarg1 - oldArr address 3463 * c_rarg2 - newIdx 3464 * c_rarg3 - shiftCount 3465 * not Win64 3466 * c_rarg4 - numIter 3467 * Win64 3468 * rsp40 - numIter 3469 */ 3470 address StubGenerator::generate_bigIntegerLeftShift() { 3471 __ align(CodeEntryAlignment); 3472 StubCodeMark mark(this, "StubRoutines", "bigIntegerLeftShiftWorker"); 3473 address start = __ pc(); 3474 3475 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3476 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3477 const Register newArr = rdi; 3478 const Register oldArr = rsi; 3479 const Register newIdx = rdx; 3480 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3481 const Register totalNumIter = r8; 3482 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3483 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3484 const Register tmp1 = r11; // Caller save. 3485 const Register tmp2 = rax; // Caller save. 3486 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3487 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3488 const Register tmp5 = r14; // Callee save. 3489 3490 const XMMRegister x0 = xmm0; 3491 const XMMRegister x1 = xmm1; 3492 const XMMRegister x2 = xmm2; 3493 BLOCK_COMMENT("Entry:"); 3494 __ enter(); // required for proper stackwalking of RuntimeStub frame 3495 3496 #ifdef _WIN64 3497 setup_arg_regs(4); 3498 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3499 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3500 // Save callee save registers. 3501 __ push(tmp3); 3502 __ push(tmp4); 3503 #endif 3504 __ push(tmp5); 3505 3506 // Rename temps used throughout the code 3507 const Register idx = tmp1; 3508 const Register numIterTmp = tmp2; 3509 3510 // Start idx from zero. 3511 __ xorl(idx, idx); 3512 // Compute interior pointer for new array. We do this so that we can use same index for both old and new arrays. 3513 __ lea(newArr, Address(newArr, newIdx, Address::times_4)); 3514 __ movl(numIterTmp, totalNumIter); 3515 3516 // If vectorization is enabled, check if the number of iterations is at least 64 3517 // If not, then go to ShiftTwo shifting two numbers at a time 3518 if (VM_Version::supports_avx512_vbmi2()) { 3519 __ cmpl(totalNumIter, (AVX3Threshold/64)); 3520 __ jcc(Assembler::less, ShiftTwo); 3521 3522 if (AVX3Threshold < 16 * 64) { 3523 __ cmpl(totalNumIter, 16); 3524 __ jcc(Assembler::less, ShiftTwo); 3525 } 3526 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3527 __ subl(numIterTmp, 16); 3528 __ BIND(Shift512Loop); 3529 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3530 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 0x4), Assembler::AVX_512bit); 3531 __ vpshldvd(x1, x2, x0, Assembler::AVX_512bit); 3532 __ evmovdqul(Address(newArr, idx, Address::times_4), x1, Assembler::AVX_512bit); 3533 __ addl(idx, 16); 3534 __ subl(numIterTmp, 16); 3535 __ jcc(Assembler::greaterEqual, Shift512Loop); 3536 __ addl(numIterTmp, 16); 3537 } 3538 __ BIND(ShiftTwo); 3539 __ cmpl(totalNumIter, 1); 3540 __ jcc(Assembler::less, Exit); 3541 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3542 __ subl(numIterTmp, 2); 3543 __ jcc(Assembler::less, ShiftOne); 3544 3545 __ BIND(ShiftTwoLoop); 3546 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3547 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 0x8)); 3548 __ shldl(tmp3, tmp4); 3549 __ shldl(tmp4, tmp5); 3550 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3551 __ movl(Address(newArr, idx, Address::times_4, 0x4), tmp4); 3552 __ movl(tmp3, tmp5); 3553 __ addl(idx, 2); 3554 __ subl(numIterTmp, 2); 3555 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3556 3557 // Do the last iteration 3558 __ BIND(ShiftOne); 3559 __ addl(numIterTmp, 2); 3560 __ cmpl(numIterTmp, 1); 3561 __ jcc(Assembler::less, Exit); 3562 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3563 __ shldl(tmp3, tmp4); 3564 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3565 3566 __ BIND(Exit); 3567 __ vzeroupper(); 3568 // Restore callee save registers. 3569 __ pop(tmp5); 3570 #ifdef _WIN64 3571 __ pop(tmp4); 3572 __ pop(tmp3); 3573 restore_arg_regs(); 3574 #endif 3575 __ leave(); // required for proper stackwalking of RuntimeStub frame 3576 __ ret(0); 3577 3578 return start; 3579 } 3580 3581 void StubGenerator::generate_libm_stubs() { 3582 if (UseLibmIntrinsic && InlineIntrinsics) { 3583 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) { 3584 StubRoutines::_dsin = generate_libmSin(); // from stubGenerator_x86_64_sin.cpp 3585 } 3586 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { 3587 StubRoutines::_dcos = generate_libmCos(); // from stubGenerator_x86_64_cos.cpp 3588 } 3589 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) { 3590 StubRoutines::_dtan = generate_libmTan(); // from stubGenerator_x86_64_tan.cpp 3591 } 3592 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) { 3593 StubRoutines::_dexp = generate_libmExp(); // from stubGenerator_x86_64_exp.cpp 3594 } 3595 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) { 3596 StubRoutines::_dpow = generate_libmPow(); // from stubGenerator_x86_64_pow.cpp 3597 } 3598 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) { 3599 StubRoutines::_dlog = generate_libmLog(); // from stubGenerator_x86_64_log.cpp 3600 } 3601 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) { 3602 StubRoutines::_dlog10 = generate_libmLog10(); // from stubGenerator_x86_64_log.cpp 3603 } 3604 } 3605 } 3606 3607 /** 3608 * Arguments: 3609 * 3610 * Input: 3611 * c_rarg0 - float16 jshort 3612 * 3613 * Output: 3614 * xmm0 - float 3615 */ 3616 address StubGenerator::generate_float16ToFloat() { 3617 StubCodeMark mark(this, "StubRoutines", "float16ToFloat"); 3618 3619 address start = __ pc(); 3620 3621 BLOCK_COMMENT("Entry:"); 3622 // No need for RuntimeStub frame since it is called only during JIT compilation 3623 3624 // Load value into xmm0 and convert 3625 __ flt16_to_flt(xmm0, c_rarg0); 3626 3627 __ ret(0); 3628 3629 return start; 3630 } 3631 3632 /** 3633 * Arguments: 3634 * 3635 * Input: 3636 * xmm0 - float 3637 * 3638 * Output: 3639 * rax - float16 jshort 3640 */ 3641 address StubGenerator::generate_floatToFloat16() { 3642 StubCodeMark mark(this, "StubRoutines", "floatToFloat16"); 3643 3644 address start = __ pc(); 3645 3646 BLOCK_COMMENT("Entry:"); 3647 // No need for RuntimeStub frame since it is called only during JIT compilation 3648 3649 // Convert and put result into rax 3650 __ flt_to_flt16(rax, xmm0, xmm1); 3651 3652 __ ret(0); 3653 3654 return start; 3655 } 3656 3657 address StubGenerator::generate_cont_thaw(const char* label, Continuation::thaw_kind kind) { 3658 if (!Continuations::enabled()) return nullptr; 3659 3660 bool return_barrier = Continuation::is_thaw_return_barrier(kind); 3661 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); 3662 3663 StubCodeMark mark(this, "StubRoutines", label); 3664 address start = __ pc(); 3665 3666 // TODO: Handle Valhalla return types. May require generating different return barriers. 3667 3668 if (!return_barrier) { 3669 // Pop return address. If we don't do this, we get a drift, 3670 // where the bottom-most frozen frame continuously grows. 3671 __ pop(c_rarg3); 3672 } else { 3673 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3674 } 3675 3676 #ifdef ASSERT 3677 { 3678 Label L_good_sp; 3679 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3680 __ jcc(Assembler::equal, L_good_sp); 3681 __ stop("Incorrect rsp at thaw entry"); 3682 __ BIND(L_good_sp); 3683 } 3684 #endif // ASSERT 3685 3686 if (return_barrier) { 3687 // Preserve possible return value from a method returning to the return barrier. 3688 __ push(rax); 3689 __ push_d(xmm0); 3690 } 3691 3692 __ movptr(c_rarg0, r15_thread); 3693 __ movptr(c_rarg1, (return_barrier ? 1 : 0)); 3694 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2); 3695 __ movptr(rbx, rax); 3696 3697 if (return_barrier) { 3698 // Restore return value from a method returning to the return barrier. 3699 // No safepoint in the call to thaw, so even an oop return value should be OK. 3700 __ pop_d(xmm0); 3701 __ pop(rax); 3702 } 3703 3704 #ifdef ASSERT 3705 { 3706 Label L_good_sp; 3707 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3708 __ jcc(Assembler::equal, L_good_sp); 3709 __ stop("Incorrect rsp after prepare thaw"); 3710 __ BIND(L_good_sp); 3711 } 3712 #endif // ASSERT 3713 3714 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames 3715 Label L_thaw_success; 3716 __ testptr(rbx, rbx); 3717 __ jccb(Assembler::notZero, L_thaw_success); 3718 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry())); 3719 __ bind(L_thaw_success); 3720 3721 // Make room for the thawed frames and align the stack. 3722 __ subptr(rsp, rbx); 3723 __ andptr(rsp, -StackAlignmentInBytes); 3724 3725 if (return_barrier) { 3726 // Preserve possible return value from a method returning to the return barrier. (Again.) 3727 __ push(rax); 3728 __ push_d(xmm0); 3729 } 3730 3731 // If we want, we can templatize thaw by kind, and have three different entries. 3732 __ movptr(c_rarg0, r15_thread); 3733 __ movptr(c_rarg1, kind); 3734 __ call_VM_leaf(Continuation::thaw_entry(), 2); 3735 __ movptr(rbx, rax); 3736 3737 if (return_barrier) { 3738 // Restore return value from a method returning to the return barrier. (Again.) 3739 // No safepoint in the call to thaw, so even an oop return value should be OK. 3740 __ pop_d(xmm0); 3741 __ pop(rax); 3742 } else { 3743 // Return 0 (success) from doYield. 3744 __ xorptr(rax, rax); 3745 } 3746 3747 // After thawing, rbx is the SP of the yielding frame. 3748 // Move there, and then to saved RBP slot. 3749 __ movptr(rsp, rbx); 3750 __ subptr(rsp, 2*wordSize); 3751 3752 if (return_barrier_exception) { 3753 __ movptr(c_rarg0, r15_thread); 3754 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address 3755 3756 // rax still holds the original exception oop, save it before the call 3757 __ push(rax); 3758 3759 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2); 3760 __ movptr(rbx, rax); 3761 3762 // Continue at exception handler: 3763 // rax: exception oop 3764 // rbx: exception handler 3765 // rdx: exception pc 3766 __ pop(rax); 3767 __ verify_oop(rax); 3768 __ pop(rbp); // pop out RBP here too 3769 __ pop(rdx); 3770 __ jmp(rbx); 3771 } else { 3772 // We are "returning" into the topmost thawed frame; see Thaw::push_return_frame 3773 __ pop(rbp); 3774 __ ret(0); 3775 } 3776 3777 return start; 3778 } 3779 3780 address StubGenerator::generate_cont_thaw() { 3781 return generate_cont_thaw("Cont thaw", Continuation::thaw_top); 3782 } 3783 3784 // TODO: will probably need multiple return barriers depending on return type 3785 3786 address StubGenerator::generate_cont_returnBarrier() { 3787 return generate_cont_thaw("Cont thaw return barrier", Continuation::thaw_return_barrier); 3788 } 3789 3790 address StubGenerator::generate_cont_returnBarrier_exception() { 3791 return generate_cont_thaw("Cont thaw return barrier exception", Continuation::thaw_return_barrier_exception); 3792 } 3793 3794 // exception handler for upcall stubs 3795 address StubGenerator::generate_upcall_stub_exception_handler() { 3796 StubCodeMark mark(this, "StubRoutines", "upcall stub exception handler"); 3797 address start = __ pc(); 3798 3799 // native caller has no idea how to handle exceptions 3800 // we just crash here. Up to callee to catch exceptions. 3801 __ verify_oop(rax); 3802 __ vzeroupper(); 3803 __ mov(c_rarg0, rax); 3804 __ andptr(rsp, -StackAlignmentInBytes); // align stack as required by ABI 3805 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 3806 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception))); 3807 __ should_not_reach_here(); 3808 3809 return start; 3810 } 3811 3812 address StubGenerator::generate_lookup_secondary_supers_table_stub(u1 super_klass_index) { 3813 StubCodeMark mark(this, "StubRoutines", "lookup_secondary_supers_table"); 3814 3815 address start = __ pc(); 3816 3817 const Register 3818 r_super_klass = rax, 3819 r_sub_klass = rsi, 3820 result = rdi; 3821 3822 __ lookup_secondary_supers_table(r_sub_klass, r_super_klass, 3823 rdx, rcx, rbx, r11, // temps 3824 result, 3825 super_klass_index); 3826 __ ret(0); 3827 3828 return start; 3829 } 3830 3831 // Slow path implementation for UseSecondarySupersTable. 3832 address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { 3833 StubCodeMark mark(this, "StubRoutines", "lookup_secondary_supers_table"); 3834 3835 address start = __ pc(); 3836 3837 const Register 3838 r_super_klass = rax, 3839 r_array_base = rbx, 3840 r_array_index = rdx, 3841 r_sub_klass = rsi, 3842 r_bitmap = r11, 3843 result = rdi; 3844 3845 Label L_success; 3846 __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, 3847 rcx, rdi, // temps 3848 &L_success); 3849 // bind(L_failure); 3850 __ movl(result, 1); 3851 __ ret(0); 3852 3853 __ bind(L_success); 3854 __ movl(result, 0); 3855 __ ret(0); 3856 3857 return start; 3858 } 3859 3860 void StubGenerator::create_control_words() { 3861 // Round to nearest, 64-bit mode, exceptions masked, flags specialized 3862 StubRoutines::x86::_mxcsr_std = EnableX86ECoreOpts ? 0x1FBF : 0x1F80; 3863 // Round to zero, 64-bit mode, exceptions masked, flags specialized 3864 StubRoutines::x86::_mxcsr_rz = EnableX86ECoreOpts ? 0x7FBF : 0x7F80; 3865 } 3866 3867 // Initialization 3868 void StubGenerator::generate_initial_stubs() { 3869 // Generates all stubs and initializes the entry points 3870 3871 // This platform-specific settings are needed by generate_call_stub() 3872 create_control_words(); 3873 3874 // Initialize table for unsafe copy memeory check. 3875 if (UnsafeMemoryAccess::_table == nullptr) { 3876 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory 3877 } 3878 3879 // entry points that exist in all platforms Note: This is code 3880 // that could be shared among different platforms - however the 3881 // benefit seems to be smaller than the disadvantage of having a 3882 // much more complicated generator structure. See also comment in 3883 // stubRoutines.hpp. 3884 3885 StubRoutines::_forward_exception_entry = generate_forward_exception(); 3886 3887 // Generate these first because they are called from other stubs 3888 if (InlineTypeReturnedAsFields) { 3889 StubRoutines::_load_inline_type_fields_in_regs = 3890 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), 3891 "load_inline_type_fields_in_regs", false); 3892 StubRoutines::_store_inline_type_fields_to_buf = 3893 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), 3894 "store_inline_type_fields_to_buf", true); 3895 } 3896 3897 StubRoutines::_call_stub_entry = 3898 generate_call_stub(StubRoutines::_call_stub_return_address); 3899 3900 // is referenced by megamorphic call 3901 StubRoutines::_catch_exception_entry = generate_catch_exception(); 3902 3903 // atomic calls 3904 StubRoutines::_fence_entry = generate_orderaccess_fence(); 3905 3906 // platform dependent 3907 StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp(); 3908 3909 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr(); 3910 3911 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); 3912 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); 3913 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup(); 3914 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup(); 3915 3916 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); 3917 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); 3918 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); 3919 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); 3920 3921 if (UseCRC32Intrinsics) { 3922 // set table address before stub generation which use it 3923 StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table; 3924 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32(); 3925 } 3926 3927 if (UseCRC32CIntrinsics) { 3928 bool supports_clmul = VM_Version::supports_clmul(); 3929 StubRoutines::x86::generate_CRC32C_table(supports_clmul); 3930 StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table; 3931 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul); 3932 } 3933 3934 if (VM_Version::supports_float16()) { 3935 // For results consistency both intrinsics should be enabled. 3936 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here. 3937 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) && 3938 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) { 3939 StubRoutines::_hf2f = generate_float16ToFloat(); 3940 StubRoutines::_f2hf = generate_floatToFloat16(); 3941 } 3942 } 3943 3944 generate_libm_stubs(); 3945 3946 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp 3947 } 3948 3949 // Call here from the interpreter or compiled code to either load 3950 // multiple returned values from the inline type instance being 3951 // returned to registers or to store returned values to a newly 3952 // allocated inline type instance. 3953 // Register is a class, but it would be assigned numerical value. 3954 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull. 3955 PRAGMA_DIAG_PUSH 3956 PRAGMA_NONNULL_IGNORED 3957 address StubGenerator::generate_return_value_stub(address destination, const char* name, bool has_res) { 3958 // We need to save all registers the calling convention may use so 3959 // the runtime calls read or update those registers. This needs to 3960 // be in sync with SharedRuntime::java_return_convention(). 3961 enum layout { 3962 pad_off = frame::arg_reg_save_area_bytes/BytesPerInt, pad_off_2, 3963 rax_off, rax_off_2, 3964 j_rarg5_off, j_rarg5_2, 3965 j_rarg4_off, j_rarg4_2, 3966 j_rarg3_off, j_rarg3_2, 3967 j_rarg2_off, j_rarg2_2, 3968 j_rarg1_off, j_rarg1_2, 3969 j_rarg0_off, j_rarg0_2, 3970 j_farg0_off, j_farg0_2, 3971 j_farg1_off, j_farg1_2, 3972 j_farg2_off, j_farg2_2, 3973 j_farg3_off, j_farg3_2, 3974 j_farg4_off, j_farg4_2, 3975 j_farg5_off, j_farg5_2, 3976 j_farg6_off, j_farg6_2, 3977 j_farg7_off, j_farg7_2, 3978 rbp_off, rbp_off_2, 3979 return_off, return_off_2, 3980 3981 framesize 3982 }; 3983 3984 CodeBuffer buffer(name, 1000, 512); 3985 MacroAssembler* _masm = new MacroAssembler(&buffer); 3986 3987 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16); 3988 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned"); 3989 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt; 3990 int frame_size_in_words = frame_size_in_bytes / wordSize; 3991 3992 OopMapSet *oop_maps = new OopMapSet(); 3993 OopMap* map = new OopMap(frame_size_in_slots, 0); 3994 3995 map->set_callee_saved(VMRegImpl::stack2reg(rax_off), rax->as_VMReg()); 3996 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg()); 3997 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg()); 3998 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg()); 3999 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg()); 4000 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg()); 4001 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg()); 4002 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg()); 4003 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg()); 4004 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg()); 4005 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg()); 4006 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg()); 4007 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg()); 4008 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg()); 4009 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg()); 4010 4011 int start = __ offset(); 4012 4013 __ subptr(rsp, frame_size_in_bytes - 8 /* return address*/); 4014 4015 __ movptr(Address(rsp, rbp_off * BytesPerInt), rbp); 4016 __ movdbl(Address(rsp, j_farg7_off * BytesPerInt), j_farg7); 4017 __ movdbl(Address(rsp, j_farg6_off * BytesPerInt), j_farg6); 4018 __ movdbl(Address(rsp, j_farg5_off * BytesPerInt), j_farg5); 4019 __ movdbl(Address(rsp, j_farg4_off * BytesPerInt), j_farg4); 4020 __ movdbl(Address(rsp, j_farg3_off * BytesPerInt), j_farg3); 4021 __ movdbl(Address(rsp, j_farg2_off * BytesPerInt), j_farg2); 4022 __ movdbl(Address(rsp, j_farg1_off * BytesPerInt), j_farg1); 4023 __ movdbl(Address(rsp, j_farg0_off * BytesPerInt), j_farg0); 4024 4025 __ movptr(Address(rsp, j_rarg0_off * BytesPerInt), j_rarg0); 4026 __ movptr(Address(rsp, j_rarg1_off * BytesPerInt), j_rarg1); 4027 __ movptr(Address(rsp, j_rarg2_off * BytesPerInt), j_rarg2); 4028 __ movptr(Address(rsp, j_rarg3_off * BytesPerInt), j_rarg3); 4029 __ movptr(Address(rsp, j_rarg4_off * BytesPerInt), j_rarg4); 4030 __ movptr(Address(rsp, j_rarg5_off * BytesPerInt), j_rarg5); 4031 __ movptr(Address(rsp, rax_off * BytesPerInt), rax); 4032 4033 int frame_complete = __ offset(); 4034 4035 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); 4036 4037 __ mov(c_rarg0, r15_thread); 4038 __ mov(c_rarg1, rax); 4039 4040 __ call(RuntimeAddress(destination)); 4041 4042 // Set an oopmap for the call site. 4043 4044 oop_maps->add_gc_map( __ offset() - start, map); 4045 4046 // clear last_Java_sp 4047 __ reset_last_Java_frame(false); 4048 4049 __ movptr(rbp, Address(rsp, rbp_off * BytesPerInt)); 4050 __ movdbl(j_farg7, Address(rsp, j_farg7_off * BytesPerInt)); 4051 __ movdbl(j_farg6, Address(rsp, j_farg6_off * BytesPerInt)); 4052 __ movdbl(j_farg5, Address(rsp, j_farg5_off * BytesPerInt)); 4053 __ movdbl(j_farg4, Address(rsp, j_farg4_off * BytesPerInt)); 4054 __ movdbl(j_farg3, Address(rsp, j_farg3_off * BytesPerInt)); 4055 __ movdbl(j_farg2, Address(rsp, j_farg2_off * BytesPerInt)); 4056 __ movdbl(j_farg1, Address(rsp, j_farg1_off * BytesPerInt)); 4057 __ movdbl(j_farg0, Address(rsp, j_farg0_off * BytesPerInt)); 4058 4059 __ movptr(j_rarg0, Address(rsp, j_rarg0_off * BytesPerInt)); 4060 __ movptr(j_rarg1, Address(rsp, j_rarg1_off * BytesPerInt)); 4061 __ movptr(j_rarg2, Address(rsp, j_rarg2_off * BytesPerInt)); 4062 __ movptr(j_rarg3, Address(rsp, j_rarg3_off * BytesPerInt)); 4063 __ movptr(j_rarg4, Address(rsp, j_rarg4_off * BytesPerInt)); 4064 __ movptr(j_rarg5, Address(rsp, j_rarg5_off * BytesPerInt)); 4065 __ movptr(rax, Address(rsp, rax_off * BytesPerInt)); 4066 4067 __ addptr(rsp, frame_size_in_bytes-8); 4068 4069 // check for pending exceptions 4070 Label pending; 4071 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD); 4072 __ jcc(Assembler::notEqual, pending); 4073 4074 if (has_res) { 4075 __ get_vm_result(rax, r15_thread); 4076 } 4077 4078 __ ret(0); 4079 4080 __ bind(pending); 4081 4082 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 4083 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 4084 4085 // ------------- 4086 // make sure all code is generated 4087 _masm->flush(); 4088 4089 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false); 4090 return stub->entry_point(); 4091 } 4092 4093 void StubGenerator::generate_continuation_stubs() { 4094 // Continuation stubs: 4095 StubRoutines::_cont_thaw = generate_cont_thaw(); 4096 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); 4097 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); 4098 } 4099 4100 void StubGenerator::generate_final_stubs() { 4101 // Generates the rest of stubs and initializes the entry points 4102 4103 // support for verify_oop (must happen after universe_init) 4104 if (VerifyOops) { 4105 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); 4106 } 4107 4108 // data cache line writeback 4109 StubRoutines::_data_cache_writeback = generate_data_cache_writeback(); 4110 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync(); 4111 4112 // arraycopy stubs used by compilers 4113 generate_arraycopy_stubs(); 4114 4115 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 4116 if (bs_nm != nullptr) { 4117 StubRoutines::_method_entry_barrier = generate_method_entry_barrier(); 4118 } 4119 4120 if (UseVectorizedMismatchIntrinsic) { 4121 StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch(); 4122 } 4123 4124 StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler(); 4125 } 4126 4127 void StubGenerator::generate_compiler_stubs() { 4128 #if COMPILER2_OR_JVMCI 4129 4130 // Entry points that are C2 compiler specific. 4131 4132 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF); 4133 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000); 4134 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF); 4135 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000); 4136 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF); 4137 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x0000000100000001); 4138 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff); 4139 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask"); 4140 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff); 4141 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff); 4142 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit, 4143 0xFFFFFFFF, 0, 0, 0); 4144 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit, 4145 0xFFFFFFFF, 0xFFFFFFFF, 0, 0); 4146 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100); 4147 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask"); 4148 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100); 4149 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000); 4150 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000); 4151 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices"); 4152 StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut("count_leading_zeros_lut"); 4153 StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut("reverse_bit_lut"); 4154 StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long("perm_mask_long"); 4155 StubRoutines::x86::_vector_reverse_byte_perm_mask_int = generate_vector_reverse_byte_perm_mask_int("perm_mask_int"); 4156 StubRoutines::x86::_vector_reverse_byte_perm_mask_short = generate_vector_reverse_byte_perm_mask_short("perm_mask_short"); 4157 4158 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512vl()) { 4159 StubRoutines::x86::_compress_perm_table32 = generate_compress_perm_table("compress_perm_table32", 32); 4160 StubRoutines::x86::_compress_perm_table64 = generate_compress_perm_table("compress_perm_table64", 64); 4161 StubRoutines::x86::_expand_perm_table32 = generate_expand_perm_table("expand_perm_table32", 32); 4162 StubRoutines::x86::_expand_perm_table64 = generate_expand_perm_table("expand_perm_table64", 64); 4163 } 4164 4165 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) { 4166 // lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight. 4167 StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut("popcount_lut"); 4168 } 4169 4170 generate_aes_stubs(); 4171 4172 generate_ghash_stubs(); 4173 4174 generate_chacha_stubs(); 4175 4176 #ifdef COMPILER2 4177 if ((UseAVX == 2) && EnableX86ECoreOpts) { 4178 generate_string_indexof(StubRoutines::_string_indexof_array); 4179 } 4180 #endif 4181 4182 if (UseAdler32Intrinsics) { 4183 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32(); 4184 } 4185 4186 if (UsePoly1305Intrinsics) { 4187 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks(); 4188 } 4189 4190 if (UseIntPolyIntrinsics) { 4191 StubRoutines::_intpoly_montgomeryMult_P256 = generate_intpoly_montgomeryMult_P256(); 4192 StubRoutines::_intpoly_assign = generate_intpoly_assign(); 4193 } 4194 4195 if (UseMD5Intrinsics) { 4196 StubRoutines::_md5_implCompress = generate_md5_implCompress(false, "md5_implCompress"); 4197 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(true, "md5_implCompressMB"); 4198 } 4199 4200 if (UseSHA1Intrinsics) { 4201 StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask(); 4202 StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask(); 4203 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress"); 4204 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB"); 4205 } 4206 4207 if (UseSHA256Intrinsics) { 4208 StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; 4209 char* dst = (char*)StubRoutines::x86::_k256_W; 4210 char* src = (char*)StubRoutines::x86::_k256; 4211 for (int ii = 0; ii < 16; ++ii) { 4212 memcpy(dst + 32 * ii, src + 16 * ii, 16); 4213 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); 4214 } 4215 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; 4216 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); 4217 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress"); 4218 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB"); 4219 } 4220 4221 if (UseSHA512Intrinsics) { 4222 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; 4223 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); 4224 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress"); 4225 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB"); 4226 } 4227 4228 if (UseBASE64Intrinsics) { 4229 if(VM_Version::supports_avx2()) { 4230 StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr(); 4231 StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr(); 4232 StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr(); 4233 StubRoutines::x86::_avx2_decode_tables_base64 = base64_AVX2_decode_tables_addr(); 4234 StubRoutines::x86::_avx2_decode_lut_tables_base64 = base64_AVX2_decode_LUT_tables_addr(); 4235 } 4236 StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr(); 4237 if (VM_Version::supports_avx512_vbmi()) { 4238 StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr(); 4239 StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr(); 4240 StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr(); 4241 StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr(); 4242 StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr(); 4243 StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr(); 4244 StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr(); 4245 StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr(); 4246 StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr(); 4247 } 4248 StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr(); 4249 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock(); 4250 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); 4251 } 4252 4253 #ifdef COMPILER2 4254 if (UseMultiplyToLenIntrinsic) { 4255 StubRoutines::_multiplyToLen = generate_multiplyToLen(); 4256 } 4257 if (UseSquareToLenIntrinsic) { 4258 StubRoutines::_squareToLen = generate_squareToLen(); 4259 } 4260 if (UseMulAddIntrinsic) { 4261 StubRoutines::_mulAdd = generate_mulAdd(); 4262 } 4263 if (VM_Version::supports_avx512_vbmi2()) { 4264 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); 4265 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); 4266 } 4267 if (UseSecondarySupersTable) { 4268 StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub(); 4269 if (! InlineSecondarySupersTest) { 4270 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { 4271 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = generate_lookup_secondary_supers_table_stub(slot); 4272 } 4273 } 4274 } 4275 if (UseMontgomeryMultiplyIntrinsic) { 4276 StubRoutines::_montgomeryMultiply 4277 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply); 4278 } 4279 if (UseMontgomerySquareIntrinsic) { 4280 StubRoutines::_montgomerySquare 4281 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square); 4282 } 4283 4284 // Load x86_64_sort library on supported hardware to enable SIMD sort and partition intrinsics 4285 4286 if (VM_Version::is_intel() && (VM_Version::supports_avx512dq() || VM_Version::supports_avx2())) { 4287 void *libsimdsort = nullptr; 4288 char ebuf_[1024]; 4289 char dll_name_simd_sort[JVM_MAXPATHLEN]; 4290 if (os::dll_locate_lib(dll_name_simd_sort, sizeof(dll_name_simd_sort), Arguments::get_dll_dir(), "simdsort")) { 4291 libsimdsort = os::dll_load(dll_name_simd_sort, ebuf_, sizeof ebuf_); 4292 } 4293 // Get addresses for SIMD sort and partition routines 4294 if (libsimdsort != nullptr) { 4295 log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "simdsort" JNI_LIB_SUFFIX, p2i(libsimdsort)); 4296 4297 snprintf(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512dq() ? "avx512_sort" : "avx2_sort"); 4298 StubRoutines::_array_sort = (address)os::dll_lookup(libsimdsort, ebuf_); 4299 4300 snprintf(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512dq() ? "avx512_partition" : "avx2_partition"); 4301 StubRoutines::_array_partition = (address)os::dll_lookup(libsimdsort, ebuf_); 4302 } 4303 } 4304 4305 // Get svml stub routine addresses 4306 void *libjsvml = nullptr; 4307 char ebuf[1024]; 4308 char dll_name[JVM_MAXPATHLEN]; 4309 if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "jsvml")) { 4310 libjsvml = os::dll_load(dll_name, ebuf, sizeof ebuf); 4311 } 4312 if (libjsvml != nullptr) { 4313 // SVML method naming convention 4314 // All the methods are named as __jsvml_op<T><N>_ha_<VV> 4315 // Where: 4316 // ha stands for high accuracy 4317 // <T> is optional to indicate float/double 4318 // Set to f for vector float operation 4319 // Omitted for vector double operation 4320 // <N> is the number of elements in the vector 4321 // 1, 2, 4, 8, 16 4322 // e.g. 128 bit float vector has 4 float elements 4323 // <VV> indicates the avx/sse level: 4324 // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 4325 // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns 4326 // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns 4327 4328 log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "jsvml" JNI_LIB_SUFFIX, p2i(libjsvml)); 4329 if (UseAVX > 2) { 4330 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { 4331 int vop = VectorSupport::VECTOR_OP_SVML_START + op; 4332 if ((!VM_Version::supports_avx512dq()) && 4333 (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { 4334 continue; 4335 } 4336 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::svmlname[op]); 4337 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); 4338 4339 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::svmlname[op]); 4340 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); 4341 } 4342 } 4343 const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex"); 4344 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { 4345 int vop = VectorSupport::VECTOR_OP_SVML_START + op; 4346 if (vop == VectorSupport::VECTOR_OP_POW) { 4347 continue; 4348 } 4349 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4350 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); 4351 4352 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4353 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); 4354 4355 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4356 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); 4357 4358 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4359 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); 4360 4361 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4362 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); 4363 4364 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); 4365 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); 4366 } 4367 } 4368 4369 #endif // COMPILER2 4370 #endif // COMPILER2_OR_JVMCI 4371 } 4372 4373 StubGenerator::StubGenerator(CodeBuffer* code, StubsKind kind) : StubCodeGenerator(code) { 4374 DEBUG_ONLY( _regs_in_thread = false; ) 4375 switch(kind) { 4376 case Initial_stubs: 4377 generate_initial_stubs(); 4378 break; 4379 case Continuation_stubs: 4380 generate_continuation_stubs(); 4381 break; 4382 case Compiler_stubs: 4383 generate_compiler_stubs(); 4384 break; 4385 case Final_stubs: 4386 generate_final_stubs(); 4387 break; 4388 default: 4389 fatal("unexpected stubs kind: %d", kind); 4390 break; 4391 }; 4392 } 4393 4394 void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind) { 4395 StubGenerator g(code, kind); 4396 } 4397 4398 #undef __