1 /* 2 * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "asm/macroAssembler.hpp" 26 #include "classfile/javaClasses.hpp" 27 #include "classfile/vmIntrinsics.hpp" 28 #include "code/aotCodeCache.hpp" 29 #include "compiler/oopMap.hpp" 30 #include "gc/shared/barrierSet.hpp" 31 #include "gc/shared/barrierSetAssembler.hpp" 32 #include "gc/shared/barrierSetNMethod.hpp" 33 #include "gc/shared/gc_globals.hpp" 34 #include "memory/universe.hpp" 35 #include "prims/jvmtiExport.hpp" 36 #include "prims/upcallLinker.hpp" 37 #include "runtime/arguments.hpp" 38 #include "runtime/continuationEntry.hpp" 39 #include "runtime/javaThread.hpp" 40 #include "runtime/sharedRuntime.hpp" 41 #include "runtime/stubRoutines.hpp" 42 #include "stubGenerator_x86_64.hpp" 43 #ifdef COMPILER2 44 #include "opto/runtime.hpp" 45 #include "opto/c2_globals.hpp" 46 #endif 47 #if INCLUDE_JVMCI 48 #include "jvmci/jvmci_globals.hpp" 49 #endif 50 51 // For a more detailed description of the stub routine structure 52 // see the comment in stubRoutines.hpp 53 54 #define __ _masm-> 55 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) 56 57 #ifdef PRODUCT 58 #define BLOCK_COMMENT(str) /* nothing */ 59 #else 60 #define BLOCK_COMMENT(str) __ block_comment(str) 61 #endif // PRODUCT 62 63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 64 65 // 66 // Linux Arguments: 67 // c_rarg0: call wrapper address address 68 // c_rarg1: result address 69 // c_rarg2: result type BasicType 70 // c_rarg3: method Method* 71 // c_rarg4: (interpreter) entry point address 72 // c_rarg5: parameters intptr_t* 73 // 16(rbp): parameter size (in words) int 74 // 24(rbp): thread Thread* 75 // 76 // [ return_from_Java ] <--- rsp 77 // [ argument word n ] 78 // ... 79 // -12 [ argument word 1 ] 80 // -11 [ saved r15 ] <--- rsp_after_call 81 // -10 [ saved r14 ] 82 // -9 [ saved r13 ] 83 // -8 [ saved r12 ] 84 // -7 [ saved rbx ] 85 // -6 [ call wrapper ] 86 // -5 [ result ] 87 // -4 [ result type ] 88 // -3 [ method ] 89 // -2 [ entry point ] 90 // -1 [ parameters ] 91 // 0 [ saved rbp ] <--- rbp 92 // 1 [ return address ] 93 // 2 [ parameter size ] 94 // 3 [ thread ] 95 // 96 // Windows Arguments: 97 // c_rarg0: call wrapper address address 98 // c_rarg1: result address 99 // c_rarg2: result type BasicType 100 // c_rarg3: method Method* 101 // 48(rbp): (interpreter) entry point address 102 // 56(rbp): parameters intptr_t* 103 // 64(rbp): parameter size (in words) int 104 // 72(rbp): thread Thread* 105 // 106 // [ return_from_Java ] <--- rsp 107 // [ argument word n ] 108 // ... 109 // -28 [ argument word 1 ] 110 // -27 [ saved xmm15 ] <--- rsp after_call 111 // [ saved xmm7-xmm14 ] 112 // -9 [ saved xmm6 ] (each xmm register takes 2 slots) 113 // -7 [ saved r15 ] 114 // -6 [ saved r14 ] 115 // -5 [ saved r13 ] 116 // -4 [ saved r12 ] 117 // -3 [ saved rdi ] 118 // -2 [ saved rsi ] 119 // -1 [ saved rbx ] 120 // 0 [ saved rbp ] <--- rbp 121 // 1 [ return address ] 122 // 2 [ call wrapper ] 123 // 3 [ result ] 124 // 4 [ result type ] 125 // 5 [ method ] 126 // 6 [ entry point ] 127 // 7 [ parameters ] 128 // 8 [ parameter size ] 129 // 9 [ thread ] 130 // 131 // Windows reserves the callers stack space for arguments 1-4. 132 // We spill c_rarg0-c_rarg3 to this space. 133 134 // Call stub stack layout word offsets from rbp 135 #ifdef _WIN64 136 enum call_stub_layout { 137 xmm_save_first = 6, // save from xmm6 138 xmm_save_last = 15, // to xmm15 139 xmm_save_base = -9, 140 rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27 141 r15_off = -7, 142 r14_off = -6, 143 r13_off = -5, 144 r12_off = -4, 145 rdi_off = -3, 146 rsi_off = -2, 147 rbx_off = -1, 148 rbp_off = 0, 149 retaddr_off = 1, 150 call_wrapper_off = 2, 151 result_off = 3, 152 result_type_off = 4, 153 method_off = 5, 154 entry_point_off = 6, 155 parameters_off = 7, 156 parameter_size_off = 8, 157 thread_off = 9 158 }; 159 160 static Address xmm_save(int reg) { 161 assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range"); 162 return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize); 163 } 164 #else // !_WIN64 165 enum call_stub_layout { 166 rsp_after_call_off = -12, 167 mxcsr_off = rsp_after_call_off, 168 r15_off = -11, 169 r14_off = -10, 170 r13_off = -9, 171 r12_off = -8, 172 rbx_off = -7, 173 call_wrapper_off = -6, 174 result_off = -5, 175 result_type_off = -4, 176 method_off = -3, 177 entry_point_off = -2, 178 parameters_off = -1, 179 rbp_off = 0, 180 retaddr_off = 1, 181 parameter_size_off = 2, 182 thread_off = 3 183 }; 184 #endif // _WIN64 185 186 address StubGenerator::generate_call_stub(address& return_address) { 187 188 assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && 189 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, 190 "adjust this code"); 191 StubId stub_id = StubId::stubgen_call_stub_id; 192 StubCodeMark mark(this, stub_id); 193 address start = __ pc(); 194 195 // same as in generate_catch_exception()! 196 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 197 198 const Address call_wrapper (rbp, call_wrapper_off * wordSize); 199 const Address result (rbp, result_off * wordSize); 200 const Address result_type (rbp, result_type_off * wordSize); 201 const Address method (rbp, method_off * wordSize); 202 const Address entry_point (rbp, entry_point_off * wordSize); 203 const Address parameters (rbp, parameters_off * wordSize); 204 const Address parameter_size(rbp, parameter_size_off * wordSize); 205 206 // same as in generate_catch_exception()! 207 const Address thread (rbp, thread_off * wordSize); 208 209 const Address r15_save(rbp, r15_off * wordSize); 210 const Address r14_save(rbp, r14_off * wordSize); 211 const Address r13_save(rbp, r13_off * wordSize); 212 const Address r12_save(rbp, r12_off * wordSize); 213 const Address rbx_save(rbp, rbx_off * wordSize); 214 215 // stub code 216 __ enter(); 217 __ subptr(rsp, -rsp_after_call_off * wordSize); 218 219 // save register parameters 220 #ifndef _WIN64 221 __ movptr(parameters, c_rarg5); // parameters 222 __ movptr(entry_point, c_rarg4); // entry_point 223 #endif 224 225 __ movptr(method, c_rarg3); // method 226 __ movl(result_type, c_rarg2); // result type 227 __ movptr(result, c_rarg1); // result 228 __ movptr(call_wrapper, c_rarg0); // call wrapper 229 230 // save regs belonging to calling function 231 __ movptr(rbx_save, rbx); 232 __ movptr(r12_save, r12); 233 __ movptr(r13_save, r13); 234 __ movptr(r14_save, r14); 235 __ movptr(r15_save, r15); 236 237 #ifdef _WIN64 238 int last_reg = 15; 239 for (int i = xmm_save_first; i <= last_reg; i++) { 240 __ movdqu(xmm_save(i), as_XMMRegister(i)); 241 } 242 243 const Address rdi_save(rbp, rdi_off * wordSize); 244 const Address rsi_save(rbp, rsi_off * wordSize); 245 246 __ movptr(rsi_save, rsi); 247 __ movptr(rdi_save, rdi); 248 #else 249 const Address mxcsr_save(rbp, mxcsr_off * wordSize); 250 { 251 Label skip_ldmx; 252 __ cmp32_mxcsr_std(mxcsr_save, rax, rscratch1); 253 __ jcc(Assembler::equal, skip_ldmx); 254 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 255 __ ldmxcsr(mxcsr_std, rscratch1); 256 __ bind(skip_ldmx); 257 } 258 #endif 259 260 // Load up thread register 261 __ movptr(r15_thread, thread); 262 __ reinit_heapbase(); 263 264 #ifdef ASSERT 265 // make sure we have no pending exceptions 266 { 267 Label L; 268 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 269 __ jcc(Assembler::equal, L); 270 __ stop("StubRoutines::call_stub: entered with pending exception"); 271 __ bind(L); 272 } 273 #endif 274 275 // pass parameters if any 276 BLOCK_COMMENT("pass parameters if any"); 277 Label parameters_done; 278 __ movl(c_rarg3, parameter_size); 279 __ testl(c_rarg3, c_rarg3); 280 __ jcc(Assembler::zero, parameters_done); 281 282 Label loop; 283 __ movptr(c_rarg2, parameters); // parameter pointer 284 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 285 __ BIND(loop); 286 __ movptr(rax, Address(c_rarg2, 0));// get parameter 287 __ addptr(c_rarg2, wordSize); // advance to next parameter 288 __ decrementl(c_rarg1); // decrement counter 289 __ push(rax); // pass parameter 290 __ jcc(Assembler::notZero, loop); 291 292 // call Java function 293 __ BIND(parameters_done); 294 __ movptr(rbx, method); // get Method* 295 __ movptr(c_rarg1, entry_point); // get entry_point 296 __ mov(r13, rsp); // set sender sp 297 BLOCK_COMMENT("call Java function"); 298 __ call(c_rarg1); 299 300 BLOCK_COMMENT("call_stub_return_address:"); 301 return_address = __ pc(); 302 303 // store result depending on type (everything that is not 304 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) 305 __ movptr(c_rarg0, result); 306 Label is_long, is_float, is_double, exit; 307 __ movl(c_rarg1, result_type); 308 __ cmpl(c_rarg1, T_OBJECT); 309 __ jcc(Assembler::equal, is_long); 310 __ cmpl(c_rarg1, T_LONG); 311 __ jcc(Assembler::equal, is_long); 312 __ cmpl(c_rarg1, T_FLOAT); 313 __ jcc(Assembler::equal, is_float); 314 __ cmpl(c_rarg1, T_DOUBLE); 315 __ jcc(Assembler::equal, is_double); 316 #ifdef ASSERT 317 // make sure the type is INT 318 { 319 Label L; 320 __ cmpl(c_rarg1, T_INT); 321 __ jcc(Assembler::equal, L); 322 __ stop("StubRoutines::call_stub: unexpected result type"); 323 __ bind(L); 324 } 325 #endif 326 327 // handle T_INT case 328 __ movl(Address(c_rarg0, 0), rax); 329 330 __ BIND(exit); 331 332 // pop parameters 333 __ lea(rsp, rsp_after_call); 334 335 #ifdef ASSERT 336 // verify that threads correspond 337 { 338 Label L1, L2, L3; 339 __ cmpptr(r15_thread, thread); 340 __ jcc(Assembler::equal, L1); 341 __ stop("StubRoutines::call_stub: r15_thread is corrupted"); 342 __ bind(L1); 343 __ get_thread_slow(rbx); 344 __ cmpptr(r15_thread, thread); 345 __ jcc(Assembler::equal, L2); 346 __ stop("StubRoutines::call_stub: r15_thread is modified by call"); 347 __ bind(L2); 348 __ cmpptr(r15_thread, rbx); 349 __ jcc(Assembler::equal, L3); 350 __ stop("StubRoutines::call_stub: threads must correspond"); 351 __ bind(L3); 352 } 353 #endif 354 355 __ pop_cont_fastpath(); 356 357 // restore regs belonging to calling function 358 #ifdef _WIN64 359 // emit the restores for xmm regs 360 for (int i = xmm_save_first; i <= last_reg; i++) { 361 __ movdqu(as_XMMRegister(i), xmm_save(i)); 362 } 363 #endif 364 __ movptr(r15, r15_save); 365 __ movptr(r14, r14_save); 366 __ movptr(r13, r13_save); 367 __ movptr(r12, r12_save); 368 __ movptr(rbx, rbx_save); 369 370 #ifdef _WIN64 371 __ movptr(rdi, rdi_save); 372 __ movptr(rsi, rsi_save); 373 #else 374 __ ldmxcsr(mxcsr_save); 375 #endif 376 377 // restore rsp 378 __ addptr(rsp, -rsp_after_call_off * wordSize); 379 380 // return 381 __ vzeroupper(); 382 __ pop(rbp); 383 __ ret(0); 384 385 // handle return types different from T_INT 386 __ BIND(is_long); 387 __ movq(Address(c_rarg0, 0), rax); 388 __ jmp(exit); 389 390 __ BIND(is_float); 391 __ movflt(Address(c_rarg0, 0), xmm0); 392 __ jmp(exit); 393 394 __ BIND(is_double); 395 __ movdbl(Address(c_rarg0, 0), xmm0); 396 __ jmp(exit); 397 398 return start; 399 } 400 401 // Return point for a Java call if there's an exception thrown in 402 // Java code. The exception is caught and transformed into a 403 // pending exception stored in JavaThread that can be tested from 404 // within the VM. 405 // 406 // Note: Usually the parameters are removed by the callee. In case 407 // of an exception crossing an activation frame boundary, that is 408 // not the case if the callee is compiled code => need to setup the 409 // rsp. 410 // 411 // rax: exception oop 412 413 address StubGenerator::generate_catch_exception() { 414 StubId stub_id = StubId::stubgen_catch_exception_id; 415 StubCodeMark mark(this, stub_id); 416 address start = __ pc(); 417 418 // same as in generate_call_stub(): 419 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 420 const Address thread (rbp, thread_off * wordSize); 421 422 #ifdef ASSERT 423 // verify that threads correspond 424 { 425 Label L1, L2, L3; 426 __ cmpptr(r15_thread, thread); 427 __ jcc(Assembler::equal, L1); 428 __ stop("StubRoutines::catch_exception: r15_thread is corrupted"); 429 __ bind(L1); 430 __ get_thread_slow(rbx); 431 __ cmpptr(r15_thread, thread); 432 __ jcc(Assembler::equal, L2); 433 __ stop("StubRoutines::catch_exception: r15_thread is modified by call"); 434 __ bind(L2); 435 __ cmpptr(r15_thread, rbx); 436 __ jcc(Assembler::equal, L3); 437 __ stop("StubRoutines::catch_exception: threads must correspond"); 438 __ bind(L3); 439 } 440 #endif 441 442 // set pending exception 443 __ verify_oop(rax); 444 445 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); 446 __ lea(rscratch1, ExternalAddress((address)__FILE__)); 447 __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); 448 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); 449 450 // complete return to VM 451 assert(StubRoutines::_call_stub_return_address != nullptr, 452 "_call_stub_return_address must have been generated before"); 453 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); 454 455 return start; 456 } 457 458 // Continuation point for runtime calls returning with a pending 459 // exception. The pending exception check happened in the runtime 460 // or native call stub. The pending exception in Thread is 461 // converted into a Java-level exception. 462 // 463 // Contract with Java-level exception handlers: 464 // rax: exception 465 // rdx: throwing pc 466 // 467 // NOTE: At entry of this stub, exception-pc must be on stack !! 468 469 address StubGenerator::generate_forward_exception() { 470 StubId stub_id = StubId::stubgen_forward_exception_id; 471 StubCodeMark mark(this, stub_id); 472 address start = __ pc(); 473 474 // Upon entry, the sp points to the return address returning into 475 // Java (interpreted or compiled) code; i.e., the return address 476 // becomes the throwing pc. 477 // 478 // Arguments pushed before the runtime call are still on the stack 479 // but the exception handler will reset the stack pointer -> 480 // ignore them. A potential result in registers can be ignored as 481 // well. 482 483 #ifdef ASSERT 484 // make sure this code is only executed if there is a pending exception 485 { 486 Label L; 487 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 488 __ jcc(Assembler::notEqual, L); 489 __ stop("StubRoutines::forward exception: no pending exception (1)"); 490 __ bind(L); 491 } 492 #endif 493 494 // compute exception handler into rbx 495 __ movptr(c_rarg0, Address(rsp, 0)); 496 BLOCK_COMMENT("call exception_handler_for_return_address"); 497 __ call_VM_leaf(CAST_FROM_FN_PTR(address, 498 SharedRuntime::exception_handler_for_return_address), 499 r15_thread, c_rarg0); 500 __ mov(rbx, rax); 501 502 // setup rax & rdx, remove return address & clear pending exception 503 __ pop(rdx); 504 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 505 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 506 507 #ifdef ASSERT 508 // make sure exception is set 509 { 510 Label L; 511 __ testptr(rax, rax); 512 __ jcc(Assembler::notEqual, L); 513 __ stop("StubRoutines::forward exception: no pending exception (2)"); 514 __ bind(L); 515 } 516 #endif 517 518 // continue at exception handler (return address removed) 519 // rax: exception 520 // rbx: exception handler 521 // rdx: throwing pc 522 __ verify_oop(rax); 523 __ jmp(rbx); 524 525 return start; 526 } 527 528 // Support for intptr_t OrderAccess::fence() 529 // 530 // Arguments : 531 // 532 // Result: 533 address StubGenerator::generate_orderaccess_fence() { 534 StubId stub_id = StubId::stubgen_fence_id; 535 StubCodeMark mark(this, stub_id); 536 address start = __ pc(); 537 538 __ membar(Assembler::StoreLoad); 539 __ ret(0); 540 541 return start; 542 } 543 544 545 //---------------------------------------------------------------------------------------------------- 546 // Support for void verify_mxcsr() 547 // 548 // This routine is used with -Xcheck:jni to verify that native 549 // JNI code does not return to Java code without restoring the 550 // MXCSR register to our expected state. 551 552 address StubGenerator::generate_verify_mxcsr() { 553 StubId stub_id = StubId::stubgen_verify_mxcsr_id; 554 StubCodeMark mark(this, stub_id); 555 address start = __ pc(); 556 557 const Address mxcsr_save(rsp, 0); 558 559 if (CheckJNICalls) { 560 Label ok_ret; 561 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 562 __ push_ppx(rax); 563 __ subptr(rsp, wordSize); // allocate a temp location 564 __ cmp32_mxcsr_std(mxcsr_save, rax, rscratch1); 565 __ jcc(Assembler::equal, ok_ret); 566 567 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall"); 568 569 __ ldmxcsr(mxcsr_std, rscratch1); 570 571 __ bind(ok_ret); 572 __ addptr(rsp, wordSize); 573 __ pop_ppx(rax); 574 } 575 576 __ ret(0); 577 578 return start; 579 } 580 581 address StubGenerator::generate_f2i_fixup() { 582 StubId stub_id = StubId::stubgen_f2i_fixup_id; 583 StubCodeMark mark(this, stub_id); 584 Address inout(rsp, 5 * wordSize); // return address + 4 saves 585 586 address start = __ pc(); 587 588 Label L; 589 590 __ push_ppx(rax); 591 __ push_ppx(c_rarg3); 592 __ push_ppx(c_rarg2); 593 __ push_ppx(c_rarg1); 594 595 __ movl(rax, 0x7f800000); 596 __ xorl(c_rarg3, c_rarg3); 597 __ movl(c_rarg2, inout); 598 __ movl(c_rarg1, c_rarg2); 599 __ andl(c_rarg1, 0x7fffffff); 600 __ cmpl(rax, c_rarg1); // NaN? -> 0 601 __ jcc(Assembler::negative, L); 602 __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint 603 __ movl(c_rarg3, 0x80000000); 604 __ movl(rax, 0x7fffffff); 605 __ cmovl(Assembler::positive, c_rarg3, rax); 606 607 __ bind(L); 608 __ movptr(inout, c_rarg3); 609 610 __ pop_ppx(c_rarg1); 611 __ pop_ppx(c_rarg2); 612 __ pop_ppx(c_rarg3); 613 __ pop_ppx(rax); 614 615 __ ret(0); 616 617 return start; 618 } 619 620 address StubGenerator::generate_f2l_fixup() { 621 StubId stub_id = StubId::stubgen_f2l_fixup_id; 622 StubCodeMark mark(this, stub_id); 623 Address inout(rsp, 5 * wordSize); // return address + 4 saves 624 address start = __ pc(); 625 626 Label L; 627 628 __ push_ppx(rax); 629 __ push_ppx(c_rarg3); 630 __ push_ppx(c_rarg2); 631 __ push_ppx(c_rarg1); 632 633 __ movl(rax, 0x7f800000); 634 __ xorl(c_rarg3, c_rarg3); 635 __ movl(c_rarg2, inout); 636 __ movl(c_rarg1, c_rarg2); 637 __ andl(c_rarg1, 0x7fffffff); 638 __ cmpl(rax, c_rarg1); // NaN? -> 0 639 __ jcc(Assembler::negative, L); 640 __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong 641 __ mov64(c_rarg3, 0x8000000000000000); 642 __ mov64(rax, 0x7fffffffffffffff); 643 __ cmov(Assembler::positive, c_rarg3, rax); 644 645 __ bind(L); 646 __ movptr(inout, c_rarg3); 647 648 __ pop_ppx(c_rarg1); 649 __ pop_ppx(c_rarg2); 650 __ pop_ppx(c_rarg3); 651 __ pop_ppx(rax); 652 653 __ ret(0); 654 655 return start; 656 } 657 658 address StubGenerator::generate_d2i_fixup() { 659 StubId stub_id = StubId::stubgen_d2i_fixup_id; 660 StubCodeMark mark(this, stub_id); 661 Address inout(rsp, 6 * wordSize); // return address + 5 saves 662 663 address start = __ pc(); 664 665 Label L; 666 667 __ push_ppx(rax); 668 __ push_ppx(c_rarg3); 669 __ push_ppx(c_rarg2); 670 __ push_ppx(c_rarg1); 671 __ push_ppx(c_rarg0); 672 673 __ movl(rax, 0x7ff00000); 674 __ movq(c_rarg2, inout); 675 __ movl(c_rarg3, c_rarg2); 676 __ mov(c_rarg1, c_rarg2); 677 __ mov(c_rarg0, c_rarg2); 678 __ negl(c_rarg3); 679 __ shrptr(c_rarg1, 0x20); 680 __ orl(c_rarg3, c_rarg2); 681 __ andl(c_rarg1, 0x7fffffff); 682 __ xorl(c_rarg2, c_rarg2); 683 __ shrl(c_rarg3, 0x1f); 684 __ orl(c_rarg1, c_rarg3); 685 __ cmpl(rax, c_rarg1); 686 __ jcc(Assembler::negative, L); // NaN -> 0 687 __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint 688 __ movl(c_rarg2, 0x80000000); 689 __ movl(rax, 0x7fffffff); 690 __ cmov(Assembler::positive, c_rarg2, rax); 691 692 __ bind(L); 693 __ movptr(inout, c_rarg2); 694 695 __ pop_ppx(c_rarg0); 696 __ pop_ppx(c_rarg1); 697 __ pop_ppx(c_rarg2); 698 __ pop_ppx(c_rarg3); 699 __ pop_ppx(rax); 700 701 __ ret(0); 702 703 return start; 704 } 705 706 address StubGenerator::generate_d2l_fixup() { 707 StubId stub_id = StubId::stubgen_d2l_fixup_id; 708 StubCodeMark mark(this, stub_id); 709 Address inout(rsp, 6 * wordSize); // return address + 5 saves 710 711 address start = __ pc(); 712 713 Label L; 714 715 __ push_ppx(rax); 716 __ push_ppx(c_rarg3); 717 __ push_ppx(c_rarg2); 718 __ push_ppx(c_rarg1); 719 __ push_ppx(c_rarg0); 720 721 __ movl(rax, 0x7ff00000); 722 __ movq(c_rarg2, inout); 723 __ movl(c_rarg3, c_rarg2); 724 __ mov(c_rarg1, c_rarg2); 725 __ mov(c_rarg0, c_rarg2); 726 __ negl(c_rarg3); 727 __ shrptr(c_rarg1, 0x20); 728 __ orl(c_rarg3, c_rarg2); 729 __ andl(c_rarg1, 0x7fffffff); 730 __ xorl(c_rarg2, c_rarg2); 731 __ shrl(c_rarg3, 0x1f); 732 __ orl(c_rarg1, c_rarg3); 733 __ cmpl(rax, c_rarg1); 734 __ jcc(Assembler::negative, L); // NaN -> 0 735 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong 736 __ mov64(c_rarg2, 0x8000000000000000); 737 __ mov64(rax, 0x7fffffffffffffff); 738 __ cmovq(Assembler::positive, c_rarg2, rax); 739 740 __ bind(L); 741 __ movq(inout, c_rarg2); 742 743 __ pop_ppx(c_rarg0); 744 __ pop_ppx(c_rarg1); 745 __ pop_ppx(c_rarg2); 746 __ pop_ppx(c_rarg3); 747 __ pop_ppx(rax); 748 749 __ ret(0); 750 751 return start; 752 } 753 754 address StubGenerator::generate_count_leading_zeros_lut() { 755 __ align64(); 756 StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id; 757 StubCodeMark mark(this, stub_id); 758 address start = __ pc(); 759 760 __ emit_data64(0x0101010102020304, relocInfo::none); 761 __ emit_data64(0x0000000000000000, relocInfo::none); 762 __ emit_data64(0x0101010102020304, relocInfo::none); 763 __ emit_data64(0x0000000000000000, relocInfo::none); 764 __ emit_data64(0x0101010102020304, relocInfo::none); 765 __ emit_data64(0x0000000000000000, relocInfo::none); 766 __ emit_data64(0x0101010102020304, relocInfo::none); 767 __ emit_data64(0x0000000000000000, relocInfo::none); 768 769 return start; 770 } 771 772 address StubGenerator::generate_popcount_avx_lut() { 773 __ align64(); 774 StubId stub_id = StubId::stubgen_vector_popcount_lut_id; 775 StubCodeMark mark(this, stub_id); 776 address start = __ pc(); 777 778 __ emit_data64(0x0302020102010100, relocInfo::none); 779 __ emit_data64(0x0403030203020201, relocInfo::none); 780 __ emit_data64(0x0302020102010100, relocInfo::none); 781 __ emit_data64(0x0403030203020201, relocInfo::none); 782 __ emit_data64(0x0302020102010100, relocInfo::none); 783 __ emit_data64(0x0403030203020201, relocInfo::none); 784 __ emit_data64(0x0302020102010100, relocInfo::none); 785 __ emit_data64(0x0403030203020201, relocInfo::none); 786 787 return start; 788 } 789 790 address StubGenerator::generate_iota_indices() { 791 __ align(CodeEntryAlignment); 792 StubId stub_id = StubId::stubgen_vector_iota_indices_id; 793 StubCodeMark mark(this, stub_id); 794 address start = __ pc(); 795 // B 796 __ emit_data64(0x0706050403020100, relocInfo::none); 797 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); 798 __ emit_data64(0x1716151413121110, relocInfo::none); 799 __ emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none); 800 __ emit_data64(0x2726252423222120, relocInfo::none); 801 __ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none); 802 __ emit_data64(0x3736353433323130, relocInfo::none); 803 __ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none); 804 // W 805 __ emit_data64(0x0003000200010000, relocInfo::none); 806 __ emit_data64(0x0007000600050004, relocInfo::none); 807 __ emit_data64(0x000B000A00090008, relocInfo::none); 808 __ emit_data64(0x000F000E000D000C, relocInfo::none); 809 __ emit_data64(0x0013001200110010, relocInfo::none); 810 __ emit_data64(0x0017001600150014, relocInfo::none); 811 __ emit_data64(0x001B001A00190018, relocInfo::none); 812 __ emit_data64(0x001F001E001D001C, relocInfo::none); 813 // D 814 __ emit_data64(0x0000000100000000, relocInfo::none); 815 __ emit_data64(0x0000000300000002, relocInfo::none); 816 __ emit_data64(0x0000000500000004, relocInfo::none); 817 __ emit_data64(0x0000000700000006, relocInfo::none); 818 __ emit_data64(0x0000000900000008, relocInfo::none); 819 __ emit_data64(0x0000000B0000000A, relocInfo::none); 820 __ emit_data64(0x0000000D0000000C, relocInfo::none); 821 __ emit_data64(0x0000000F0000000E, relocInfo::none); 822 // Q 823 __ emit_data64(0x0000000000000000, relocInfo::none); 824 __ emit_data64(0x0000000000000001, relocInfo::none); 825 __ emit_data64(0x0000000000000002, relocInfo::none); 826 __ emit_data64(0x0000000000000003, relocInfo::none); 827 __ emit_data64(0x0000000000000004, relocInfo::none); 828 __ emit_data64(0x0000000000000005, relocInfo::none); 829 __ emit_data64(0x0000000000000006, relocInfo::none); 830 __ emit_data64(0x0000000000000007, relocInfo::none); 831 // D - FP 832 __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f 833 __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f 834 __ emit_data64(0x40A0000040800000, relocInfo::none); // 4.0f, 5.0f 835 __ emit_data64(0x40E0000040C00000, relocInfo::none); // 6.0f, 7.0f 836 __ emit_data64(0x4110000041000000, relocInfo::none); // 8.0f, 9.0f 837 __ emit_data64(0x4130000041200000, relocInfo::none); // 10.0f, 11.0f 838 __ emit_data64(0x4150000041400000, relocInfo::none); // 12.0f, 13.0f 839 __ emit_data64(0x4170000041600000, relocInfo::none); // 14.0f, 15.0f 840 // Q - FP 841 __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d 842 __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d 843 __ emit_data64(0x4000000000000000, relocInfo::none); // 2.0d 844 __ emit_data64(0x4008000000000000, relocInfo::none); // 3.0d 845 __ emit_data64(0x4010000000000000, relocInfo::none); // 4.0d 846 __ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d 847 __ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d 848 __ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d 849 return start; 850 } 851 852 address StubGenerator::generate_vector_reverse_bit_lut() { 853 __ align(CodeEntryAlignment); 854 StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id; 855 StubCodeMark mark(this, stub_id); 856 address start = __ pc(); 857 858 __ emit_data64(0x0E060A020C040800, relocInfo::none); 859 __ emit_data64(0x0F070B030D050901, relocInfo::none); 860 __ emit_data64(0x0E060A020C040800, relocInfo::none); 861 __ emit_data64(0x0F070B030D050901, relocInfo::none); 862 __ emit_data64(0x0E060A020C040800, relocInfo::none); 863 __ emit_data64(0x0F070B030D050901, relocInfo::none); 864 __ emit_data64(0x0E060A020C040800, relocInfo::none); 865 __ emit_data64(0x0F070B030D050901, relocInfo::none); 866 867 return start; 868 } 869 870 address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { 871 __ align(CodeEntryAlignment); 872 StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id; 873 StubCodeMark mark(this, stub_id); 874 address start = __ pc(); 875 876 __ emit_data64(0x0001020304050607, relocInfo::none); 877 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 878 __ emit_data64(0x0001020304050607, relocInfo::none); 879 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 880 __ emit_data64(0x0001020304050607, relocInfo::none); 881 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 882 __ emit_data64(0x0001020304050607, relocInfo::none); 883 __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); 884 885 return start; 886 } 887 888 address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { 889 __ align(CodeEntryAlignment); 890 StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id; 891 StubCodeMark mark(this, stub_id); 892 address start = __ pc(); 893 894 __ emit_data64(0x0405060700010203, relocInfo::none); 895 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 896 __ emit_data64(0x0405060700010203, relocInfo::none); 897 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 898 __ emit_data64(0x0405060700010203, relocInfo::none); 899 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 900 __ emit_data64(0x0405060700010203, relocInfo::none); 901 __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); 902 903 return start; 904 } 905 906 address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { 907 __ align(CodeEntryAlignment); 908 StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id; 909 StubCodeMark mark(this, stub_id); 910 address start = __ pc(); 911 912 __ emit_data64(0x0607040502030001, relocInfo::none); 913 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 914 __ emit_data64(0x0607040502030001, relocInfo::none); 915 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 916 __ emit_data64(0x0607040502030001, relocInfo::none); 917 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 918 __ emit_data64(0x0607040502030001, relocInfo::none); 919 __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); 920 921 return start; 922 } 923 924 address StubGenerator::generate_vector_byte_shuffle_mask() { 925 __ align(CodeEntryAlignment); 926 StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id; 927 StubCodeMark mark(this, stub_id); 928 address start = __ pc(); 929 930 __ emit_data64(0x7070707070707070, relocInfo::none); 931 __ emit_data64(0x7070707070707070, relocInfo::none); 932 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 933 __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); 934 935 return start; 936 } 937 938 address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { 939 __ align(CodeEntryAlignment); 940 StubCodeMark mark(this, stub_id); 941 address start = __ pc(); 942 943 __ emit_data64( mask, relocInfo::none ); 944 __ emit_data64( mask, relocInfo::none ); 945 946 return start; 947 } 948 949 address StubGenerator::generate_compress_perm_table(StubId stub_id) { 950 int esize; 951 switch (stub_id) { 952 case StubId::stubgen_compress_perm_table32_id: 953 esize = 32; 954 break; 955 case StubId::stubgen_compress_perm_table64_id: 956 esize = 64; 957 break; 958 default: 959 ShouldNotReachHere(); 960 } 961 __ align(CodeEntryAlignment); 962 StubCodeMark mark(this, stub_id); 963 address start = __ pc(); 964 if (esize == 32) { 965 // Loop to generate 256 x 8 int compression permute index table. A row is 966 // accessed using 8 bit index computed using vector mask. An entry in 967 // a row holds either a valid permute index corresponding to set bit position 968 // or a -1 (default) value. 969 for (int mask = 0; mask < 256; mask++) { 970 int ctr = 0; 971 for (int j = 0; j < 8; j++) { 972 if (mask & (1 << j)) { 973 __ emit_data(j, relocInfo::none); 974 ctr++; 975 } 976 } 977 for (; ctr < 8; ctr++) { 978 __ emit_data(-1, relocInfo::none); 979 } 980 } 981 } else { 982 assert(esize == 64, ""); 983 // Loop to generate 16 x 4 long compression permute index table. A row is 984 // accessed using 4 bit index computed using vector mask. An entry in 985 // a row holds either a valid permute index pair for a quadword corresponding 986 // to set bit position or a -1 (default) value. 987 for (int mask = 0; mask < 16; mask++) { 988 int ctr = 0; 989 for (int j = 0; j < 4; j++) { 990 if (mask & (1 << j)) { 991 __ emit_data(2 * j, relocInfo::none); 992 __ emit_data(2 * j + 1, relocInfo::none); 993 ctr++; 994 } 995 } 996 for (; ctr < 4; ctr++) { 997 __ emit_data64(-1L, relocInfo::none); 998 } 999 } 1000 } 1001 return start; 1002 } 1003 1004 address StubGenerator::generate_expand_perm_table(StubId stub_id) { 1005 int esize; 1006 switch (stub_id) { 1007 case StubId::stubgen_expand_perm_table32_id: 1008 esize = 32; 1009 break; 1010 case StubId::stubgen_expand_perm_table64_id: 1011 esize = 64; 1012 break; 1013 default: 1014 ShouldNotReachHere(); 1015 } 1016 __ align(CodeEntryAlignment); 1017 StubCodeMark mark(this, stub_id); 1018 address start = __ pc(); 1019 if (esize == 32) { 1020 // Loop to generate 256 x 8 int expand permute index table. A row is accessed 1021 // using 8 bit index computed using vector mask. An entry in a row holds either 1022 // a valid permute index (starting from least significant lane) placed at poisition 1023 // corresponding to set bit position or a -1 (default) value. 1024 for (int mask = 0; mask < 256; mask++) { 1025 int ctr = 0; 1026 for (int j = 0; j < 8; j++) { 1027 if (mask & (1 << j)) { 1028 __ emit_data(ctr++, relocInfo::none); 1029 } else { 1030 __ emit_data(-1, relocInfo::none); 1031 } 1032 } 1033 } 1034 } else { 1035 assert(esize == 64, ""); 1036 // Loop to generate 16 x 4 long expand permute index table. A row is accessed 1037 // using 4 bit index computed using vector mask. An entry in a row holds either 1038 // a valid doubleword permute index pair representing a quadword index (starting 1039 // from least significant lane) placed at poisition corresponding to set bit 1040 // position or a -1 (default) value. 1041 for (int mask = 0; mask < 16; mask++) { 1042 int ctr = 0; 1043 for (int j = 0; j < 4; j++) { 1044 if (mask & (1 << j)) { 1045 __ emit_data(2 * ctr, relocInfo::none); 1046 __ emit_data(2 * ctr + 1, relocInfo::none); 1047 ctr++; 1048 } else { 1049 __ emit_data64(-1L, relocInfo::none); 1050 } 1051 } 1052 } 1053 } 1054 return start; 1055 } 1056 1057 address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { 1058 __ align(CodeEntryAlignment); 1059 StubCodeMark mark(this, stub_id); 1060 address start = __ pc(); 1061 1062 __ emit_data64(mask, relocInfo::none); 1063 __ emit_data64(mask, relocInfo::none); 1064 __ emit_data64(mask, relocInfo::none); 1065 __ emit_data64(mask, relocInfo::none); 1066 __ emit_data64(mask, relocInfo::none); 1067 __ emit_data64(mask, relocInfo::none); 1068 __ emit_data64(mask, relocInfo::none); 1069 __ emit_data64(mask, relocInfo::none); 1070 1071 return start; 1072 } 1073 1074 address StubGenerator::generate_vector_byte_perm_mask() { 1075 __ align(CodeEntryAlignment); 1076 StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id; 1077 StubCodeMark mark(this, stub_id); 1078 address start = __ pc(); 1079 1080 __ emit_data64(0x0000000000000001, relocInfo::none); 1081 __ emit_data64(0x0000000000000003, relocInfo::none); 1082 __ emit_data64(0x0000000000000005, relocInfo::none); 1083 __ emit_data64(0x0000000000000007, relocInfo::none); 1084 __ emit_data64(0x0000000000000000, relocInfo::none); 1085 __ emit_data64(0x0000000000000002, relocInfo::none); 1086 __ emit_data64(0x0000000000000004, relocInfo::none); 1087 __ emit_data64(0x0000000000000006, relocInfo::none); 1088 1089 return start; 1090 } 1091 1092 address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { 1093 __ align(CodeEntryAlignment); 1094 StubCodeMark mark(this, stub_id); 1095 address start = __ pc(); 1096 1097 __ emit_data64(mask, relocInfo::none); 1098 __ emit_data64(mask, relocInfo::none); 1099 __ emit_data64(mask, relocInfo::none); 1100 __ emit_data64(mask, relocInfo::none); 1101 __ emit_data64(mask, relocInfo::none); 1102 __ emit_data64(mask, relocInfo::none); 1103 __ emit_data64(mask, relocInfo::none); 1104 __ emit_data64(mask, relocInfo::none); 1105 1106 return start; 1107 } 1108 1109 address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::AvxVectorLen len, 1110 int32_t val0, int32_t val1, int32_t val2, int32_t val3, 1111 int32_t val4, int32_t val5, int32_t val6, int32_t val7, 1112 int32_t val8, int32_t val9, int32_t val10, int32_t val11, 1113 int32_t val12, int32_t val13, int32_t val14, int32_t val15) { 1114 __ align(CodeEntryAlignment); 1115 StubCodeMark mark(this, stub_id); 1116 address start = __ pc(); 1117 1118 assert(len != Assembler::AVX_NoVec, "vector len must be specified"); 1119 __ emit_data(val0, relocInfo::none, 0); 1120 __ emit_data(val1, relocInfo::none, 0); 1121 __ emit_data(val2, relocInfo::none, 0); 1122 __ emit_data(val3, relocInfo::none, 0); 1123 if (len >= Assembler::AVX_256bit) { 1124 __ emit_data(val4, relocInfo::none, 0); 1125 __ emit_data(val5, relocInfo::none, 0); 1126 __ emit_data(val6, relocInfo::none, 0); 1127 __ emit_data(val7, relocInfo::none, 0); 1128 if (len >= Assembler::AVX_512bit) { 1129 __ emit_data(val8, relocInfo::none, 0); 1130 __ emit_data(val9, relocInfo::none, 0); 1131 __ emit_data(val10, relocInfo::none, 0); 1132 __ emit_data(val11, relocInfo::none, 0); 1133 __ emit_data(val12, relocInfo::none, 0); 1134 __ emit_data(val13, relocInfo::none, 0); 1135 __ emit_data(val14, relocInfo::none, 0); 1136 __ emit_data(val15, relocInfo::none, 0); 1137 } 1138 } 1139 return start; 1140 } 1141 1142 // Non-destructive plausibility checks for oops 1143 // 1144 // Arguments: 1145 // all args on stack! 1146 // 1147 // Stack after saving c_rarg3: 1148 // [tos + 0]: saved c_rarg3 1149 // [tos + 1]: saved c_rarg2 1150 // [tos + 2]: saved r12 (several TemplateTable methods use it) 1151 // [tos + 3]: saved flags 1152 // [tos + 4]: return address 1153 // * [tos + 5]: error message (char*) 1154 // * [tos + 6]: object to verify (oop) 1155 // * [tos + 7]: saved rax - saved by caller and bashed 1156 // * [tos + 8]: saved r10 (rscratch1) - saved by caller 1157 // * = popped on exit 1158 address StubGenerator::generate_verify_oop() { 1159 StubId stub_id = StubId::stubgen_verify_oop_id; 1160 StubCodeMark mark(this, stub_id); 1161 address start = __ pc(); 1162 1163 Label exit, error; 1164 1165 __ pushf(); 1166 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()), rscratch1); 1167 1168 __ push_ppx(r12); 1169 1170 // save c_rarg2 and c_rarg3 1171 __ push_ppx(c_rarg2); 1172 __ push_ppx(c_rarg3); 1173 1174 enum { 1175 // After previous pushes. 1176 oop_to_verify = 6 * wordSize, 1177 saved_rax = 7 * wordSize, 1178 saved_r10 = 8 * wordSize, 1179 1180 // Before the call to MacroAssembler::debug(), see below. 1181 return_addr = 16 * wordSize, 1182 error_msg = 17 * wordSize 1183 }; 1184 1185 // get object 1186 __ movptr(rax, Address(rsp, oop_to_verify)); 1187 1188 // make sure object is 'reasonable' 1189 __ testptr(rax, rax); 1190 __ jcc(Assembler::zero, exit); // if obj is null it is OK 1191 1192 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); 1193 bs_asm->check_oop(_masm, rax, c_rarg2, c_rarg3, error); 1194 1195 // return if everything seems ok 1196 __ bind(exit); 1197 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1198 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1199 __ pop_ppx(c_rarg3); // restore c_rarg3 1200 __ pop_ppx(c_rarg2); // restore c_rarg2 1201 __ pop_ppx(r12); // restore r12 1202 __ popf(); // restore flags 1203 __ ret(4 * wordSize); // pop caller saved stuff 1204 1205 // handle errors 1206 __ bind(error); 1207 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back 1208 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back 1209 __ pop_ppx(c_rarg3); // get saved c_rarg3 back 1210 __ pop_ppx(c_rarg2); // get saved c_rarg2 back 1211 __ pop_ppx(r12); // get saved r12 back 1212 __ popf(); // get saved flags off stack -- 1213 // will be ignored 1214 1215 __ pusha(); // push registers 1216 // (rip is already 1217 // already pushed) 1218 // debug(char* msg, int64_t pc, int64_t regs[]) 1219 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and 1220 // pushed all the registers, so now the stack looks like: 1221 // [tos + 0] 16 saved registers 1222 // [tos + 16] return address 1223 // * [tos + 17] error message (char*) 1224 // * [tos + 18] object to verify (oop) 1225 // * [tos + 19] saved rax - saved by caller and bashed 1226 // * [tos + 20] saved r10 (rscratch1) - saved by caller 1227 // * = popped on exit 1228 1229 __ movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message 1230 __ movptr(c_rarg1, Address(rsp, return_addr)); // pass return address 1231 __ movq(c_rarg2, rsp); // pass address of regs on stack 1232 __ mov(r12, rsp); // remember rsp 1233 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 1234 __ andptr(rsp, -16); // align stack as required by ABI 1235 BLOCK_COMMENT("call MacroAssembler::debug"); 1236 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 1237 __ hlt(); 1238 1239 return start; 1240 } 1241 1242 1243 // Shuffle first three arg regs on Windows into Linux/Solaris locations. 1244 // 1245 // Outputs: 1246 // rdi - rcx 1247 // rsi - rdx 1248 // rdx - r8 1249 // rcx - r9 1250 // 1251 // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter 1252 // are non-volatile. r9 and r10 should not be used by the caller. 1253 // 1254 void StubGenerator::setup_arg_regs(int nargs) { 1255 const Register saved_rdi = r9; 1256 const Register saved_rsi = r10; 1257 assert(nargs == 3 || nargs == 4, "else fix"); 1258 #ifdef _WIN64 1259 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1260 "unexpected argument registers"); 1261 if (nargs == 4) { 1262 __ mov(rax, r9); // r9 is also saved_rdi 1263 } 1264 __ movptr(saved_rdi, rdi); 1265 __ movptr(saved_rsi, rsi); 1266 __ mov(rdi, rcx); // c_rarg0 1267 __ mov(rsi, rdx); // c_rarg1 1268 __ mov(rdx, r8); // c_rarg2 1269 if (nargs == 4) { 1270 __ mov(rcx, rax); // c_rarg3 (via rax) 1271 } 1272 #else 1273 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1274 "unexpected argument registers"); 1275 #endif 1276 DEBUG_ONLY(_regs_in_thread = false;) 1277 } 1278 1279 1280 void StubGenerator::restore_arg_regs() { 1281 assert(!_regs_in_thread, "wrong call to restore_arg_regs"); 1282 const Register saved_rdi = r9; 1283 const Register saved_rsi = r10; 1284 #ifdef _WIN64 1285 __ movptr(rdi, saved_rdi); 1286 __ movptr(rsi, saved_rsi); 1287 #endif 1288 } 1289 1290 1291 // This is used in places where r10 is a scratch register, and can 1292 // be adapted if r9 is needed also. 1293 void StubGenerator::setup_arg_regs_using_thread(int nargs) { 1294 const Register saved_r15 = r9; 1295 assert(nargs == 3 || nargs == 4, "else fix"); 1296 #ifdef _WIN64 1297 if (nargs == 4) { 1298 __ mov(rax, r9); // r9 is also saved_r15 1299 } 1300 __ mov(saved_r15, r15); // r15 is callee saved and needs to be restored 1301 __ get_thread_slow(r15_thread); 1302 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1303 "unexpected argument registers"); 1304 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi); 1305 __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi); 1306 1307 __ mov(rdi, rcx); // c_rarg0 1308 __ mov(rsi, rdx); // c_rarg1 1309 __ mov(rdx, r8); // c_rarg2 1310 if (nargs == 4) { 1311 __ mov(rcx, rax); // c_rarg3 (via rax) 1312 } 1313 #else 1314 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1315 "unexpected argument registers"); 1316 #endif 1317 DEBUG_ONLY(_regs_in_thread = true;) 1318 } 1319 1320 1321 void StubGenerator::restore_arg_regs_using_thread() { 1322 assert(_regs_in_thread, "wrong call to restore_arg_regs"); 1323 const Register saved_r15 = r9; 1324 #ifdef _WIN64 1325 __ get_thread_slow(r15_thread); 1326 __ movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset()))); 1327 __ movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset()))); 1328 __ mov(r15, saved_r15); // r15 is callee saved and needs to be restored 1329 #endif 1330 } 1331 1332 1333 void StubGenerator::setup_argument_regs(BasicType type) { 1334 if (type == T_BYTE || type == T_SHORT) { 1335 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1336 // r9 and r10 may be used to save non-volatile registers 1337 } else { 1338 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx 1339 // r9 is used to save r15_thread 1340 } 1341 } 1342 1343 1344 void StubGenerator::restore_argument_regs(BasicType type) { 1345 if (type == T_BYTE || type == T_SHORT) { 1346 restore_arg_regs(); 1347 } else { 1348 restore_arg_regs_using_thread(); 1349 } 1350 } 1351 1352 address StubGenerator::generate_data_cache_writeback() { 1353 const Register src = c_rarg0; // source address 1354 1355 __ align(CodeEntryAlignment); 1356 1357 StubId stub_id = StubId::stubgen_data_cache_writeback_id; 1358 StubCodeMark mark(this, stub_id); 1359 1360 address start = __ pc(); 1361 1362 __ enter(); 1363 __ cache_wb(Address(src, 0)); 1364 __ leave(); 1365 __ ret(0); 1366 1367 return start; 1368 } 1369 1370 address StubGenerator::generate_data_cache_writeback_sync() { 1371 const Register is_pre = c_rarg0; // pre or post sync 1372 1373 __ align(CodeEntryAlignment); 1374 1375 StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; 1376 StubCodeMark mark(this, stub_id); 1377 1378 // pre wbsync is a no-op 1379 // post wbsync translates to an sfence 1380 1381 Label skip; 1382 address start = __ pc(); 1383 1384 __ enter(); 1385 __ cmpl(is_pre, 0); 1386 __ jcc(Assembler::notEqual, skip); 1387 __ cache_wbsync(false); 1388 __ bind(skip); 1389 __ leave(); 1390 __ ret(0); 1391 1392 return start; 1393 } 1394 1395 // ofs and limit are use for multi-block byte array. 1396 // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs) 1397 address StubGenerator::generate_md5_implCompress(StubId stub_id) { 1398 bool multi_block; 1399 switch (stub_id) { 1400 case StubId::stubgen_md5_implCompress_id: 1401 multi_block = false; 1402 break; 1403 case StubId::stubgen_md5_implCompressMB_id: 1404 multi_block = true; 1405 break; 1406 default: 1407 ShouldNotReachHere(); 1408 } 1409 __ align(CodeEntryAlignment); 1410 StubCodeMark mark(this, stub_id); 1411 address start = __ pc(); 1412 1413 const Register buf_param = r15; 1414 const Address state_param(rsp, 0 * wordSize); 1415 const Address ofs_param (rsp, 1 * wordSize ); 1416 const Address limit_param(rsp, 1 * wordSize + 4); 1417 1418 __ enter(); 1419 __ push_ppx(rbx); 1420 __ push_ppx(rdi); 1421 __ push_ppx(rsi); 1422 __ push_ppx(r15); 1423 __ subptr(rsp, 2 * wordSize); 1424 1425 __ movptr(buf_param, c_rarg0); 1426 __ movptr(state_param, c_rarg1); 1427 if (multi_block) { 1428 __ movl(ofs_param, c_rarg2); 1429 __ movl(limit_param, c_rarg3); 1430 } 1431 __ fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block); 1432 1433 __ addptr(rsp, 2 * wordSize); 1434 __ pop_ppx(r15); 1435 __ pop_ppx(rsi); 1436 __ pop_ppx(rdi); 1437 __ pop_ppx(rbx); 1438 __ leave(); 1439 __ ret(0); 1440 1441 return start; 1442 } 1443 1444 address StubGenerator::generate_upper_word_mask() { 1445 __ align64(); 1446 StubId stub_id = StubId::stubgen_upper_word_mask_id; 1447 StubCodeMark mark(this, stub_id); 1448 address start = __ pc(); 1449 1450 __ emit_data64(0x0000000000000000, relocInfo::none); 1451 __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); 1452 1453 return start; 1454 } 1455 1456 address StubGenerator::generate_shuffle_byte_flip_mask() { 1457 __ align64(); 1458 StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id; 1459 StubCodeMark mark(this, stub_id); 1460 address start = __ pc(); 1461 1462 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1463 __ emit_data64(0x0001020304050607, relocInfo::none); 1464 1465 return start; 1466 } 1467 1468 // ofs and limit are use for multi-block byte array. 1469 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1470 address StubGenerator::generate_sha1_implCompress(StubId stub_id) { 1471 bool multi_block; 1472 switch (stub_id) { 1473 case StubId::stubgen_sha1_implCompress_id: 1474 multi_block = false; 1475 break; 1476 case StubId::stubgen_sha1_implCompressMB_id: 1477 multi_block = true; 1478 break; 1479 default: 1480 ShouldNotReachHere(); 1481 } 1482 __ align(CodeEntryAlignment); 1483 StubCodeMark mark(this, stub_id); 1484 address start = __ pc(); 1485 1486 Register buf = c_rarg0; 1487 Register state = c_rarg1; 1488 Register ofs = c_rarg2; 1489 Register limit = c_rarg3; 1490 1491 const XMMRegister abcd = xmm0; 1492 const XMMRegister e0 = xmm1; 1493 const XMMRegister e1 = xmm2; 1494 const XMMRegister msg0 = xmm3; 1495 1496 const XMMRegister msg1 = xmm4; 1497 const XMMRegister msg2 = xmm5; 1498 const XMMRegister msg3 = xmm6; 1499 const XMMRegister shuf_mask = xmm7; 1500 1501 __ enter(); 1502 1503 __ subptr(rsp, 4 * wordSize); 1504 1505 __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask, 1506 buf, state, ofs, limit, rsp, multi_block); 1507 1508 __ addptr(rsp, 4 * wordSize); 1509 1510 __ leave(); 1511 __ ret(0); 1512 1513 return start; 1514 } 1515 1516 address StubGenerator::generate_pshuffle_byte_flip_mask() { 1517 __ align64(); 1518 StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_id; 1519 StubCodeMark mark(this, stub_id); 1520 address start = __ pc(); 1521 1522 __ emit_data64(0x0405060700010203, relocInfo::none); 1523 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1524 1525 if (VM_Version::supports_avx2()) { 1526 __ emit_data64(0x0405060700010203, relocInfo::none); // second copy 1527 __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); 1528 // _SHUF_00BA 1529 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1530 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1531 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1532 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1533 // _SHUF_DC00 1534 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1535 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1536 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1537 __ emit_data64(0x0b0a090803020100, relocInfo::none); 1538 } 1539 1540 return start; 1541 } 1542 1543 //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 1544 address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() { 1545 __ align32(); 1546 StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_sha512_id; 1547 StubCodeMark mark(this, stub_id); 1548 address start = __ pc(); 1549 1550 if (VM_Version::supports_avx2()) { 1551 __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK 1552 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); 1553 __ emit_data64(0x1011121314151617, relocInfo::none); 1554 __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); 1555 __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO 1556 __ emit_data64(0x0000000000000000, relocInfo::none); 1557 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1558 __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); 1559 } 1560 1561 return start; 1562 } 1563 1564 // ofs and limit are use for multi-block byte array. 1565 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 1566 address StubGenerator::generate_sha256_implCompress(StubId stub_id) { 1567 bool multi_block; 1568 switch (stub_id) { 1569 case StubId::stubgen_sha256_implCompress_id: 1570 multi_block = false; 1571 break; 1572 case StubId::stubgen_sha256_implCompressMB_id: 1573 multi_block = true; 1574 break; 1575 default: 1576 ShouldNotReachHere(); 1577 } 1578 assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); 1579 __ align(CodeEntryAlignment); 1580 StubCodeMark mark(this, stub_id); 1581 address start = __ pc(); 1582 1583 Register buf = c_rarg0; 1584 Register state = c_rarg1; 1585 Register ofs = c_rarg2; 1586 Register limit = c_rarg3; 1587 1588 const XMMRegister msg = xmm0; 1589 const XMMRegister state0 = xmm1; 1590 const XMMRegister state1 = xmm2; 1591 const XMMRegister msgtmp0 = xmm3; 1592 1593 const XMMRegister msgtmp1 = xmm4; 1594 const XMMRegister msgtmp2 = xmm5; 1595 const XMMRegister msgtmp3 = xmm6; 1596 const XMMRegister msgtmp4 = xmm7; 1597 1598 const XMMRegister shuf_mask = xmm8; 1599 1600 __ enter(); 1601 1602 __ subptr(rsp, 4 * wordSize); 1603 1604 if (VM_Version::supports_sha()) { 1605 __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1606 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1607 } else if (VM_Version::supports_avx2()) { 1608 __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1609 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1610 } 1611 __ addptr(rsp, 4 * wordSize); 1612 __ vzeroupper(); 1613 __ leave(); 1614 __ ret(0); 1615 1616 return start; 1617 } 1618 1619 address StubGenerator::generate_sha512_implCompress(StubId stub_id) { 1620 bool multi_block; 1621 switch (stub_id) { 1622 case StubId::stubgen_sha512_implCompress_id: 1623 multi_block = false; 1624 break; 1625 case StubId::stubgen_sha512_implCompressMB_id: 1626 multi_block = true; 1627 break; 1628 default: 1629 ShouldNotReachHere(); 1630 } 1631 assert(VM_Version::supports_avx2(), ""); 1632 assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), ""); 1633 __ align(CodeEntryAlignment); 1634 StubCodeMark mark(this, stub_id); 1635 address start = __ pc(); 1636 1637 Register buf = c_rarg0; 1638 Register state = c_rarg1; 1639 Register ofs = c_rarg2; 1640 Register limit = c_rarg3; 1641 1642 __ enter(); 1643 1644 if (VM_Version::supports_sha512()) { 1645 __ sha512_update_ni_x1(state, buf, ofs, limit, multi_block); 1646 } else { 1647 const XMMRegister msg = xmm0; 1648 const XMMRegister state0 = xmm1; 1649 const XMMRegister state1 = xmm2; 1650 const XMMRegister msgtmp0 = xmm3; 1651 const XMMRegister msgtmp1 = xmm4; 1652 const XMMRegister msgtmp2 = xmm5; 1653 const XMMRegister msgtmp3 = xmm6; 1654 const XMMRegister msgtmp4 = xmm7; 1655 1656 const XMMRegister shuf_mask = xmm8; 1657 __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, 1658 buf, state, ofs, limit, rsp, multi_block, shuf_mask); 1659 } 1660 __ vzeroupper(); 1661 __ leave(); 1662 __ ret(0); 1663 1664 return start; 1665 } 1666 1667 address StubGenerator::base64_shuffle_addr() { 1668 __ align64(); 1669 StubId stub_id = StubId::stubgen_shuffle_base64_id; 1670 StubCodeMark mark(this, stub_id); 1671 address start = __ pc(); 1672 1673 assert(((unsigned long long)start & 0x3f) == 0, 1674 "Alignment problem (0x%08llx)", (unsigned long long)start); 1675 __ emit_data64(0x0405030401020001, relocInfo::none); 1676 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1677 __ emit_data64(0x10110f100d0e0c0d, relocInfo::none); 1678 __ emit_data64(0x1617151613141213, relocInfo::none); 1679 __ emit_data64(0x1c1d1b1c191a1819, relocInfo::none); 1680 __ emit_data64(0x222321221f201e1f, relocInfo::none); 1681 __ emit_data64(0x2829272825262425, relocInfo::none); 1682 __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); 1683 1684 return start; 1685 } 1686 1687 address StubGenerator::base64_avx2_shuffle_addr() { 1688 __ align32(); 1689 StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id; 1690 StubCodeMark mark(this, stub_id); 1691 address start = __ pc(); 1692 1693 __ emit_data64(0x0809070805060405, relocInfo::none); 1694 __ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none); 1695 __ emit_data64(0x0405030401020001, relocInfo::none); 1696 __ emit_data64(0x0a0b090a07080607, relocInfo::none); 1697 1698 return start; 1699 } 1700 1701 address StubGenerator::base64_avx2_input_mask_addr() { 1702 __ align32(); 1703 StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id; 1704 StubCodeMark mark(this, stub_id); 1705 address start = __ pc(); 1706 1707 __ emit_data64(0x8000000000000000, relocInfo::none); 1708 __ emit_data64(0x8000000080000000, relocInfo::none); 1709 __ emit_data64(0x8000000080000000, relocInfo::none); 1710 __ emit_data64(0x8000000080000000, relocInfo::none); 1711 1712 return start; 1713 } 1714 1715 address StubGenerator::base64_avx2_lut_addr() { 1716 __ align32(); 1717 StubId stub_id = StubId::stubgen_avx2_lut_base64_id; 1718 StubCodeMark mark(this, stub_id); 1719 address start = __ pc(); 1720 1721 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1722 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1723 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1724 __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); 1725 1726 // URL LUT 1727 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1728 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1729 __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); 1730 __ emit_data64(0x000020effcfcfcfc, relocInfo::none); 1731 1732 return start; 1733 } 1734 1735 address StubGenerator::base64_encoding_table_addr() { 1736 __ align64(); 1737 StubId stub_id = StubId::stubgen_encoding_table_base64_id; 1738 StubCodeMark mark(this, stub_id); 1739 address start = __ pc(); 1740 1741 assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); 1742 __ emit_data64(0x4847464544434241, relocInfo::none); 1743 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1744 __ emit_data64(0x5857565554535251, relocInfo::none); 1745 __ emit_data64(0x6665646362615a59, relocInfo::none); 1746 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1747 __ emit_data64(0x767574737271706f, relocInfo::none); 1748 __ emit_data64(0x333231307a797877, relocInfo::none); 1749 __ emit_data64(0x2f2b393837363534, relocInfo::none); 1750 1751 // URL table 1752 __ emit_data64(0x4847464544434241, relocInfo::none); 1753 __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); 1754 __ emit_data64(0x5857565554535251, relocInfo::none); 1755 __ emit_data64(0x6665646362615a59, relocInfo::none); 1756 __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none); 1757 __ emit_data64(0x767574737271706f, relocInfo::none); 1758 __ emit_data64(0x333231307a797877, relocInfo::none); 1759 __ emit_data64(0x5f2d393837363534, relocInfo::none); 1760 1761 return start; 1762 } 1763 1764 // Code for generating Base64 encoding. 1765 // Intrinsic function prototype in Base64.java: 1766 // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, 1767 // boolean isURL) { 1768 address StubGenerator::generate_base64_encodeBlock() 1769 { 1770 __ align(CodeEntryAlignment); 1771 StubId stub_id = StubId::stubgen_base64_encodeBlock_id; 1772 StubCodeMark mark(this, stub_id); 1773 address start = __ pc(); 1774 1775 __ enter(); 1776 1777 // Save callee-saved registers before using them 1778 __ push_ppx(r12); 1779 __ push_ppx(r13); 1780 __ push_ppx(r14); 1781 __ push_ppx(r15); 1782 1783 // arguments 1784 const Register source = c_rarg0; // Source Array 1785 const Register start_offset = c_rarg1; // start offset 1786 const Register end_offset = c_rarg2; // end offset 1787 const Register dest = c_rarg3; // destination array 1788 1789 #ifndef _WIN64 1790 const Register dp = c_rarg4; // Position for writing to dest array 1791 const Register isURL = c_rarg5; // Base64 or URL character set 1792 #else 1793 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 1794 const Address isURL_mem(rbp, 7 * wordSize); 1795 const Register isURL = r10; // pick the volatile windows register 1796 const Register dp = r12; 1797 __ movl(dp, dp_mem); 1798 __ movl(isURL, isURL_mem); 1799 #endif 1800 1801 const Register length = r14; 1802 const Register encode_table = r13; 1803 Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop; 1804 1805 // calculate length from offsets 1806 __ movl(length, end_offset); 1807 __ subl(length, start_offset); 1808 __ jcc(Assembler::lessEqual, L_exit); 1809 1810 // Code for 512-bit VBMI encoding. Encodes 48 input bytes into 64 1811 // output bytes. We read 64 input bytes and ignore the last 16, so be 1812 // sure not to read past the end of the input buffer. 1813 if (VM_Version::supports_avx512_vbmi()) { 1814 __ cmpl(length, 64); // Do not overrun input buffer. 1815 __ jcc(Assembler::below, L_not512); 1816 1817 __ shll(isURL, 6); // index into decode table based on isURL 1818 __ lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 1819 __ addptr(encode_table, isURL); 1820 __ shrl(isURL, 6); // restore isURL 1821 1822 __ mov64(rax, 0x3036242a1016040aull); // Shifts 1823 __ evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15); 1824 __ evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit); 1825 __ evpbroadcastq(xmm1, rax, Assembler::AVX_512bit); 1826 1827 __ align32(); 1828 __ BIND(L_vbmiLoop); 1829 1830 __ vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit); 1831 __ subl(length, 48); 1832 1833 // Put the input bytes into the proper lanes for writing, then 1834 // encode them. 1835 __ evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit); 1836 __ vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit); 1837 1838 // Write to destination 1839 __ evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit); 1840 1841 __ addptr(dest, 64); 1842 __ addptr(source, 48); 1843 __ cmpl(length, 64); 1844 __ jcc(Assembler::aboveEqual, L_vbmiLoop); 1845 1846 __ vzeroupper(); 1847 } 1848 1849 __ BIND(L_not512); 1850 if (VM_Version::supports_avx2()) { 1851 /* 1852 ** This AVX2 encoder is based off the paper at: 1853 ** https://dl.acm.org/doi/10.1145/3132709 1854 ** 1855 ** We use AVX2 SIMD instructions to encode 24 bytes into 32 1856 ** output bytes. 1857 ** 1858 */ 1859 // Lengths under 32 bytes are done with scalar routine 1860 __ cmpl(length, 31); 1861 __ jcc(Assembler::belowEqual, L_process3); 1862 1863 // Set up supporting constant table data 1864 __ vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax); 1865 // 6-bit mask for 2nd and 4th (and multiples) 6-bit values 1866 __ movl(rax, 0x0fc0fc00); 1867 __ movdl(xmm8, rax); 1868 __ vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax); 1869 __ vpbroadcastd(xmm8, xmm8, Assembler::AVX_256bit); 1870 1871 // Multiplication constant for "shifting" right by 6 and 10 1872 // bits 1873 __ movl(rax, 0x04000040); 1874 1875 __ subl(length, 24); 1876 __ movdl(xmm7, rax); 1877 __ vpbroadcastd(xmm7, xmm7, Assembler::AVX_256bit); 1878 1879 // For the first load, we mask off reading of the first 4 1880 // bytes into the register. This is so we can get 4 3-byte 1881 // chunks into each lane of the register, avoiding having to 1882 // handle end conditions. We then shuffle these bytes into a 1883 // specific order so that manipulation is easier. 1884 // 1885 // The initial read loads the XMM register like this: 1886 // 1887 // Lower 128-bit lane: 1888 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1889 // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1 1890 // | C2 | D0 | D1 | D2 | 1891 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1892 // 1893 // Upper 128-bit lane: 1894 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1895 // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2 1896 // | XX | XX | XX | XX | 1897 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ 1898 // 1899 // Where A0 is the first input byte, B0 is the fourth, etc. 1900 // The alphabetical significance denotes the 3 bytes to be 1901 // consumed and encoded into 4 bytes. 1902 // 1903 // We then shuffle the register so each 32-bit word contains 1904 // the sequence: 1905 // A1 A0 A2 A1, B1, B0, B2, B1, etc. 1906 // Each of these byte sequences are then manipulated into 4 1907 // 6-bit values ready for encoding. 1908 // 1909 // If we focus on one set of 3-byte chunks, changing the 1910 // nomenclature such that A0 => a, A1 => b, and A2 => c, we 1911 // shuffle such that each 24-bit chunk contains: 1912 // 1913 // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6 1914 // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0 1915 // Explain this step. 1916 // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4 1917 // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2 1918 // 1919 // W first and off all but bits 4-9 and 16-21 (c5..c0 and 1920 // a5..a0) and shift them using a vector multiplication 1921 // operation (vpmulhuw) which effectively shifts c right by 6 1922 // bits and a right by 10 bits. We similarly mask bits 10-15 1923 // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4 1924 // bits respectively. This is done using vpmullw. We end up 1925 // with 4 6-bit values, thus splitting the 3 input bytes, 1926 // ready for encoding: 1927 // 0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0 1928 // 1929 // For translation, we recognize that there are 5 distinct 1930 // ranges of legal Base64 characters as below: 1931 // 1932 // +-------------+-------------+------------+ 1933 // | 6-bit value | ASCII range | offset | 1934 // +-------------+-------------+------------+ 1935 // | 0..25 | A..Z | 65 | 1936 // | 26..51 | a..z | 71 | 1937 // | 52..61 | 0..9 | -4 | 1938 // | 62 | + or - | -19 or -17 | 1939 // | 63 | / or _ | -16 or 32 | 1940 // +-------------+-------------+------------+ 1941 // 1942 // We note that vpshufb does a parallel lookup in a 1943 // destination register using the lower 4 bits of bytes from a 1944 // source register. If we use a saturated subtraction and 1945 // subtract 51 from each 6-bit value, bytes from [0,51] 1946 // saturate to 0, and [52,63] map to a range of [1,12]. We 1947 // distinguish the [0,25] and [26,51] ranges by assigning a 1948 // value of 13 for all 6-bit values less than 26. We end up 1949 // with: 1950 // 1951 // +-------------+-------------+------------+ 1952 // | 6-bit value | Reduced | offset | 1953 // +-------------+-------------+------------+ 1954 // | 0..25 | 13 | 65 | 1955 // | 26..51 | 0 | 71 | 1956 // | 52..61 | 0..9 | -4 | 1957 // | 62 | 11 | -19 or -17 | 1958 // | 63 | 12 | -16 or 32 | 1959 // +-------------+-------------+------------+ 1960 // 1961 // We then use a final vpshufb to add the appropriate offset, 1962 // translating the bytes. 1963 // 1964 // Load input bytes - only 28 bytes. Mask the first load to 1965 // not load into the full register. 1966 __ vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit); 1967 1968 // Move 3-byte chunks of input (12 bytes) into 16 bytes, 1969 // ordering by: 1970 // 1, 0, 2, 1; 4, 3, 5, 4; etc. This groups 6-bit chunks 1971 // for easy masking 1972 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 1973 1974 __ addl(start_offset, 24); 1975 1976 // Load masking register for first and third (and multiples) 1977 // 6-bit values. 1978 __ movl(rax, 0x003f03f0); 1979 __ movdl(xmm6, rax); 1980 __ vpbroadcastd(xmm6, xmm6, Assembler::AVX_256bit); 1981 // Multiplication constant for "shifting" left by 4 and 8 bits 1982 __ movl(rax, 0x01000010); 1983 __ movdl(xmm5, rax); 1984 __ vpbroadcastd(xmm5, xmm5, Assembler::AVX_256bit); 1985 1986 // Isolate 6-bit chunks of interest 1987 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 1988 1989 // Load constants for encoding 1990 __ movl(rax, 0x19191919); 1991 __ movdl(xmm3, rax); 1992 __ vpbroadcastd(xmm3, xmm3, Assembler::AVX_256bit); 1993 __ movl(rax, 0x33333333); 1994 __ movdl(xmm4, rax); 1995 __ vpbroadcastd(xmm4, xmm4, Assembler::AVX_256bit); 1996 1997 // Shift output bytes 0 and 2 into proper lanes 1998 __ vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit); 1999 2000 // Mask and shift output bytes 1 and 3 into proper lanes and 2001 // combine 2002 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 2003 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 2004 __ vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 2005 2006 // Find out which are 0..25. This indicates which input 2007 // values fall in the range of 'A'-'Z', which require an 2008 // additional offset (see comments above) 2009 __ vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit); 2010 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 2011 __ vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit); 2012 2013 // Load the proper lookup table 2014 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr())); 2015 __ movl(r15, isURL); 2016 __ shll(r15, 5); 2017 __ vmovdqu(xmm2, Address(r11, r15)); 2018 2019 // Shuffle the offsets based on the range calculation done 2020 // above. This allows us to add the correct offset to the 2021 // 6-bit value corresponding to the range documented above. 2022 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 2023 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 2024 2025 // Store the encoded bytes 2026 __ vmovdqu(Address(dest, dp), xmm0); 2027 __ addl(dp, 32); 2028 2029 __ cmpl(length, 31); 2030 __ jcc(Assembler::belowEqual, L_process3); 2031 2032 __ align32(); 2033 __ BIND(L_32byteLoop); 2034 2035 // Get next 32 bytes 2036 __ vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4)); 2037 2038 __ subl(length, 24); 2039 __ addl(start_offset, 24); 2040 2041 // This logic is identical to the above, with only constant 2042 // register loads removed. Shuffle the input, mask off 6-bit 2043 // chunks, shift them into place, then add the offset to 2044 // encode. 2045 __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); 2046 2047 __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); 2048 __ vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit); 2049 __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); 2050 __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); 2051 __ vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 2052 __ vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit); 2053 __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); 2054 __ vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit); 2055 __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); 2056 __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); 2057 2058 // Store the encoded bytes 2059 __ vmovdqu(Address(dest, dp), xmm0); 2060 __ addl(dp, 32); 2061 2062 __ cmpl(length, 31); 2063 __ jcc(Assembler::above, L_32byteLoop); 2064 2065 __ BIND(L_process3); 2066 __ vzeroupper(); 2067 } else { 2068 __ BIND(L_process3); 2069 } 2070 2071 __ cmpl(length, 3); 2072 __ jcc(Assembler::below, L_exit); 2073 2074 // Load the encoding table based on isURL 2075 __ lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); 2076 __ movl(r15, isURL); 2077 __ shll(r15, 6); 2078 __ addptr(r11, r15); 2079 2080 __ BIND(L_processdata); 2081 2082 // Load 3 bytes 2083 __ load_unsigned_byte(r15, Address(source, start_offset)); 2084 __ load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1)); 2085 __ load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2)); 2086 2087 // Build a 32-bit word with bytes 1, 2, 0, 1 2088 __ movl(rax, r10); 2089 __ shll(r10, 24); 2090 __ orl(rax, r10); 2091 2092 __ subl(length, 3); 2093 2094 __ shll(r15, 8); 2095 __ shll(r13, 16); 2096 __ orl(rax, r15); 2097 2098 __ addl(start_offset, 3); 2099 2100 __ orl(rax, r13); 2101 // At this point, rax contains | byte1 | byte2 | byte0 | byte1 2102 // r13 has byte2 << 16 - need low-order 6 bits to translate. 2103 // This translated byte is the fourth output byte. 2104 __ shrl(r13, 16); 2105 __ andl(r13, 0x3f); 2106 2107 // The high-order 6 bits of r15 (byte0) is translated. 2108 // The translated byte is the first output byte. 2109 __ shrl(r15, 10); 2110 2111 __ load_unsigned_byte(r13, Address(r11, r13)); 2112 __ load_unsigned_byte(r15, Address(r11, r15)); 2113 2114 __ movb(Address(dest, dp, Address::times_1, 3), r13); 2115 2116 // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0. 2117 // This translated byte is the second output byte. 2118 __ shrl(rax, 4); 2119 __ movl(r10, rax); 2120 __ andl(rax, 0x3f); 2121 2122 __ movb(Address(dest, dp, Address::times_1, 0), r15); 2123 2124 __ load_unsigned_byte(rax, Address(r11, rax)); 2125 2126 // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2. 2127 // This translated byte is the third output byte. 2128 __ shrl(r10, 18); 2129 __ andl(r10, 0x3f); 2130 2131 __ load_unsigned_byte(r10, Address(r11, r10)); 2132 2133 __ movb(Address(dest, dp, Address::times_1, 1), rax); 2134 __ movb(Address(dest, dp, Address::times_1, 2), r10); 2135 2136 __ addl(dp, 4); 2137 __ cmpl(length, 3); 2138 __ jcc(Assembler::aboveEqual, L_processdata); 2139 2140 __ BIND(L_exit); 2141 __ pop_ppx(r15); 2142 __ pop_ppx(r14); 2143 __ pop_ppx(r13); 2144 __ pop_ppx(r12); 2145 __ leave(); 2146 __ ret(0); 2147 2148 return start; 2149 } 2150 2151 // base64 AVX512vbmi tables 2152 address StubGenerator::base64_vbmi_lookup_lo_addr() { 2153 __ align64(); 2154 StubId stub_id = StubId::stubgen_lookup_lo_base64_id; 2155 StubCodeMark mark(this, stub_id); 2156 address start = __ pc(); 2157 2158 assert(((unsigned long long)start & 0x3f) == 0, 2159 "Alignment problem (0x%08llx)", (unsigned long long)start); 2160 __ emit_data64(0x8080808080808080, relocInfo::none); 2161 __ emit_data64(0x8080808080808080, relocInfo::none); 2162 __ emit_data64(0x8080808080808080, relocInfo::none); 2163 __ emit_data64(0x8080808080808080, relocInfo::none); 2164 __ emit_data64(0x8080808080808080, relocInfo::none); 2165 __ emit_data64(0x3f8080803e808080, relocInfo::none); 2166 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2167 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2168 2169 return start; 2170 } 2171 2172 address StubGenerator::base64_vbmi_lookup_hi_addr() { 2173 __ align64(); 2174 StubId stub_id = StubId::stubgen_lookup_hi_base64_id; 2175 StubCodeMark mark(this, stub_id); 2176 address start = __ pc(); 2177 2178 assert(((unsigned long long)start & 0x3f) == 0, 2179 "Alignment problem (0x%08llx)", (unsigned long long)start); 2180 __ emit_data64(0x0605040302010080, relocInfo::none); 2181 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2182 __ emit_data64(0x161514131211100f, relocInfo::none); 2183 __ emit_data64(0x8080808080191817, relocInfo::none); 2184 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2185 __ emit_data64(0x2827262524232221, relocInfo::none); 2186 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2187 __ emit_data64(0x8080808080333231, relocInfo::none); 2188 2189 return start; 2190 } 2191 address StubGenerator::base64_vbmi_lookup_lo_url_addr() { 2192 __ align64(); 2193 StubId stub_id = StubId::stubgen_lookup_lo_base64url_id; 2194 StubCodeMark mark(this, stub_id); 2195 address start = __ pc(); 2196 2197 assert(((unsigned long long)start & 0x3f) == 0, 2198 "Alignment problem (0x%08llx)", (unsigned long long)start); 2199 __ emit_data64(0x8080808080808080, relocInfo::none); 2200 __ emit_data64(0x8080808080808080, relocInfo::none); 2201 __ emit_data64(0x8080808080808080, relocInfo::none); 2202 __ emit_data64(0x8080808080808080, relocInfo::none); 2203 __ emit_data64(0x8080808080808080, relocInfo::none); 2204 __ emit_data64(0x80803e8080808080, relocInfo::none); 2205 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2206 __ emit_data64(0x8080808080803d3c, relocInfo::none); 2207 2208 return start; 2209 } 2210 2211 address StubGenerator::base64_vbmi_lookup_hi_url_addr() { 2212 __ align64(); 2213 StubId stub_id = StubId::stubgen_lookup_hi_base64url_id; 2214 StubCodeMark mark(this, stub_id); 2215 address start = __ pc(); 2216 2217 assert(((unsigned long long)start & 0x3f) == 0, 2218 "Alignment problem (0x%08llx)", (unsigned long long)start); 2219 __ emit_data64(0x0605040302010080, relocInfo::none); 2220 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2221 __ emit_data64(0x161514131211100f, relocInfo::none); 2222 __ emit_data64(0x3f80808080191817, relocInfo::none); 2223 __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); 2224 __ emit_data64(0x2827262524232221, relocInfo::none); 2225 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2226 __ emit_data64(0x8080808080333231, relocInfo::none); 2227 2228 return start; 2229 } 2230 2231 address StubGenerator::base64_vbmi_pack_vec_addr() { 2232 __ align64(); 2233 StubId stub_id = StubId::stubgen_pack_vec_base64_id; 2234 StubCodeMark mark(this, stub_id); 2235 address start = __ pc(); 2236 2237 assert(((unsigned long long)start & 0x3f) == 0, 2238 "Alignment problem (0x%08llx)", (unsigned long long)start); 2239 __ emit_data64(0x090a040506000102, relocInfo::none); 2240 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2241 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2242 __ emit_data64(0x292a242526202122, relocInfo::none); 2243 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2244 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2245 __ emit_data64(0x0000000000000000, relocInfo::none); 2246 __ emit_data64(0x0000000000000000, relocInfo::none); 2247 2248 return start; 2249 } 2250 2251 address StubGenerator::base64_vbmi_join_0_1_addr() { 2252 __ align64(); 2253 StubId stub_id = StubId::stubgen_join_0_1_base64_id; 2254 StubCodeMark mark(this, stub_id); 2255 address start = __ pc(); 2256 2257 assert(((unsigned long long)start & 0x3f) == 0, 2258 "Alignment problem (0x%08llx)", (unsigned long long)start); 2259 __ emit_data64(0x090a040506000102, relocInfo::none); 2260 __ emit_data64(0x161011120c0d0e08, relocInfo::none); 2261 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2262 __ emit_data64(0x292a242526202122, relocInfo::none); 2263 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2264 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2265 __ emit_data64(0x494a444546404142, relocInfo::none); 2266 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2267 2268 return start; 2269 } 2270 2271 address StubGenerator::base64_vbmi_join_1_2_addr() { 2272 __ align64(); 2273 StubId stub_id = StubId::stubgen_join_1_2_base64_id; 2274 StubCodeMark mark(this, stub_id); 2275 address start = __ pc(); 2276 2277 assert(((unsigned long long)start & 0x3f) == 0, 2278 "Alignment problem (0x%08llx)", (unsigned long long)start); 2279 __ emit_data64(0x1c1d1e18191a1415, relocInfo::none); 2280 __ emit_data64(0x292a242526202122, relocInfo::none); 2281 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2282 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2283 __ emit_data64(0x494a444546404142, relocInfo::none); 2284 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2285 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2286 __ emit_data64(0x696a646566606162, relocInfo::none); 2287 2288 return start; 2289 } 2290 2291 address StubGenerator::base64_vbmi_join_2_3_addr() { 2292 __ align64(); 2293 StubId stub_id = StubId::stubgen_join_2_3_base64_id; 2294 StubCodeMark mark(this, stub_id); 2295 address start = __ pc(); 2296 2297 assert(((unsigned long long)start & 0x3f) == 0, 2298 "Alignment problem (0x%08llx)", (unsigned long long)start); 2299 __ emit_data64(0x363031322c2d2e28, relocInfo::none); 2300 __ emit_data64(0x3c3d3e38393a3435, relocInfo::none); 2301 __ emit_data64(0x494a444546404142, relocInfo::none); 2302 __ emit_data64(0x565051524c4d4e48, relocInfo::none); 2303 __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); 2304 __ emit_data64(0x696a646566606162, relocInfo::none); 2305 __ emit_data64(0x767071726c6d6e68, relocInfo::none); 2306 __ emit_data64(0x7c7d7e78797a7475, relocInfo::none); 2307 2308 return start; 2309 } 2310 2311 address StubGenerator::base64_AVX2_decode_tables_addr() { 2312 __ align64(); 2313 StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id; 2314 StubCodeMark mark(this, stub_id); 2315 address start = __ pc(); 2316 2317 assert(((unsigned long long)start & 0x3f) == 0, 2318 "Alignment problem (0x%08llx)", (unsigned long long)start); 2319 __ emit_data(0x2f2f2f2f, relocInfo::none, 0); 2320 __ emit_data(0x5f5f5f5f, relocInfo::none, 0); // for URL 2321 2322 __ emit_data(0xffffffff, relocInfo::none, 0); 2323 __ emit_data(0xfcfcfcfc, relocInfo::none, 0); // for URL 2324 2325 // Permute table 2326 __ emit_data64(0x0000000100000000, relocInfo::none); 2327 __ emit_data64(0x0000000400000002, relocInfo::none); 2328 __ emit_data64(0x0000000600000005, relocInfo::none); 2329 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2330 2331 // Shuffle table 2332 __ emit_data64(0x090a040506000102, relocInfo::none); 2333 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2334 __ emit_data64(0x090a040506000102, relocInfo::none); 2335 __ emit_data64(0xffffffff0c0d0e08, relocInfo::none); 2336 2337 // merge table 2338 __ emit_data(0x01400140, relocInfo::none, 0); 2339 2340 // merge multiplier 2341 __ emit_data(0x00011000, relocInfo::none, 0); 2342 2343 return start; 2344 } 2345 2346 address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { 2347 __ align64(); 2348 StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id; 2349 StubCodeMark mark(this, stub_id); 2350 address start = __ pc(); 2351 2352 assert(((unsigned long long)start & 0x3f) == 0, 2353 "Alignment problem (0x%08llx)", (unsigned long long)start); 2354 // lut_lo 2355 __ emit_data64(0x1111111111111115, relocInfo::none); 2356 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2357 __ emit_data64(0x1111111111111115, relocInfo::none); 2358 __ emit_data64(0x1a1b1b1b1a131111, relocInfo::none); 2359 2360 // lut_roll 2361 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2362 __ emit_data64(0x0000000000000000, relocInfo::none); 2363 __ emit_data64(0xb9b9bfbf04131000, relocInfo::none); 2364 __ emit_data64(0x0000000000000000, relocInfo::none); 2365 2366 // lut_lo URL 2367 __ emit_data64(0x1111111111111115, relocInfo::none); 2368 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2369 __ emit_data64(0x1111111111111115, relocInfo::none); 2370 __ emit_data64(0x1b1b1a1b1b131111, relocInfo::none); 2371 2372 // lut_roll URL 2373 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2374 __ emit_data64(0x0000000000000000, relocInfo::none); 2375 __ emit_data64(0xb9b9bfbf0411e000, relocInfo::none); 2376 __ emit_data64(0x0000000000000000, relocInfo::none); 2377 2378 // lut_hi 2379 __ emit_data64(0x0804080402011010, relocInfo::none); 2380 __ emit_data64(0x1010101010101010, relocInfo::none); 2381 __ emit_data64(0x0804080402011010, relocInfo::none); 2382 __ emit_data64(0x1010101010101010, relocInfo::none); 2383 2384 return start; 2385 } 2386 2387 address StubGenerator::base64_decoding_table_addr() { 2388 StubId stub_id = StubId::stubgen_decoding_table_base64_id; 2389 StubCodeMark mark(this, stub_id); 2390 address start = __ pc(); 2391 2392 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2393 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2394 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2395 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2396 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2397 __ emit_data64(0x3fffffff3effffff, relocInfo::none); 2398 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2399 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2400 __ emit_data64(0x06050403020100ff, relocInfo::none); 2401 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2402 __ emit_data64(0x161514131211100f, relocInfo::none); 2403 __ emit_data64(0xffffffffff191817, relocInfo::none); 2404 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2405 __ emit_data64(0x2827262524232221, relocInfo::none); 2406 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2407 __ emit_data64(0xffffffffff333231, relocInfo::none); 2408 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2409 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2410 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2411 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2412 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2413 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2414 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2415 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2416 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2417 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2418 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2419 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2420 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2421 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2422 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2423 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2424 2425 // URL table 2426 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2427 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2428 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2429 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2430 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2431 __ emit_data64(0xffff3effffffffff, relocInfo::none); 2432 __ emit_data64(0x3b3a393837363534, relocInfo::none); 2433 __ emit_data64(0xffffffffffff3d3c, relocInfo::none); 2434 __ emit_data64(0x06050403020100ff, relocInfo::none); 2435 __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none); 2436 __ emit_data64(0x161514131211100f, relocInfo::none); 2437 __ emit_data64(0x3fffffffff191817, relocInfo::none); 2438 __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); 2439 __ emit_data64(0x2827262524232221, relocInfo::none); 2440 __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); 2441 __ emit_data64(0xffffffffff333231, relocInfo::none); 2442 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2443 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2444 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2445 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2446 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2447 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2448 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2449 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2450 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2451 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2452 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2453 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2454 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2455 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2456 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2457 __ emit_data64(0xffffffffffffffff, relocInfo::none); 2458 2459 return start; 2460 } 2461 2462 2463 // Code for generating Base64 decoding. 2464 // 2465 // Based on the article (and associated code) from https://arxiv.org/abs/1910.05109. 2466 // 2467 // Intrinsic function prototype in Base64.java: 2468 // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { 2469 address StubGenerator::generate_base64_decodeBlock() { 2470 __ align(CodeEntryAlignment); 2471 StubId stub_id = StubId::stubgen_base64_decodeBlock_id; 2472 StubCodeMark mark(this, stub_id); 2473 address start = __ pc(); 2474 2475 __ enter(); 2476 2477 // Save callee-saved registers before using them 2478 __ push_ppx(r12); 2479 __ push_ppx(r13); 2480 __ push_ppx(r14); 2481 __ push_ppx(r15); 2482 __ push_ppx(rbx); 2483 2484 // arguments 2485 const Register source = c_rarg0; // Source Array 2486 const Register start_offset = c_rarg1; // start offset 2487 const Register end_offset = c_rarg2; // end offset 2488 const Register dest = c_rarg3; // destination array 2489 const Register isMIME = rbx; 2490 2491 #ifndef _WIN64 2492 const Register dp = c_rarg4; // Position for writing to dest array 2493 const Register isURL = c_rarg5;// Base64 or URL character set 2494 __ movl(isMIME, Address(rbp, 2 * wordSize)); 2495 #else 2496 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 2497 const Address isURL_mem(rbp, 7 * wordSize); 2498 const Register isURL = r10; // pick the volatile windows register 2499 const Register dp = r12; 2500 __ movl(dp, dp_mem); 2501 __ movl(isURL, isURL_mem); 2502 __ movl(isMIME, Address(rbp, 8 * wordSize)); 2503 #endif 2504 2505 const XMMRegister lookup_lo = xmm5; 2506 const XMMRegister lookup_hi = xmm6; 2507 const XMMRegister errorvec = xmm7; 2508 const XMMRegister pack16_op = xmm9; 2509 const XMMRegister pack32_op = xmm8; 2510 const XMMRegister input0 = xmm3; 2511 const XMMRegister input1 = xmm20; 2512 const XMMRegister input2 = xmm21; 2513 const XMMRegister input3 = xmm19; 2514 const XMMRegister join01 = xmm12; 2515 const XMMRegister join12 = xmm11; 2516 const XMMRegister join23 = xmm10; 2517 const XMMRegister translated0 = xmm2; 2518 const XMMRegister translated1 = xmm1; 2519 const XMMRegister translated2 = xmm0; 2520 const XMMRegister translated3 = xmm4; 2521 2522 const XMMRegister merged0 = xmm2; 2523 const XMMRegister merged1 = xmm1; 2524 const XMMRegister merged2 = xmm0; 2525 const XMMRegister merged3 = xmm4; 2526 const XMMRegister merge_ab_bc0 = xmm2; 2527 const XMMRegister merge_ab_bc1 = xmm1; 2528 const XMMRegister merge_ab_bc2 = xmm0; 2529 const XMMRegister merge_ab_bc3 = xmm4; 2530 2531 const XMMRegister pack24bits = xmm4; 2532 2533 const Register length = r14; 2534 const Register output_size = r13; 2535 const Register output_mask = r15; 2536 const KRegister input_mask = k1; 2537 2538 const XMMRegister input_initial_valid_b64 = xmm0; 2539 const XMMRegister tmp = xmm10; 2540 const XMMRegister mask = xmm0; 2541 const XMMRegister invalid_b64 = xmm1; 2542 2543 Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL; 2544 Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce; 2545 Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero, L_lastChunk; 2546 2547 // calculate length from offsets 2548 __ movl(length, end_offset); 2549 __ subl(length, start_offset); 2550 __ push_ppx(dest); // Save for return value calc 2551 2552 // If AVX512 VBMI not supported, just compile non-AVX code 2553 if(VM_Version::supports_avx512_vbmi() && 2554 VM_Version::supports_avx512bw()) { 2555 __ cmpl(length, 31); // 32-bytes is break-even for AVX-512 2556 __ jcc(Assembler::lessEqual, L_lastChunk); 2557 2558 __ cmpl(isMIME, 0); 2559 __ jcc(Assembler::notEqual, L_lastChunk); 2560 2561 // Load lookup tables based on isURL 2562 __ cmpl(isURL, 0); 2563 __ jcc(Assembler::notZero, L_loadURL); 2564 2565 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13); 2566 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13); 2567 2568 __ BIND(L_continue); 2569 2570 __ movl(r15, 0x01400140); 2571 __ evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit); 2572 2573 __ movl(r15, 0x00011000); 2574 __ evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit); 2575 2576 __ cmpl(length, 0xff); 2577 __ jcc(Assembler::lessEqual, L_process64); 2578 2579 // load masks required for decoding data 2580 __ BIND(L_processdata); 2581 __ evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13); 2582 __ evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13); 2583 __ evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13); 2584 2585 __ align32(); 2586 __ BIND(L_process256); 2587 // Grab input data 2588 __ evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit); 2589 __ evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit); 2590 __ evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit); 2591 __ evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit); 2592 2593 // Copy the low part of the lookup table into the destination of the permutation 2594 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2595 __ evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit); 2596 __ evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit); 2597 __ evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit); 2598 2599 // Translate the base64 input into "decoded" bytes 2600 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2601 __ evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit); 2602 __ evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit); 2603 __ evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit); 2604 2605 // OR all of the translations together to check for errors (high-order bit of byte set) 2606 __ vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit); 2607 2608 __ vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit); 2609 __ vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit); 2610 __ vpor(errorvec, input3, input0, Assembler::AVX_512bit); 2611 2612 // Check if there was an error - if so, try 64-byte chunks 2613 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2614 __ kortestql(k3, k3); 2615 __ jcc(Assembler::notZero, L_process64); 2616 2617 // The merging and shuffling happens here 2618 // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa] 2619 // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd] 2620 // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40 2621 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2622 __ vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit); 2623 __ vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit); 2624 __ vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit); 2625 2626 // Now do the same with packed 16-bit values. 2627 // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb] 2628 // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12 2629 // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd] 2630 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2631 __ vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit); 2632 __ vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit); 2633 __ vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit); 2634 2635 // The join vectors specify which byte from which vector goes into the outputs 2636 // One of every 4 bytes in the extended vector is zero, so we pack them into their 2637 // final positions in the register for storing (256 bytes in, 192 bytes out) 2638 __ evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit); 2639 __ evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit); 2640 __ evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit); 2641 2642 // Store result 2643 __ evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit); 2644 __ evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit); 2645 __ evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit); 2646 2647 __ addptr(source, 0x100); 2648 __ addptr(dest, 0xc0); 2649 __ subl(length, 0x100); 2650 __ cmpl(length, 64 * 4); 2651 __ jcc(Assembler::greaterEqual, L_process256); 2652 2653 // At this point, we've decoded 64 * 4 * n bytes. 2654 // The remaining length will be <= 64 * 4 - 1. 2655 // UNLESS there was an error decoding the first 256-byte chunk. In this 2656 // case, the length will be arbitrarily long. 2657 // 2658 // Note that this will be the path for MIME-encoded strings. 2659 2660 __ BIND(L_process64); 2661 2662 __ evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13); 2663 2664 __ cmpl(length, 63); 2665 __ jcc(Assembler::lessEqual, L_finalBit); 2666 2667 __ mov64(rax, 0x0000ffffffffffff); 2668 __ kmovql(k2, rax); 2669 2670 __ align32(); 2671 __ BIND(L_process64Loop); 2672 2673 // Handle first 64-byte block 2674 2675 __ evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit); 2676 __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); 2677 __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); 2678 2679 __ vpor(errorvec, translated0, input0, Assembler::AVX_512bit); 2680 2681 // Check for error and bomb out before updating dest 2682 __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit); 2683 __ kortestql(k3, k3); 2684 __ jcc(Assembler::notZero, L_exit); 2685 2686 // Pack output register, selecting correct byte ordering 2687 __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); 2688 __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); 2689 __ vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit); 2690 2691 __ evmovdqub(Address(dest, dp), k2, merged0, true, Assembler::AVX_512bit); 2692 2693 __ subl(length, 64); 2694 __ addptr(source, 64); 2695 __ addptr(dest, 48); 2696 2697 __ cmpl(length, 64); 2698 __ jcc(Assembler::greaterEqual, L_process64Loop); 2699 2700 __ cmpl(length, 0); 2701 __ jcc(Assembler::lessEqual, L_exit); 2702 2703 __ BIND(L_finalBit); 2704 // Now have 1 to 63 bytes left to decode 2705 2706 // I was going to let Java take care of the final fragment 2707 // however it will repeatedly call this routine for every 4 bytes 2708 // of input data, so handle the rest here. 2709 __ movq(rax, -1); 2710 __ bzhiq(rax, rax, length); // Input mask in rax 2711 2712 __ movl(output_size, length); 2713 __ shrl(output_size, 2); // Find (len / 4) * 3 (output length) 2714 __ lea(output_size, Address(output_size, output_size, Address::times_2, 0)); 2715 // output_size in r13 2716 2717 // Strip pad characters, if any, and adjust length and mask 2718 __ addq(length, start_offset); 2719 __ cmpb(Address(source, length, Address::times_1, -1), '='); 2720 __ jcc(Assembler::equal, L_padding); 2721 2722 __ BIND(L_donePadding); 2723 __ subq(length, start_offset); 2724 2725 // Output size is (64 - output_size), output mask is (all 1s >> output_size). 2726 __ kmovql(input_mask, rax); 2727 __ movq(output_mask, -1); 2728 __ bzhiq(output_mask, output_mask, output_size); 2729 2730 // Load initial input with all valid base64 characters. Will be used 2731 // in merging source bytes to avoid masking when determining if an error occurred. 2732 __ movl(rax, 0x61616161); 2733 __ evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit); 2734 2735 // A register containing all invalid base64 decoded values 2736 __ movl(rax, 0x80808080); 2737 __ evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit); 2738 2739 // input_mask is in k1 2740 // output_size is in r13 2741 // output_mask is in r15 2742 // zmm0 - free 2743 // zmm1 - 0x00011000 2744 // zmm2 - 0x01400140 2745 // zmm3 - errorvec 2746 // zmm4 - pack vector 2747 // zmm5 - lookup_lo 2748 // zmm6 - lookup_hi 2749 // zmm7 - errorvec 2750 // zmm8 - 0x61616161 2751 // zmm9 - 0x80808080 2752 2753 // Load only the bytes from source, merging into our "fully-valid" register 2754 __ evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit); 2755 2756 // Decode all bytes within our merged input 2757 __ evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit); 2758 __ evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit); 2759 __ evporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit); 2760 2761 // Check for error. Compare (decoded | initial) to all invalid. 2762 // If any bytes have their high-order bit set, then we have an error. 2763 __ evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit); 2764 __ kortestql(k2, k2); 2765 2766 // If we have an error, use the brute force loop to decode what we can (4-byte chunks). 2767 __ jcc(Assembler::notZero, L_bruteForce); 2768 2769 // Shuffle output bytes 2770 __ vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit); 2771 __ vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit); 2772 2773 __ vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit); 2774 __ kmovql(k1, output_mask); 2775 __ evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit); 2776 2777 __ addptr(dest, output_size); 2778 2779 __ BIND(L_exit); 2780 __ vzeroupper(); 2781 __ pop_ppx(rax); // Get original dest value 2782 __ subptr(dest, rax); // Number of bytes converted 2783 __ movptr(rax, dest); 2784 __ pop_ppx(rbx); 2785 __ pop_ppx(r15); 2786 __ pop_ppx(r14); 2787 __ pop_ppx(r13); 2788 __ pop_ppx(r12); 2789 __ leave(); 2790 __ ret(0); 2791 2792 __ BIND(L_loadURL); 2793 __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13); 2794 __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13); 2795 __ jmp(L_continue); 2796 2797 __ BIND(L_padding); 2798 __ decrementq(output_size, 1); 2799 __ shrq(rax, 1); 2800 2801 __ cmpb(Address(source, length, Address::times_1, -2), '='); 2802 __ jcc(Assembler::notEqual, L_donePadding); 2803 2804 __ decrementq(output_size, 1); 2805 __ shrq(rax, 1); 2806 __ jmp(L_donePadding); 2807 2808 __ align32(); 2809 __ BIND(L_bruteForce); 2810 } // End of if(avx512_vbmi) 2811 2812 if (VM_Version::supports_avx2()) { 2813 Label L_tailProc, L_topLoop, L_enterLoop; 2814 2815 __ cmpl(isMIME, 0); 2816 __ jcc(Assembler::notEqual, L_lastChunk); 2817 2818 // Check for buffer too small (for algorithm) 2819 __ subl(length, 0x2c); 2820 __ jcc(Assembler::less, L_tailProc); 2821 2822 __ shll(isURL, 2); 2823 2824 // Algorithm adapted from https://arxiv.org/abs/1704.00605, "Faster Base64 2825 // Encoding and Decoding using AVX2 Instructions". URL modifications added. 2826 2827 // Set up constants 2828 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_tables_addr())); 2829 __ vpbroadcastd(xmm4, Address(r13, isURL, Address::times_1), Assembler::AVX_256bit); // 2F or 5F 2830 __ vpbroadcastd(xmm10, Address(r13, isURL, Address::times_1, 0x08), Assembler::AVX_256bit); // -1 or -4 2831 __ vmovdqu(xmm12, Address(r13, 0x10)); // permute 2832 __ vmovdqu(xmm13, Address(r13, 0x30)); // shuffle 2833 __ vpbroadcastd(xmm7, Address(r13, 0x50), Assembler::AVX_256bit); // merge 2834 __ vpbroadcastd(xmm6, Address(r13, 0x54), Assembler::AVX_256bit); // merge mult 2835 2836 __ lea(r13, ExternalAddress(StubRoutines::x86::base64_AVX2_decode_LUT_tables_addr())); 2837 __ shll(isURL, 4); 2838 __ vmovdqu(xmm11, Address(r13, isURL, Address::times_1, 0x00)); // lut_lo 2839 __ vmovdqu(xmm8, Address(r13, isURL, Address::times_1, 0x20)); // lut_roll 2840 __ shrl(isURL, 6); // restore isURL 2841 __ vmovdqu(xmm9, Address(r13, 0x80)); // lut_hi 2842 __ jmp(L_enterLoop); 2843 2844 __ align32(); 2845 __ bind(L_topLoop); 2846 // Add in the offset value (roll) to get 6-bit out values 2847 __ vpaddb(xmm0, xmm0, xmm2, Assembler::AVX_256bit); 2848 // Merge and permute the output bits into appropriate output byte lanes 2849 __ vpmaddubsw(xmm0, xmm0, xmm7, Assembler::AVX_256bit); 2850 __ vpmaddwd(xmm0, xmm0, xmm6, Assembler::AVX_256bit); 2851 __ vpshufb(xmm0, xmm0, xmm13, Assembler::AVX_256bit); 2852 __ vpermd(xmm0, xmm12, xmm0, Assembler::AVX_256bit); 2853 // Store the output bytes 2854 __ vmovdqu(Address(dest, dp, Address::times_1, 0), xmm0); 2855 __ addptr(source, 0x20); 2856 __ addptr(dest, 0x18); 2857 __ subl(length, 0x20); 2858 __ jcc(Assembler::less, L_tailProc); 2859 2860 __ bind(L_enterLoop); 2861 2862 // Load in encoded string (32 bytes) 2863 __ vmovdqu(xmm2, Address(source, start_offset, Address::times_1, 0x0)); 2864 // Extract the high nibble for indexing into the lut tables. High 4 bits are don't care. 2865 __ vpsrld(xmm1, xmm2, 0x4, Assembler::AVX_256bit); 2866 __ vpand(xmm1, xmm4, xmm1, Assembler::AVX_256bit); 2867 // Extract the low nibble. 5F/2F will isolate the low-order 4 bits. High 4 bits are don't care. 2868 __ vpand(xmm3, xmm2, xmm4, Assembler::AVX_256bit); 2869 // Check for special-case (0x2F or 0x5F (URL)) 2870 __ vpcmpeqb(xmm0, xmm4, xmm2, Assembler::AVX_256bit); 2871 // Get the bitset based on the low nibble. vpshufb uses low-order 4 bits only. 2872 __ vpshufb(xmm3, xmm11, xmm3, Assembler::AVX_256bit); 2873 // Get the bit value of the high nibble 2874 __ vpshufb(xmm5, xmm9, xmm1, Assembler::AVX_256bit); 2875 // Make sure 2F / 5F shows as valid 2876 __ vpandn(xmm3, xmm0, xmm3, Assembler::AVX_256bit); 2877 // Make adjustment for roll index. For non-URL, this is a no-op, 2878 // for URL, this adjusts by -4. This is to properly index the 2879 // roll value for 2F / 5F. 2880 __ vpand(xmm0, xmm0, xmm10, Assembler::AVX_256bit); 2881 // If the and of the two is non-zero, we have an invalid input character 2882 __ vptest(xmm3, xmm5); 2883 // Extract the "roll" value - value to add to the input to get 6-bit out value 2884 __ vpaddb(xmm0, xmm0, xmm1, Assembler::AVX_256bit); // Handle 2F / 5F 2885 __ vpshufb(xmm0, xmm8, xmm0, Assembler::AVX_256bit); 2886 __ jcc(Assembler::equal, L_topLoop); // Fall through on error 2887 2888 __ bind(L_tailProc); 2889 2890 __ addl(length, 0x2c); 2891 2892 __ vzeroupper(); 2893 } 2894 2895 // Use non-AVX code to decode 4-byte chunks into 3 bytes of output 2896 2897 // Register state (Linux): 2898 // r12-15 - saved on stack 2899 // rdi - src 2900 // rsi - sp 2901 // rdx - sl 2902 // rcx - dst 2903 // r8 - dp 2904 // r9 - isURL 2905 2906 // Register state (Windows): 2907 // r12-15 - saved on stack 2908 // rcx - src 2909 // rdx - sp 2910 // r8 - sl 2911 // r9 - dst 2912 // r12 - dp 2913 // r10 - isURL 2914 2915 // Registers (common): 2916 // length (r14) - bytes in src 2917 2918 const Register decode_table = r11; 2919 const Register out_byte_count = rbx; 2920 const Register byte1 = r13; 2921 const Register byte2 = r15; 2922 const Register byte3 = WIN64_ONLY(r8) NOT_WIN64(rdx); 2923 const Register byte4 = WIN64_ONLY(r10) NOT_WIN64(r9); 2924 2925 __ bind(L_lastChunk); 2926 2927 __ shrl(length, 2); // Multiple of 4 bytes only - length is # 4-byte chunks 2928 __ cmpl(length, 0); 2929 __ jcc(Assembler::lessEqual, L_exit_no_vzero); 2930 2931 __ shll(isURL, 8); // index into decode table based on isURL 2932 __ lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr())); 2933 __ addptr(decode_table, isURL); 2934 2935 __ jmp(L_bottomLoop); 2936 2937 __ align32(); 2938 __ BIND(L_forceLoop); 2939 __ shll(byte1, 18); 2940 __ shll(byte2, 12); 2941 __ shll(byte3, 6); 2942 __ orl(byte1, byte2); 2943 __ orl(byte1, byte3); 2944 __ orl(byte1, byte4); 2945 2946 __ addptr(source, 4); 2947 2948 __ movb(Address(dest, dp, Address::times_1, 2), byte1); 2949 __ shrl(byte1, 8); 2950 __ movb(Address(dest, dp, Address::times_1, 1), byte1); 2951 __ shrl(byte1, 8); 2952 __ movb(Address(dest, dp, Address::times_1, 0), byte1); 2953 2954 __ addptr(dest, 3); 2955 __ decrementl(length, 1); 2956 __ jcc(Assembler::zero, L_exit_no_vzero); 2957 2958 __ BIND(L_bottomLoop); 2959 __ load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00)); 2960 __ load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01)); 2961 __ load_signed_byte(byte1, Address(decode_table, byte1)); 2962 __ load_signed_byte(byte2, Address(decode_table, byte2)); 2963 __ load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02)); 2964 __ load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03)); 2965 __ load_signed_byte(byte3, Address(decode_table, byte3)); 2966 __ load_signed_byte(byte4, Address(decode_table, byte4)); 2967 2968 __ mov(rax, byte1); 2969 __ orl(rax, byte2); 2970 __ orl(rax, byte3); 2971 __ orl(rax, byte4); 2972 __ jcc(Assembler::positive, L_forceLoop); 2973 2974 __ BIND(L_exit_no_vzero); 2975 __ pop_ppx(rax); // Get original dest value 2976 __ subptr(dest, rax); // Number of bytes converted 2977 __ movptr(rax, dest); 2978 __ pop_ppx(rbx); 2979 __ pop_ppx(r15); 2980 __ pop_ppx(r14); 2981 __ pop_ppx(r13); 2982 __ pop_ppx(r12); 2983 __ leave(); 2984 __ ret(0); 2985 2986 return start; 2987 } 2988 2989 2990 /** 2991 * Arguments: 2992 * 2993 * Inputs: 2994 * c_rarg0 - int crc 2995 * c_rarg1 - byte* buf 2996 * c_rarg2 - int length 2997 * 2998 * Output: 2999 * rax - int crc result 3000 */ 3001 address StubGenerator::generate_updateBytesCRC32() { 3002 assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions"); 3003 3004 __ align(CodeEntryAlignment); 3005 StubId stub_id = StubId::stubgen_updateBytesCRC32_id; 3006 StubCodeMark mark(this, stub_id); 3007 3008 address start = __ pc(); 3009 3010 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3011 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3012 // rscratch1: r10 3013 const Register crc = c_rarg0; // crc 3014 const Register buf = c_rarg1; // source java byte array address 3015 const Register len = c_rarg2; // length 3016 const Register table = c_rarg3; // crc_table address (reuse register) 3017 const Register tmp1 = r11; 3018 const Register tmp2 = r10; 3019 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax); 3020 3021 BLOCK_COMMENT("Entry:"); 3022 __ enter(); // required for proper stackwalking of RuntimeStub frame 3023 3024 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 3025 VM_Version::supports_avx512bw() && 3026 VM_Version::supports_avx512vl()) { 3027 // The constants used in the CRC32 algorithm requires the 1's compliment of the initial crc value. 3028 // However, the constant table for CRC32-C assumes the original crc value. Account for this 3029 // difference before calling and after returning. 3030 __ lea(table, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr())); 3031 __ notl(crc); 3032 __ kernel_crc32_avx512(crc, buf, len, table, tmp1, tmp2); 3033 __ notl(crc); 3034 } else { 3035 __ kernel_crc32(crc, buf, len, table, tmp1); 3036 } 3037 3038 __ movl(rax, crc); 3039 __ vzeroupper(); 3040 __ leave(); // required for proper stackwalking of RuntimeStub frame 3041 __ ret(0); 3042 3043 return start; 3044 } 3045 3046 /** 3047 * Arguments: 3048 * 3049 * Inputs: 3050 * c_rarg0 - int crc 3051 * c_rarg1 - byte* buf 3052 * c_rarg2 - long length 3053 * c_rarg3 - table_start - optional (present only when doing a library_call, 3054 * not used by x86 algorithm) 3055 * 3056 * Output: 3057 * rax - int crc result 3058 */ 3059 address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { 3060 assert(UseCRC32CIntrinsics, "need SSE4_2"); 3061 __ align(CodeEntryAlignment); 3062 StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; 3063 StubCodeMark mark(this, stub_id); 3064 address start = __ pc(); 3065 3066 //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs 3067 //Windows RCX RDX R8 R9 none none XMM0..XMM3 3068 //Lin / Sol RDI RSI RDX RCX R8 R9 XMM0..XMM7 3069 const Register crc = c_rarg0; // crc 3070 const Register buf = c_rarg1; // source java byte array address 3071 const Register len = c_rarg2; // length 3072 const Register a = rax; 3073 const Register j = r9; 3074 const Register k = r10; 3075 const Register l = r11; 3076 #ifdef _WIN64 3077 const Register y = rdi; 3078 const Register z = rsi; 3079 #else 3080 const Register y = rcx; 3081 const Register z = r8; 3082 #endif 3083 assert_different_registers(crc, buf, len, a, j, k, l, y, z); 3084 3085 BLOCK_COMMENT("Entry:"); 3086 __ enter(); // required for proper stackwalking of RuntimeStub frame 3087 Label L_continue; 3088 3089 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && 3090 VM_Version::supports_avx512bw() && 3091 VM_Version::supports_avx512vl()) { 3092 Label L_doSmall; 3093 3094 __ cmpl(len, 384); 3095 __ jcc(Assembler::lessEqual, L_doSmall); 3096 3097 __ lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr())); 3098 __ kernel_crc32_avx512(crc, buf, len, j, l, k); 3099 3100 __ jmp(L_continue); 3101 3102 __ bind(L_doSmall); 3103 } 3104 #ifdef _WIN64 3105 __ push_ppx(y); 3106 __ push_ppx(z); 3107 #endif 3108 __ crc32c_ipl_alg2_alt2(crc, buf, len, 3109 a, j, k, 3110 l, y, z, 3111 c_farg0, c_farg1, c_farg2, 3112 is_pclmulqdq_supported); 3113 #ifdef _WIN64 3114 __ pop_ppx(z); 3115 __ pop_ppx(y); 3116 #endif 3117 3118 __ bind(L_continue); 3119 __ movl(rax, crc); 3120 __ vzeroupper(); 3121 __ leave(); // required for proper stackwalking of RuntimeStub frame 3122 __ ret(0); 3123 3124 return start; 3125 } 3126 3127 3128 /** 3129 * Arguments: 3130 * 3131 * Input: 3132 * c_rarg0 - x address 3133 * c_rarg1 - x length 3134 * c_rarg2 - y address 3135 * c_rarg3 - y length 3136 * not Win64 3137 * c_rarg4 - z address 3138 * Win64 3139 * rsp+40 - z address 3140 */ 3141 address StubGenerator::generate_multiplyToLen() { 3142 __ align(CodeEntryAlignment); 3143 StubId stub_id = StubId::stubgen_multiplyToLen_id; 3144 StubCodeMark mark(this, stub_id); 3145 address start = __ pc(); 3146 3147 if (AOTCodeCache::load_stub(this, vmIntrinsics::_multiplyToLen, "multiplyToLen", start)) { 3148 return start; 3149 } 3150 3151 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3152 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3153 const Register x = rdi; 3154 const Register xlen = rax; 3155 const Register y = rsi; 3156 const Register ylen = rcx; 3157 const Register z = r8; 3158 3159 // Next registers will be saved on stack in multiply_to_len(). 3160 const Register tmp0 = r11; 3161 const Register tmp1 = r12; 3162 const Register tmp2 = r13; 3163 const Register tmp3 = r14; 3164 const Register tmp4 = r15; 3165 const Register tmp5 = rbx; 3166 3167 BLOCK_COMMENT("Entry:"); 3168 __ enter(); // required for proper stackwalking of RuntimeStub frame 3169 3170 setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx 3171 // ylen => rcx, z => r8 3172 // r9 and r10 may be used to save non-volatile registers 3173 #ifdef _WIN64 3174 // last argument (#4) is on stack on Win64 3175 __ movptr(z, Address(rsp, 6 * wordSize)); 3176 #endif 3177 3178 __ movptr(xlen, rsi); 3179 __ movptr(y, rdx); 3180 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 3181 3182 restore_arg_regs(); 3183 3184 __ leave(); // required for proper stackwalking of RuntimeStub frame 3185 __ ret(0); 3186 3187 AOTCodeCache::store_stub(this, vmIntrinsics::_multiplyToLen, "multiplyToLen", start); 3188 return start; 3189 } 3190 3191 /** 3192 * Arguments: 3193 * 3194 * Input: 3195 * c_rarg0 - obja address 3196 * c_rarg1 - objb address 3197 * c_rarg3 - length length 3198 * c_rarg4 - scale log2_array_indxscale 3199 * 3200 * Output: 3201 * rax - int >= mismatched index, < 0 bitwise complement of tail 3202 */ 3203 address StubGenerator::generate_vectorizedMismatch() { 3204 __ align(CodeEntryAlignment); 3205 StubId stub_id = StubId::stubgen_vectorizedMismatch_id; 3206 StubCodeMark mark(this, stub_id); 3207 address start = __ pc(); 3208 3209 BLOCK_COMMENT("Entry:"); 3210 __ enter(); 3211 3212 #ifdef _WIN64 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3213 const Register scale = c_rarg0; //rcx, will exchange with r9 3214 const Register objb = c_rarg1; //rdx 3215 const Register length = c_rarg2; //r8 3216 const Register obja = c_rarg3; //r9 3217 __ xchgq(obja, scale); //now obja and scale contains the correct contents 3218 3219 const Register tmp1 = r10; 3220 const Register tmp2 = r11; 3221 #endif 3222 #ifndef _WIN64 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3223 const Register obja = c_rarg0; //U:rdi 3224 const Register objb = c_rarg1; //U:rsi 3225 const Register length = c_rarg2; //U:rdx 3226 const Register scale = c_rarg3; //U:rcx 3227 const Register tmp1 = r8; 3228 const Register tmp2 = r9; 3229 #endif 3230 const Register result = rax; //return value 3231 const XMMRegister vec0 = xmm0; 3232 const XMMRegister vec1 = xmm1; 3233 const XMMRegister vec2 = xmm2; 3234 3235 __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2); 3236 3237 __ vzeroupper(); 3238 __ leave(); 3239 __ ret(0); 3240 3241 return start; 3242 } 3243 3244 /** 3245 * Arguments: 3246 * 3247 // Input: 3248 // c_rarg0 - x address 3249 // c_rarg1 - x length 3250 // c_rarg2 - z address 3251 // c_rarg3 - z length 3252 * 3253 */ 3254 address StubGenerator::generate_squareToLen() { 3255 3256 __ align(CodeEntryAlignment); 3257 StubId stub_id = StubId::stubgen_squareToLen_id; 3258 StubCodeMark mark(this, stub_id); 3259 address start = __ pc(); 3260 3261 if (AOTCodeCache::load_stub(this, vmIntrinsics::_squareToLen, "squareToLen", start)) { 3262 return start; 3263 } 3264 3265 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3266 // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...) 3267 const Register x = rdi; 3268 const Register len = rsi; 3269 const Register z = r8; 3270 const Register zlen = rcx; 3271 3272 const Register tmp1 = r12; 3273 const Register tmp2 = r13; 3274 const Register tmp3 = r14; 3275 const Register tmp4 = r15; 3276 const Register tmp5 = rbx; 3277 3278 BLOCK_COMMENT("Entry:"); 3279 __ enter(); // required for proper stackwalking of RuntimeStub frame 3280 3281 setup_arg_regs(4); // x => rdi, len => rsi, z => rdx 3282 // zlen => rcx 3283 // r9 and r10 may be used to save non-volatile registers 3284 __ movptr(r8, rdx); 3285 __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3286 3287 restore_arg_regs(); 3288 3289 __ leave(); // required for proper stackwalking of RuntimeStub frame 3290 __ ret(0); 3291 3292 AOTCodeCache::store_stub(this, vmIntrinsics::_squareToLen, "squareToLen", start); 3293 return start; 3294 } 3295 3296 address StubGenerator::generate_method_entry_barrier() { 3297 __ align(CodeEntryAlignment); 3298 StubId stub_id = StubId::stubgen_method_entry_barrier_id; 3299 StubCodeMark mark(this, stub_id); 3300 address start = __ pc(); 3301 3302 Label deoptimize_label; 3303 3304 __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing 3305 3306 BLOCK_COMMENT("Entry:"); 3307 __ enter(); // save rbp 3308 3309 // save c_rarg0, because we want to use that value. 3310 // We could do without it but then we depend on the number of slots used by pusha 3311 __ push_ppx(c_rarg0); 3312 3313 __ lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address 3314 3315 __ pusha(); 3316 3317 // The method may have floats as arguments, and we must spill them before calling 3318 // the VM runtime. 3319 assert(Argument::n_float_register_parameters_j == 8, "Assumption"); 3320 const int xmm_size = wordSize * 2; 3321 const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j; 3322 __ subptr(rsp, xmm_spill_size); 3323 __ movdqu(Address(rsp, xmm_size * 7), xmm7); 3324 __ movdqu(Address(rsp, xmm_size * 6), xmm6); 3325 __ movdqu(Address(rsp, xmm_size * 5), xmm5); 3326 __ movdqu(Address(rsp, xmm_size * 4), xmm4); 3327 __ movdqu(Address(rsp, xmm_size * 3), xmm3); 3328 __ movdqu(Address(rsp, xmm_size * 2), xmm2); 3329 __ movdqu(Address(rsp, xmm_size * 1), xmm1); 3330 __ movdqu(Address(rsp, xmm_size * 0), xmm0); 3331 3332 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), 1); 3333 3334 __ movdqu(xmm0, Address(rsp, xmm_size * 0)); 3335 __ movdqu(xmm1, Address(rsp, xmm_size * 1)); 3336 __ movdqu(xmm2, Address(rsp, xmm_size * 2)); 3337 __ movdqu(xmm3, Address(rsp, xmm_size * 3)); 3338 __ movdqu(xmm4, Address(rsp, xmm_size * 4)); 3339 __ movdqu(xmm5, Address(rsp, xmm_size * 5)); 3340 __ movdqu(xmm6, Address(rsp, xmm_size * 6)); 3341 __ movdqu(xmm7, Address(rsp, xmm_size * 7)); 3342 __ addptr(rsp, xmm_spill_size); 3343 3344 __ cmpl(rax, 1); // 1 means deoptimize 3345 __ jcc(Assembler::equal, deoptimize_label); 3346 3347 __ popa(); 3348 __ pop_ppx(c_rarg0); 3349 3350 __ leave(); 3351 3352 __ addptr(rsp, 1 * wordSize); // cookie 3353 __ ret(0); 3354 3355 3356 __ BIND(deoptimize_label); 3357 3358 __ popa(); 3359 __ pop_ppx(c_rarg0); 3360 3361 __ leave(); 3362 3363 // this can be taken out, but is good for verification purposes. getting a SIGSEGV 3364 // here while still having a correct stack is valuable 3365 __ testptr(rsp, Address(rsp, 0)); 3366 3367 __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier 3368 __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point 3369 3370 return start; 3371 } 3372 3373 /** 3374 * Arguments: 3375 * 3376 * Input: 3377 * c_rarg0 - out address 3378 * c_rarg1 - in address 3379 * c_rarg2 - offset 3380 * c_rarg3 - len 3381 * not Win64 3382 * c_rarg4 - k 3383 * Win64 3384 * rsp+40 - k 3385 */ 3386 address StubGenerator::generate_mulAdd() { 3387 __ align(CodeEntryAlignment); 3388 StubId stub_id = StubId::stubgen_mulAdd_id; 3389 StubCodeMark mark(this, stub_id); 3390 address start = __ pc(); 3391 3392 if (AOTCodeCache::load_stub(this, vmIntrinsics::_mulAdd, "mulAdd", start)) { 3393 return start; 3394 } 3395 3396 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) 3397 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) 3398 const Register out = rdi; 3399 const Register in = rsi; 3400 const Register offset = r11; 3401 const Register len = rcx; 3402 const Register k = r8; 3403 3404 // Next registers will be saved on stack in mul_add(). 3405 const Register tmp1 = r12; 3406 const Register tmp2 = r13; 3407 const Register tmp3 = r14; 3408 const Register tmp4 = r15; 3409 const Register tmp5 = rbx; 3410 3411 BLOCK_COMMENT("Entry:"); 3412 __ enter(); // required for proper stackwalking of RuntimeStub frame 3413 3414 setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx 3415 // len => rcx, k => r8 3416 // r9 and r10 may be used to save non-volatile registers 3417 #ifdef _WIN64 3418 // last argument is on stack on Win64 3419 __ movl(k, Address(rsp, 6 * wordSize)); 3420 #endif 3421 __ movptr(r11, rdx); // move offset in rdx to offset(r11) 3422 __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); 3423 3424 restore_arg_regs(); 3425 3426 __ leave(); // required for proper stackwalking of RuntimeStub frame 3427 __ ret(0); 3428 3429 AOTCodeCache::store_stub(this, vmIntrinsics::_mulAdd, "mulAdd", start); 3430 return start; 3431 } 3432 3433 address StubGenerator::generate_bigIntegerRightShift() { 3434 __ align(CodeEntryAlignment); 3435 StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; 3436 StubCodeMark mark(this, stub_id); 3437 address start = __ pc(); 3438 3439 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3440 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3441 const Register newArr = rdi; 3442 const Register oldArr = rsi; 3443 const Register newIdx = rdx; 3444 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3445 const Register totalNumIter = r8; 3446 3447 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3448 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3449 const Register tmp1 = r11; // Caller save. 3450 const Register tmp2 = rax; // Caller save. 3451 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3452 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3453 const Register tmp5 = r14; // Callee save. 3454 const Register tmp6 = r15; 3455 3456 const XMMRegister x0 = xmm0; 3457 const XMMRegister x1 = xmm1; 3458 const XMMRegister x2 = xmm2; 3459 3460 BLOCK_COMMENT("Entry:"); 3461 __ enter(); // required for proper stackwalking of RuntimeStub frame 3462 3463 #ifdef _WIN64 3464 setup_arg_regs(4); 3465 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3466 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3467 // Save callee save registers. 3468 __ push_ppx(tmp3); 3469 __ push_ppx(tmp4); 3470 #endif 3471 __ push_ppx(tmp5); 3472 3473 // Rename temps used throughout the code. 3474 const Register idx = tmp1; 3475 const Register nIdx = tmp2; 3476 3477 __ xorl(idx, idx); 3478 3479 // Start right shift from end of the array. 3480 // For example, if #iteration = 4 and newIdx = 1 3481 // then dest[4] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3482 // if #iteration = 4 and newIdx = 0 3483 // then dest[3] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) 3484 __ movl(idx, totalNumIter); 3485 __ movl(nIdx, idx); 3486 __ addl(nIdx, newIdx); 3487 3488 // If vectorization is enabled, check if the number of iterations is at least 64 3489 // If not, then go to ShifTwo processing 2 iterations 3490 if (VM_Version::supports_avx512_vbmi2()) { 3491 __ cmpptr(totalNumIter, (AVX3Threshold/64)); 3492 __ jcc(Assembler::less, ShiftTwo); 3493 3494 if (AVX3Threshold < 16 * 64) { 3495 __ cmpl(totalNumIter, 16); 3496 __ jcc(Assembler::less, ShiftTwo); 3497 } 3498 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3499 __ subl(idx, 16); 3500 __ subl(nIdx, 16); 3501 __ BIND(Shift512Loop); 3502 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 4), Assembler::AVX_512bit); 3503 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3504 __ vpshrdvd(x2, x1, x0, Assembler::AVX_512bit); 3505 __ evmovdqul(Address(newArr, nIdx, Address::times_4), x2, Assembler::AVX_512bit); 3506 __ subl(nIdx, 16); 3507 __ subl(idx, 16); 3508 __ jcc(Assembler::greaterEqual, Shift512Loop); 3509 __ addl(idx, 16); 3510 __ addl(nIdx, 16); 3511 } 3512 __ BIND(ShiftTwo); 3513 __ cmpl(idx, 2); 3514 __ jcc(Assembler::less, ShiftOne); 3515 __ subl(idx, 2); 3516 __ subl(nIdx, 2); 3517 __ BIND(ShiftTwoLoop); 3518 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 8)); 3519 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3520 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3521 __ shrdl(tmp5, tmp4); 3522 __ shrdl(tmp4, tmp3); 3523 __ movl(Address(newArr, nIdx, Address::times_4, 4), tmp5); 3524 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3525 __ subl(nIdx, 2); 3526 __ subl(idx, 2); 3527 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3528 __ addl(idx, 2); 3529 __ addl(nIdx, 2); 3530 3531 // Do the last iteration 3532 __ BIND(ShiftOne); 3533 __ cmpl(idx, 1); 3534 __ jcc(Assembler::less, Exit); 3535 __ subl(idx, 1); 3536 __ subl(nIdx, 1); 3537 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); 3538 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3539 __ shrdl(tmp4, tmp3); 3540 __ movl(Address(newArr, nIdx, Address::times_4), tmp4); 3541 __ BIND(Exit); 3542 __ vzeroupper(); 3543 // Restore callee save registers. 3544 __ pop_ppx(tmp5); 3545 #ifdef _WIN64 3546 __ pop_ppx(tmp4); 3547 __ pop_ppx(tmp3); 3548 restore_arg_regs(); 3549 #endif 3550 __ leave(); // required for proper stackwalking of RuntimeStub frame 3551 __ ret(0); 3552 3553 return start; 3554 } 3555 3556 /** 3557 * Arguments: 3558 * 3559 * Input: 3560 * c_rarg0 - newArr address 3561 * c_rarg1 - oldArr address 3562 * c_rarg2 - newIdx 3563 * c_rarg3 - shiftCount 3564 * not Win64 3565 * c_rarg4 - numIter 3566 * Win64 3567 * rsp40 - numIter 3568 */ 3569 address StubGenerator::generate_bigIntegerLeftShift() { 3570 __ align(CodeEntryAlignment); 3571 StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; 3572 StubCodeMark mark(this, stub_id); 3573 address start = __ pc(); 3574 3575 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; 3576 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. 3577 const Register newArr = rdi; 3578 const Register oldArr = rsi; 3579 const Register newIdx = rdx; 3580 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. 3581 const Register totalNumIter = r8; 3582 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. 3583 // For everything else, we prefer using r9 and r10 since we do not have to save them before use. 3584 const Register tmp1 = r11; // Caller save. 3585 const Register tmp2 = rax; // Caller save. 3586 const Register tmp3 = WIN64_ONLY(r12) NOT_WIN64(r9); // Windows: Callee save. Linux: Caller save. 3587 const Register tmp4 = WIN64_ONLY(r13) NOT_WIN64(r10); // Windows: Callee save. Linux: Caller save. 3588 const Register tmp5 = r14; // Callee save. 3589 3590 const XMMRegister x0 = xmm0; 3591 const XMMRegister x1 = xmm1; 3592 const XMMRegister x2 = xmm2; 3593 BLOCK_COMMENT("Entry:"); 3594 __ enter(); // required for proper stackwalking of RuntimeStub frame 3595 3596 #ifdef _WIN64 3597 setup_arg_regs(4); 3598 // For windows, since last argument is on stack, we need to move it to the appropriate register. 3599 __ movl(totalNumIter, Address(rsp, 6 * wordSize)); 3600 // Save callee save registers. 3601 __ push_ppx(tmp3); 3602 __ push_ppx(tmp4); 3603 #endif 3604 __ push_ppx(tmp5); 3605 3606 // Rename temps used throughout the code 3607 const Register idx = tmp1; 3608 const Register numIterTmp = tmp2; 3609 3610 // Start idx from zero. 3611 __ xorl(idx, idx); 3612 // Compute interior pointer for new array. We do this so that we can use same index for both old and new arrays. 3613 __ lea(newArr, Address(newArr, newIdx, Address::times_4)); 3614 __ movl(numIterTmp, totalNumIter); 3615 3616 // If vectorization is enabled, check if the number of iterations is at least 64 3617 // If not, then go to ShiftTwo shifting two numbers at a time 3618 if (VM_Version::supports_avx512_vbmi2()) { 3619 __ cmpl(totalNumIter, (AVX3Threshold/64)); 3620 __ jcc(Assembler::less, ShiftTwo); 3621 3622 if (AVX3Threshold < 16 * 64) { 3623 __ cmpl(totalNumIter, 16); 3624 __ jcc(Assembler::less, ShiftTwo); 3625 } 3626 __ evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); 3627 __ subl(numIterTmp, 16); 3628 __ BIND(Shift512Loop); 3629 __ evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); 3630 __ evmovdqul(x2, Address(oldArr, idx, Address::times_4, 0x4), Assembler::AVX_512bit); 3631 __ vpshldvd(x1, x2, x0, Assembler::AVX_512bit); 3632 __ evmovdqul(Address(newArr, idx, Address::times_4), x1, Assembler::AVX_512bit); 3633 __ addl(idx, 16); 3634 __ subl(numIterTmp, 16); 3635 __ jcc(Assembler::greaterEqual, Shift512Loop); 3636 __ addl(numIterTmp, 16); 3637 } 3638 __ BIND(ShiftTwo); 3639 __ cmpl(totalNumIter, 1); 3640 __ jcc(Assembler::less, Exit); 3641 __ movl(tmp3, Address(oldArr, idx, Address::times_4)); 3642 __ subl(numIterTmp, 2); 3643 __ jcc(Assembler::less, ShiftOne); 3644 3645 __ BIND(ShiftTwoLoop); 3646 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3647 __ movl(tmp5, Address(oldArr, idx, Address::times_4, 0x8)); 3648 __ shldl(tmp3, tmp4); 3649 __ shldl(tmp4, tmp5); 3650 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3651 __ movl(Address(newArr, idx, Address::times_4, 0x4), tmp4); 3652 __ movl(tmp3, tmp5); 3653 __ addl(idx, 2); 3654 __ subl(numIterTmp, 2); 3655 __ jcc(Assembler::greaterEqual, ShiftTwoLoop); 3656 3657 // Do the last iteration 3658 __ BIND(ShiftOne); 3659 __ addl(numIterTmp, 2); 3660 __ cmpl(numIterTmp, 1); 3661 __ jcc(Assembler::less, Exit); 3662 __ movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); 3663 __ shldl(tmp3, tmp4); 3664 __ movl(Address(newArr, idx, Address::times_4), tmp3); 3665 3666 __ BIND(Exit); 3667 __ vzeroupper(); 3668 // Restore callee save registers. 3669 __ pop_ppx(tmp5); 3670 #ifdef _WIN64 3671 __ pop_ppx(tmp4); 3672 __ pop_ppx(tmp3); 3673 restore_arg_regs(); 3674 #endif 3675 __ leave(); // required for proper stackwalking of RuntimeStub frame 3676 __ ret(0); 3677 3678 return start; 3679 } 3680 3681 void StubGenerator::generate_libm_stubs() { 3682 if (UseLibmIntrinsic && InlineIntrinsics) { 3683 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) { 3684 StubRoutines::_dsin = generate_libmSin(); // from stubGenerator_x86_64_sin.cpp 3685 } 3686 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { 3687 StubRoutines::_dcos = generate_libmCos(); // from stubGenerator_x86_64_cos.cpp 3688 } 3689 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) { 3690 StubRoutines::_dtan = generate_libmTan(); // from stubGenerator_x86_64_tan.cpp 3691 } 3692 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsinh)) { 3693 StubRoutines::_dsinh = generate_libmSinh(); // from stubGenerator_x86_64_sinh.cpp 3694 } 3695 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtanh)) { 3696 StubRoutines::_dtanh = generate_libmTanh(); // from stubGenerator_x86_64_tanh.cpp 3697 } 3698 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcbrt)) { 3699 StubRoutines::_dcbrt = generate_libmCbrt(); // from stubGenerator_x86_64_cbrt.cpp 3700 } 3701 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) { 3702 StubRoutines::_dexp = generate_libmExp(); // from stubGenerator_x86_64_exp.cpp 3703 } 3704 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) { 3705 StubRoutines::_dpow = generate_libmPow(); // from stubGenerator_x86_64_pow.cpp 3706 } 3707 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) { 3708 StubRoutines::_dlog = generate_libmLog(); // from stubGenerator_x86_64_log.cpp 3709 } 3710 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) { 3711 StubRoutines::_dlog10 = generate_libmLog10(); // from stubGenerator_x86_64_log.cpp 3712 } 3713 } 3714 } 3715 3716 /** 3717 * Arguments: 3718 * 3719 * Input: 3720 * c_rarg0 - float16 jshort 3721 * 3722 * Output: 3723 * xmm0 - float 3724 */ 3725 address StubGenerator::generate_float16ToFloat() { 3726 StubId stub_id = StubId::stubgen_hf2f_id; 3727 StubCodeMark mark(this, stub_id); 3728 3729 address start = __ pc(); 3730 3731 BLOCK_COMMENT("Entry:"); 3732 // No need for RuntimeStub frame since it is called only during JIT compilation 3733 3734 // Load value into xmm0 and convert 3735 __ flt16_to_flt(xmm0, c_rarg0); 3736 3737 __ ret(0); 3738 3739 return start; 3740 } 3741 3742 /** 3743 * Arguments: 3744 * 3745 * Input: 3746 * xmm0 - float 3747 * 3748 * Output: 3749 * rax - float16 jshort 3750 */ 3751 address StubGenerator::generate_floatToFloat16() { 3752 StubId stub_id = StubId::stubgen_f2hf_id; 3753 StubCodeMark mark(this, stub_id); 3754 3755 address start = __ pc(); 3756 3757 BLOCK_COMMENT("Entry:"); 3758 // No need for RuntimeStub frame since it is called only during JIT compilation 3759 3760 // Convert and put result into rax 3761 __ flt_to_flt16(rax, xmm0, xmm1); 3762 3763 __ ret(0); 3764 3765 return start; 3766 } 3767 3768 address StubGenerator::generate_cont_thaw(StubId stub_id) { 3769 if (!Continuations::enabled()) return nullptr; 3770 3771 bool return_barrier; 3772 bool return_barrier_exception; 3773 Continuation::thaw_kind kind; 3774 3775 switch (stub_id) { 3776 case StubId::stubgen_cont_thaw_id: 3777 return_barrier = false; 3778 return_barrier_exception = false; 3779 kind = Continuation::thaw_top; 3780 break; 3781 case StubId::stubgen_cont_returnBarrier_id: 3782 return_barrier = true; 3783 return_barrier_exception = false; 3784 kind = Continuation::thaw_return_barrier; 3785 break; 3786 case StubId::stubgen_cont_returnBarrierExc_id: 3787 return_barrier = true; 3788 return_barrier_exception = true; 3789 kind = Continuation::thaw_return_barrier_exception; 3790 break; 3791 default: 3792 ShouldNotReachHere(); 3793 } 3794 StubCodeMark mark(this, stub_id); 3795 address start = __ pc(); 3796 3797 // TODO: Handle Valhalla return types. May require generating different return barriers. 3798 3799 if (!return_barrier) { 3800 // Pop return address. If we don't do this, we get a drift, 3801 // where the bottom-most frozen frame continuously grows. 3802 __ pop(c_rarg3); 3803 } else { 3804 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3805 } 3806 3807 #ifdef ASSERT 3808 { 3809 Label L_good_sp; 3810 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3811 __ jcc(Assembler::equal, L_good_sp); 3812 __ stop("Incorrect rsp at thaw entry"); 3813 __ BIND(L_good_sp); 3814 } 3815 #endif // ASSERT 3816 3817 if (return_barrier) { 3818 // Preserve possible return value from a method returning to the return barrier. 3819 __ push_ppx(rax); 3820 __ push_d(xmm0); 3821 } 3822 3823 __ movptr(c_rarg0, r15_thread); 3824 __ movptr(c_rarg1, (return_barrier ? 1 : 0)); 3825 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2); 3826 __ movptr(rbx, rax); 3827 3828 if (return_barrier) { 3829 // Restore return value from a method returning to the return barrier. 3830 // No safepoint in the call to thaw, so even an oop return value should be OK. 3831 __ pop_d(xmm0); 3832 __ pop_ppx(rax); 3833 } 3834 3835 #ifdef ASSERT 3836 { 3837 Label L_good_sp; 3838 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3839 __ jcc(Assembler::equal, L_good_sp); 3840 __ stop("Incorrect rsp after prepare thaw"); 3841 __ BIND(L_good_sp); 3842 } 3843 #endif // ASSERT 3844 3845 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames 3846 Label L_thaw_success; 3847 __ testptr(rbx, rbx); 3848 __ jccb(Assembler::notZero, L_thaw_success); 3849 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry())); 3850 __ bind(L_thaw_success); 3851 3852 // Make room for the thawed frames and align the stack. 3853 __ subptr(rsp, rbx); 3854 __ andptr(rsp, -StackAlignmentInBytes); 3855 3856 if (return_barrier) { 3857 // Preserve possible return value from a method returning to the return barrier. (Again.) 3858 __ push_ppx(rax); 3859 __ push_d(xmm0); 3860 } 3861 3862 // If we want, we can templatize thaw by kind, and have three different entries. 3863 __ movptr(c_rarg0, r15_thread); 3864 __ movptr(c_rarg1, kind); 3865 __ call_VM_leaf(Continuation::thaw_entry(), 2); 3866 __ movptr(rbx, rax); 3867 3868 if (return_barrier) { 3869 // Restore return value from a method returning to the return barrier. (Again.) 3870 // No safepoint in the call to thaw, so even an oop return value should be OK. 3871 __ pop_d(xmm0); 3872 __ pop_ppx(rax); 3873 } else { 3874 // Return 0 (success) from doYield. 3875 __ xorptr(rax, rax); 3876 } 3877 3878 // After thawing, rbx is the SP of the yielding frame. 3879 // Move there, and then to saved RBP slot. 3880 __ movptr(rsp, rbx); 3881 __ subptr(rsp, 2*wordSize); 3882 3883 if (return_barrier_exception) { 3884 __ movptr(c_rarg0, r15_thread); 3885 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address 3886 3887 // rax still holds the original exception oop, save it before the call 3888 __ push_ppx(rax); 3889 3890 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2); 3891 __ movptr(rbx, rax); 3892 3893 // Continue at exception handler: 3894 // rax: exception oop 3895 // rbx: exception handler 3896 // rdx: exception pc 3897 __ pop_ppx(rax); 3898 __ verify_oop(rax); 3899 __ pop(rbp); // pop out RBP here too 3900 __ pop(rdx); 3901 __ jmp(rbx); 3902 } else { 3903 // We are "returning" into the topmost thawed frame; see Thaw::push_return_frame 3904 __ pop(rbp); 3905 __ ret(0); 3906 } 3907 3908 return start; 3909 } 3910 3911 address StubGenerator::generate_cont_thaw() { 3912 return generate_cont_thaw(StubId::stubgen_cont_thaw_id); 3913 } 3914 3915 // TODO: will probably need multiple return barriers depending on return type 3916 3917 address StubGenerator::generate_cont_returnBarrier() { 3918 return generate_cont_thaw(StubId::stubgen_cont_returnBarrier_id); 3919 } 3920 3921 address StubGenerator::generate_cont_returnBarrier_exception() { 3922 return generate_cont_thaw(StubId::stubgen_cont_returnBarrierExc_id); 3923 } 3924 3925 address StubGenerator::generate_cont_preempt_stub() { 3926 if (!Continuations::enabled()) return nullptr; 3927 StubId stub_id = StubId::stubgen_cont_preempt_id; 3928 StubCodeMark mark(this, stub_id); 3929 address start = __ pc(); 3930 3931 __ reset_last_Java_frame(true); 3932 3933 // Set rsp to enterSpecial frame, i.e. remove all frames copied into the heap. 3934 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 3935 3936 Label preemption_cancelled; 3937 __ movbool(rscratch1, Address(r15_thread, JavaThread::preemption_cancelled_offset())); 3938 __ testbool(rscratch1); 3939 __ jcc(Assembler::notZero, preemption_cancelled); 3940 3941 // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. 3942 SharedRuntime::continuation_enter_cleanup(_masm); 3943 __ pop(rbp); 3944 __ ret(0); 3945 3946 // We acquired the monitor after freezing the frames so call thaw to continue execution. 3947 __ bind(preemption_cancelled); 3948 __ movbool(Address(r15_thread, JavaThread::preemption_cancelled_offset()), false); 3949 __ lea(rbp, Address(rsp, checked_cast<int32_t>(ContinuationEntry::size()))); 3950 __ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); 3951 __ jmp(rscratch1); 3952 3953 return start; 3954 } 3955 3956 // exception handler for upcall stubs 3957 address StubGenerator::generate_upcall_stub_exception_handler() { 3958 StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; 3959 StubCodeMark mark(this, stub_id); 3960 address start = __ pc(); 3961 3962 // native caller has no idea how to handle exceptions 3963 // we just crash here. Up to callee to catch exceptions. 3964 __ verify_oop(rax); 3965 __ vzeroupper(); 3966 __ mov(c_rarg0, rax); 3967 __ andptr(rsp, -StackAlignmentInBytes); // align stack as required by ABI 3968 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 3969 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception))); 3970 __ should_not_reach_here(); 3971 3972 return start; 3973 } 3974 3975 // load Method* target of MethodHandle 3976 // j_rarg0 = jobject receiver 3977 // rbx = result 3978 address StubGenerator::generate_upcall_stub_load_target() { 3979 StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; 3980 StubCodeMark mark(this, stub_id); 3981 address start = __ pc(); 3982 3983 __ resolve_global_jobject(j_rarg0, rscratch1); 3984 // Load target method from receiver 3985 __ load_heap_oop(rbx, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1); 3986 __ load_heap_oop(rbx, Address(rbx, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1); 3987 __ load_heap_oop(rbx, Address(rbx, java_lang_invoke_MemberName::method_offset()), rscratch1); 3988 __ access_load_at(T_ADDRESS, IN_HEAP, rbx, 3989 Address(rbx, java_lang_invoke_ResolvedMethodName::vmtarget_offset()), 3990 noreg); 3991 __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized 3992 3993 __ ret(0); 3994 3995 return start; 3996 } 3997 3998 void StubGenerator::generate_lookup_secondary_supers_table_stub() { 3999 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; 4000 StubCodeMark mark(this, stub_id); 4001 4002 const Register 4003 r_super_klass = rax, 4004 r_sub_klass = rsi, 4005 result = rdi; 4006 4007 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { 4008 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); 4009 __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, 4010 rdx, rcx, rbx, r11, // temps 4011 result, 4012 slot); 4013 __ ret(0); 4014 } 4015 } 4016 4017 // Slow path implementation for UseSecondarySupersTable. 4018 address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { 4019 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; 4020 StubCodeMark mark(this, stub_id); 4021 4022 address start = __ pc(); 4023 4024 const Register 4025 r_super_klass = rax, 4026 r_array_base = rbx, 4027 r_array_index = rdx, 4028 r_sub_klass = rsi, 4029 r_bitmap = r11, 4030 result = rdi; 4031 4032 Label L_success; 4033 __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, 4034 rcx, rdi, // temps 4035 &L_success); 4036 // bind(L_failure); 4037 __ movl(result, 1); 4038 __ ret(0); 4039 4040 __ bind(L_success); 4041 __ movl(result, 0); 4042 __ ret(0); 4043 4044 return start; 4045 } 4046 4047 void StubGenerator::create_control_words() { 4048 // Round to nearest, 64-bit mode, exceptions masked, flags specialized 4049 StubRoutines::x86::_mxcsr_std = EnableX86ECoreOpts ? 0x1FBF : 0x1F80; 4050 // Round to zero, 64-bit mode, exceptions masked, flags specialized 4051 StubRoutines::x86::_mxcsr_rz = EnableX86ECoreOpts ? 0x7FBF : 0x7F80; 4052 } 4053 4054 // Initialization 4055 void StubGenerator::generate_preuniverse_stubs() { 4056 // atomic calls 4057 StubRoutines::_fence_entry = generate_orderaccess_fence(); 4058 } 4059 4060 void StubGenerator::generate_initial_stubs() { 4061 // Generates all stubs and initializes the entry points 4062 4063 // This platform-specific settings are needed by generate_call_stub() 4064 create_control_words(); 4065 4066 // Initialize table for unsafe copy memeory check. 4067 if (UnsafeMemoryAccess::_table == nullptr) { 4068 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory 4069 } 4070 4071 // entry points that exist in all platforms Note: This is code 4072 // that could be shared among different platforms - however the 4073 // benefit seems to be smaller than the disadvantage of having a 4074 // much more complicated generator structure. See also comment in 4075 // stubRoutines.hpp. 4076 4077 StubRoutines::_forward_exception_entry = generate_forward_exception(); 4078 4079 StubRoutines::_call_stub_entry = 4080 generate_call_stub(StubRoutines::_call_stub_return_address); 4081 4082 // is referenced by megamorphic call 4083 StubRoutines::_catch_exception_entry = generate_catch_exception(); 4084 4085 // platform dependent 4086 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr(); 4087 4088 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); 4089 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); 4090 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup(); 4091 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup(); 4092 4093 StubRoutines::x86::_float_sign_mask = generate_fp_mask(StubId::stubgen_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF); 4094 StubRoutines::x86::_float_sign_flip = generate_fp_mask(StubId::stubgen_float_sign_flip_id, 0x8000000080000000); 4095 StubRoutines::x86::_double_sign_mask = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF); 4096 StubRoutines::x86::_double_sign_flip = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000); 4097 4098 if (UseCRC32Intrinsics) { 4099 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32(); 4100 } 4101 4102 if (UseCRC32CIntrinsics) { 4103 bool supports_clmul = VM_Version::supports_clmul(); 4104 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul); 4105 } 4106 4107 if (VM_Version::supports_float16()) { 4108 // For results consistency both intrinsics should be enabled. 4109 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here. 4110 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) && 4111 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) { 4112 StubRoutines::_hf2f = generate_float16ToFloat(); 4113 StubRoutines::_f2hf = generate_floatToFloat16(); 4114 } 4115 } 4116 4117 generate_libm_stubs(); 4118 4119 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp 4120 } 4121 4122 void StubGenerator::generate_continuation_stubs() { 4123 // Continuation stubs: 4124 StubRoutines::_cont_thaw = generate_cont_thaw(); 4125 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); 4126 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); 4127 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); 4128 } 4129 4130 void StubGenerator::generate_final_stubs() { 4131 // Generates the rest of stubs and initializes the entry points 4132 4133 // support for verify_oop (must happen after universe_init) 4134 if (VerifyOops) { 4135 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); 4136 } 4137 4138 // arraycopy stubs used by compilers 4139 generate_arraycopy_stubs(); 4140 4141 StubRoutines::_method_entry_barrier = generate_method_entry_barrier(); 4142 4143 #ifdef COMPILER2 4144 if (UseSecondarySupersTable) { 4145 StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub(); 4146 if (! InlineSecondarySupersTest) { 4147 generate_lookup_secondary_supers_table_stub(); 4148 } 4149 } 4150 #endif // COMPILER2 4151 4152 if (UseVectorizedMismatchIntrinsic) { 4153 StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch(); 4154 } 4155 4156 StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler(); 4157 StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target(); 4158 } 4159 4160 void StubGenerator::generate_compiler_stubs() { 4161 #if COMPILER2_OR_JVMCI 4162 4163 // Entry points that are C2 compiler specific. 4164 4165 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask(StubId::stubgen_vector_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF); 4166 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask(StubId::stubgen_vector_float_sign_flip_id, 0x8000000080000000); 4167 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask(StubId::stubgen_vector_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF); 4168 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask(StubId::stubgen_vector_double_sign_flip_id, 0x8000000000000000); 4169 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask(StubId::stubgen_vector_all_bits_set_id, 0xFFFFFFFFFFFFFFFF); 4170 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask(StubId::stubgen_vector_int_mask_cmp_bits_id, 0x0000000100000001); 4171 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask(StubId::stubgen_vector_short_to_byte_mask_id, 0x00ff00ff00ff00ff); 4172 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask(); 4173 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask(StubId::stubgen_vector_int_to_byte_mask_id, 0x000000ff000000ff); 4174 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask(StubId::stubgen_vector_int_to_short_mask_id, 0x0000ffff0000ffff); 4175 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32(StubId::stubgen_vector_32_bit_mask_id, Assembler::AVX_512bit, 4176 0xFFFFFFFF, 0, 0, 0); 4177 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32(StubId::stubgen_vector_64_bit_mask_id, Assembler::AVX_512bit, 4178 0xFFFFFFFF, 0xFFFFFFFF, 0, 0); 4179 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_int_shuffle_mask_id, 0x0302010003020100); 4180 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask(); 4181 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_short_shuffle_mask_id, 0x0100010001000100); 4182 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_long_shuffle_mask_id, 0x0000000100000000); 4183 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask(StubId::stubgen_vector_long_sign_mask_id, 0x8000000000000000); 4184 StubRoutines::x86::_vector_iota_indices = generate_iota_indices(); 4185 StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut(); 4186 StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut(); 4187 StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long(); 4188 StubRoutines::x86::_vector_reverse_byte_perm_mask_int = generate_vector_reverse_byte_perm_mask_int(); 4189 StubRoutines::x86::_vector_reverse_byte_perm_mask_short = generate_vector_reverse_byte_perm_mask_short(); 4190 4191 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512vl()) { 4192 StubRoutines::x86::_compress_perm_table32 = generate_compress_perm_table(StubId::stubgen_compress_perm_table32_id); 4193 StubRoutines::x86::_compress_perm_table64 = generate_compress_perm_table(StubId::stubgen_compress_perm_table64_id); 4194 StubRoutines::x86::_expand_perm_table32 = generate_expand_perm_table(StubId::stubgen_expand_perm_table32_id); 4195 StubRoutines::x86::_expand_perm_table64 = generate_expand_perm_table(StubId::stubgen_expand_perm_table64_id); 4196 } 4197 4198 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) { 4199 // lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight. 4200 StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut(); 4201 } 4202 4203 generate_aes_stubs(); 4204 4205 generate_ghash_stubs(); 4206 4207 generate_chacha_stubs(); 4208 4209 generate_kyber_stubs(); 4210 4211 generate_dilithium_stubs(); 4212 4213 generate_sha3_stubs(); 4214 4215 // data cache line writeback 4216 StubRoutines::_data_cache_writeback = generate_data_cache_writeback(); 4217 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync(); 4218 4219 #ifdef COMPILER2 4220 if ((UseAVX == 2) && EnableX86ECoreOpts) { 4221 generate_string_indexof(StubRoutines::_string_indexof_array); 4222 } 4223 #endif 4224 4225 if (UseAdler32Intrinsics) { 4226 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32(); 4227 } 4228 4229 if (UsePoly1305Intrinsics) { 4230 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks(); 4231 } 4232 4233 if (UseIntPolyIntrinsics) { 4234 StubRoutines::_intpoly_montgomeryMult_P256 = generate_intpoly_montgomeryMult_P256(); 4235 StubRoutines::_intpoly_assign = generate_intpoly_assign(); 4236 } 4237 4238 if (UseMD5Intrinsics) { 4239 StubRoutines::_md5_implCompress = generate_md5_implCompress(StubId::stubgen_md5_implCompress_id); 4240 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubId::stubgen_md5_implCompressMB_id); 4241 } 4242 4243 if (UseSHA1Intrinsics) { 4244 StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask(); 4245 StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask(); 4246 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(StubId::stubgen_sha1_implCompress_id); 4247 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(StubId::stubgen_sha1_implCompressMB_id); 4248 } 4249 4250 if (UseSHA256Intrinsics) { 4251 StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; 4252 char* dst = (char*)StubRoutines::x86::_k256_W; 4253 char* src = (char*)StubRoutines::x86::_k256; 4254 for (int ii = 0; ii < 16; ++ii) { 4255 memcpy(dst + 32 * ii, src + 16 * ii, 16); 4256 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); 4257 } 4258 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; 4259 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); 4260 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id); 4261 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id); 4262 } 4263 4264 if (UseSHA512Intrinsics) { 4265 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; 4266 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); 4267 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id); 4268 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id); 4269 } 4270 4271 if (UseBASE64Intrinsics) { 4272 if(VM_Version::supports_avx2()) { 4273 StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr(); 4274 StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr(); 4275 StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr(); 4276 StubRoutines::x86::_avx2_decode_tables_base64 = base64_AVX2_decode_tables_addr(); 4277 StubRoutines::x86::_avx2_decode_lut_tables_base64 = base64_AVX2_decode_LUT_tables_addr(); 4278 } 4279 StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr(); 4280 if (VM_Version::supports_avx512_vbmi()) { 4281 StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr(); 4282 StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr(); 4283 StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr(); 4284 StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr(); 4285 StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr(); 4286 StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr(); 4287 StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr(); 4288 StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr(); 4289 StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr(); 4290 } 4291 StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr(); 4292 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock(); 4293 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); 4294 } 4295 4296 #ifdef COMPILER2 4297 if (UseMultiplyToLenIntrinsic) { 4298 StubRoutines::_multiplyToLen = generate_multiplyToLen(); 4299 } 4300 if (UseSquareToLenIntrinsic) { 4301 StubRoutines::_squareToLen = generate_squareToLen(); 4302 } 4303 if (UseMulAddIntrinsic) { 4304 StubRoutines::_mulAdd = generate_mulAdd(); 4305 } 4306 if (VM_Version::supports_avx512_vbmi2()) { 4307 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); 4308 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); 4309 } 4310 if (UseMontgomeryMultiplyIntrinsic) { 4311 StubRoutines::_montgomeryMultiply 4312 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply); 4313 } 4314 if (UseMontgomerySquareIntrinsic) { 4315 StubRoutines::_montgomerySquare 4316 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square); 4317 } 4318 4319 // Load x86_64_sort library on supported hardware to enable SIMD sort and partition intrinsics 4320 4321 if (VM_Version::supports_avx512dq() || VM_Version::supports_avx2()) { 4322 void *libsimdsort = nullptr; 4323 char ebuf_[1024]; 4324 char dll_name_simd_sort[JVM_MAXPATHLEN]; 4325 if (os::dll_locate_lib(dll_name_simd_sort, sizeof(dll_name_simd_sort), Arguments::get_dll_dir(), "simdsort")) { 4326 libsimdsort = os::dll_load(dll_name_simd_sort, ebuf_, sizeof ebuf_); 4327 } 4328 // Get addresses for SIMD sort and partition routines 4329 if (libsimdsort != nullptr) { 4330 log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "simdsort" JNI_LIB_SUFFIX, p2i(libsimdsort)); 4331 4332 os::snprintf_checked(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512_simd_sort() ? "avx512_sort" : "avx2_sort"); 4333 StubRoutines::_array_sort = (address)os::dll_lookup(libsimdsort, ebuf_); 4334 4335 os::snprintf_checked(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512_simd_sort() ? "avx512_partition" : "avx2_partition"); 4336 StubRoutines::_array_partition = (address)os::dll_lookup(libsimdsort, ebuf_); 4337 } 4338 } 4339 4340 #endif // COMPILER2 4341 #endif // COMPILER2_OR_JVMCI 4342 } 4343 4344 StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { 4345 switch(blob_id) { 4346 case BlobId::stubgen_preuniverse_id: 4347 generate_preuniverse_stubs(); 4348 break; 4349 case BlobId::stubgen_initial_id: 4350 generate_initial_stubs(); 4351 break; 4352 case BlobId::stubgen_continuation_id: 4353 generate_continuation_stubs(); 4354 break; 4355 case BlobId::stubgen_compiler_id: 4356 generate_compiler_stubs(); 4357 break; 4358 case BlobId::stubgen_final_id: 4359 generate_final_stubs(); 4360 break; 4361 default: 4362 fatal("unexpected blob id: %s", StubInfo::name(blob_id)); 4363 break; 4364 }; 4365 } 4366 4367 void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { 4368 StubGenerator g(code, blob_id); 4369 } 4370 4371 #undef __ --- EOF ---