1 /* 2 * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef _WINDOWS 26 #include "alloca.h" 27 #endif 28 #include "asm/macroAssembler.hpp" 29 #include "asm/macroAssembler.inline.hpp" 30 #include "classfile/symbolTable.hpp" 31 #include "code/aotCodeCache.hpp" 32 #include "code/compiledIC.hpp" 33 #include "code/debugInfoRec.hpp" 34 #include "code/nativeInst.hpp" 35 #include "code/vtableStubs.hpp" 36 #include "compiler/oopMap.hpp" 37 #include "gc/shared/collectedHeap.hpp" 38 #include "gc/shared/gcLocker.hpp" 39 #include "gc/shared/barrierSet.hpp" 40 #include "gc/shared/barrierSetAssembler.hpp" 41 #include "interpreter/interpreter.hpp" 42 #include "logging/log.hpp" 43 #include "memory/resourceArea.hpp" 44 #include "memory/universe.hpp" 45 #include "oops/klass.inline.hpp" 46 #include "oops/method.inline.hpp" 47 #include "prims/methodHandles.hpp" 48 #include "runtime/continuation.hpp" 49 #include "runtime/continuationEntry.inline.hpp" 50 #include "runtime/globals.hpp" 51 #include "runtime/jniHandles.hpp" 52 #include "runtime/safepointMechanism.hpp" 53 #include "runtime/sharedRuntime.hpp" 54 #include "runtime/signature.hpp" 55 #include "runtime/stubRoutines.hpp" 56 #include "runtime/timerTrace.hpp" 57 #include "runtime/vframeArray.hpp" 58 #include "runtime/vm_version.hpp" 59 #include "utilities/align.hpp" 60 #include "utilities/checkedCast.hpp" 61 #include "utilities/formatBuffer.hpp" 62 #include "vmreg_x86.inline.hpp" 63 #ifdef COMPILER1 64 #include "c1/c1_Runtime1.hpp" 65 #endif 66 #ifdef COMPILER2 67 #include "opto/runtime.hpp" 68 #endif 69 #if INCLUDE_JVMCI 70 #include "jvmci/jvmciJavaClasses.hpp" 71 #endif 72 73 #define __ masm-> 74 75 #ifdef PRODUCT 76 #define BLOCK_COMMENT(str) /* nothing */ 77 #else 78 #define BLOCK_COMMENT(str) __ block_comment(str) 79 #endif // PRODUCT 80 81 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size; 82 83 class RegisterSaver { 84 // Capture info about frame layout. Layout offsets are in jint 85 // units because compiler frame slots are jints. 86 #define XSAVE_AREA_BEGIN 160 87 #define XSAVE_AREA_YMM_BEGIN 576 88 #define XSAVE_AREA_EGPRS 960 89 #define XSAVE_AREA_OPMASK_BEGIN 1088 90 #define XSAVE_AREA_ZMM_BEGIN 1152 91 #define XSAVE_AREA_UPPERBANK 1664 92 #define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off 93 #define DEF_YMM_OFFS(regnum) ymm ## regnum ## _off = ymm_off + (regnum)*16/BytesPerInt, ymm ## regnum ## H_off 94 #define DEF_ZMM_OFFS(regnum) zmm ## regnum ## _off = zmm_off + (regnum)*32/BytesPerInt, zmm ## regnum ## H_off 95 #define DEF_OPMASK_OFFS(regnum) opmask ## regnum ## _off = opmask_off + (regnum)*8/BytesPerInt, opmask ## regnum ## H_off 96 #define DEF_ZMM_UPPER_OFFS(regnum) zmm ## regnum ## _off = zmm_upper_off + (regnum-16)*64/BytesPerInt, zmm ## regnum ## H_off 97 enum layout { 98 fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt, // fxsave save area 99 xmm_off = fpu_state_off + XSAVE_AREA_BEGIN/BytesPerInt, // offset in fxsave save area 100 DEF_XMM_OFFS(0), 101 DEF_XMM_OFFS(1), 102 // 2..15 are implied in range usage 103 ymm_off = xmm_off + (XSAVE_AREA_YMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt, 104 DEF_YMM_OFFS(0), 105 DEF_YMM_OFFS(1), 106 r16_off = xmm_off + (XSAVE_AREA_EGPRS - XSAVE_AREA_BEGIN)/BytesPerInt, 107 r16H_off, 108 r17_off, r17H_off, 109 r18_off, r18H_off, 110 r19_off, r19H_off, 111 r20_off, r20H_off, 112 r21_off, r21H_off, 113 r22_off, r22H_off, 114 r23_off, r23H_off, 115 r24_off, r24H_off, 116 r25_off, r25H_off, 117 r26_off, r26H_off, 118 r27_off, r27H_off, 119 r28_off, r28H_off, 120 r29_off, r29H_off, 121 r30_off, r30H_off, 122 r31_off, r31H_off, 123 opmask_off = xmm_off + (XSAVE_AREA_OPMASK_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt, 124 DEF_OPMASK_OFFS(0), 125 DEF_OPMASK_OFFS(1), 126 // 2..7 are implied in range usage 127 zmm_off = xmm_off + (XSAVE_AREA_ZMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt, 128 DEF_ZMM_OFFS(0), 129 DEF_ZMM_OFFS(1), 130 zmm_upper_off = xmm_off + (XSAVE_AREA_UPPERBANK - XSAVE_AREA_BEGIN)/BytesPerInt, 131 DEF_ZMM_UPPER_OFFS(16), 132 DEF_ZMM_UPPER_OFFS(17), 133 // 18..31 are implied in range usage 134 fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt), 135 fpu_stateH_end, 136 r15_off, r15H_off, 137 r14_off, r14H_off, 138 r13_off, r13H_off, 139 r12_off, r12H_off, 140 r11_off, r11H_off, 141 r10_off, r10H_off, 142 r9_off, r9H_off, 143 r8_off, r8H_off, 144 rdi_off, rdiH_off, 145 rsi_off, rsiH_off, 146 ignore_off, ignoreH_off, // extra copy of rbp 147 rsp_off, rspH_off, 148 rbx_off, rbxH_off, 149 rdx_off, rdxH_off, 150 rcx_off, rcxH_off, 151 rax_off, raxH_off, 152 // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state 153 align_off, alignH_off, 154 flags_off, flagsH_off, 155 // The frame sender code expects that rbp will be in the "natural" place and 156 // will override any oopMap setting for it. We must therefore force the layout 157 // so that it agrees with the frame sender code. 158 rbp_off, rbpH_off, // copy of rbp we will restore 159 return_off, returnH_off, // slot for return address 160 reg_save_size // size in compiler stack slots 161 }; 162 163 public: 164 static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors); 165 static void restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors = false); 166 167 // Offsets into the register save area 168 // Used by deoptimization when it is managing result register 169 // values on its own 170 171 static int rax_offset_in_bytes(void) { return BytesPerInt * rax_off; } 172 static int rdx_offset_in_bytes(void) { return BytesPerInt * rdx_off; } 173 static int rbx_offset_in_bytes(void) { return BytesPerInt * rbx_off; } 174 static int r15_offset_in_bytes(void) { return BytesPerInt * r15_off; } 175 static int xmm0_offset_in_bytes(void) { return BytesPerInt * xmm0_off; } 176 static int return_offset_in_bytes(void) { return BytesPerInt * return_off; } 177 178 // During deoptimization only the result registers need to be restored, 179 // all the other values have already been extracted. 180 static void restore_result_registers(MacroAssembler* masm); 181 }; 182 183 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors) { 184 int off = 0; 185 int num_xmm_regs = XMMRegister::available_xmm_registers(); 186 #if COMPILER2_OR_JVMCI 187 if (save_wide_vectors && UseAVX == 0) { 188 save_wide_vectors = false; // vectors larger than 16 byte long are supported only with AVX 189 } 190 assert(!save_wide_vectors || MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported"); 191 #else 192 save_wide_vectors = false; // vectors are generated only by C2 and JVMCI 193 #endif 194 195 // Always make the frame size 16-byte aligned, both vector and non vector stacks are always allocated 196 int frame_size_in_bytes = align_up(reg_save_size*BytesPerInt, num_xmm_regs); 197 // OopMap frame size is in compiler stack slots (jint's) not bytes or words 198 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt; 199 // CodeBlob frame size is in words. 200 int frame_size_in_words = frame_size_in_bytes / wordSize; 201 *total_frame_words = frame_size_in_words; 202 203 // Save registers, fpu state, and flags. 204 // We assume caller has already pushed the return address onto the 205 // stack, so rsp is 8-byte aligned here. 206 // We push rpb twice in this sequence because we want the real rbp 207 // to be under the return like a normal enter. 208 209 __ enter(); // rsp becomes 16-byte aligned here 210 __ pushf(); 211 // Make sure rsp stays 16-byte aligned 212 __ subq(rsp, 8); 213 // Push CPU state in multiple of 16 bytes 214 __ save_legacy_gprs(); 215 __ push_FPU_state(); 216 217 218 // push cpu state handles this on EVEX enabled targets 219 if (save_wide_vectors) { 220 // Save upper half of YMM registers(0..15) 221 int base_addr = XSAVE_AREA_YMM_BEGIN; 222 for (int n = 0; n < 16; n++) { 223 __ vextractf128_high(Address(rsp, base_addr+n*16), as_XMMRegister(n)); 224 } 225 if (VM_Version::supports_evex()) { 226 // Save upper half of ZMM registers(0..15) 227 base_addr = XSAVE_AREA_ZMM_BEGIN; 228 for (int n = 0; n < 16; n++) { 229 __ vextractf64x4_high(Address(rsp, base_addr+n*32), as_XMMRegister(n)); 230 } 231 // Save full ZMM registers(16..num_xmm_regs) 232 base_addr = XSAVE_AREA_UPPERBANK; 233 off = 0; 234 int vector_len = Assembler::AVX_512bit; 235 for (int n = 16; n < num_xmm_regs; n++) { 236 __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len); 237 } 238 #if COMPILER2_OR_JVMCI 239 base_addr = XSAVE_AREA_OPMASK_BEGIN; 240 off = 0; 241 for(int n = 0; n < KRegister::number_of_registers; n++) { 242 __ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n)); 243 } 244 #endif 245 } 246 } else { 247 if (VM_Version::supports_evex()) { 248 // Save upper bank of XMM registers(16..31) for scalar or 16-byte vector usage 249 int base_addr = XSAVE_AREA_UPPERBANK; 250 off = 0; 251 int vector_len = VM_Version::supports_avx512vl() ? Assembler::AVX_128bit : Assembler::AVX_512bit; 252 for (int n = 16; n < num_xmm_regs; n++) { 253 __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len); 254 } 255 #if COMPILER2_OR_JVMCI 256 base_addr = XSAVE_AREA_OPMASK_BEGIN; 257 off = 0; 258 for(int n = 0; n < KRegister::number_of_registers; n++) { 259 __ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n)); 260 } 261 #endif 262 } 263 } 264 265 #if COMPILER2_OR_JVMCI 266 if (UseAPX) { 267 int base_addr = XSAVE_AREA_EGPRS; 268 off = 0; 269 for (int n = 16; n < Register::number_of_registers; n++) { 270 __ movq(Address(rsp, base_addr+(off++*8)), as_Register(n)); 271 } 272 } 273 #endif 274 275 __ vzeroupper(); 276 if (frame::arg_reg_save_area_bytes != 0) { 277 // Allocate argument register save area 278 __ subptr(rsp, frame::arg_reg_save_area_bytes); 279 } 280 281 // Set an oopmap for the call site. This oopmap will map all 282 // oop-registers and debug-info registers as callee-saved. This 283 // will allow deoptimization at this safepoint to find all possible 284 // debug-info recordings, as well as let GC find all oops. 285 286 OopMapSet *oop_maps = new OopMapSet(); 287 OopMap* map = new OopMap(frame_size_in_slots, 0); 288 289 #define STACK_OFFSET(x) VMRegImpl::stack2reg((x)) 290 291 map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg()); 292 map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg()); 293 map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg()); 294 map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg()); 295 // rbp location is known implicitly by the frame sender code, needs no oopmap 296 // and the location where rbp was saved by is ignored 297 map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg()); 298 map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg()); 299 map->set_callee_saved(STACK_OFFSET( r8_off ), r8->as_VMReg()); 300 map->set_callee_saved(STACK_OFFSET( r9_off ), r9->as_VMReg()); 301 map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg()); 302 map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg()); 303 map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg()); 304 map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg()); 305 map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg()); 306 map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg()); 307 308 if (UseAPX) { 309 map->set_callee_saved(STACK_OFFSET( r16_off ), r16->as_VMReg()); 310 map->set_callee_saved(STACK_OFFSET( r17_off ), r17->as_VMReg()); 311 map->set_callee_saved(STACK_OFFSET( r18_off ), r18->as_VMReg()); 312 map->set_callee_saved(STACK_OFFSET( r19_off ), r19->as_VMReg()); 313 map->set_callee_saved(STACK_OFFSET( r20_off ), r20->as_VMReg()); 314 map->set_callee_saved(STACK_OFFSET( r21_off ), r21->as_VMReg()); 315 map->set_callee_saved(STACK_OFFSET( r22_off ), r22->as_VMReg()); 316 map->set_callee_saved(STACK_OFFSET( r23_off ), r23->as_VMReg()); 317 map->set_callee_saved(STACK_OFFSET( r24_off ), r24->as_VMReg()); 318 map->set_callee_saved(STACK_OFFSET( r25_off ), r25->as_VMReg()); 319 map->set_callee_saved(STACK_OFFSET( r26_off ), r26->as_VMReg()); 320 map->set_callee_saved(STACK_OFFSET( r27_off ), r27->as_VMReg()); 321 map->set_callee_saved(STACK_OFFSET( r28_off ), r28->as_VMReg()); 322 map->set_callee_saved(STACK_OFFSET( r29_off ), r29->as_VMReg()); 323 map->set_callee_saved(STACK_OFFSET( r30_off ), r30->as_VMReg()); 324 map->set_callee_saved(STACK_OFFSET( r31_off ), r31->as_VMReg()); 325 } 326 // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15, 327 // on EVEX enabled targets, we get it included in the xsave area 328 off = xmm0_off; 329 int delta = xmm1_off - off; 330 for (int n = 0; n < 16; n++) { 331 XMMRegister xmm_name = as_XMMRegister(n); 332 map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()); 333 off += delta; 334 } 335 if (UseAVX > 2) { 336 // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets 337 off = zmm16_off; 338 delta = zmm17_off - off; 339 for (int n = 16; n < num_xmm_regs; n++) { 340 XMMRegister zmm_name = as_XMMRegister(n); 341 map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()); 342 off += delta; 343 } 344 } 345 346 #if COMPILER2_OR_JVMCI 347 if (save_wide_vectors) { 348 // Save upper half of YMM registers(0..15) 349 off = ymm0_off; 350 delta = ymm1_off - ymm0_off; 351 for (int n = 0; n < 16; n++) { 352 XMMRegister ymm_name = as_XMMRegister(n); 353 map->set_callee_saved(STACK_OFFSET(off), ymm_name->as_VMReg()->next(4)); 354 off += delta; 355 } 356 if (VM_Version::supports_evex()) { 357 // Save upper half of ZMM registers(0..15) 358 off = zmm0_off; 359 delta = zmm1_off - zmm0_off; 360 for (int n = 0; n < 16; n++) { 361 XMMRegister zmm_name = as_XMMRegister(n); 362 map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next(8)); 363 off += delta; 364 } 365 } 366 } 367 #endif // COMPILER2_OR_JVMCI 368 369 // %%% These should all be a waste but we'll keep things as they were for now 370 if (true) { 371 map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next()); 372 map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next()); 373 map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next()); 374 map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next()); 375 // rbp location is known implicitly by the frame sender code, needs no oopmap 376 map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next()); 377 map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next()); 378 map->set_callee_saved(STACK_OFFSET( r8H_off ), r8->as_VMReg()->next()); 379 map->set_callee_saved(STACK_OFFSET( r9H_off ), r9->as_VMReg()->next()); 380 map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next()); 381 map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next()); 382 map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next()); 383 map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next()); 384 map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next()); 385 map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next()); 386 if (UseAPX) { 387 map->set_callee_saved(STACK_OFFSET( r16H_off ), r16->as_VMReg()->next()); 388 map->set_callee_saved(STACK_OFFSET( r17H_off ), r17->as_VMReg()->next()); 389 map->set_callee_saved(STACK_OFFSET( r18H_off ), r18->as_VMReg()->next()); 390 map->set_callee_saved(STACK_OFFSET( r19H_off ), r19->as_VMReg()->next()); 391 map->set_callee_saved(STACK_OFFSET( r20H_off ), r20->as_VMReg()->next()); 392 map->set_callee_saved(STACK_OFFSET( r21H_off ), r21->as_VMReg()->next()); 393 map->set_callee_saved(STACK_OFFSET( r22H_off ), r22->as_VMReg()->next()); 394 map->set_callee_saved(STACK_OFFSET( r23H_off ), r23->as_VMReg()->next()); 395 map->set_callee_saved(STACK_OFFSET( r24H_off ), r24->as_VMReg()->next()); 396 map->set_callee_saved(STACK_OFFSET( r25H_off ), r25->as_VMReg()->next()); 397 map->set_callee_saved(STACK_OFFSET( r26H_off ), r26->as_VMReg()->next()); 398 map->set_callee_saved(STACK_OFFSET( r27H_off ), r27->as_VMReg()->next()); 399 map->set_callee_saved(STACK_OFFSET( r28H_off ), r28->as_VMReg()->next()); 400 map->set_callee_saved(STACK_OFFSET( r29H_off ), r29->as_VMReg()->next()); 401 map->set_callee_saved(STACK_OFFSET( r30H_off ), r30->as_VMReg()->next()); 402 map->set_callee_saved(STACK_OFFSET( r31H_off ), r31->as_VMReg()->next()); 403 } 404 // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15, 405 // on EVEX enabled targets, we get it included in the xsave area 406 off = xmm0H_off; 407 delta = xmm1H_off - off; 408 for (int n = 0; n < 16; n++) { 409 XMMRegister xmm_name = as_XMMRegister(n); 410 map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()->next()); 411 off += delta; 412 } 413 if (UseAVX > 2) { 414 // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets 415 off = zmm16H_off; 416 delta = zmm17H_off - off; 417 for (int n = 16; n < num_xmm_regs; n++) { 418 XMMRegister zmm_name = as_XMMRegister(n); 419 map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next()); 420 off += delta; 421 } 422 } 423 } 424 425 return map; 426 } 427 428 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors) { 429 int num_xmm_regs = XMMRegister::available_xmm_registers(); 430 if (frame::arg_reg_save_area_bytes != 0) { 431 // Pop arg register save area 432 __ addptr(rsp, frame::arg_reg_save_area_bytes); 433 } 434 435 #if COMPILER2_OR_JVMCI 436 if (restore_wide_vectors) { 437 assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX"); 438 assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported"); 439 } 440 #else 441 assert(!restore_wide_vectors, "vectors are generated only by C2"); 442 #endif 443 444 __ vzeroupper(); 445 446 // On EVEX enabled targets everything is handled in pop fpu state 447 if (restore_wide_vectors) { 448 // Restore upper half of YMM registers (0..15) 449 int base_addr = XSAVE_AREA_YMM_BEGIN; 450 for (int n = 0; n < 16; n++) { 451 __ vinsertf128_high(as_XMMRegister(n), Address(rsp, base_addr+n*16)); 452 } 453 if (VM_Version::supports_evex()) { 454 // Restore upper half of ZMM registers (0..15) 455 base_addr = XSAVE_AREA_ZMM_BEGIN; 456 for (int n = 0; n < 16; n++) { 457 __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, base_addr+n*32)); 458 } 459 // Restore full ZMM registers(16..num_xmm_regs) 460 base_addr = XSAVE_AREA_UPPERBANK; 461 int vector_len = Assembler::AVX_512bit; 462 int off = 0; 463 for (int n = 16; n < num_xmm_regs; n++) { 464 __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len); 465 } 466 #if COMPILER2_OR_JVMCI 467 base_addr = XSAVE_AREA_OPMASK_BEGIN; 468 off = 0; 469 for (int n = 0; n < KRegister::number_of_registers; n++) { 470 __ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8))); 471 } 472 #endif 473 } 474 } else { 475 if (VM_Version::supports_evex()) { 476 // Restore upper bank of XMM registers(16..31) for scalar or 16-byte vector usage 477 int base_addr = XSAVE_AREA_UPPERBANK; 478 int off = 0; 479 int vector_len = VM_Version::supports_avx512vl() ? Assembler::AVX_128bit : Assembler::AVX_512bit; 480 for (int n = 16; n < num_xmm_regs; n++) { 481 __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len); 482 } 483 #if COMPILER2_OR_JVMCI 484 base_addr = XSAVE_AREA_OPMASK_BEGIN; 485 off = 0; 486 for (int n = 0; n < KRegister::number_of_registers; n++) { 487 __ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8))); 488 } 489 #endif 490 } 491 } 492 493 #if COMPILER2_OR_JVMCI 494 if (UseAPX) { 495 int base_addr = XSAVE_AREA_EGPRS; 496 int off = 0; 497 for (int n = 16; n < Register::number_of_registers; n++) { 498 __ movq(as_Register(n), Address(rsp, base_addr+(off++*8))); 499 } 500 } 501 #endif 502 503 // Recover CPU state 504 __ pop_FPU_state(); 505 __ restore_legacy_gprs(); 506 __ addq(rsp, 8); 507 __ popf(); 508 // Get the rbp described implicitly by the calling convention (no oopMap) 509 __ pop(rbp); 510 } 511 512 void RegisterSaver::restore_result_registers(MacroAssembler* masm) { 513 514 // Just restore result register. Only used by deoptimization. By 515 // now any callee save register that needs to be restored to a c2 516 // caller of the deoptee has been extracted into the vframeArray 517 // and will be stuffed into the c2i adapter we create for later 518 // restoration so only result registers need to be restored here. 519 520 // Restore fp result register 521 __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes())); 522 // Restore integer result register 523 __ movptr(rax, Address(rsp, rax_offset_in_bytes())); 524 __ movptr(rdx, Address(rsp, rdx_offset_in_bytes())); 525 526 // Pop all of the register save are off the stack except the return address 527 __ addptr(rsp, return_offset_in_bytes()); 528 } 529 530 // Is vector's size (in bytes) bigger than a size saved by default? 531 // 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions. 532 bool SharedRuntime::is_wide_vector(int size) { 533 return size > 16; 534 } 535 536 // --------------------------------------------------------------------------- 537 // Read the array of BasicTypes from a signature, and compute where the 538 // arguments should go. Values in the VMRegPair regs array refer to 4-byte 539 // quantities. Values less than VMRegImpl::stack0 are registers, those above 540 // refer to 4-byte stack slots. All stack slots are based off of the stack pointer 541 // as framesizes are fixed. 542 // VMRegImpl::stack0 refers to the first slot 0(sp). 543 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. 544 // Register up to Register::number_of_registers are the 64-bit 545 // integer registers. 546 547 // Note: the INPUTS in sig_bt are in units of Java argument words, which are 548 // either 32-bit or 64-bit depending on the build. The OUTPUTS are in 32-bit 549 // units regardless of build. Of course for i486 there is no 64 bit build 550 551 // The Java calling convention is a "shifted" version of the C ABI. 552 // By skipping the first C ABI register we can call non-static jni methods 553 // with small numbers of arguments without having to shuffle the arguments 554 // at all. Since we control the java ABI we ought to at least get some 555 // advantage out of it. 556 557 int SharedRuntime::java_calling_convention(const BasicType *sig_bt, 558 VMRegPair *regs, 559 int total_args_passed) { 560 561 // Create the mapping between argument positions and 562 // registers. 563 static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = { 564 j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5 565 }; 566 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_j] = { 567 j_farg0, j_farg1, j_farg2, j_farg3, 568 j_farg4, j_farg5, j_farg6, j_farg7 569 }; 570 571 572 uint int_args = 0; 573 uint fp_args = 0; 574 uint stk_args = 0; 575 576 for (int i = 0; i < total_args_passed; i++) { 577 switch (sig_bt[i]) { 578 case T_BOOLEAN: 579 case T_CHAR: 580 case T_BYTE: 581 case T_SHORT: 582 case T_INT: 583 if (int_args < Argument::n_int_register_parameters_j) { 584 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg()); 585 } else { 586 stk_args = align_up(stk_args, 2); 587 regs[i].set1(VMRegImpl::stack2reg(stk_args)); 588 stk_args += 1; 589 } 590 break; 591 case T_VOID: 592 // halves of T_LONG or T_DOUBLE 593 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); 594 regs[i].set_bad(); 595 break; 596 case T_LONG: 597 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); 598 // fall through 599 case T_OBJECT: 600 case T_ARRAY: 601 case T_ADDRESS: 602 if (int_args < Argument::n_int_register_parameters_j) { 603 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg()); 604 } else { 605 stk_args = align_up(stk_args, 2); 606 regs[i].set2(VMRegImpl::stack2reg(stk_args)); 607 stk_args += 2; 608 } 609 break; 610 case T_FLOAT: 611 if (fp_args < Argument::n_float_register_parameters_j) { 612 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg()); 613 } else { 614 stk_args = align_up(stk_args, 2); 615 regs[i].set1(VMRegImpl::stack2reg(stk_args)); 616 stk_args += 1; 617 } 618 break; 619 case T_DOUBLE: 620 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); 621 if (fp_args < Argument::n_float_register_parameters_j) { 622 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg()); 623 } else { 624 stk_args = align_up(stk_args, 2); 625 regs[i].set2(VMRegImpl::stack2reg(stk_args)); 626 stk_args += 2; 627 } 628 break; 629 default: 630 ShouldNotReachHere(); 631 break; 632 } 633 } 634 635 return stk_args; 636 } 637 638 // Same as java_calling_convention() but for multiple return 639 // values. There's no way to store them on the stack so if we don't 640 // have enough registers, multiple values can't be returned. 641 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j+1; 642 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j; 643 int SharedRuntime::java_return_convention(const BasicType *sig_bt, 644 VMRegPair *regs, 645 int total_args_passed) { 646 // Create the mapping between argument positions and 647 // registers. 648 static const Register INT_ArgReg[java_return_convention_max_int] = { 649 rax, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0 650 }; 651 static const XMMRegister FP_ArgReg[java_return_convention_max_float] = { 652 j_farg0, j_farg1, j_farg2, j_farg3, 653 j_farg4, j_farg5, j_farg6, j_farg7 654 }; 655 656 657 uint int_args = 0; 658 uint fp_args = 0; 659 660 for (int i = 0; i < total_args_passed; i++) { 661 switch (sig_bt[i]) { 662 case T_BOOLEAN: 663 case T_CHAR: 664 case T_BYTE: 665 case T_SHORT: 666 case T_INT: 667 if (int_args < Argument::n_int_register_parameters_j+1) { 668 regs[i].set1(INT_ArgReg[int_args]->as_VMReg()); 669 int_args++; 670 } else { 671 return -1; 672 } 673 break; 674 case T_VOID: 675 // halves of T_LONG or T_DOUBLE 676 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); 677 regs[i].set_bad(); 678 break; 679 case T_LONG: 680 assert(sig_bt[i + 1] == T_VOID, "expecting half"); 681 // fall through 682 case T_OBJECT: 683 case T_ARRAY: 684 case T_ADDRESS: 685 case T_METADATA: 686 if (int_args < Argument::n_int_register_parameters_j+1) { 687 regs[i].set2(INT_ArgReg[int_args]->as_VMReg()); 688 int_args++; 689 } else { 690 return -1; 691 } 692 break; 693 case T_FLOAT: 694 if (fp_args < Argument::n_float_register_parameters_j) { 695 regs[i].set1(FP_ArgReg[fp_args]->as_VMReg()); 696 fp_args++; 697 } else { 698 return -1; 699 } 700 break; 701 case T_DOUBLE: 702 assert(sig_bt[i + 1] == T_VOID, "expecting half"); 703 if (fp_args < Argument::n_float_register_parameters_j) { 704 regs[i].set2(FP_ArgReg[fp_args]->as_VMReg()); 705 fp_args++; 706 } else { 707 return -1; 708 } 709 break; 710 default: 711 ShouldNotReachHere(); 712 break; 713 } 714 } 715 716 return int_args + fp_args; 717 } 718 719 // Patch the callers callsite with entry to compiled code if it exists. 720 static void patch_callers_callsite(MacroAssembler *masm) { 721 Label L; 722 __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD); 723 __ jcc(Assembler::equal, L); 724 725 // Save the current stack pointer 726 __ mov(r13, rsp); 727 // Schedule the branch target address early. 728 // Call into the VM to patch the caller, then jump to compiled callee 729 // rax isn't live so capture return address while we easily can 730 __ movptr(rax, Address(rsp, 0)); 731 732 // align stack so push_CPU_state doesn't fault 733 __ andptr(rsp, -(StackAlignmentInBytes)); 734 __ push_CPU_state(); 735 __ vzeroupper(); 736 // VM needs caller's callsite 737 // VM needs target method 738 // This needs to be a long call since we will relocate this adapter to 739 // the codeBuffer and it may not reach 740 741 // Allocate argument register save area 742 if (frame::arg_reg_save_area_bytes != 0) { 743 __ subptr(rsp, frame::arg_reg_save_area_bytes); 744 } 745 __ mov(c_rarg0, rbx); 746 __ mov(c_rarg1, rax); 747 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite))); 748 749 // De-allocate argument register save area 750 if (frame::arg_reg_save_area_bytes != 0) { 751 __ addptr(rsp, frame::arg_reg_save_area_bytes); 752 } 753 754 __ vzeroupper(); 755 __ pop_CPU_state(); 756 // restore sp 757 __ mov(rsp, r13); 758 __ bind(L); 759 } 760 761 // For each inline type argument, sig includes the list of fields of 762 // the inline type. This utility function computes the number of 763 // arguments for the call if inline types are passed by reference (the 764 // calling convention the interpreter expects). 765 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) { 766 int total_args_passed = 0; 767 if (InlineTypePassFieldsAsArgs) { 768 for (int i = 0; i < sig_extended->length(); i++) { 769 BasicType bt = sig_extended->at(i)._bt; 770 if (bt == T_METADATA) { 771 // In sig_extended, an inline type argument starts with: 772 // T_METADATA, followed by the types of the fields of the 773 // inline type and T_VOID to mark the end of the value 774 // type. Inline types are flattened so, for instance, in the 775 // case of an inline type with an int field and an inline type 776 // field that itself has 2 fields, an int and a long: 777 // T_METADATA T_INT T_METADATA T_INT T_LONG T_VOID (second 778 // slot for the T_LONG) T_VOID (inner inline type) T_VOID 779 // (outer inline type) 780 total_args_passed++; 781 int vt = 1; 782 do { 783 i++; 784 BasicType bt = sig_extended->at(i)._bt; 785 BasicType prev_bt = sig_extended->at(i-1)._bt; 786 if (bt == T_METADATA) { 787 vt++; 788 } else if (bt == T_VOID && 789 prev_bt != T_LONG && 790 prev_bt != T_DOUBLE) { 791 vt--; 792 } 793 } while (vt != 0); 794 } else { 795 total_args_passed++; 796 } 797 } 798 } else { 799 total_args_passed = sig_extended->length(); 800 } 801 return total_args_passed; 802 } 803 804 805 static void gen_c2i_adapter_helper(MacroAssembler* masm, 806 BasicType bt, 807 BasicType prev_bt, 808 size_t size_in_bytes, 809 const VMRegPair& reg_pair, 810 const Address& to, 811 int extraspace, 812 bool is_oop) { 813 if (bt == T_VOID) { 814 assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half"); 815 return; 816 } 817 818 // Say 4 args: 819 // i st_off 820 // 0 32 T_LONG 821 // 1 24 T_VOID 822 // 2 16 T_OBJECT 823 // 3 8 T_BOOL 824 // - 0 return address 825 // 826 // However to make thing extra confusing. Because we can fit a long/double in 827 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter 828 // leaves one slot empty and only stores to a single slot. In this case the 829 // slot that is occupied is the T_VOID slot. See I said it was confusing. 830 831 bool wide = (size_in_bytes == wordSize); 832 VMReg r_1 = reg_pair.first(); 833 VMReg r_2 = reg_pair.second(); 834 assert(r_2->is_valid() == wide, "invalid size"); 835 if (!r_1->is_valid()) { 836 assert(!r_2->is_valid(), "must be invalid"); 837 return; 838 } 839 840 if (!r_1->is_XMMRegister()) { 841 Register val = rax; 842 if (r_1->is_stack()) { 843 int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace; 844 __ load_sized_value(val, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false); 845 } else { 846 val = r_1->as_Register(); 847 } 848 assert_different_registers(to.base(), val, rscratch1); 849 if (is_oop) { 850 __ push(r13); 851 __ push(rbx); 852 __ store_heap_oop(to, val, rscratch1, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED); 853 __ pop(rbx); 854 __ pop(r13); 855 } else { 856 __ store_sized_value(to, val, size_in_bytes); 857 } 858 } else { 859 if (wide) { 860 __ movdbl(to, r_1->as_XMMRegister()); 861 } else { 862 __ movflt(to, r_1->as_XMMRegister()); 863 } 864 } 865 } 866 867 static void gen_c2i_adapter(MacroAssembler *masm, 868 const GrowableArray<SigEntry>* sig_extended, 869 const VMRegPair *regs, 870 bool requires_clinit_barrier, 871 address& c2i_no_clinit_check_entry, 872 Label& skip_fixup, 873 address start, 874 OopMapSet* oop_maps, 875 int& frame_complete, 876 int& frame_size_in_words, 877 bool alloc_inline_receiver) { 878 if (requires_clinit_barrier && VM_Version::supports_fast_class_init_checks()) { 879 Label L_skip_barrier; 880 Register method = rbx; 881 882 { // Bypass the barrier for non-static methods 883 Register flags = rscratch1; 884 __ load_unsigned_short(flags, Address(method, Method::access_flags_offset())); 885 __ testl(flags, JVM_ACC_STATIC); 886 __ jcc(Assembler::zero, L_skip_barrier); // non-static 887 } 888 889 Register klass = rscratch1; 890 __ load_method_holder(klass, method); 891 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/); 892 893 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path 894 895 __ bind(L_skip_barrier); 896 c2i_no_clinit_check_entry = __ pc(); 897 } 898 899 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 900 bs->c2i_entry_barrier(masm); 901 902 // Before we get into the guts of the C2I adapter, see if we should be here 903 // at all. We've come from compiled code and are attempting to jump to the 904 // interpreter, which means the caller made a static call to get here 905 // (vcalls always get a compiled target if there is one). Check for a 906 // compiled target. If there is one, we need to patch the caller's call. 907 patch_callers_callsite(masm); 908 909 __ bind(skip_fixup); 910 911 if (InlineTypePassFieldsAsArgs) { 912 // Is there an inline type argument? 913 bool has_inline_argument = false; 914 for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) { 915 has_inline_argument = (sig_extended->at(i)._bt == T_METADATA); 916 } 917 if (has_inline_argument) { 918 // There is at least an inline type argument: we're coming from 919 // compiled code so we have no buffers to back the inline types. 920 // Allocate the buffers here with a runtime call. 921 OopMap* map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_vectors*/ false); 922 923 frame_complete = __ offset(); 924 925 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); 926 927 __ mov(c_rarg0, r15_thread); 928 __ mov(c_rarg1, rbx); 929 __ mov64(c_rarg2, (int64_t)alloc_inline_receiver); 930 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types))); 931 932 oop_maps->add_gc_map((int)(__ pc() - start), map); 933 __ reset_last_Java_frame(false); 934 935 RegisterSaver::restore_live_registers(masm); 936 937 Label no_exception; 938 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 939 __ jcc(Assembler::equal, no_exception); 940 941 __ movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD); 942 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 943 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 944 945 __ bind(no_exception); 946 947 // We get an array of objects from the runtime call 948 __ get_vm_result_oop(rscratch2); // Use rscratch2 (r11) as temporary because rscratch1 (r10) is trashed by movptr() 949 __ get_vm_result_metadata(rbx); // TODO: required to keep the callee Method live? 950 } 951 } 952 953 // Since all args are passed on the stack, total_args_passed * 954 // Interpreter::stackElementSize is the space we need. 955 int total_args_passed = compute_total_args_passed_int(sig_extended); 956 assert(total_args_passed >= 0, "total_args_passed is %d", total_args_passed); 957 958 int extraspace = (total_args_passed * Interpreter::stackElementSize); 959 960 // stack is aligned, keep it that way 961 // This is not currently needed or enforced by the interpreter, but 962 // we might as well conform to the ABI. 963 extraspace = align_up(extraspace, 2*wordSize); 964 965 // set senderSP value 966 __ lea(r13, Address(rsp, wordSize)); 967 968 #ifdef ASSERT 969 __ check_stack_alignment(r13, "sender stack not aligned"); 970 #endif 971 if (extraspace > 0) { 972 // Pop the return address 973 __ pop(rax); 974 975 __ subptr(rsp, extraspace); 976 977 // Push the return address 978 __ push(rax); 979 980 // Account for the return address location since we store it first rather 981 // than hold it in a register across all the shuffling 982 extraspace += wordSize; 983 } 984 985 #ifdef ASSERT 986 __ check_stack_alignment(rsp, "callee stack not aligned", wordSize, rax); 987 #endif 988 989 // Now write the args into the outgoing interpreter space 990 991 // next_arg_comp is the next argument from the compiler point of 992 // view (inline type fields are passed in registers/on the stack). In 993 // sig_extended, an inline type argument starts with: T_METADATA, 994 // followed by the types of the fields of the inline type and T_VOID 995 // to mark the end of the inline type. ignored counts the number of 996 // T_METADATA/T_VOID. next_vt_arg is the next inline type argument: 997 // used to get the buffer for that argument from the pool of buffers 998 // we allocated above and want to pass to the 999 // interpreter. next_arg_int is the next argument from the 1000 // interpreter point of view (inline types are passed by reference). 1001 for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0; 1002 next_arg_comp < sig_extended->length(); next_arg_comp++) { 1003 assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments"); 1004 assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?"); 1005 BasicType bt = sig_extended->at(next_arg_comp)._bt; 1006 int st_off = (total_args_passed - next_arg_int) * Interpreter::stackElementSize; 1007 if (!InlineTypePassFieldsAsArgs || bt != T_METADATA) { 1008 int next_off = st_off - Interpreter::stackElementSize; 1009 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off; 1010 const VMRegPair reg_pair = regs[next_arg_comp-ignored]; 1011 size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4; 1012 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL, 1013 size_in_bytes, reg_pair, Address(rsp, offset), extraspace, false); 1014 next_arg_int++; 1015 #ifdef ASSERT 1016 if (bt == T_LONG || bt == T_DOUBLE) { 1017 // Overwrite the unused slot with known junk 1018 __ mov64(rax, CONST64(0xdeadffffdeadaaaa)); 1019 __ movptr(Address(rsp, st_off), rax); 1020 } 1021 #endif /* ASSERT */ 1022 } else { 1023 ignored++; 1024 // get the buffer from the just allocated pool of buffers 1025 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_OBJECT); 1026 __ load_heap_oop(r14, Address(rscratch2, index)); 1027 next_vt_arg++; next_arg_int++; 1028 int vt = 1; 1029 // write fields we get from compiled code in registers/stack 1030 // slots to the buffer: we know we are done with that inline type 1031 // argument when we hit the T_VOID that acts as an end of inline 1032 // type delimiter for this inline type. Inline types are flattened 1033 // so we might encounter embedded inline types. Each entry in 1034 // sig_extended contains a field offset in the buffer. 1035 Label L_null; 1036 do { 1037 next_arg_comp++; 1038 BasicType bt = sig_extended->at(next_arg_comp)._bt; 1039 BasicType prev_bt = sig_extended->at(next_arg_comp-1)._bt; 1040 if (bt == T_METADATA) { 1041 vt++; 1042 ignored++; 1043 } else if (bt == T_VOID && 1044 prev_bt != T_LONG && 1045 prev_bt != T_DOUBLE) { 1046 vt--; 1047 ignored++; 1048 } else { 1049 int off = sig_extended->at(next_arg_comp)._offset; 1050 if (off == -1) { 1051 // Nullable inline type argument, emit null check 1052 VMReg reg = regs[next_arg_comp-ignored].first(); 1053 Label L_notNull; 1054 if (reg->is_stack()) { 1055 int ld_off = reg->reg2stack() * VMRegImpl::stack_slot_size + extraspace; 1056 __ testb(Address(rsp, ld_off), 1); 1057 } else { 1058 __ testb(reg->as_Register(), 1); 1059 } 1060 __ jcc(Assembler::notZero, L_notNull); 1061 __ movptr(Address(rsp, st_off), 0); 1062 __ jmp(L_null); 1063 __ bind(L_notNull); 1064 continue; 1065 } 1066 assert(off > 0, "offset in object should be positive"); 1067 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize; 1068 bool is_oop = is_reference_type(bt); 1069 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL, 1070 size_in_bytes, regs[next_arg_comp-ignored], Address(r14, off), extraspace, is_oop); 1071 } 1072 } while (vt != 0); 1073 // pass the buffer to the interpreter 1074 __ movptr(Address(rsp, st_off), r14); 1075 __ bind(L_null); 1076 } 1077 } 1078 1079 // Schedule the branch target address early. 1080 __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset()))); 1081 __ jmp(rcx); 1082 } 1083 1084 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, 1085 int comp_args_on_stack, 1086 const GrowableArray<SigEntry>* sig, 1087 const VMRegPair *regs) { 1088 1089 // Note: r13 contains the senderSP on entry. We must preserve it since 1090 // we may do a i2c -> c2i transition if we lose a race where compiled 1091 // code goes non-entrant while we get args ready. 1092 // In addition we use r13 to locate all the interpreter args as 1093 // we must align the stack to 16 bytes on an i2c entry else we 1094 // lose alignment we expect in all compiled code and register 1095 // save code can segv when fxsave instructions find improperly 1096 // aligned stack pointer. 1097 1098 // Adapters can be frameless because they do not require the caller 1099 // to perform additional cleanup work, such as correcting the stack pointer. 1100 // An i2c adapter is frameless because the *caller* frame, which is interpreted, 1101 // routinely repairs its own stack pointer (from interpreter_frame_last_sp), 1102 // even if a callee has modified the stack pointer. 1103 // A c2i adapter is frameless because the *callee* frame, which is interpreted, 1104 // routinely repairs its caller's stack pointer (from sender_sp, which is set 1105 // up via the senderSP register). 1106 // In other words, if *either* the caller or callee is interpreted, we can 1107 // get the stack pointer repaired after a call. 1108 // This is why c2i and i2c adapters cannot be indefinitely composed. 1109 // In particular, if a c2i adapter were to somehow call an i2c adapter, 1110 // both caller and callee would be compiled methods, and neither would 1111 // clean up the stack pointer changes performed by the two adapters. 1112 // If this happens, control eventually transfers back to the compiled 1113 // caller, but with an uncorrected stack, causing delayed havoc. 1114 1115 // Must preserve original SP for loading incoming arguments because 1116 // we need to align the outgoing SP for compiled code. 1117 __ movptr(r11, rsp); 1118 1119 // Pick up the return address 1120 __ pop(rax); 1121 1122 // Convert 4-byte c2 stack slots to words. 1123 int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord; 1124 1125 if (comp_args_on_stack) { 1126 __ subptr(rsp, comp_words_on_stack * wordSize); 1127 } 1128 1129 // Ensure compiled code always sees stack at proper alignment 1130 __ andptr(rsp, -16); 1131 1132 // push the return address and misalign the stack that youngest frame always sees 1133 // as far as the placement of the call instruction 1134 __ push(rax); 1135 1136 // Put saved SP in another register 1137 const Register saved_sp = rax; 1138 __ movptr(saved_sp, r11); 1139 1140 // Will jump to the compiled code just as if compiled code was doing it. 1141 // Pre-load the register-jump target early, to schedule it better. 1142 __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_inline_offset()))); 1143 1144 #if INCLUDE_JVMCI 1145 if (EnableJVMCI) { 1146 // check if this call should be routed towards a specific entry point 1147 __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0); 1148 Label no_alternative_target; 1149 __ jcc(Assembler::equal, no_alternative_target); 1150 __ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset()))); 1151 __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0); 1152 __ bind(no_alternative_target); 1153 } 1154 #endif // INCLUDE_JVMCI 1155 1156 int total_args_passed = sig->length(); 1157 1158 // Now generate the shuffle code. Pick up all register args and move the 1159 // rest through the floating point stack top. 1160 for (int i = 0; i < total_args_passed; i++) { 1161 BasicType bt = sig->at(i)._bt; 1162 if (bt == T_VOID) { 1163 // Longs and doubles are passed in native word order, but misaligned 1164 // in the 32-bit build. 1165 BasicType prev_bt = (i > 0) ? sig->at(i-1)._bt : T_ILLEGAL; 1166 assert(i > 0 && (prev_bt == T_LONG || prev_bt == T_DOUBLE), "missing half"); 1167 continue; 1168 } 1169 1170 // Pick up 0, 1 or 2 words from SP+offset. 1171 1172 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), 1173 "scrambled load targets?"); 1174 // Load in argument order going down. 1175 int ld_off = (total_args_passed - i)*Interpreter::stackElementSize; 1176 // Point to interpreter value (vs. tag) 1177 int next_off = ld_off - Interpreter::stackElementSize; 1178 // 1179 // 1180 // 1181 VMReg r_1 = regs[i].first(); 1182 VMReg r_2 = regs[i].second(); 1183 if (!r_1->is_valid()) { 1184 assert(!r_2->is_valid(), ""); 1185 continue; 1186 } 1187 if (r_1->is_stack()) { 1188 // Convert stack slot to an SP offset (+ wordSize to account for return address ) 1189 int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize; 1190 1191 // We can use r13 as a temp here because compiled code doesn't need r13 as an input 1192 // and if we end up going thru a c2i because of a miss a reasonable value of r13 1193 // will be generated. 1194 if (!r_2->is_valid()) { 1195 // sign extend??? 1196 __ movl(r13, Address(saved_sp, ld_off)); 1197 __ movptr(Address(rsp, st_off), r13); 1198 } else { 1199 // 1200 // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE 1201 // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case 1202 // So we must adjust where to pick up the data to match the interpreter. 1203 // 1204 // Interpreter local[n] == MSW, local[n+1] == LSW however locals 1205 // are accessed as negative so LSW is at LOW address 1206 1207 // ld_off is MSW so get LSW 1208 const int offset = (bt==T_LONG||bt==T_DOUBLE)? 1209 next_off : ld_off; 1210 __ movq(r13, Address(saved_sp, offset)); 1211 // st_off is LSW (i.e. reg.first()) 1212 __ movq(Address(rsp, st_off), r13); 1213 } 1214 } else if (r_1->is_Register()) { // Register argument 1215 Register r = r_1->as_Register(); 1216 assert(r != rax, "must be different"); 1217 if (r_2->is_valid()) { 1218 // 1219 // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE 1220 // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case 1221 // So we must adjust where to pick up the data to match the interpreter. 1222 1223 const int offset = (bt==T_LONG||bt==T_DOUBLE)? 1224 next_off : ld_off; 1225 1226 // this can be a misaligned move 1227 __ movq(r, Address(saved_sp, offset)); 1228 } else { 1229 // sign extend and use a full word? 1230 __ movl(r, Address(saved_sp, ld_off)); 1231 } 1232 } else { 1233 if (!r_2->is_valid()) { 1234 __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off)); 1235 } else { 1236 __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off)); 1237 } 1238 } 1239 } 1240 1241 __ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about 1242 1243 // 6243940 We might end up in handle_wrong_method if 1244 // the callee is deoptimized as we race thru here. If that 1245 // happens we don't want to take a safepoint because the 1246 // caller frame will look interpreted and arguments are now 1247 // "compiled" so it is much better to make this transition 1248 // invisible to the stack walking code. Unfortunately if 1249 // we try and find the callee by normal means a safepoint 1250 // is possible. So we stash the desired callee in the thread 1251 // and the vm will find there should this case occur. 1252 1253 __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); 1254 1255 // put Method* where a c2i would expect should we end up there 1256 // only needed because of c2 resolve stubs return Method* as a result in 1257 // rax 1258 __ mov(rax, rbx); 1259 __ jmp(r11); 1260 } 1261 1262 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) { 1263 Register data = rax; 1264 __ ic_check(1 /* end_alignment */); 1265 __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset())); 1266 1267 // Method might have been compiled since the call site was patched to 1268 // interpreted if that is the case treat it as a miss so we can get 1269 // the call site corrected. 1270 __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD); 1271 __ jcc(Assembler::equal, skip_fixup); 1272 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1273 } 1274 1275 // --------------------------------------------------------------- 1276 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm, 1277 int comp_args_on_stack, 1278 const GrowableArray<SigEntry>* sig, 1279 const VMRegPair* regs, 1280 const GrowableArray<SigEntry>* sig_cc, 1281 const VMRegPair* regs_cc, 1282 const GrowableArray<SigEntry>* sig_cc_ro, 1283 const VMRegPair* regs_cc_ro, 1284 address entry_address[AdapterBlob::ENTRY_COUNT], 1285 AdapterBlob*& new_adapter, 1286 bool allocate_code_blob) { 1287 entry_address[AdapterBlob::I2C] = __ pc(); 1288 gen_i2c_adapter(masm, comp_args_on_stack, sig, regs); 1289 1290 // ------------------------------------------------------------------------- 1291 // Generate a C2I adapter. On entry we know rbx holds the Method* during calls 1292 // to the interpreter. The args start out packed in the compiled layout. They 1293 // need to be unpacked into the interpreter layout. This will almost always 1294 // require some stack space. We grow the current (compiled) stack, then repack 1295 // the args. We finally end in a jump to the generic interpreter entry point. 1296 // On exit from the interpreter, the interpreter will restore our SP (lest the 1297 // compiled code, which relies solely on SP and not RBP, get sick). 1298 1299 entry_address[AdapterBlob::C2I_Unverified] = __ pc(); 1300 entry_address[AdapterBlob::C2I_Unverified_Inline] = __ pc(); 1301 Label skip_fixup; 1302 1303 gen_inline_cache_check(masm, skip_fixup); 1304 1305 OopMapSet* oop_maps = new OopMapSet(); 1306 int frame_complete = CodeOffsets::frame_never_safe; 1307 int frame_size_in_words = 0; 1308 1309 // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver) 1310 entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr; 1311 entry_address[AdapterBlob::C2I_Inline_RO] = __ pc(); 1312 if (regs_cc != regs_cc_ro) { 1313 // No class init barrier needed because method is guaranteed to be non-static 1314 gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, /* requires_clinit_barrier = */ false, entry_address[AdapterBlob::C2I_No_Clinit_Check], 1315 skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false); 1316 skip_fixup.reset(); 1317 } 1318 1319 // Scalarized c2i adapter 1320 entry_address[AdapterBlob::C2I] = __ pc(); 1321 entry_address[AdapterBlob::C2I_Inline] = __ pc(); 1322 gen_c2i_adapter(masm, sig_cc, regs_cc, /* requires_clinit_barrier = */ true, entry_address[AdapterBlob::C2I_No_Clinit_Check], 1323 skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ true); 1324 1325 // Non-scalarized c2i adapter 1326 if (regs != regs_cc) { 1327 entry_address[AdapterBlob::C2I_Unverified_Inline] = __ pc(); 1328 Label inline_entry_skip_fixup; 1329 gen_inline_cache_check(masm, inline_entry_skip_fixup); 1330 1331 entry_address[AdapterBlob::C2I_Inline] = __ pc(); 1332 gen_c2i_adapter(masm, sig, regs, /* requires_clinit_barrier = */ true, entry_address[AdapterBlob::C2I_No_Clinit_Check], 1333 inline_entry_skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false); 1334 } 1335 1336 // The c2i adapters might safepoint and trigger a GC. The caller must make sure that 1337 // the GC knows about the location of oop argument locations passed to the c2i adapter. 1338 if (allocate_code_blob) { 1339 bool caller_must_gc_arguments = (regs != regs_cc); 1340 int entry_offset[AdapterHandlerEntry::ENTRIES_COUNT]; 1341 assert(AdapterHandlerEntry::ENTRIES_COUNT == 7, "sanity"); 1342 AdapterHandlerLibrary::address_to_offset(entry_address, entry_offset); 1343 new_adapter = AdapterBlob::create(masm->code(), entry_offset, frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments); 1344 } 1345 } 1346 1347 int SharedRuntime::c_calling_convention(const BasicType *sig_bt, 1348 VMRegPair *regs, 1349 int total_args_passed) { 1350 1351 // We return the amount of VMRegImpl stack slots we need to reserve for all 1352 // the arguments NOT counting out_preserve_stack_slots. 1353 1354 // NOTE: These arrays will have to change when c1 is ported 1355 #ifdef _WIN64 1356 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = { 1357 c_rarg0, c_rarg1, c_rarg2, c_rarg3 1358 }; 1359 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = { 1360 c_farg0, c_farg1, c_farg2, c_farg3 1361 }; 1362 #else 1363 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = { 1364 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5 1365 }; 1366 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = { 1367 c_farg0, c_farg1, c_farg2, c_farg3, 1368 c_farg4, c_farg5, c_farg6, c_farg7 1369 }; 1370 #endif // _WIN64 1371 1372 1373 uint int_args = 0; 1374 uint fp_args = 0; 1375 uint stk_args = 0; // inc by 2 each time 1376 1377 for (int i = 0; i < total_args_passed; i++) { 1378 switch (sig_bt[i]) { 1379 case T_BOOLEAN: 1380 case T_CHAR: 1381 case T_BYTE: 1382 case T_SHORT: 1383 case T_INT: 1384 if (int_args < Argument::n_int_register_parameters_c) { 1385 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg()); 1386 #ifdef _WIN64 1387 fp_args++; 1388 // Allocate slots for callee to stuff register args the stack. 1389 stk_args += 2; 1390 #endif 1391 } else { 1392 regs[i].set1(VMRegImpl::stack2reg(stk_args)); 1393 stk_args += 2; 1394 } 1395 break; 1396 case T_LONG: 1397 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); 1398 // fall through 1399 case T_OBJECT: 1400 case T_ARRAY: 1401 case T_ADDRESS: 1402 case T_METADATA: 1403 if (int_args < Argument::n_int_register_parameters_c) { 1404 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg()); 1405 #ifdef _WIN64 1406 fp_args++; 1407 stk_args += 2; 1408 #endif 1409 } else { 1410 regs[i].set2(VMRegImpl::stack2reg(stk_args)); 1411 stk_args += 2; 1412 } 1413 break; 1414 case T_FLOAT: 1415 if (fp_args < Argument::n_float_register_parameters_c) { 1416 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg()); 1417 #ifdef _WIN64 1418 int_args++; 1419 // Allocate slots for callee to stuff register args the stack. 1420 stk_args += 2; 1421 #endif 1422 } else { 1423 regs[i].set1(VMRegImpl::stack2reg(stk_args)); 1424 stk_args += 2; 1425 } 1426 break; 1427 case T_DOUBLE: 1428 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); 1429 if (fp_args < Argument::n_float_register_parameters_c) { 1430 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg()); 1431 #ifdef _WIN64 1432 int_args++; 1433 // Allocate slots for callee to stuff register args the stack. 1434 stk_args += 2; 1435 #endif 1436 } else { 1437 regs[i].set2(VMRegImpl::stack2reg(stk_args)); 1438 stk_args += 2; 1439 } 1440 break; 1441 case T_VOID: // Halves of longs and doubles 1442 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); 1443 regs[i].set_bad(); 1444 break; 1445 default: 1446 ShouldNotReachHere(); 1447 break; 1448 } 1449 } 1450 #ifdef _WIN64 1451 // windows abi requires that we always allocate enough stack space 1452 // for 4 64bit registers to be stored down. 1453 if (stk_args < 8) { 1454 stk_args = 8; 1455 } 1456 #endif // _WIN64 1457 1458 return stk_args; 1459 } 1460 1461 int SharedRuntime::vector_calling_convention(VMRegPair *regs, 1462 uint num_bits, 1463 uint total_args_passed) { 1464 assert(num_bits == 64 || num_bits == 128 || num_bits == 256 || num_bits == 512, 1465 "only certain vector sizes are supported for now"); 1466 1467 static const XMMRegister VEC_ArgReg[32] = { 1468 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, 1469 xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 1470 xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm22, xmm23, 1471 xmm24, xmm25, xmm26, xmm27, xmm28, xmm29, xmm30, xmm31 1472 }; 1473 1474 uint stk_args = 0; 1475 uint fp_args = 0; 1476 1477 for (uint i = 0; i < total_args_passed; i++) { 1478 VMReg vmreg = VEC_ArgReg[fp_args++]->as_VMReg(); 1479 int next_val = num_bits == 64 ? 1 : (num_bits == 128 ? 3 : (num_bits == 256 ? 7 : 15)); 1480 regs[i].set_pair(vmreg->next(next_val), vmreg); 1481 } 1482 1483 return stk_args; 1484 } 1485 1486 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { 1487 // We always ignore the frame_slots arg and just use the space just below frame pointer 1488 // which by this time is free to use 1489 switch (ret_type) { 1490 case T_FLOAT: 1491 __ movflt(Address(rbp, -wordSize), xmm0); 1492 break; 1493 case T_DOUBLE: 1494 __ movdbl(Address(rbp, -wordSize), xmm0); 1495 break; 1496 case T_VOID: break; 1497 default: { 1498 __ movptr(Address(rbp, -wordSize), rax); 1499 } 1500 } 1501 } 1502 1503 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { 1504 // We always ignore the frame_slots arg and just use the space just below frame pointer 1505 // which by this time is free to use 1506 switch (ret_type) { 1507 case T_FLOAT: 1508 __ movflt(xmm0, Address(rbp, -wordSize)); 1509 break; 1510 case T_DOUBLE: 1511 __ movdbl(xmm0, Address(rbp, -wordSize)); 1512 break; 1513 case T_VOID: break; 1514 default: { 1515 __ movptr(rax, Address(rbp, -wordSize)); 1516 } 1517 } 1518 } 1519 1520 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { 1521 for ( int i = first_arg ; i < arg_count ; i++ ) { 1522 if (args[i].first()->is_Register()) { 1523 __ push(args[i].first()->as_Register()); 1524 } else if (args[i].first()->is_XMMRegister()) { 1525 __ subptr(rsp, 2*wordSize); 1526 __ movdbl(Address(rsp, 0), args[i].first()->as_XMMRegister()); 1527 } 1528 } 1529 } 1530 1531 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { 1532 for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) { 1533 if (args[i].first()->is_Register()) { 1534 __ pop(args[i].first()->as_Register()); 1535 } else if (args[i].first()->is_XMMRegister()) { 1536 __ movdbl(args[i].first()->as_XMMRegister(), Address(rsp, 0)); 1537 __ addptr(rsp, 2*wordSize); 1538 } 1539 } 1540 } 1541 1542 static void verify_oop_args(MacroAssembler* masm, 1543 const methodHandle& method, 1544 const BasicType* sig_bt, 1545 const VMRegPair* regs) { 1546 Register temp_reg = rbx; // not part of any compiled calling seq 1547 if (VerifyOops) { 1548 for (int i = 0; i < method->size_of_parameters(); i++) { 1549 if (is_reference_type(sig_bt[i])) { 1550 VMReg r = regs[i].first(); 1551 assert(r->is_valid(), "bad oop arg"); 1552 if (r->is_stack()) { 1553 __ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize)); 1554 __ verify_oop(temp_reg); 1555 } else { 1556 __ verify_oop(r->as_Register()); 1557 } 1558 } 1559 } 1560 } 1561 } 1562 1563 static void check_continuation_enter_argument(VMReg actual_vmreg, 1564 Register expected_reg, 1565 const char* name) { 1566 assert(!actual_vmreg->is_stack(), "%s cannot be on stack", name); 1567 assert(actual_vmreg->as_Register() == expected_reg, 1568 "%s is in unexpected register: %s instead of %s", 1569 name, actual_vmreg->as_Register()->name(), expected_reg->name()); 1570 } 1571 1572 1573 //---------------------------- continuation_enter_setup --------------------------- 1574 // 1575 // Arguments: 1576 // None. 1577 // 1578 // Results: 1579 // rsp: pointer to blank ContinuationEntry 1580 // 1581 // Kills: 1582 // rax 1583 // 1584 static OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) { 1585 assert(ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, ""); 1586 assert(in_bytes(ContinuationEntry::cont_offset()) % VMRegImpl::stack_slot_size == 0, ""); 1587 assert(in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, ""); 1588 1589 stack_slots += checked_cast<int>(ContinuationEntry::size()) / wordSize; 1590 __ subptr(rsp, checked_cast<int32_t>(ContinuationEntry::size())); 1591 1592 int frame_size = (checked_cast<int>(ContinuationEntry::size()) + wordSize) / VMRegImpl::stack_slot_size; 1593 OopMap* map = new OopMap(frame_size, 0); 1594 1595 __ movptr(rax, Address(r15_thread, JavaThread::cont_entry_offset())); 1596 __ movptr(Address(rsp, ContinuationEntry::parent_offset()), rax); 1597 __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rsp); 1598 1599 return map; 1600 } 1601 1602 //---------------------------- fill_continuation_entry --------------------------- 1603 // 1604 // Arguments: 1605 // rsp: pointer to blank Continuation entry 1606 // reg_cont_obj: pointer to the continuation 1607 // reg_flags: flags 1608 // 1609 // Results: 1610 // rsp: pointer to filled out ContinuationEntry 1611 // 1612 // Kills: 1613 // rax 1614 // 1615 static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj, Register reg_flags) { 1616 assert_different_registers(rax, reg_cont_obj, reg_flags); 1617 #ifdef ASSERT 1618 __ movl(Address(rsp, ContinuationEntry::cookie_offset()), ContinuationEntry::cookie_value()); 1619 #endif 1620 __ movptr(Address(rsp, ContinuationEntry::cont_offset()), reg_cont_obj); 1621 __ movl (Address(rsp, ContinuationEntry::flags_offset()), reg_flags); 1622 __ movptr(Address(rsp, ContinuationEntry::chunk_offset()), 0); 1623 __ movl(Address(rsp, ContinuationEntry::argsize_offset()), 0); 1624 __ movl(Address(rsp, ContinuationEntry::pin_count_offset()), 0); 1625 1626 __ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset())); 1627 __ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax); 1628 __ movq(rax, Address(r15_thread, JavaThread::held_monitor_count_offset())); 1629 __ movq(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax); 1630 1631 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0); 1632 __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), 0); 1633 } 1634 1635 //---------------------------- continuation_enter_cleanup --------------------------- 1636 // 1637 // Arguments: 1638 // rsp: pointer to the ContinuationEntry 1639 // 1640 // Results: 1641 // rsp: pointer to the spilled rbp in the entry frame 1642 // 1643 // Kills: 1644 // rbx 1645 // 1646 static void continuation_enter_cleanup(MacroAssembler* masm) { 1647 #ifdef ASSERT 1648 Label L_good_sp; 1649 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 1650 __ jcc(Assembler::equal, L_good_sp); 1651 __ stop("Incorrect rsp at continuation_enter_cleanup"); 1652 __ bind(L_good_sp); 1653 #endif 1654 __ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset())); 1655 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx); 1656 1657 if (CheckJNICalls) { 1658 // Check if this is a virtual thread continuation 1659 Label L_skip_vthread_code; 1660 __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0); 1661 __ jcc(Assembler::equal, L_skip_vthread_code); 1662 1663 // If the held monitor count is > 0 and this vthread is terminating then 1664 // it failed to release a JNI monitor. So we issue the same log message 1665 // that JavaThread::exit does. 1666 __ cmpptr(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0); 1667 __ jcc(Assembler::equal, L_skip_vthread_code); 1668 1669 // rax may hold an exception oop, save it before the call 1670 __ push(rax); 1671 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held)); 1672 __ pop(rax); 1673 1674 // For vthreads we have to explicitly zero the JNI monitor count of the carrier 1675 // on termination. The held count is implicitly zeroed below when we restore from 1676 // the parent held count (which has to be zero). 1677 __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0); 1678 1679 __ bind(L_skip_vthread_code); 1680 } 1681 #ifdef ASSERT 1682 else { 1683 // Check if this is a virtual thread continuation 1684 Label L_skip_vthread_code; 1685 __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0); 1686 __ jcc(Assembler::equal, L_skip_vthread_code); 1687 1688 // See comment just above. If not checking JNI calls the JNI count is only 1689 // needed for assertion checking. 1690 __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0); 1691 1692 __ bind(L_skip_vthread_code); 1693 } 1694 #endif 1695 1696 __ movq(rbx, Address(rsp, ContinuationEntry::parent_held_monitor_count_offset())); 1697 __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), rbx); 1698 1699 __ movptr(rbx, Address(rsp, ContinuationEntry::parent_offset())); 1700 __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rbx); 1701 __ addptr(rsp, checked_cast<int32_t>(ContinuationEntry::size())); 1702 } 1703 1704 static void gen_continuation_enter(MacroAssembler* masm, 1705 const VMRegPair* regs, 1706 int& exception_offset, 1707 OopMapSet* oop_maps, 1708 int& frame_complete, 1709 int& stack_slots, 1710 int& interpreted_entry_offset, 1711 int& compiled_entry_offset) { 1712 1713 // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) 1714 int pos_cont_obj = 0; 1715 int pos_is_cont = 1; 1716 int pos_is_virtual = 2; 1717 1718 // The platform-specific calling convention may present the arguments in various registers. 1719 // To simplify the rest of the code, we expect the arguments to reside at these known 1720 // registers, and we additionally check the placement here in case calling convention ever 1721 // changes. 1722 Register reg_cont_obj = c_rarg1; 1723 Register reg_is_cont = c_rarg2; 1724 Register reg_is_virtual = c_rarg3; 1725 1726 check_continuation_enter_argument(regs[pos_cont_obj].first(), reg_cont_obj, "Continuation object"); 1727 check_continuation_enter_argument(regs[pos_is_cont].first(), reg_is_cont, "isContinue"); 1728 check_continuation_enter_argument(regs[pos_is_virtual].first(), reg_is_virtual, "isVirtualThread"); 1729 1730 // Utility methods kill rax, make sure there are no collisions 1731 assert_different_registers(rax, reg_cont_obj, reg_is_cont, reg_is_virtual); 1732 1733 AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), 1734 relocInfo::static_call_type); 1735 1736 address start = __ pc(); 1737 1738 Label L_thaw, L_exit; 1739 1740 // i2i entry used at interp_only_mode only 1741 interpreted_entry_offset = __ pc() - start; 1742 { 1743 #ifdef ASSERT 1744 Label is_interp_only; 1745 __ cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0); 1746 __ jcc(Assembler::notEqual, is_interp_only); 1747 __ stop("enterSpecial interpreter entry called when not in interp_only_mode"); 1748 __ bind(is_interp_only); 1749 #endif 1750 1751 __ pop(rax); // return address 1752 // Read interpreter arguments into registers (this is an ad-hoc i2c adapter) 1753 __ movptr(c_rarg1, Address(rsp, Interpreter::stackElementSize*2)); 1754 __ movl(c_rarg2, Address(rsp, Interpreter::stackElementSize*1)); 1755 __ movl(c_rarg3, Address(rsp, Interpreter::stackElementSize*0)); 1756 __ andptr(rsp, -16); // Ensure compiled code always sees stack at proper alignment 1757 __ push(rax); // return address 1758 __ push_cont_fastpath(); 1759 1760 __ enter(); 1761 1762 stack_slots = 2; // will be adjusted in setup 1763 OopMap* map = continuation_enter_setup(masm, stack_slots); 1764 // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe, 1765 // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway. 1766 1767 __ verify_oop(reg_cont_obj); 1768 1769 fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); 1770 1771 // If continuation, call to thaw. Otherwise, resolve the call and exit. 1772 __ testptr(reg_is_cont, reg_is_cont); 1773 __ jcc(Assembler::notZero, L_thaw); 1774 1775 // --- Resolve path 1776 1777 // Make sure the call is patchable 1778 __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset); 1779 // Emit stub for static call 1780 address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc()); 1781 if (stub == nullptr) { 1782 fatal("CodeCache is full at gen_continuation_enter"); 1783 } 1784 __ call(resolve); 1785 oop_maps->add_gc_map(__ pc() - start, map); 1786 __ post_call_nop(); 1787 1788 __ jmp(L_exit); 1789 } 1790 1791 // compiled entry 1792 __ align(CodeEntryAlignment); 1793 compiled_entry_offset = __ pc() - start; 1794 __ enter(); 1795 1796 stack_slots = 2; // will be adjusted in setup 1797 OopMap* map = continuation_enter_setup(masm, stack_slots); 1798 1799 // Frame is now completed as far as size and linkage. 1800 frame_complete = __ pc() - start; 1801 1802 __ verify_oop(reg_cont_obj); 1803 1804 fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); 1805 1806 // If isContinue, call to thaw. Otherwise, call Continuation.enter(Continuation c, boolean isContinue) 1807 __ testptr(reg_is_cont, reg_is_cont); 1808 __ jccb(Assembler::notZero, L_thaw); 1809 1810 // --- call Continuation.enter(Continuation c, boolean isContinue) 1811 1812 // Make sure the call is patchable 1813 __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset); 1814 1815 // Emit stub for static call 1816 address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc()); 1817 if (stub == nullptr) { 1818 fatal("CodeCache is full at gen_continuation_enter"); 1819 } 1820 1821 // The call needs to be resolved. There's a special case for this in 1822 // SharedRuntime::find_callee_info_helper() which calls 1823 // LinkResolver::resolve_continuation_enter() which resolves the call to 1824 // Continuation.enter(Continuation c, boolean isContinue). 1825 __ call(resolve); 1826 1827 oop_maps->add_gc_map(__ pc() - start, map); 1828 __ post_call_nop(); 1829 1830 __ jmpb(L_exit); 1831 1832 // --- Thawing path 1833 1834 __ bind(L_thaw); 1835 1836 ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; 1837 __ call(RuntimeAddress(StubRoutines::cont_thaw())); 1838 1839 ContinuationEntry::_return_pc_offset = __ pc() - start; 1840 oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); 1841 __ post_call_nop(); 1842 1843 // --- Normal exit (resolve/thawing) 1844 1845 __ bind(L_exit); 1846 ContinuationEntry::_cleanup_offset = __ pc() - start; 1847 continuation_enter_cleanup(masm); 1848 __ pop(rbp); 1849 __ ret(0); 1850 1851 // --- Exception handling path 1852 1853 exception_offset = __ pc() - start; 1854 1855 continuation_enter_cleanup(masm); 1856 __ pop(rbp); 1857 1858 __ movptr(c_rarg0, r15_thread); 1859 __ movptr(c_rarg1, Address(rsp, 0)); // return address 1860 1861 // rax still holds the original exception oop, save it before the call 1862 __ push(rax); 1863 1864 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2); 1865 __ movptr(rbx, rax); 1866 1867 // Continue at exception handler: 1868 // rax: exception oop 1869 // rbx: exception handler 1870 // rdx: exception pc 1871 __ pop(rax); 1872 __ verify_oop(rax); 1873 __ pop(rdx); 1874 __ jmp(rbx); 1875 } 1876 1877 static void gen_continuation_yield(MacroAssembler* masm, 1878 const VMRegPair* regs, 1879 OopMapSet* oop_maps, 1880 int& frame_complete, 1881 int& stack_slots, 1882 int& compiled_entry_offset) { 1883 enum layout { 1884 rbp_off, 1885 rbpH_off, 1886 return_off, 1887 return_off2, 1888 framesize // inclusive of return address 1889 }; 1890 stack_slots = framesize / VMRegImpl::slots_per_word; 1891 assert(stack_slots == 2, "recheck layout"); 1892 1893 address start = __ pc(); 1894 compiled_entry_offset = __ pc() - start; 1895 __ enter(); 1896 address the_pc = __ pc(); 1897 1898 frame_complete = the_pc - start; 1899 1900 // This nop must be exactly at the PC we push into the frame info. 1901 // We use this nop for fast CodeBlob lookup, associate the OopMap 1902 // with it right away. 1903 __ post_call_nop(); 1904 OopMap* map = new OopMap(framesize, 1); 1905 oop_maps->add_gc_map(frame_complete, map); 1906 1907 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1); 1908 __ movptr(c_rarg0, r15_thread); 1909 __ movptr(c_rarg1, rsp); 1910 __ call_VM_leaf(Continuation::freeze_entry(), 2); 1911 __ reset_last_Java_frame(true); 1912 1913 Label L_pinned; 1914 1915 __ testptr(rax, rax); 1916 __ jcc(Assembler::notZero, L_pinned); 1917 1918 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); 1919 continuation_enter_cleanup(masm); 1920 __ pop(rbp); 1921 __ ret(0); 1922 1923 __ bind(L_pinned); 1924 1925 // Pinned, return to caller 1926 1927 // handle pending exception thrown by freeze 1928 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 1929 Label ok; 1930 __ jcc(Assembler::equal, ok); 1931 __ leave(); 1932 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1933 __ bind(ok); 1934 1935 __ leave(); 1936 __ ret(0); 1937 } 1938 1939 void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { 1940 ::continuation_enter_cleanup(masm); 1941 } 1942 1943 static void gen_special_dispatch(MacroAssembler* masm, 1944 const methodHandle& method, 1945 const BasicType* sig_bt, 1946 const VMRegPair* regs) { 1947 verify_oop_args(masm, method, sig_bt, regs); 1948 vmIntrinsics::ID iid = method->intrinsic_id(); 1949 1950 // Now write the args into the outgoing interpreter space 1951 bool has_receiver = false; 1952 Register receiver_reg = noreg; 1953 int member_arg_pos = -1; 1954 Register member_reg = noreg; 1955 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid); 1956 if (ref_kind != 0) { 1957 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument 1958 member_reg = rbx; // known to be free at this point 1959 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind); 1960 } else if (iid == vmIntrinsics::_invokeBasic) { 1961 has_receiver = true; 1962 } else if (iid == vmIntrinsics::_linkToNative) { 1963 member_arg_pos = method->size_of_parameters() - 1; // trailing NativeEntryPoint argument 1964 member_reg = rbx; // known to be free at this point 1965 } else { 1966 fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid)); 1967 } 1968 1969 if (member_reg != noreg) { 1970 // Load the member_arg into register, if necessary. 1971 SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs); 1972 VMReg r = regs[member_arg_pos].first(); 1973 if (r->is_stack()) { 1974 __ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize)); 1975 } else { 1976 // no data motion is needed 1977 member_reg = r->as_Register(); 1978 } 1979 } 1980 1981 if (has_receiver) { 1982 // Make sure the receiver is loaded into a register. 1983 assert(method->size_of_parameters() > 0, "oob"); 1984 assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object"); 1985 VMReg r = regs[0].first(); 1986 assert(r->is_valid(), "bad receiver arg"); 1987 if (r->is_stack()) { 1988 // Porting note: This assumes that compiled calling conventions always 1989 // pass the receiver oop in a register. If this is not true on some 1990 // platform, pick a temp and load the receiver from stack. 1991 fatal("receiver always in a register"); 1992 receiver_reg = j_rarg0; // known to be free at this point 1993 __ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize)); 1994 } else { 1995 // no data motion is needed 1996 receiver_reg = r->as_Register(); 1997 } 1998 } 1999 2000 // Figure out which address we are really jumping to: 2001 MethodHandles::generate_method_handle_dispatch(masm, iid, 2002 receiver_reg, member_reg, /*for_compiler_entry:*/ true); 2003 } 2004 2005 // --------------------------------------------------------------------------- 2006 // Generate a native wrapper for a given method. The method takes arguments 2007 // in the Java compiled code convention, marshals them to the native 2008 // convention (handlizes oops, etc), transitions to native, makes the call, 2009 // returns to java state (possibly blocking), unhandlizes any result and 2010 // returns. 2011 // 2012 // Critical native functions are a shorthand for the use of 2013 // GetPrimtiveArrayCritical and disallow the use of any other JNI 2014 // functions. The wrapper is expected to unpack the arguments before 2015 // passing them to the callee. Critical native functions leave the state _in_Java, 2016 // since they cannot stop for GC. 2017 // Some other parts of JNI setup are skipped like the tear down of the JNI handle 2018 // block and the check for pending exceptions it's impossible for them 2019 // to be thrown. 2020 // 2021 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, 2022 const methodHandle& method, 2023 int compile_id, 2024 BasicType* in_sig_bt, 2025 VMRegPair* in_regs, 2026 BasicType ret_type) { 2027 if (method->is_continuation_native_intrinsic()) { 2028 int exception_offset = -1; 2029 OopMapSet* oop_maps = new OopMapSet(); 2030 int frame_complete = -1; 2031 int stack_slots = -1; 2032 int interpreted_entry_offset = -1; 2033 int vep_offset = -1; 2034 if (method->is_continuation_enter_intrinsic()) { 2035 gen_continuation_enter(masm, 2036 in_regs, 2037 exception_offset, 2038 oop_maps, 2039 frame_complete, 2040 stack_slots, 2041 interpreted_entry_offset, 2042 vep_offset); 2043 } else if (method->is_continuation_yield_intrinsic()) { 2044 gen_continuation_yield(masm, 2045 in_regs, 2046 oop_maps, 2047 frame_complete, 2048 stack_slots, 2049 vep_offset); 2050 } else { 2051 guarantee(false, "Unknown Continuation native intrinsic"); 2052 } 2053 2054 #ifdef ASSERT 2055 if (method->is_continuation_enter_intrinsic()) { 2056 assert(interpreted_entry_offset != -1, "Must be set"); 2057 assert(exception_offset != -1, "Must be set"); 2058 } else { 2059 assert(interpreted_entry_offset == -1, "Must be unset"); 2060 assert(exception_offset == -1, "Must be unset"); 2061 } 2062 assert(frame_complete != -1, "Must be set"); 2063 assert(stack_slots != -1, "Must be set"); 2064 assert(vep_offset != -1, "Must be set"); 2065 #endif 2066 2067 __ flush(); 2068 nmethod* nm = nmethod::new_native_nmethod(method, 2069 compile_id, 2070 masm->code(), 2071 vep_offset, 2072 frame_complete, 2073 stack_slots, 2074 in_ByteSize(-1), 2075 in_ByteSize(-1), 2076 oop_maps, 2077 exception_offset); 2078 if (nm == nullptr) return nm; 2079 if (method->is_continuation_enter_intrinsic()) { 2080 ContinuationEntry::set_enter_code(nm, interpreted_entry_offset); 2081 } else if (method->is_continuation_yield_intrinsic()) { 2082 _cont_doYield_stub = nm; 2083 } 2084 return nm; 2085 } 2086 2087 if (method->is_method_handle_intrinsic()) { 2088 vmIntrinsics::ID iid = method->intrinsic_id(); 2089 intptr_t start = (intptr_t)__ pc(); 2090 int vep_offset = ((intptr_t)__ pc()) - start; 2091 gen_special_dispatch(masm, 2092 method, 2093 in_sig_bt, 2094 in_regs); 2095 int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period 2096 __ flush(); 2097 int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually 2098 return nmethod::new_native_nmethod(method, 2099 compile_id, 2100 masm->code(), 2101 vep_offset, 2102 frame_complete, 2103 stack_slots / VMRegImpl::slots_per_word, 2104 in_ByteSize(-1), 2105 in_ByteSize(-1), 2106 nullptr); 2107 } 2108 address native_func = method->native_function(); 2109 assert(native_func != nullptr, "must have function"); 2110 2111 // An OopMap for lock (and class if static) 2112 OopMapSet *oop_maps = new OopMapSet(); 2113 intptr_t start = (intptr_t)__ pc(); 2114 2115 // We have received a description of where all the java arg are located 2116 // on entry to the wrapper. We need to convert these args to where 2117 // the jni function will expect them. To figure out where they go 2118 // we convert the java signature to a C signature by inserting 2119 // the hidden arguments as arg[0] and possibly arg[1] (static method) 2120 2121 const int total_in_args = method->size_of_parameters(); 2122 int total_c_args = total_in_args + (method->is_static() ? 2 : 1); 2123 2124 BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args); 2125 VMRegPair* out_regs = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args); 2126 2127 int argc = 0; 2128 out_sig_bt[argc++] = T_ADDRESS; 2129 if (method->is_static()) { 2130 out_sig_bt[argc++] = T_OBJECT; 2131 } 2132 2133 for (int i = 0; i < total_in_args ; i++ ) { 2134 out_sig_bt[argc++] = in_sig_bt[i]; 2135 } 2136 2137 // Now figure out where the args must be stored and how much stack space 2138 // they require. 2139 int out_arg_slots; 2140 out_arg_slots = c_calling_convention(out_sig_bt, out_regs, total_c_args); 2141 2142 // Compute framesize for the wrapper. We need to handlize all oops in 2143 // incoming registers 2144 2145 // Calculate the total number of stack slots we will need. 2146 2147 // First count the abi requirement plus all of the outgoing args 2148 int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots; 2149 2150 // Now the space for the inbound oop handle area 2151 int total_save_slots = 6 * VMRegImpl::slots_per_word; // 6 arguments passed in registers 2152 2153 int oop_handle_offset = stack_slots; 2154 stack_slots += total_save_slots; 2155 2156 // Now any space we need for handlizing a klass if static method 2157 2158 int klass_slot_offset = 0; 2159 int klass_offset = -1; 2160 int lock_slot_offset = 0; 2161 bool is_static = false; 2162 2163 if (method->is_static()) { 2164 klass_slot_offset = stack_slots; 2165 stack_slots += VMRegImpl::slots_per_word; 2166 klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size; 2167 is_static = true; 2168 } 2169 2170 // Plus a lock if needed 2171 2172 if (method->is_synchronized()) { 2173 lock_slot_offset = stack_slots; 2174 stack_slots += VMRegImpl::slots_per_word; 2175 } 2176 2177 // Now a place (+2) to save return values or temp during shuffling 2178 // + 4 for return address (which we own) and saved rbp 2179 stack_slots += 6; 2180 2181 // Ok The space we have allocated will look like: 2182 // 2183 // 2184 // FP-> | | 2185 // |---------------------| 2186 // | 2 slots for moves | 2187 // |---------------------| 2188 // | lock box (if sync) | 2189 // |---------------------| <- lock_slot_offset 2190 // | klass (if static) | 2191 // |---------------------| <- klass_slot_offset 2192 // | oopHandle area | 2193 // |---------------------| <- oop_handle_offset (6 java arg registers) 2194 // | outbound memory | 2195 // | based arguments | 2196 // | | 2197 // |---------------------| 2198 // | | 2199 // SP-> | out_preserved_slots | 2200 // 2201 // 2202 2203 2204 // Now compute actual number of stack words we need rounding to make 2205 // stack properly aligned. 2206 stack_slots = align_up(stack_slots, StackAlignmentInSlots); 2207 2208 int stack_size = stack_slots * VMRegImpl::stack_slot_size; 2209 2210 // First thing make an ic check to see if we should even be here 2211 2212 // We are free to use all registers as temps without saving them and 2213 // restoring them except rbp. rbp is the only callee save register 2214 // as far as the interpreter and the compiler(s) are concerned. 2215 2216 const Register receiver = j_rarg0; 2217 2218 Label exception_pending; 2219 2220 assert_different_registers(receiver, rscratch1, rscratch2); 2221 __ verify_oop(receiver); 2222 __ ic_check(8 /* end_alignment */); 2223 2224 int vep_offset = ((intptr_t)__ pc()) - start; 2225 2226 if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) { 2227 Label L_skip_barrier; 2228 Register klass = r10; 2229 __ mov_metadata(klass, method->method_holder()); // InstanceKlass* 2230 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/); 2231 2232 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path 2233 2234 __ bind(L_skip_barrier); 2235 } 2236 2237 #ifdef COMPILER1 2238 // For Object.hashCode, System.identityHashCode try to pull hashCode from object header if available. 2239 if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) { 2240 inline_check_hashcode_from_object_header(masm, method, j_rarg0 /*obj_reg*/, rax /*result*/); 2241 } 2242 #endif // COMPILER1 2243 2244 // The instruction at the verified entry point must be 5 bytes or longer 2245 // because it can be patched on the fly by make_non_entrant. The stack bang 2246 // instruction fits that requirement. 2247 2248 // Generate stack overflow check 2249 __ bang_stack_with_offset((int)StackOverflow::stack_shadow_zone_size()); 2250 2251 // Generate a new frame for the wrapper. 2252 __ enter(); 2253 // -2 because return address is already present and so is saved rbp 2254 __ subptr(rsp, stack_size - 2*wordSize); 2255 2256 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 2257 // native wrapper is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub 2258 bs->nmethod_entry_barrier(masm, nullptr /* slow_path */, nullptr /* continuation */); 2259 2260 // Frame is now completed as far as size and linkage. 2261 int frame_complete = ((intptr_t)__ pc()) - start; 2262 2263 #ifdef ASSERT 2264 __ check_stack_alignment(rsp, "improperly aligned stack"); 2265 #endif /* ASSERT */ 2266 2267 2268 // We use r14 as the oop handle for the receiver/klass 2269 // It is callee save so it survives the call to native 2270 2271 const Register oop_handle_reg = r14; 2272 2273 // 2274 // We immediately shuffle the arguments so that any vm call we have to 2275 // make from here on out (sync slow path, jvmti, etc.) we will have 2276 // captured the oops from our caller and have a valid oopMap for 2277 // them. 2278 2279 // ----------------- 2280 // The Grand Shuffle 2281 2282 // The Java calling convention is either equal (linux) or denser (win64) than the 2283 // c calling convention. However the because of the jni_env argument the c calling 2284 // convention always has at least one more (and two for static) arguments than Java. 2285 // Therefore if we move the args from java -> c backwards then we will never have 2286 // a register->register conflict and we don't have to build a dependency graph 2287 // and figure out how to break any cycles. 2288 // 2289 2290 // Record esp-based slot for receiver on stack for non-static methods 2291 int receiver_offset = -1; 2292 2293 // This is a trick. We double the stack slots so we can claim 2294 // the oops in the caller's frame. Since we are sure to have 2295 // more args than the caller doubling is enough to make 2296 // sure we can capture all the incoming oop args from the 2297 // caller. 2298 // 2299 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/); 2300 2301 // Mark location of rbp (someday) 2302 // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rbp)); 2303 2304 // Use eax, ebx as temporaries during any memory-memory moves we have to do 2305 // All inbound args are referenced based on rbp and all outbound args via rsp. 2306 2307 2308 #ifdef ASSERT 2309 bool reg_destroyed[Register::number_of_registers]; 2310 bool freg_destroyed[XMMRegister::number_of_registers]; 2311 for ( int r = 0 ; r < Register::number_of_registers ; r++ ) { 2312 reg_destroyed[r] = false; 2313 } 2314 for ( int f = 0 ; f < XMMRegister::number_of_registers ; f++ ) { 2315 freg_destroyed[f] = false; 2316 } 2317 2318 #endif /* ASSERT */ 2319 2320 // For JNI natives the incoming and outgoing registers are offset upwards. 2321 GrowableArray<int> arg_order(2 * total_in_args); 2322 2323 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) { 2324 arg_order.push(i); 2325 arg_order.push(c_arg); 2326 } 2327 2328 for (int ai = 0; ai < arg_order.length(); ai += 2) { 2329 int i = arg_order.at(ai); 2330 int c_arg = arg_order.at(ai + 1); 2331 __ block_comment(err_msg("move %d -> %d", i, c_arg)); 2332 #ifdef ASSERT 2333 if (in_regs[i].first()->is_Register()) { 2334 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!"); 2335 } else if (in_regs[i].first()->is_XMMRegister()) { 2336 assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!"); 2337 } 2338 if (out_regs[c_arg].first()->is_Register()) { 2339 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true; 2340 } else if (out_regs[c_arg].first()->is_XMMRegister()) { 2341 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true; 2342 } 2343 #endif /* ASSERT */ 2344 switch (in_sig_bt[i]) { 2345 case T_ARRAY: 2346 case T_OBJECT: 2347 __ object_move(map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg], 2348 ((i == 0) && (!is_static)), 2349 &receiver_offset); 2350 break; 2351 case T_VOID: 2352 break; 2353 2354 case T_FLOAT: 2355 __ float_move(in_regs[i], out_regs[c_arg]); 2356 break; 2357 2358 case T_DOUBLE: 2359 assert( i + 1 < total_in_args && 2360 in_sig_bt[i + 1] == T_VOID && 2361 out_sig_bt[c_arg+1] == T_VOID, "bad arg list"); 2362 __ double_move(in_regs[i], out_regs[c_arg]); 2363 break; 2364 2365 case T_LONG : 2366 __ long_move(in_regs[i], out_regs[c_arg]); 2367 break; 2368 2369 case T_ADDRESS: assert(false, "found T_ADDRESS in java args"); 2370 2371 default: 2372 __ move32_64(in_regs[i], out_regs[c_arg]); 2373 } 2374 } 2375 2376 int c_arg; 2377 2378 // Pre-load a static method's oop into r14. Used both by locking code and 2379 // the normal JNI call code. 2380 // point c_arg at the first arg that is already loaded in case we 2381 // need to spill before we call out 2382 c_arg = total_c_args - total_in_args; 2383 2384 if (method->is_static()) { 2385 2386 // load oop into a register 2387 __ movoop(oop_handle_reg, JNIHandles::make_local(method->method_holder()->java_mirror())); 2388 2389 // Now handlize the static class mirror it's known not-null. 2390 __ movptr(Address(rsp, klass_offset), oop_handle_reg); 2391 map->set_oop(VMRegImpl::stack2reg(klass_slot_offset)); 2392 2393 // Now get the handle 2394 __ lea(oop_handle_reg, Address(rsp, klass_offset)); 2395 // store the klass handle as second argument 2396 __ movptr(c_rarg1, oop_handle_reg); 2397 // and protect the arg if we must spill 2398 c_arg--; 2399 } 2400 2401 // Change state to native (we save the return address in the thread, since it might not 2402 // be pushed on the stack when we do a stack traversal). It is enough that the pc() 2403 // points into the right code segment. It does not have to be the correct return pc. 2404 // We use the same pc/oopMap repeatedly when we call out 2405 2406 Label native_return; 2407 if (method->is_object_wait0()) { 2408 // For convenience we use the pc we want to resume to in case of preemption on Object.wait. 2409 __ set_last_Java_frame(rsp, noreg, native_return, rscratch1); 2410 } else { 2411 intptr_t the_pc = (intptr_t) __ pc(); 2412 oop_maps->add_gc_map(the_pc - start, map); 2413 2414 __ set_last_Java_frame(rsp, noreg, __ pc(), rscratch1); 2415 } 2416 2417 // We have all of the arguments setup at this point. We must not touch any register 2418 // argument registers at this point (what if we save/restore them there are no oop? 2419 2420 if (DTraceMethodProbes) { 2421 // protect the args we've loaded 2422 save_args(masm, total_c_args, c_arg, out_regs); 2423 __ mov_metadata(c_rarg1, method()); 2424 __ call_VM_leaf( 2425 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), 2426 r15_thread, c_rarg1); 2427 restore_args(masm, total_c_args, c_arg, out_regs); 2428 } 2429 2430 // RedefineClasses() tracing support for obsolete method entry 2431 if (log_is_enabled(Trace, redefine, class, obsolete)) { 2432 // protect the args we've loaded 2433 save_args(masm, total_c_args, c_arg, out_regs); 2434 __ mov_metadata(c_rarg1, method()); 2435 __ call_VM_leaf( 2436 CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 2437 r15_thread, c_rarg1); 2438 restore_args(masm, total_c_args, c_arg, out_regs); 2439 } 2440 2441 // Lock a synchronized method 2442 2443 // Register definitions used by locking and unlocking 2444 2445 const Register swap_reg = rax; // Must use rax for cmpxchg instruction 2446 const Register obj_reg = rbx; // Will contain the oop 2447 const Register lock_reg = r13; // Address of compiler lock object (BasicLock) 2448 2449 Label slow_path_lock; 2450 Label lock_done; 2451 2452 if (method->is_synchronized()) { 2453 // Get the handle (the 2nd argument) 2454 __ mov(oop_handle_reg, c_rarg1); 2455 2456 // Get address of the box 2457 2458 __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size)); 2459 2460 // Load the oop from the handle 2461 __ movptr(obj_reg, Address(oop_handle_reg, 0)); 2462 2463 __ lightweight_lock(lock_reg, obj_reg, swap_reg, rscratch1, slow_path_lock); 2464 2465 // Slow path will re-enter here 2466 __ bind(lock_done); 2467 } 2468 2469 // Finally just about ready to make the JNI call 2470 2471 // get JNIEnv* which is first argument to native 2472 __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset()))); 2473 2474 // Now set thread in native 2475 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native); 2476 2477 __ call(RuntimeAddress(native_func)); 2478 2479 // Verify or restore cpu control state after JNI call 2480 __ restore_cpu_control_state_after_jni(rscratch1); 2481 2482 // Unpack native results. 2483 switch (ret_type) { 2484 case T_BOOLEAN: __ c2bool(rax); break; 2485 case T_CHAR : __ movzwl(rax, rax); break; 2486 case T_BYTE : __ sign_extend_byte (rax); break; 2487 case T_SHORT : __ sign_extend_short(rax); break; 2488 case T_INT : /* nothing to do */ break; 2489 case T_DOUBLE : 2490 case T_FLOAT : 2491 // Result is in xmm0 we'll save as needed 2492 break; 2493 case T_ARRAY: // Really a handle 2494 case T_OBJECT: // Really a handle 2495 break; // can't de-handlize until after safepoint check 2496 case T_VOID: break; 2497 case T_LONG: break; 2498 default : ShouldNotReachHere(); 2499 } 2500 2501 // Switch thread to "native transition" state before reading the synchronization state. 2502 // This additional state is necessary because reading and testing the synchronization 2503 // state is not atomic w.r.t. GC, as this scenario demonstrates: 2504 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted. 2505 // VM thread changes sync state to synchronizing and suspends threads for GC. 2506 // Thread A is resumed to finish this native method, but doesn't block here since it 2507 // didn't see any synchronization is progress, and escapes. 2508 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans); 2509 2510 // Force this write out before the read below 2511 if (!UseSystemMemoryBarrier) { 2512 __ membar(Assembler::Membar_mask_bits( 2513 Assembler::LoadLoad | Assembler::LoadStore | 2514 Assembler::StoreLoad | Assembler::StoreStore)); 2515 } 2516 2517 // check for safepoint operation in progress and/or pending suspend requests 2518 { 2519 Label Continue; 2520 Label slow_path; 2521 2522 __ safepoint_poll(slow_path, true /* at_return */, false /* in_nmethod */); 2523 2524 __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0); 2525 __ jcc(Assembler::equal, Continue); 2526 __ bind(slow_path); 2527 2528 // Don't use call_VM as it will see a possible pending exception and forward it 2529 // and never return here preventing us from clearing _last_native_pc down below. 2530 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are 2531 // preserved and correspond to the bcp/locals pointers. So we do a runtime call 2532 // by hand. 2533 // 2534 __ vzeroupper(); 2535 save_native_result(masm, ret_type, stack_slots); 2536 __ mov(c_rarg0, r15_thread); 2537 __ mov(r12, rsp); // remember sp 2538 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 2539 __ andptr(rsp, -16); // align stack as required by ABI 2540 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans))); 2541 __ mov(rsp, r12); // restore sp 2542 __ reinit_heapbase(); 2543 // Restore any method result value 2544 restore_native_result(masm, ret_type, stack_slots); 2545 __ bind(Continue); 2546 } 2547 2548 // change thread state 2549 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java); 2550 2551 if (method->is_object_wait0()) { 2552 // Check preemption for Object.wait() 2553 __ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset())); 2554 __ cmpptr(rscratch1, NULL_WORD); 2555 __ jccb(Assembler::equal, native_return); 2556 __ movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); 2557 __ jmp(rscratch1); 2558 __ bind(native_return); 2559 2560 intptr_t the_pc = (intptr_t) __ pc(); 2561 oop_maps->add_gc_map(the_pc - start, map); 2562 } 2563 2564 2565 Label reguard; 2566 Label reguard_done; 2567 __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled); 2568 __ jcc(Assembler::equal, reguard); 2569 __ bind(reguard_done); 2570 2571 // native result if any is live 2572 2573 // Unlock 2574 Label slow_path_unlock; 2575 Label unlock_done; 2576 if (method->is_synchronized()) { 2577 2578 Label fast_done; 2579 2580 // Get locked oop from the handle we passed to jni 2581 __ movptr(obj_reg, Address(oop_handle_reg, 0)); 2582 2583 // Must save rax if it is live now because cmpxchg must use it 2584 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) { 2585 save_native_result(masm, ret_type, stack_slots); 2586 } 2587 2588 __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock); 2589 2590 // slow path re-enters here 2591 __ bind(unlock_done); 2592 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) { 2593 restore_native_result(masm, ret_type, stack_slots); 2594 } 2595 2596 __ bind(fast_done); 2597 } 2598 if (DTraceMethodProbes) { 2599 save_native_result(masm, ret_type, stack_slots); 2600 __ mov_metadata(c_rarg1, method()); 2601 __ call_VM_leaf( 2602 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), 2603 r15_thread, c_rarg1); 2604 restore_native_result(masm, ret_type, stack_slots); 2605 } 2606 2607 __ reset_last_Java_frame(false); 2608 2609 // Unbox oop result, e.g. JNIHandles::resolve value. 2610 if (is_reference_type(ret_type)) { 2611 __ resolve_jobject(rax /* value */, 2612 rcx /* tmp */); 2613 } 2614 2615 if (CheckJNICalls) { 2616 // clear_pending_jni_exception_check 2617 __ movptr(Address(r15_thread, JavaThread::pending_jni_exception_check_fn_offset()), NULL_WORD); 2618 } 2619 2620 // reset handle block 2621 __ movptr(rcx, Address(r15_thread, JavaThread::active_handles_offset())); 2622 __ movl(Address(rcx, JNIHandleBlock::top_offset()), NULL_WORD); 2623 2624 // pop our frame 2625 2626 __ leave(); 2627 2628 #if INCLUDE_JFR 2629 // We need to do a poll test after unwind in case the sampler 2630 // managed to sample the native frame after returning to Java. 2631 Label L_return; 2632 address poll_test_pc = __ pc(); 2633 __ relocate(relocInfo::poll_return_type); 2634 __ testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 2635 __ jccb(Assembler::zero, L_return); 2636 __ lea(rscratch1, InternalAddress(poll_test_pc)); 2637 __ movptr(Address(r15_thread, JavaThread::saved_exception_pc_offset()), rscratch1); 2638 assert(SharedRuntime::polling_page_return_handler_blob() != nullptr, 2639 "polling page return stub not created yet"); 2640 address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); 2641 __ jump(RuntimeAddress(stub)); 2642 __ bind(L_return); 2643 #endif // INCLUDE_JFR 2644 2645 // Any exception pending? 2646 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD); 2647 __ jcc(Assembler::notEqual, exception_pending); 2648 2649 // Return 2650 2651 __ ret(0); 2652 2653 // Unexpected paths are out of line and go here 2654 2655 // forward the exception 2656 __ bind(exception_pending); 2657 2658 // and forward the exception 2659 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 2660 2661 // Slow path locking & unlocking 2662 if (method->is_synchronized()) { 2663 2664 // BEGIN Slow path lock 2665 __ bind(slow_path_lock); 2666 2667 // has last_Java_frame setup. No exceptions so do vanilla call not call_VM 2668 // args are (oop obj, BasicLock* lock, JavaThread* thread) 2669 2670 // protect the args we've loaded 2671 save_args(masm, total_c_args, c_arg, out_regs); 2672 2673 __ mov(c_rarg0, obj_reg); 2674 __ mov(c_rarg1, lock_reg); 2675 __ mov(c_rarg2, r15_thread); 2676 2677 // Not a leaf but we have last_Java_frame setup as we want. 2678 // We don't want to unmount in case of contention since that would complicate preserving 2679 // the arguments that had already been marshalled into the native convention. So we force 2680 // the freeze slow path to find this native wrapper frame (see recurse_freeze_native_frame()) 2681 // and pin the vthread. Otherwise the fast path won't find it since we don't walk the stack. 2682 __ push_cont_fastpath(); 2683 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); 2684 __ pop_cont_fastpath(); 2685 restore_args(masm, total_c_args, c_arg, out_regs); 2686 2687 #ifdef ASSERT 2688 { Label L; 2689 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD); 2690 __ jcc(Assembler::equal, L); 2691 __ stop("no pending exception allowed on exit from monitorenter"); 2692 __ bind(L); 2693 } 2694 #endif 2695 __ jmp(lock_done); 2696 2697 // END Slow path lock 2698 2699 // BEGIN Slow path unlock 2700 __ bind(slow_path_unlock); 2701 2702 // If we haven't already saved the native result we must save it now as xmm registers 2703 // are still exposed. 2704 __ vzeroupper(); 2705 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) { 2706 save_native_result(masm, ret_type, stack_slots); 2707 } 2708 2709 __ lea(c_rarg1, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size)); 2710 2711 __ mov(c_rarg0, obj_reg); 2712 __ mov(c_rarg2, r15_thread); 2713 __ mov(r12, rsp); // remember sp 2714 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 2715 __ andptr(rsp, -16); // align stack as required by ABI 2716 2717 // Save pending exception around call to VM (which contains an EXCEPTION_MARK) 2718 // NOTE that obj_reg == rbx currently 2719 __ movptr(rbx, Address(r15_thread, in_bytes(Thread::pending_exception_offset()))); 2720 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD); 2721 2722 // args are (oop obj, BasicLock* lock, JavaThread* thread) 2723 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C))); 2724 __ mov(rsp, r12); // restore sp 2725 __ reinit_heapbase(); 2726 #ifdef ASSERT 2727 { 2728 Label L; 2729 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD); 2730 __ jcc(Assembler::equal, L); 2731 __ stop("no pending exception allowed on exit complete_monitor_unlocking_C"); 2732 __ bind(L); 2733 } 2734 #endif /* ASSERT */ 2735 2736 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), rbx); 2737 2738 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) { 2739 restore_native_result(masm, ret_type, stack_slots); 2740 } 2741 __ jmp(unlock_done); 2742 2743 // END Slow path unlock 2744 2745 } // synchronized 2746 2747 // SLOW PATH Reguard the stack if needed 2748 2749 __ bind(reguard); 2750 __ vzeroupper(); 2751 save_native_result(masm, ret_type, stack_slots); 2752 __ mov(r12, rsp); // remember sp 2753 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows 2754 __ andptr(rsp, -16); // align stack as required by ABI 2755 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages))); 2756 __ mov(rsp, r12); // restore sp 2757 __ reinit_heapbase(); 2758 restore_native_result(masm, ret_type, stack_slots); 2759 // and continue 2760 __ jmp(reguard_done); 2761 2762 2763 2764 __ flush(); 2765 2766 nmethod *nm = nmethod::new_native_nmethod(method, 2767 compile_id, 2768 masm->code(), 2769 vep_offset, 2770 frame_complete, 2771 stack_slots / VMRegImpl::slots_per_word, 2772 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)), 2773 in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size), 2774 oop_maps); 2775 2776 return nm; 2777 } 2778 2779 // this function returns the adjust size (in number of words) to a c2i adapter 2780 // activation for use during deoptimization 2781 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) { 2782 return (callee_locals - callee_parameters) * Interpreter::stackElementWords; 2783 } 2784 2785 2786 uint SharedRuntime::out_preserve_stack_slots() { 2787 return 0; 2788 } 2789 2790 2791 // Number of stack slots between incoming argument block and the start of 2792 // a new frame. The PROLOG must add this many slots to the stack. The 2793 // EPILOG must remove this many slots. amd64 needs two slots for 2794 // return address. 2795 uint SharedRuntime::in_preserve_stack_slots() { 2796 return 4 + 2 * VerifyStackAtCalls; 2797 } 2798 2799 VMReg SharedRuntime::thread_register() { 2800 return r15_thread->as_VMReg(); 2801 } 2802 2803 //------------------------------generate_deopt_blob---------------------------- 2804 void SharedRuntime::generate_deopt_blob() { 2805 // Allocate space for the code 2806 ResourceMark rm; 2807 // Setup code generation tools 2808 int pad = 0; 2809 if (UseAVX > 2) { 2810 pad += 1024; 2811 } 2812 if (UseAPX) { 2813 pad += 1024; 2814 } 2815 #if INCLUDE_JVMCI 2816 if (EnableJVMCI) { 2817 pad += 512; // Increase the buffer size when compiling for JVMCI 2818 } 2819 #endif 2820 const char* name = SharedRuntime::stub_name(StubId::shared_deopt_id); 2821 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, BlobId::shared_deopt_id); 2822 if (blob != nullptr) { 2823 _deopt_blob = blob->as_deoptimization_blob(); 2824 return; 2825 } 2826 2827 CodeBuffer buffer(name, 2560+pad, 1024); 2828 MacroAssembler* masm = new MacroAssembler(&buffer); 2829 int frame_size_in_words; 2830 OopMap* map = nullptr; 2831 OopMapSet *oop_maps = new OopMapSet(); 2832 2833 // ------------- 2834 // This code enters when returning to a de-optimized nmethod. A return 2835 // address has been pushed on the stack, and return values are in 2836 // registers. 2837 // If we are doing a normal deopt then we were called from the patched 2838 // nmethod from the point we returned to the nmethod. So the return 2839 // address on the stack is wrong by NativeCall::instruction_size 2840 // We will adjust the value so it looks like we have the original return 2841 // address on the stack (like when we eagerly deoptimized). 2842 // In the case of an exception pending when deoptimizing, we enter 2843 // with a return address on the stack that points after the call we patched 2844 // into the exception handler. We have the following register state from, 2845 // e.g., the forward exception stub (see stubGenerator_x86_64.cpp). 2846 // rax: exception oop 2847 // rbx: exception handler 2848 // rdx: throwing pc 2849 // So in this case we simply jam rdx into the useless return address and 2850 // the stack looks just like we want. 2851 // 2852 // At this point we need to de-opt. We save the argument return 2853 // registers. We call the first C routine, fetch_unroll_info(). This 2854 // routine captures the return values and returns a structure which 2855 // describes the current frame size and the sizes of all replacement frames. 2856 // The current frame is compiled code and may contain many inlined 2857 // functions, each with their own JVM state. We pop the current frame, then 2858 // push all the new frames. Then we call the C routine unpack_frames() to 2859 // populate these frames. Finally unpack_frames() returns us the new target 2860 // address. Notice that callee-save registers are BLOWN here; they have 2861 // already been captured in the vframeArray at the time the return PC was 2862 // patched. 2863 address start = __ pc(); 2864 Label cont; 2865 2866 // Prolog for non exception case! 2867 2868 // Save everything in sight. 2869 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true); 2870 2871 // Normal deoptimization. Save exec mode for unpack_frames. 2872 __ movl(r14, Deoptimization::Unpack_deopt); // callee-saved 2873 __ jmp(cont); 2874 2875 int reexecute_offset = __ pc() - start; 2876 #if INCLUDE_JVMCI && !defined(COMPILER1) 2877 if (UseJVMCICompiler) { 2878 // JVMCI does not use this kind of deoptimization 2879 __ should_not_reach_here(); 2880 } 2881 #endif 2882 2883 // Reexecute case 2884 // return address is the pc describes what bci to do re-execute at 2885 2886 // No need to update map as each call to save_live_registers will produce identical oopmap 2887 (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true); 2888 2889 __ movl(r14, Deoptimization::Unpack_reexecute); // callee-saved 2890 __ jmp(cont); 2891 2892 #if INCLUDE_JVMCI 2893 Label after_fetch_unroll_info_call; 2894 int implicit_exception_uncommon_trap_offset = 0; 2895 int uncommon_trap_offset = 0; 2896 2897 if (EnableJVMCI) { 2898 implicit_exception_uncommon_trap_offset = __ pc() - start; 2899 2900 __ pushptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset()))); 2901 __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())), NULL_WORD); 2902 2903 uncommon_trap_offset = __ pc() - start; 2904 2905 // Save everything in sight. 2906 RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true); 2907 // fetch_unroll_info needs to call last_java_frame() 2908 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); 2909 2910 __ movl(c_rarg1, Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset()))); 2911 __ movl(Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())), -1); 2912 2913 __ movl(r14, Deoptimization::Unpack_reexecute); 2914 __ mov(c_rarg0, r15_thread); 2915 __ movl(c_rarg2, r14); // exec mode 2916 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap))); 2917 oop_maps->add_gc_map( __ pc()-start, map->deep_copy()); 2918 2919 __ reset_last_Java_frame(false); 2920 2921 __ jmp(after_fetch_unroll_info_call); 2922 } // EnableJVMCI 2923 #endif // INCLUDE_JVMCI 2924 2925 int exception_offset = __ pc() - start; 2926 2927 // Prolog for exception case 2928 2929 // all registers are dead at this entry point, except for rax, and 2930 // rdx which contain the exception oop and exception pc 2931 // respectively. Set them in TLS and fall thru to the 2932 // unpack_with_exception_in_tls entry point. 2933 2934 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx); 2935 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), rax); 2936 2937 int exception_in_tls_offset = __ pc() - start; 2938 2939 // new implementation because exception oop is now passed in JavaThread 2940 2941 // Prolog for exception case 2942 // All registers must be preserved because they might be used by LinearScan 2943 // Exceptiop oop and throwing PC are passed in JavaThread 2944 // tos: stack at point of call to method that threw the exception (i.e. only 2945 // args are on the stack, no return address) 2946 2947 // make room on stack for the return address 2948 // It will be patched later with the throwing pc. The correct value is not 2949 // available now because loading it from memory would destroy registers. 2950 __ push(0); 2951 2952 // Save everything in sight. 2953 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true); 2954 2955 // Now it is safe to overwrite any register 2956 2957 // Deopt during an exception. Save exec mode for unpack_frames. 2958 __ movl(r14, Deoptimization::Unpack_exception); // callee-saved 2959 2960 // load throwing pc from JavaThread and patch it as the return address 2961 // of the current frame. Then clear the field in JavaThread 2962 2963 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset())); 2964 __ movptr(Address(rbp, wordSize), rdx); 2965 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD); 2966 2967 #ifdef ASSERT 2968 // verify that there is really an exception oop in JavaThread 2969 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset())); 2970 __ verify_oop(rax); 2971 2972 // verify that there is no pending exception 2973 Label no_pending_exception; 2974 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 2975 __ testptr(rax, rax); 2976 __ jcc(Assembler::zero, no_pending_exception); 2977 __ stop("must not have pending exception here"); 2978 __ bind(no_pending_exception); 2979 #endif 2980 2981 __ bind(cont); 2982 2983 // Call C code. Need thread and this frame, but NOT official VM entry 2984 // crud. We cannot block on this call, no GC can happen. 2985 // 2986 // UnrollBlock* fetch_unroll_info(JavaThread* thread) 2987 2988 // fetch_unroll_info needs to call last_java_frame(). 2989 2990 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); 2991 #ifdef ASSERT 2992 { Label L; 2993 __ cmpptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 2994 __ jcc(Assembler::equal, L); 2995 __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared"); 2996 __ bind(L); 2997 } 2998 #endif // ASSERT 2999 __ mov(c_rarg0, r15_thread); 3000 __ movl(c_rarg1, r14); // exec_mode 3001 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info))); 3002 3003 // Need to have an oopmap that tells fetch_unroll_info where to 3004 // find any register it might need. 3005 oop_maps->add_gc_map(__ pc() - start, map); 3006 3007 __ reset_last_Java_frame(false); 3008 3009 #if INCLUDE_JVMCI 3010 if (EnableJVMCI) { 3011 __ bind(after_fetch_unroll_info_call); 3012 } 3013 #endif 3014 3015 // Load UnrollBlock* into rdi 3016 __ mov(rdi, rax); 3017 3018 __ movl(r14, Address(rdi, Deoptimization::UnrollBlock::unpack_kind_offset())); 3019 Label noException; 3020 __ cmpl(r14, Deoptimization::Unpack_exception); // Was exception pending? 3021 __ jcc(Assembler::notEqual, noException); 3022 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset())); 3023 // QQQ this is useless it was null above 3024 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset())); 3025 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD); 3026 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD); 3027 3028 __ verify_oop(rax); 3029 3030 // Overwrite the result registers with the exception results. 3031 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax); 3032 // I think this is useless 3033 __ movptr(Address(rsp, RegisterSaver::rdx_offset_in_bytes()), rdx); 3034 3035 __ bind(noException); 3036 3037 // Only register save data is on the stack. 3038 // Now restore the result registers. Everything else is either dead 3039 // or captured in the vframeArray. 3040 RegisterSaver::restore_result_registers(masm); 3041 3042 // All of the register save area has been popped of the stack. Only the 3043 // return address remains. 3044 3045 // Pop all the frames we must move/replace. 3046 // 3047 // Frame picture (youngest to oldest) 3048 // 1: self-frame (no frame link) 3049 // 2: deopting frame (no frame link) 3050 // 3: caller of deopting frame (could be compiled/interpreted). 3051 // 3052 // Note: by leaving the return address of self-frame on the stack 3053 // and using the size of frame 2 to adjust the stack 3054 // when we are done the return to frame 3 will still be on the stack. 3055 3056 // Pop deoptimized frame 3057 __ movl(rcx, Address(rdi, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset())); 3058 __ addptr(rsp, rcx); 3059 3060 // rsp should be pointing at the return address to the caller (3) 3061 3062 // Pick up the initial fp we should save 3063 // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved) 3064 __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset())); 3065 3066 #ifdef ASSERT 3067 // Compilers generate code that bang the stack by as much as the 3068 // interpreter would need. So this stack banging should never 3069 // trigger a fault. Verify that it does not on non product builds. 3070 __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::total_frame_sizes_offset())); 3071 __ bang_stack_size(rbx, rcx); 3072 #endif 3073 3074 // Load address of array of frame pcs into rcx 3075 __ movptr(rcx, Address(rdi, Deoptimization::UnrollBlock::frame_pcs_offset())); 3076 3077 // Trash the old pc 3078 __ addptr(rsp, wordSize); 3079 3080 // Load address of array of frame sizes into rsi 3081 __ movptr(rsi, Address(rdi, Deoptimization::UnrollBlock::frame_sizes_offset())); 3082 3083 // Load counter into rdx 3084 __ movl(rdx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset())); 3085 3086 // Now adjust the caller's stack to make up for the extra locals 3087 // but record the original sp so that we can save it in the skeletal interpreter 3088 // frame and the stack walking of interpreter_sender will get the unextended sp 3089 // value and not the "real" sp value. 3090 3091 const Register sender_sp = r8; 3092 3093 __ mov(sender_sp, rsp); 3094 __ movl(rbx, Address(rdi, 3095 Deoptimization::UnrollBlock:: 3096 caller_adjustment_offset())); 3097 __ subptr(rsp, rbx); 3098 3099 // Push interpreter frames in a loop 3100 Label loop; 3101 __ bind(loop); 3102 __ movptr(rbx, Address(rsi, 0)); // Load frame size 3103 __ subptr(rbx, 2*wordSize); // We'll push pc and ebp by hand 3104 __ pushptr(Address(rcx, 0)); // Save return address 3105 __ enter(); // Save old & set new ebp 3106 __ subptr(rsp, rbx); // Prolog 3107 // This value is corrected by layout_activation_impl 3108 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD); 3109 __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), sender_sp); // Make it walkable 3110 __ mov(sender_sp, rsp); // Pass sender_sp to next frame 3111 __ addptr(rsi, wordSize); // Bump array pointer (sizes) 3112 __ addptr(rcx, wordSize); // Bump array pointer (pcs) 3113 __ decrementl(rdx); // Decrement counter 3114 __ jcc(Assembler::notZero, loop); 3115 __ pushptr(Address(rcx, 0)); // Save final return address 3116 3117 // Re-push self-frame 3118 __ enter(); // Save old & set new ebp 3119 3120 // Allocate a full sized register save area. 3121 // Return address and rbp are in place, so we allocate two less words. 3122 __ subptr(rsp, (frame_size_in_words - 2) * wordSize); 3123 3124 // Restore frame locals after moving the frame 3125 __ movdbl(Address(rsp, RegisterSaver::xmm0_offset_in_bytes()), xmm0); 3126 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax); 3127 3128 // Call C code. Need thread but NOT official VM entry 3129 // crud. We cannot block on this call, no GC can happen. Call should 3130 // restore return values to their stack-slots with the new SP. 3131 // 3132 // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode) 3133 3134 // Use rbp because the frames look interpreted now 3135 // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP. 3136 // Don't need the precise return PC here, just precise enough to point into this code blob. 3137 address the_pc = __ pc(); 3138 __ set_last_Java_frame(noreg, rbp, the_pc, rscratch1); 3139 3140 __ andptr(rsp, -(StackAlignmentInBytes)); // Fix stack alignment as required by ABI 3141 __ mov(c_rarg0, r15_thread); 3142 __ movl(c_rarg1, r14); // second arg: exec_mode 3143 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames))); 3144 // Revert SP alignment after call since we're going to do some SP relative addressing below 3145 __ movptr(rsp, Address(r15_thread, JavaThread::last_Java_sp_offset())); 3146 3147 // Set an oopmap for the call site 3148 // Use the same PC we used for the last java frame 3149 oop_maps->add_gc_map(the_pc - start, 3150 new OopMap( frame_size_in_words, 0 )); 3151 3152 // Clear fp AND pc 3153 __ reset_last_Java_frame(true); 3154 3155 // Collect return values 3156 __ movdbl(xmm0, Address(rsp, RegisterSaver::xmm0_offset_in_bytes())); 3157 __ movptr(rax, Address(rsp, RegisterSaver::rax_offset_in_bytes())); 3158 // I think this is useless (throwing pc?) 3159 __ movptr(rdx, Address(rsp, RegisterSaver::rdx_offset_in_bytes())); 3160 3161 // Pop self-frame. 3162 __ leave(); // Epilog 3163 3164 // Jump to interpreter 3165 __ ret(0); 3166 3167 // Make sure all code is generated 3168 masm->flush(); 3169 3170 _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words); 3171 _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset); 3172 #if INCLUDE_JVMCI 3173 if (EnableJVMCI) { 3174 _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset); 3175 _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset); 3176 } 3177 #endif 3178 3179 AOTCodeCache::store_code_blob(*_deopt_blob, AOTCodeEntry::SharedBlob, BlobId::shared_deopt_id); 3180 } 3181 3182 //------------------------------generate_handler_blob------ 3183 // 3184 // Generate a special Compile2Runtime blob that saves all registers, 3185 // and setup oopmap. 3186 // 3187 SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr) { 3188 assert(StubRoutines::forward_exception_entry() != nullptr, 3189 "must be generated before"); 3190 assert(is_polling_page_id(id), "expected a polling page stub id"); 3191 3192 // Allocate space for the code. Setup code generation tools. 3193 const char* name = SharedRuntime::stub_name(id); 3194 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3195 if (blob != nullptr) { 3196 return blob->as_safepoint_blob(); 3197 } 3198 3199 ResourceMark rm; 3200 OopMapSet *oop_maps = new OopMapSet(); 3201 OopMap* map; 3202 CodeBuffer buffer(name, 2548, 1024); 3203 MacroAssembler* masm = new MacroAssembler(&buffer); 3204 3205 address start = __ pc(); 3206 address call_pc = nullptr; 3207 int frame_size_in_words; 3208 bool cause_return = (id == StubId::shared_polling_page_return_handler_id); 3209 bool save_wide_vectors = (id == StubId::shared_polling_page_vectors_safepoint_handler_id); 3210 3211 // Make room for return address (or push it again) 3212 if (!cause_return) { 3213 __ push(rbx); 3214 } 3215 3216 // Save registers, fpu state, and flags 3217 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, save_wide_vectors); 3218 3219 // The following is basically a call_VM. However, we need the precise 3220 // address of the call in order to generate an oopmap. Hence, we do all the 3221 // work ourselves. 3222 3223 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); // JavaFrameAnchor::capture_last_Java_pc() will get the pc from the return address, which we store next: 3224 3225 // The return address must always be correct so that frame constructor never 3226 // sees an invalid pc. 3227 3228 if (!cause_return) { 3229 // Get the return pc saved by the signal handler and stash it in its appropriate place on the stack. 3230 // Additionally, rbx is a callee saved register and we can look at it later to determine 3231 // if someone changed the return address for us! 3232 __ movptr(rbx, Address(r15_thread, JavaThread::saved_exception_pc_offset())); 3233 __ movptr(Address(rbp, wordSize), rbx); 3234 } 3235 3236 // Do the call 3237 __ mov(c_rarg0, r15_thread); 3238 __ call(RuntimeAddress(call_ptr)); 3239 3240 // Set an oopmap for the call site. This oopmap will map all 3241 // oop-registers and debug-info registers as callee-saved. This 3242 // will allow deoptimization at this safepoint to find all possible 3243 // debug-info recordings, as well as let GC find all oops. 3244 3245 oop_maps->add_gc_map( __ pc() - start, map); 3246 3247 Label noException; 3248 3249 __ reset_last_Java_frame(false); 3250 3251 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 3252 __ jcc(Assembler::equal, noException); 3253 3254 // Exception pending 3255 3256 RegisterSaver::restore_live_registers(masm, save_wide_vectors); 3257 3258 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 3259 3260 // No exception case 3261 __ bind(noException); 3262 3263 Label no_adjust; 3264 #ifdef ASSERT 3265 Label bail; 3266 #endif 3267 if (!cause_return) { 3268 Label no_prefix, not_special, check_rex_prefix; 3269 3270 // If our stashed return pc was modified by the runtime we avoid touching it 3271 __ cmpptr(rbx, Address(rbp, wordSize)); 3272 __ jcc(Assembler::notEqual, no_adjust); 3273 3274 // Skip over the poll instruction. 3275 // See NativeInstruction::is_safepoint_poll() 3276 // Possible encodings: 3277 // 85 00 test %eax,(%rax) 3278 // 85 01 test %eax,(%rcx) 3279 // 85 02 test %eax,(%rdx) 3280 // 85 03 test %eax,(%rbx) 3281 // 85 06 test %eax,(%rsi) 3282 // 85 07 test %eax,(%rdi) 3283 // 3284 // 41 85 00 test %eax,(%r8) 3285 // 41 85 01 test %eax,(%r9) 3286 // 41 85 02 test %eax,(%r10) 3287 // 41 85 03 test %eax,(%r11) 3288 // 41 85 06 test %eax,(%r14) 3289 // 41 85 07 test %eax,(%r15) 3290 // 3291 // 85 04 24 test %eax,(%rsp) 3292 // 41 85 04 24 test %eax,(%r12) 3293 // 85 45 00 test %eax,0x0(%rbp) 3294 // 41 85 45 00 test %eax,0x0(%r13) 3295 // 3296 // Notes: 3297 // Format of legacy MAP0 test instruction:- 3298 // [REX/REX2] [OPCODE] [ModRM] [SIB] [DISP] [IMM32] 3299 // o For safepoint polling instruction "test %eax,(%rax)", encoding of first register 3300 // operand and base register of memory operand is b/w [0-8), hence we do not require 3301 // additional REX prefix where REX.B bit stores MSB bit of register encoding, which 3302 // is why two bytes encoding is sufficient here. 3303 // o For safepoint polling instruction like "test %eax,(%r8)", register encoding of BASE 3304 // register of memory operand is 1000, thus we need additional REX prefix in this case, 3305 // there by adding additional byte to instruction encoding. 3306 // o In case BASE register is one of the 32 extended GPR registers available only on targets 3307 // supporting Intel APX extension, then we need to emit two bytes REX2 prefix to hold 3308 // most significant two bits of 5 bit register encoding. 3309 3310 if (VM_Version::supports_apx_f()) { 3311 __ cmpb(Address(rbx, 0), Assembler::REX2); 3312 __ jccb(Assembler::notEqual, check_rex_prefix); 3313 __ addptr(rbx, 2); 3314 __ bind(check_rex_prefix); 3315 } 3316 __ cmpb(Address(rbx, 0), NativeTstRegMem::instruction_rex_b_prefix); 3317 __ jccb(Assembler::notEqual, no_prefix); 3318 __ addptr(rbx, 1); 3319 __ bind(no_prefix); 3320 #ifdef ASSERT 3321 __ movptr(rax, rbx); // remember where 0x85 should be, for verification below 3322 #endif 3323 // r12/r13/rsp/rbp base encoding takes 3 bytes with the following register values: 3324 // r12/rsp 0x04 3325 // r13/rbp 0x05 3326 __ movzbq(rcx, Address(rbx, 1)); 3327 __ andptr(rcx, 0x07); // looking for 0x04 .. 0x05 3328 __ subptr(rcx, 4); // looking for 0x00 .. 0x01 3329 __ cmpptr(rcx, 1); 3330 __ jccb(Assembler::above, not_special); 3331 __ addptr(rbx, 1); 3332 __ bind(not_special); 3333 #ifdef ASSERT 3334 // Verify the correct encoding of the poll we're about to skip. 3335 __ cmpb(Address(rax, 0), NativeTstRegMem::instruction_code_memXregl); 3336 __ jcc(Assembler::notEqual, bail); 3337 // Mask out the modrm bits 3338 __ testb(Address(rax, 1), NativeTstRegMem::modrm_mask); 3339 // rax encodes to 0, so if the bits are nonzero it's incorrect 3340 __ jcc(Assembler::notZero, bail); 3341 #endif 3342 // Adjust return pc forward to step over the safepoint poll instruction 3343 __ addptr(rbx, 2); 3344 __ movptr(Address(rbp, wordSize), rbx); 3345 } 3346 3347 __ bind(no_adjust); 3348 // Normal exit, restore registers and exit. 3349 RegisterSaver::restore_live_registers(masm, save_wide_vectors); 3350 __ ret(0); 3351 3352 #ifdef ASSERT 3353 __ bind(bail); 3354 __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected"); 3355 #endif 3356 3357 // Make sure all code is generated 3358 masm->flush(); 3359 3360 // Fill-out other meta info 3361 SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words); 3362 3363 AOTCodeCache::store_code_blob(*sp_blob, AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3364 return sp_blob; 3365 } 3366 3367 // 3368 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss 3369 // 3370 // Generate a stub that calls into vm to find out the proper destination 3371 // of a java call. All the argument registers are live at this point 3372 // but since this is generic code we don't know what they are and the caller 3373 // must do any gc of the args. 3374 // 3375 RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination) { 3376 assert (StubRoutines::forward_exception_entry() != nullptr, "must be generated before"); 3377 assert(is_resolve_id(id), "expected a resolve stub id"); 3378 3379 const char* name = SharedRuntime::stub_name(id); 3380 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3381 if (blob != nullptr) { 3382 return blob->as_runtime_stub(); 3383 } 3384 3385 // allocate space for the code 3386 ResourceMark rm; 3387 CodeBuffer buffer(name, 1552, 512); 3388 MacroAssembler* masm = new MacroAssembler(&buffer); 3389 3390 int frame_size_in_words; 3391 3392 OopMapSet *oop_maps = new OopMapSet(); 3393 OopMap* map = nullptr; 3394 3395 int start = __ offset(); 3396 3397 // No need to save vector registers since they are caller-saved anyway. 3398 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ false); 3399 3400 int frame_complete = __ offset(); 3401 3402 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1); 3403 3404 __ mov(c_rarg0, r15_thread); 3405 3406 __ call(RuntimeAddress(destination)); 3407 3408 3409 // Set an oopmap for the call site. 3410 // We need this not only for callee-saved registers, but also for volatile 3411 // registers that the compiler might be keeping live across a safepoint. 3412 3413 oop_maps->add_gc_map( __ offset() - start, map); 3414 3415 // rax contains the address we are going to jump to assuming no exception got installed 3416 3417 // clear last_Java_sp 3418 __ reset_last_Java_frame(false); 3419 // check for pending exceptions 3420 Label pending; 3421 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 3422 __ jcc(Assembler::notEqual, pending); 3423 3424 // get the returned Method* 3425 __ get_vm_result_metadata(rbx); 3426 __ movptr(Address(rsp, RegisterSaver::rbx_offset_in_bytes()), rbx); 3427 3428 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax); 3429 3430 RegisterSaver::restore_live_registers(masm); 3431 3432 // We are back to the original state on entry and ready to go. 3433 3434 __ jmp(rax); 3435 3436 // Pending exception after the safepoint 3437 3438 __ bind(pending); 3439 3440 RegisterSaver::restore_live_registers(masm); 3441 3442 // exception pending => remove activation and forward to exception handler 3443 3444 __ movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD); 3445 3446 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); 3447 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 3448 3449 // ------------- 3450 // make sure all code is generated 3451 masm->flush(); 3452 3453 // return the blob 3454 // frame_size_words or bytes?? 3455 RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true); 3456 3457 AOTCodeCache::store_code_blob(*rs_blob, AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3458 return rs_blob; 3459 } 3460 3461 // Continuation point for throwing of implicit exceptions that are 3462 // not handled in the current activation. Fabricates an exception 3463 // oop and initiates normal exception dispatching in this 3464 // frame. Since we need to preserve callee-saved values (currently 3465 // only for C2, but done for C1 as well) we need a callee-saved oop 3466 // map and therefore have to make these stubs into RuntimeStubs 3467 // rather than BufferBlobs. If the compiler needs all registers to 3468 // be preserved between the fault point and the exception handler 3469 // then it must assume responsibility for that in 3470 // AbstractCompiler::continuation_for_implicit_null_exception or 3471 // continuation_for_implicit_division_by_zero_exception. All other 3472 // implicit exceptions (e.g., NullPointerException or 3473 // AbstractMethodError on entry) are either at call sites or 3474 // otherwise assume that stack unwinding will be initiated, so 3475 // caller saved registers were assumed volatile in the compiler. 3476 RuntimeStub* SharedRuntime::generate_throw_exception(StubId id, address runtime_entry) { 3477 assert(is_throw_id(id), "expected a throw stub id"); 3478 3479 const char* name = SharedRuntime::stub_name(id); 3480 3481 // Information about frame layout at time of blocking runtime call. 3482 // Note that we only have to preserve callee-saved registers since 3483 // the compilers are responsible for supplying a continuation point 3484 // if they expect all registers to be preserved. 3485 enum layout { 3486 rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt, 3487 rbp_off2, 3488 return_off, 3489 return_off2, 3490 framesize // inclusive of return address 3491 }; 3492 3493 int insts_size = 512; 3494 int locs_size = 64; 3495 3496 const char* timer_msg = "SharedRuntime generate_throw_exception"; 3497 TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime)); 3498 3499 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3500 if (blob != nullptr) { 3501 return blob->as_runtime_stub(); 3502 } 3503 3504 ResourceMark rm; 3505 CodeBuffer code(name, insts_size, locs_size); 3506 OopMapSet* oop_maps = new OopMapSet(); 3507 MacroAssembler* masm = new MacroAssembler(&code); 3508 3509 address start = __ pc(); 3510 3511 // This is an inlined and slightly modified version of call_VM 3512 // which has the ability to fetch the return PC out of 3513 // thread-local storage and also sets up last_Java_sp slightly 3514 // differently than the real call_VM 3515 3516 __ enter(); // required for proper stackwalking of RuntimeStub frame 3517 3518 assert(is_even(framesize/2), "sp not 16-byte aligned"); 3519 3520 // return address and rbp are already in place 3521 __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog 3522 3523 int frame_complete = __ pc() - start; 3524 3525 // Set up last_Java_sp and last_Java_fp 3526 address the_pc = __ pc(); 3527 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1); 3528 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack 3529 3530 // Call runtime 3531 __ movptr(c_rarg0, r15_thread); 3532 BLOCK_COMMENT("call runtime_entry"); 3533 __ call(RuntimeAddress(runtime_entry)); 3534 3535 // Generate oop map 3536 OopMap* map = new OopMap(framesize, 0); 3537 3538 oop_maps->add_gc_map(the_pc - start, map); 3539 3540 __ reset_last_Java_frame(true); 3541 3542 __ leave(); // required for proper stackwalking of RuntimeStub frame 3543 3544 // check for pending exceptions 3545 #ifdef ASSERT 3546 Label L; 3547 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 3548 __ jcc(Assembler::notEqual, L); 3549 __ should_not_reach_here(); 3550 __ bind(L); 3551 #endif // ASSERT 3552 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 3553 3554 3555 // codeBlob framesize is in words (not VMRegImpl::slot_size) 3556 RuntimeStub* stub = 3557 RuntimeStub::new_runtime_stub(name, 3558 &code, 3559 frame_complete, 3560 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 3561 oop_maps, false); 3562 AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id)); 3563 3564 return stub; 3565 } 3566 3567 //------------------------------Montgomery multiplication------------------------ 3568 // 3569 3570 #ifndef _WINDOWS 3571 3572 // Subtract 0:b from carry:a. Return carry. 3573 static julong 3574 sub(julong a[], julong b[], julong carry, long len) { 3575 long long i = 0, cnt = len; 3576 julong tmp; 3577 asm volatile("clc; " 3578 "0: ; " 3579 "mov (%[b], %[i], 8), %[tmp]; " 3580 "sbb %[tmp], (%[a], %[i], 8); " 3581 "inc %[i]; dec %[cnt]; " 3582 "jne 0b; " 3583 "mov %[carry], %[tmp]; sbb $0, %[tmp]; " 3584 : [i]"+r"(i), [cnt]"+r"(cnt), [tmp]"=&r"(tmp) 3585 : [a]"r"(a), [b]"r"(b), [carry]"r"(carry) 3586 : "memory"); 3587 return tmp; 3588 } 3589 3590 // Multiply (unsigned) Long A by Long B, accumulating the double- 3591 // length result into the accumulator formed of T0, T1, and T2. 3592 #define MACC(A, B, T0, T1, T2) \ 3593 do { \ 3594 unsigned long hi, lo; \ 3595 __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4" \ 3596 : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2) \ 3597 : "r"(A), "a"(B) : "cc"); \ 3598 } while(0) 3599 3600 // As above, but add twice the double-length result into the 3601 // accumulator. 3602 #define MACC2(A, B, T0, T1, T2) \ 3603 do { \ 3604 unsigned long hi, lo; \ 3605 __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4; " \ 3606 "add %%rax, %2; adc %%rdx, %3; adc $0, %4" \ 3607 : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2) \ 3608 : "r"(A), "a"(B) : "cc"); \ 3609 } while(0) 3610 3611 #else //_WINDOWS 3612 3613 static julong 3614 sub(julong a[], julong b[], julong carry, long len) { 3615 long i; 3616 julong tmp; 3617 unsigned char c = 1; 3618 for (i = 0; i < len; i++) { 3619 c = _addcarry_u64(c, a[i], ~b[i], &tmp); 3620 a[i] = tmp; 3621 } 3622 c = _addcarry_u64(c, carry, ~0, &tmp); 3623 return tmp; 3624 } 3625 3626 // Multiply (unsigned) Long A by Long B, accumulating the double- 3627 // length result into the accumulator formed of T0, T1, and T2. 3628 #define MACC(A, B, T0, T1, T2) \ 3629 do { \ 3630 julong hi, lo; \ 3631 lo = _umul128(A, B, &hi); \ 3632 unsigned char c = _addcarry_u64(0, lo, T0, &T0); \ 3633 c = _addcarry_u64(c, hi, T1, &T1); \ 3634 _addcarry_u64(c, T2, 0, &T2); \ 3635 } while(0) 3636 3637 // As above, but add twice the double-length result into the 3638 // accumulator. 3639 #define MACC2(A, B, T0, T1, T2) \ 3640 do { \ 3641 julong hi, lo; \ 3642 lo = _umul128(A, B, &hi); \ 3643 unsigned char c = _addcarry_u64(0, lo, T0, &T0); \ 3644 c = _addcarry_u64(c, hi, T1, &T1); \ 3645 _addcarry_u64(c, T2, 0, &T2); \ 3646 c = _addcarry_u64(0, lo, T0, &T0); \ 3647 c = _addcarry_u64(c, hi, T1, &T1); \ 3648 _addcarry_u64(c, T2, 0, &T2); \ 3649 } while(0) 3650 3651 #endif //_WINDOWS 3652 3653 // Fast Montgomery multiplication. The derivation of the algorithm is 3654 // in A Cryptographic Library for the Motorola DSP56000, 3655 // Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237. 3656 3657 static void NOINLINE 3658 montgomery_multiply(julong a[], julong b[], julong n[], 3659 julong m[], julong inv, int len) { 3660 julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator 3661 int i; 3662 3663 assert(inv * n[0] == ULLONG_MAX, "broken inverse in Montgomery multiply"); 3664 3665 for (i = 0; i < len; i++) { 3666 int j; 3667 for (j = 0; j < i; j++) { 3668 MACC(a[j], b[i-j], t0, t1, t2); 3669 MACC(m[j], n[i-j], t0, t1, t2); 3670 } 3671 MACC(a[i], b[0], t0, t1, t2); 3672 m[i] = t0 * inv; 3673 MACC(m[i], n[0], t0, t1, t2); 3674 3675 assert(t0 == 0, "broken Montgomery multiply"); 3676 3677 t0 = t1; t1 = t2; t2 = 0; 3678 } 3679 3680 for (i = len; i < 2*len; i++) { 3681 int j; 3682 for (j = i-len+1; j < len; j++) { 3683 MACC(a[j], b[i-j], t0, t1, t2); 3684 MACC(m[j], n[i-j], t0, t1, t2); 3685 } 3686 m[i-len] = t0; 3687 t0 = t1; t1 = t2; t2 = 0; 3688 } 3689 3690 while (t0) 3691 t0 = sub(m, n, t0, len); 3692 } 3693 3694 // Fast Montgomery squaring. This uses asymptotically 25% fewer 3695 // multiplies so it should be up to 25% faster than Montgomery 3696 // multiplication. However, its loop control is more complex and it 3697 // may actually run slower on some machines. 3698 3699 static void NOINLINE 3700 montgomery_square(julong a[], julong n[], 3701 julong m[], julong inv, int len) { 3702 julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator 3703 int i; 3704 3705 assert(inv * n[0] == ULLONG_MAX, "broken inverse in Montgomery square"); 3706 3707 for (i = 0; i < len; i++) { 3708 int j; 3709 int end = (i+1)/2; 3710 for (j = 0; j < end; j++) { 3711 MACC2(a[j], a[i-j], t0, t1, t2); 3712 MACC(m[j], n[i-j], t0, t1, t2); 3713 } 3714 if ((i & 1) == 0) { 3715 MACC(a[j], a[j], t0, t1, t2); 3716 } 3717 for (; j < i; j++) { 3718 MACC(m[j], n[i-j], t0, t1, t2); 3719 } 3720 m[i] = t0 * inv; 3721 MACC(m[i], n[0], t0, t1, t2); 3722 3723 assert(t0 == 0, "broken Montgomery square"); 3724 3725 t0 = t1; t1 = t2; t2 = 0; 3726 } 3727 3728 for (i = len; i < 2*len; i++) { 3729 int start = i-len+1; 3730 int end = start + (len - start)/2; 3731 int j; 3732 for (j = start; j < end; j++) { 3733 MACC2(a[j], a[i-j], t0, t1, t2); 3734 MACC(m[j], n[i-j], t0, t1, t2); 3735 } 3736 if ((i & 1) == 0) { 3737 MACC(a[j], a[j], t0, t1, t2); 3738 } 3739 for (; j < len; j++) { 3740 MACC(m[j], n[i-j], t0, t1, t2); 3741 } 3742 m[i-len] = t0; 3743 t0 = t1; t1 = t2; t2 = 0; 3744 } 3745 3746 while (t0) 3747 t0 = sub(m, n, t0, len); 3748 } 3749 3750 // Swap words in a longword. 3751 static julong swap(julong x) { 3752 return (x << 32) | (x >> 32); 3753 } 3754 3755 // Copy len longwords from s to d, word-swapping as we go. The 3756 // destination array is reversed. 3757 static void reverse_words(julong *s, julong *d, int len) { 3758 d += len; 3759 while(len-- > 0) { 3760 d--; 3761 *d = swap(*s); 3762 s++; 3763 } 3764 } 3765 3766 // The threshold at which squaring is advantageous was determined 3767 // experimentally on an i7-3930K (Ivy Bridge) CPU @ 3.5GHz. 3768 #define MONTGOMERY_SQUARING_THRESHOLD 64 3769 3770 void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints, 3771 jint len, jlong inv, 3772 jint *m_ints) { 3773 assert(len % 2 == 0, "array length in montgomery_multiply must be even"); 3774 int longwords = len/2; 3775 3776 // Make very sure we don't use so much space that the stack might 3777 // overflow. 512 jints corresponds to an 16384-bit integer and 3778 // will use here a total of 8k bytes of stack space. 3779 int divisor = sizeof(julong) * 4; 3780 guarantee(longwords <= 8192 / divisor, "must be"); 3781 int total_allocation = longwords * sizeof (julong) * 4; 3782 julong *scratch = (julong *)alloca(total_allocation); 3783 3784 // Local scratch arrays 3785 julong 3786 *a = scratch + 0 * longwords, 3787 *b = scratch + 1 * longwords, 3788 *n = scratch + 2 * longwords, 3789 *m = scratch + 3 * longwords; 3790 3791 reverse_words((julong *)a_ints, a, longwords); 3792 reverse_words((julong *)b_ints, b, longwords); 3793 reverse_words((julong *)n_ints, n, longwords); 3794 3795 ::montgomery_multiply(a, b, n, m, (julong)inv, longwords); 3796 3797 reverse_words(m, (julong *)m_ints, longwords); 3798 } 3799 3800 void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints, 3801 jint len, jlong inv, 3802 jint *m_ints) { 3803 assert(len % 2 == 0, "array length in montgomery_square must be even"); 3804 int longwords = len/2; 3805 3806 // Make very sure we don't use so much space that the stack might 3807 // overflow. 512 jints corresponds to an 16384-bit integer and 3808 // will use here a total of 6k bytes of stack space. 3809 int divisor = sizeof(julong) * 3; 3810 guarantee(longwords <= (8192 / divisor), "must be"); 3811 int total_allocation = longwords * sizeof (julong) * 3; 3812 julong *scratch = (julong *)alloca(total_allocation); 3813 3814 // Local scratch arrays 3815 julong 3816 *a = scratch + 0 * longwords, 3817 *n = scratch + 1 * longwords, 3818 *m = scratch + 2 * longwords; 3819 3820 reverse_words((julong *)a_ints, a, longwords); 3821 reverse_words((julong *)n_ints, n, longwords); 3822 3823 if (len >= MONTGOMERY_SQUARING_THRESHOLD) { 3824 ::montgomery_square(a, n, m, (julong)inv, longwords); 3825 } else { 3826 ::montgomery_multiply(a, a, n, m, (julong)inv, longwords); 3827 } 3828 3829 reverse_words(m, (julong *)m_ints, longwords); 3830 } 3831 3832 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) { 3833 BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K); 3834 if (buf == nullptr) { 3835 return nullptr; 3836 } 3837 CodeBuffer buffer(buf); 3838 short buffer_locs[20]; 3839 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, 3840 sizeof(buffer_locs)/sizeof(relocInfo)); 3841 3842 MacroAssembler* masm = new MacroAssembler(&buffer); 3843 3844 const Array<SigEntry>* sig_vk = vk->extended_sig(); 3845 const Array<VMRegPair>* regs = vk->return_regs(); 3846 3847 int pack_fields_jobject_off = __ offset(); 3848 // Resolve pre-allocated buffer from JNI handle. 3849 // We cannot do this in generate_call_stub() because it requires GC code to be initialized. 3850 __ movptr(rax, Address(r13, 0)); 3851 __ resolve_jobject(rax /* value */, 3852 r12 /* tmp */); 3853 __ movptr(Address(r13, 0), rax); 3854 3855 int pack_fields_off = __ offset(); 3856 3857 int j = 1; 3858 for (int i = 0; i < sig_vk->length(); i++) { 3859 BasicType bt = sig_vk->at(i)._bt; 3860 if (bt == T_METADATA) { 3861 continue; 3862 } 3863 if (bt == T_VOID) { 3864 if (sig_vk->at(i-1)._bt == T_LONG || 3865 sig_vk->at(i-1)._bt == T_DOUBLE) { 3866 j++; 3867 } 3868 continue; 3869 } 3870 int off = sig_vk->at(i)._offset; 3871 assert(off > 0, "offset in object should be positive"); 3872 VMRegPair pair = regs->at(j); 3873 VMReg r_1 = pair.first(); 3874 VMReg r_2 = pair.second(); 3875 Address to(rax, off); 3876 if (bt == T_FLOAT) { 3877 __ movflt(to, r_1->as_XMMRegister()); 3878 } else if (bt == T_DOUBLE) { 3879 __ movdbl(to, r_1->as_XMMRegister()); 3880 } else { 3881 Register val = r_1->as_Register(); 3882 assert_different_registers(to.base(), val, r14, r13, rbx, rscratch1); 3883 if (is_reference_type(bt)) { 3884 __ store_heap_oop(to, val, r14, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED); 3885 } else { 3886 __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt)); 3887 } 3888 } 3889 j++; 3890 } 3891 assert(j == regs->length(), "missed a field?"); 3892 if (vk->has_nullable_atomic_layout()) { 3893 // Set the null marker 3894 __ movb(Address(rax, vk->null_marker_offset()), 1); 3895 } 3896 __ ret(0); 3897 3898 int unpack_fields_off = __ offset(); 3899 3900 Label skip; 3901 Label not_null; 3902 __ testptr(rax, rax); 3903 __ jcc(Assembler::notZero, not_null); 3904 3905 // Return value is null. Zero oop registers to make the GC happy. 3906 j = 1; 3907 for (int i = 0; i < sig_vk->length(); i++) { 3908 BasicType bt = sig_vk->at(i)._bt; 3909 if (bt == T_METADATA) { 3910 continue; 3911 } 3912 if (bt == T_VOID) { 3913 if (sig_vk->at(i-1)._bt == T_LONG || 3914 sig_vk->at(i-1)._bt == T_DOUBLE) { 3915 j++; 3916 } 3917 continue; 3918 } 3919 if (bt == T_OBJECT || bt == T_ARRAY) { 3920 VMRegPair pair = regs->at(j); 3921 VMReg r_1 = pair.first(); 3922 __ xorq(r_1->as_Register(), r_1->as_Register()); 3923 } 3924 j++; 3925 } 3926 __ jmp(skip); 3927 __ bind(not_null); 3928 3929 j = 1; 3930 for (int i = 0; i < sig_vk->length(); i++) { 3931 BasicType bt = sig_vk->at(i)._bt; 3932 if (bt == T_METADATA) { 3933 continue; 3934 } 3935 if (bt == T_VOID) { 3936 if (sig_vk->at(i-1)._bt == T_LONG || 3937 sig_vk->at(i-1)._bt == T_DOUBLE) { 3938 j++; 3939 } 3940 continue; 3941 } 3942 int off = sig_vk->at(i)._offset; 3943 assert(off > 0, "offset in object should be positive"); 3944 VMRegPair pair = regs->at(j); 3945 VMReg r_1 = pair.first(); 3946 VMReg r_2 = pair.second(); 3947 Address from(rax, off); 3948 if (bt == T_FLOAT) { 3949 __ movflt(r_1->as_XMMRegister(), from); 3950 } else if (bt == T_DOUBLE) { 3951 __ movdbl(r_1->as_XMMRegister(), from); 3952 } else if (bt == T_OBJECT || bt == T_ARRAY) { 3953 assert_different_registers(rax, r_1->as_Register()); 3954 __ load_heap_oop(r_1->as_Register(), from); 3955 } else { 3956 assert(is_java_primitive(bt), "unexpected basic type"); 3957 assert_different_registers(rax, r_1->as_Register()); 3958 size_t size_in_bytes = type2aelembytes(bt); 3959 __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN); 3960 } 3961 j++; 3962 } 3963 assert(j == regs->length(), "missed a field?"); 3964 3965 __ bind(skip); 3966 __ ret(0); 3967 3968 __ flush(); 3969 3970 return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off); 3971 } 3972 3973 #if INCLUDE_JFR 3974 3975 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. 3976 // It returns a jobject handle to the event writer. 3977 // The handle is dereferenced and the return value is the event writer oop. 3978 RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() { 3979 enum layout { 3980 rbp_off, 3981 rbpH_off, 3982 return_off, 3983 return_off2, 3984 framesize // inclusive of return address 3985 }; 3986 3987 const char* name = SharedRuntime::stub_name(StubId::shared_jfr_write_checkpoint_id); 3988 CodeBuffer code(name, 1024, 64); 3989 MacroAssembler* masm = new MacroAssembler(&code); 3990 address start = __ pc(); 3991 3992 __ enter(); 3993 address the_pc = __ pc(); 3994 3995 int frame_complete = the_pc - start; 3996 3997 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1); 3998 __ movptr(c_rarg0, r15_thread); 3999 __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1); 4000 __ reset_last_Java_frame(true); 4001 4002 // rax is jobject handle result, unpack and process it through a barrier. 4003 __ resolve_global_jobject(rax, c_rarg0); 4004 4005 __ leave(); 4006 __ ret(0); 4007 4008 OopMapSet* oop_maps = new OopMapSet(); 4009 OopMap* map = new OopMap(framesize, 1); 4010 oop_maps->add_gc_map(frame_complete, map); 4011 4012 RuntimeStub* stub = 4013 RuntimeStub::new_runtime_stub(name, 4014 &code, 4015 frame_complete, 4016 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 4017 oop_maps, 4018 false); 4019 return stub; 4020 } 4021 4022 // For c2: call to return a leased buffer. 4023 RuntimeStub* SharedRuntime::generate_jfr_return_lease() { 4024 enum layout { 4025 rbp_off, 4026 rbpH_off, 4027 return_off, 4028 return_off2, 4029 framesize // inclusive of return address 4030 }; 4031 4032 const char* name = SharedRuntime::stub_name(StubId::shared_jfr_return_lease_id); 4033 CodeBuffer code(name, 1024, 64); 4034 MacroAssembler* masm = new MacroAssembler(&code); 4035 address start = __ pc(); 4036 4037 __ enter(); 4038 address the_pc = __ pc(); 4039 4040 int frame_complete = the_pc - start; 4041 4042 __ set_last_Java_frame(rsp, rbp, the_pc, rscratch2); 4043 __ movptr(c_rarg0, r15_thread); 4044 __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1); 4045 __ reset_last_Java_frame(true); 4046 4047 __ leave(); 4048 __ ret(0); 4049 4050 OopMapSet* oop_maps = new OopMapSet(); 4051 OopMap* map = new OopMap(framesize, 1); 4052 oop_maps->add_gc_map(frame_complete, map); 4053 4054 RuntimeStub* stub = 4055 RuntimeStub::new_runtime_stub(name, 4056 &code, 4057 frame_complete, 4058 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 4059 oop_maps, 4060 false); 4061 return stub; 4062 } 4063 4064 #endif // INCLUDE_JFR