1 /*
2 * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "code/codeCache.hpp"
31 #include "code/debugInfoRec.hpp"
32 #include "code/icBuffer.hpp"
33 #include "code/vtableStubs.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/shared/barrierSetAssembler.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "logging/log.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/compiledICHolder.hpp"
42 #include "oops/klass.inline.hpp"
43 #include "prims/methodHandles.hpp"
44 #include "runtime/jniHandles.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/signature.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/vframeArray.hpp"
50 #include "utilities/align.hpp"
51 #include "utilities/formatBuffer.hpp"
52 #include "vmreg_aarch64.inline.hpp"
53 #ifdef COMPILER1
54 #include "c1/c1_Runtime1.hpp"
55 #endif
56 #ifdef COMPILER2
57 #include "adfiles/ad_aarch64.hpp"
58 #include "opto/runtime.hpp"
59 #endif
60 #if INCLUDE_JVMCI
61 #include "jvmci/jvmciJavaClasses.hpp"
62 #endif
63
64 #define __ masm->
65
66 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
67
68 class SimpleRuntimeFrame {
69
70 public:
71
72 // Most of the runtime stubs have this simple frame layout.
73 // This class exists to make the layout shared in one place.
74 // Offsets are for compiler stack slots, which are jints.
75 enum layout {
76 // The frame sender code expects that rbp will be in the "natural" place and
77 // will override any oopMap setting for it. We must therefore force the layout
78 // so that it agrees with the frame sender code.
79 // we don't expect any arg reg save area so aarch64 asserts that
80 // frame::arg_reg_save_area_bytes == 0
81 rbp_off = 0,
82 rbp_off2,
83 return_off, return_off2,
84 framesize
85 };
86 };
87
88 // FIXME -- this is used by C1
89 class RegisterSaver {
90 const bool _save_vectors;
91 public:
92 RegisterSaver(bool save_vectors) : _save_vectors(save_vectors) {}
93
94 OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words);
95 void restore_live_registers(MacroAssembler* masm);
96
97 // Offsets into the register save area
98 // Used by deoptimization when it is managing result register
99 // values on its own
100
101 int reg_offset_in_bytes(Register r);
102 int r0_offset_in_bytes() { return reg_offset_in_bytes(r0); }
103 int rscratch1_offset_in_bytes() { return reg_offset_in_bytes(rscratch1); }
104 int v0_offset_in_bytes();
105
106 // Total stack size in bytes for saving sve predicate registers.
107 int total_sve_predicate_in_bytes();
108
109 // Capture info about frame layout
110 // Note this is only correct when not saving full vectors.
111 enum layout {
112 fpu_state_off = 0,
113 fpu_state_end = fpu_state_off + FPUStateSizeInWords - 1,
114 // The frame sender code expects that rfp will be in
115 // the "natural" place and will override any oopMap
116 // setting for it. We must therefore force the layout
117 // so that it agrees with the frame sender code.
118 r0_off = fpu_state_off + FPUStateSizeInWords,
119 rfp_off = r0_off + (RegisterImpl::number_of_registers - 2) * RegisterImpl::max_slots_per_register,
120 return_off = rfp_off + RegisterImpl::max_slots_per_register, // slot for return address
121 reg_save_size = return_off + RegisterImpl::max_slots_per_register};
122
123 };
124
125 int RegisterSaver::reg_offset_in_bytes(Register r) {
126 // The integer registers are located above the floating point
127 // registers in the stack frame pushed by save_live_registers() so the
128 // offset depends on whether we are saving full vectors, and whether
129 // those vectors are NEON or SVE.
130
131 int slots_per_vect = FloatRegisterImpl::save_slots_per_register;
132
133 #if COMPILER2_OR_JVMCI
134 if (_save_vectors) {
135 slots_per_vect = FloatRegisterImpl::slots_per_neon_register;
136
137 #ifdef COMPILER2
138 if (Matcher::supports_scalable_vector()) {
139 slots_per_vect = Matcher::scalable_vector_reg_size(T_FLOAT);
140 }
141 #endif
142 }
143 #endif
144
145 int r0_offset = v0_offset_in_bytes() + (slots_per_vect * FloatRegisterImpl::number_of_registers) * BytesPerInt;
146 return r0_offset + r->encoding() * wordSize;
147 }
148
149 int RegisterSaver::v0_offset_in_bytes() {
150 // The floating point registers are located above the predicate registers if
151 // they are present in the stack frame pushed by save_live_registers(). So the
152 // offset depends on the saved total predicate vectors in the stack frame.
153 return (total_sve_predicate_in_bytes() / VMRegImpl::stack_slot_size) * BytesPerInt;
154 }
155
156 int RegisterSaver::total_sve_predicate_in_bytes() {
157 #ifdef COMPILER2
158 if (_save_vectors && Matcher::supports_scalable_vector()) {
159 // The number of total predicate bytes is unlikely to be a multiple
160 // of 16 bytes so we manually align it up.
161 return align_up(Matcher::scalable_predicate_reg_slots() *
162 VMRegImpl::stack_slot_size *
163 PRegisterImpl::number_of_saved_registers, 16);
164 }
165 #endif
166 return 0;
167 }
168
169 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) {
170 bool use_sve = false;
171 int sve_vector_size_in_bytes = 0;
172 int sve_vector_size_in_slots = 0;
173 int sve_predicate_size_in_slots = 0;
174 int total_predicate_in_bytes = total_sve_predicate_in_bytes();
175 int total_predicate_in_slots = total_predicate_in_bytes / VMRegImpl::stack_slot_size;
176
177 #ifdef COMPILER2
178 use_sve = Matcher::supports_scalable_vector();
179 if (use_sve) {
180 sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
181 sve_vector_size_in_slots = Matcher::scalable_vector_reg_size(T_FLOAT);
182 sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots();
183 }
184 #endif
185
186 #if COMPILER2_OR_JVMCI
187 if (_save_vectors) {
188 int extra_save_slots_per_register = 0;
189 // Save upper half of vector registers
190 if (use_sve) {
191 extra_save_slots_per_register = sve_vector_size_in_slots - FloatRegisterImpl::save_slots_per_register;
192 } else {
193 extra_save_slots_per_register = FloatRegisterImpl::extra_save_slots_per_neon_register;
194 }
195 int extra_vector_bytes = extra_save_slots_per_register *
196 VMRegImpl::stack_slot_size *
197 FloatRegisterImpl::number_of_registers;
198 additional_frame_words += ((extra_vector_bytes + total_predicate_in_bytes) / wordSize);
199 }
200 #else
201 assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
202 #endif
203
204 int frame_size_in_bytes = align_up(additional_frame_words * wordSize +
205 reg_save_size * BytesPerInt, 16);
206 // OopMap frame size is in compiler stack slots (jint's) not bytes or words
207 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
208 // The caller will allocate additional_frame_words
209 int additional_frame_slots = additional_frame_words * wordSize / BytesPerInt;
210 // CodeBlob frame size is in words.
211 int frame_size_in_words = frame_size_in_bytes / wordSize;
212 *total_frame_words = frame_size_in_words;
213
214 // Save Integer and Float registers.
215 __ enter();
216 __ push_CPU_state(_save_vectors, use_sve, sve_vector_size_in_bytes, total_predicate_in_bytes);
217
218 // Set an oopmap for the call site. This oopmap will map all
219 // oop-registers and debug-info registers as callee-saved. This
220 // will allow deoptimization at this safepoint to find all possible
221 // debug-info recordings, as well as let GC find all oops.
222
223 OopMapSet *oop_maps = new OopMapSet();
224 OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
225
226 for (int i = 0; i < RegisterImpl::number_of_registers; i++) {
227 Register r = as_Register(i);
228 if (r <= rfp && r != rscratch1 && r != rscratch2) {
229 // SP offsets are in 4-byte words.
230 // Register slots are 8 bytes wide, 32 floating-point registers.
231 int sp_offset = RegisterImpl::max_slots_per_register * i +
232 FloatRegisterImpl::save_slots_per_register * FloatRegisterImpl::number_of_registers;
233 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset + additional_frame_slots), r->as_VMReg());
234 }
235 }
236
237 for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
238 FloatRegister r = as_FloatRegister(i);
239 int sp_offset = 0;
240 if (_save_vectors) {
241 sp_offset = use_sve ? (total_predicate_in_slots + sve_vector_size_in_slots * i) :
242 (FloatRegisterImpl::slots_per_neon_register * i);
243 } else {
244 sp_offset = FloatRegisterImpl::save_slots_per_register * i;
245 }
246 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
247 }
248
249 if (_save_vectors && use_sve) {
250 for (int i = 0; i < PRegisterImpl::number_of_saved_registers; i++) {
251 PRegister r = as_PRegister(i);
252 int sp_offset = sve_predicate_size_in_slots * i;
253 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
254 }
255 }
256
257 return oop_map;
258 }
259
260 void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
261 #ifdef COMPILER2
262 __ pop_CPU_state(_save_vectors, Matcher::supports_scalable_vector(),
263 Matcher::scalable_vector_reg_size(T_BYTE), total_sve_predicate_in_bytes());
264 #else
265 #if !INCLUDE_JVMCI
266 assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
267 #endif
268 __ pop_CPU_state(_save_vectors);
269 #endif
270 __ leave();
271
272 }
273
274 // Is vector's size (in bytes) bigger than a size saved by default?
275 // 8 bytes vector registers are saved by default on AArch64.
276 // The SVE supported min vector size is 8 bytes and we need to save
277 // predicate registers when the vector size is 8 bytes as well.
278 bool SharedRuntime::is_wide_vector(int size) {
279 return size > 8 || (UseSVE > 0 && size >= 8);
280 }
281
282 // The java_calling_convention describes stack locations as ideal slots on
283 // a frame with no abi restrictions. Since we must observe abi restrictions
284 // (like the placement of the register window) the slots must be biased by
285 // the following value.
286 static int reg2offset_in(VMReg r) {
287 // Account for saved rfp and lr
288 // This should really be in_preserve_stack_slots
289 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
290 }
291
292 static int reg2offset_out(VMReg r) {
293 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
294 }
295
296 // ---------------------------------------------------------------------------
297 // Read the array of BasicTypes from a signature, and compute where the
298 // arguments should go. Values in the VMRegPair regs array refer to 4-byte
299 // quantities. Values less than VMRegImpl::stack0 are registers, those above
300 // refer to 4-byte stack slots. All stack slots are based off of the stack pointer
301 // as framesizes are fixed.
302 // VMRegImpl::stack0 refers to the first slot 0(sp).
303 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. Register
304 // up to RegisterImpl::number_of_registers) are the 64-bit
305 // integer registers.
306
307 // Note: the INPUTS in sig_bt are in units of Java argument words,
308 // which are 64-bit. The OUTPUTS are in 32-bit units.
309
310 // The Java calling convention is a "shifted" version of the C ABI.
311 // By skipping the first C ABI register we can call non-static jni
312 // methods with small numbers of arguments without having to shuffle
313 // the arguments at all. Since we control the java ABI we ought to at
314 // least get some advantage out of it.
315
316 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
317 VMRegPair *regs,
318 int total_args_passed) {
319
320 // Create the mapping between argument positions and
321 // registers.
322 static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
323 j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7
324 };
325 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {
326 j_farg0, j_farg1, j_farg2, j_farg3,
327 j_farg4, j_farg5, j_farg6, j_farg7
328 };
329
330
331 uint int_args = 0;
332 uint fp_args = 0;
333 uint stk_args = 0; // inc by 2 each time
334
335 for (int i = 0; i < total_args_passed; i++) {
336 switch (sig_bt[i]) {
337 case T_BOOLEAN:
338 case T_CHAR:
339 case T_BYTE:
340 case T_SHORT:
341 case T_INT:
342 if (int_args < Argument::n_int_register_parameters_j) {
343 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
344 } else {
345 regs[i].set1(VMRegImpl::stack2reg(stk_args));
346 stk_args += 2;
347 }
348 break;
349 case T_VOID:
350 // halves of T_LONG or T_DOUBLE
351 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
352 regs[i].set_bad();
353 break;
354 case T_LONG:
355 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
356 // fall through
357 case T_OBJECT:
358 case T_ARRAY:
359 case T_ADDRESS:
360 if (int_args < Argument::n_int_register_parameters_j) {
361 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
362 } else {
363 regs[i].set2(VMRegImpl::stack2reg(stk_args));
364 stk_args += 2;
365 }
366 break;
367 case T_FLOAT:
368 if (fp_args < Argument::n_float_register_parameters_j) {
369 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
370 } else {
371 regs[i].set1(VMRegImpl::stack2reg(stk_args));
372 stk_args += 2;
373 }
374 break;
375 case T_DOUBLE:
376 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
377 if (fp_args < Argument::n_float_register_parameters_j) {
378 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
379 } else {
380 regs[i].set2(VMRegImpl::stack2reg(stk_args));
381 stk_args += 2;
382 }
383 break;
384 default:
385 ShouldNotReachHere();
386 break;
387 }
388 }
389
390 return align_up(stk_args, 2);
391 }
392
393 // Patch the callers callsite with entry to compiled code if it exists.
394 static void patch_callers_callsite(MacroAssembler *masm) {
395 Label L;
396 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
397 __ cbz(rscratch1, L);
398
399 __ enter();
400 __ push_CPU_state();
401
402 // VM needs caller's callsite
403 // VM needs target method
404 // This needs to be a long call since we will relocate this adapter to
405 // the codeBuffer and it may not reach
406
407 #ifndef PRODUCT
408 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
409 #endif
410
411 __ mov(c_rarg0, rmethod);
412 __ mov(c_rarg1, lr);
413 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
414 __ blr(rscratch1);
415
416 // Explicit isb required because fixup_callers_callsite may change the code
417 // stream.
418 __ safepoint_isb();
419
420 __ pop_CPU_state();
421 // restore sp
422 __ leave();
423 __ bind(L);
424 }
425
426 static void gen_c2i_adapter(MacroAssembler *masm,
427 int total_args_passed,
428 int comp_args_on_stack,
429 const BasicType *sig_bt,
430 const VMRegPair *regs,
431 Label& skip_fixup) {
432 // Before we get into the guts of the C2I adapter, see if we should be here
433 // at all. We've come from compiled code and are attempting to jump to the
434 // interpreter, which means the caller made a static call to get here
435 // (vcalls always get a compiled target if there is one). Check for a
436 // compiled target. If there is one, we need to patch the caller's call.
437 patch_callers_callsite(masm);
438
439 __ bind(skip_fixup);
440
441 int words_pushed = 0;
442
443 // Since all args are passed on the stack, total_args_passed *
444 // Interpreter::stackElementSize is the space we need.
445
446 int extraspace = total_args_passed * Interpreter::stackElementSize;
447
448 __ mov(r13, sp);
449
450 // stack is aligned, keep it that way
451 extraspace = align_up(extraspace, 2*wordSize);
452
453 if (extraspace)
454 __ sub(sp, sp, extraspace);
455
456 // Now write the args into the outgoing interpreter space
457 for (int i = 0; i < total_args_passed; i++) {
458 if (sig_bt[i] == T_VOID) {
459 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
460 continue;
461 }
462
463 // offset to start parameters
464 int st_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
465 int next_off = st_off - Interpreter::stackElementSize;
466
467 // Say 4 args:
468 // i st_off
469 // 0 32 T_LONG
470 // 1 24 T_VOID
471 // 2 16 T_OBJECT
472 // 3 8 T_BOOL
473 // - 0 return address
474 //
475 // However to make thing extra confusing. Because we can fit a Java long/double in
476 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
477 // leaves one slot empty and only stores to a single slot. In this case the
478 // slot that is occupied is the T_VOID slot. See I said it was confusing.
479
480 VMReg r_1 = regs[i].first();
481 VMReg r_2 = regs[i].second();
482 if (!r_1->is_valid()) {
483 assert(!r_2->is_valid(), "");
484 continue;
485 }
486 if (r_1->is_stack()) {
487 // memory to memory use rscratch1
488 int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
489 + extraspace
490 + words_pushed * wordSize);
491 if (!r_2->is_valid()) {
492 // sign extend??
493 __ ldrw(rscratch1, Address(sp, ld_off));
494 __ str(rscratch1, Address(sp, st_off));
495
496 } else {
497
498 __ ldr(rscratch1, Address(sp, ld_off));
499
500 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
501 // T_DOUBLE and T_LONG use two slots in the interpreter
502 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
503 // ld_off == LSW, ld_off+wordSize == MSW
504 // st_off == MSW, next_off == LSW
505 __ str(rscratch1, Address(sp, next_off));
506 #ifdef ASSERT
507 // Overwrite the unused slot with known junk
508 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
509 __ str(rscratch1, Address(sp, st_off));
510 #endif /* ASSERT */
511 } else {
512 __ str(rscratch1, Address(sp, st_off));
513 }
514 }
515 } else if (r_1->is_Register()) {
516 Register r = r_1->as_Register();
517 if (!r_2->is_valid()) {
518 // must be only an int (or less ) so move only 32bits to slot
519 // why not sign extend??
520 __ str(r, Address(sp, st_off));
521 } else {
522 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
523 // T_DOUBLE and T_LONG use two slots in the interpreter
524 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
525 // jlong/double in gpr
526 #ifdef ASSERT
527 // Overwrite the unused slot with known junk
528 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
529 __ str(rscratch1, Address(sp, st_off));
530 #endif /* ASSERT */
531 __ str(r, Address(sp, next_off));
532 } else {
533 __ str(r, Address(sp, st_off));
534 }
535 }
536 } else {
537 assert(r_1->is_FloatRegister(), "");
538 if (!r_2->is_valid()) {
539 // only a float use just part of the slot
540 __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
541 } else {
542 #ifdef ASSERT
543 // Overwrite the unused slot with known junk
544 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
545 __ str(rscratch1, Address(sp, st_off));
546 #endif /* ASSERT */
547 __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
548 }
549 }
550 }
551
552 __ mov(esp, sp); // Interp expects args on caller's expression stack
553
554 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
555 __ br(rscratch1);
556 }
557
558
559 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
560 int total_args_passed,
561 int comp_args_on_stack,
562 const BasicType *sig_bt,
563 const VMRegPair *regs) {
564
565 // Note: r13 contains the senderSP on entry. We must preserve it since
566 // we may do a i2c -> c2i transition if we lose a race where compiled
567 // code goes non-entrant while we get args ready.
568
569 // In addition we use r13 to locate all the interpreter args because
570 // we must align the stack to 16 bytes.
571
572 // Adapters are frameless.
573
574 // An i2c adapter is frameless because the *caller* frame, which is
575 // interpreted, routinely repairs its own esp (from
576 // interpreter_frame_last_sp), even if a callee has modified the
577 // stack pointer. It also recalculates and aligns sp.
578
579 // A c2i adapter is frameless because the *callee* frame, which is
580 // interpreted, routinely repairs its caller's sp (from sender_sp,
581 // which is set up via the senderSP register).
582
583 // In other words, if *either* the caller or callee is interpreted, we can
584 // get the stack pointer repaired after a call.
585
586 // This is why c2i and i2c adapters cannot be indefinitely composed.
587 // In particular, if a c2i adapter were to somehow call an i2c adapter,
588 // both caller and callee would be compiled methods, and neither would
589 // clean up the stack pointer changes performed by the two adapters.
590 // If this happens, control eventually transfers back to the compiled
591 // caller, but with an uncorrected stack, causing delayed havoc.
592
593 if (VerifyAdapterCalls &&
594 (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
595 #if 0
596 // So, let's test for cascading c2i/i2c adapters right now.
597 // assert(Interpreter::contains($return_addr) ||
598 // StubRoutines::contains($return_addr),
599 // "i2c adapter must return to an interpreter frame");
600 __ block_comment("verify_i2c { ");
601 Label L_ok;
602 if (Interpreter::code() != NULL)
603 range_check(masm, rax, r11,
604 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
605 L_ok);
606 if (StubRoutines::code1() != NULL)
607 range_check(masm, rax, r11,
608 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
609 L_ok);
610 if (StubRoutines::code2() != NULL)
611 range_check(masm, rax, r11,
612 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
613 L_ok);
614 const char* msg = "i2c adapter must return to an interpreter frame";
615 __ block_comment(msg);
616 __ stop(msg);
617 __ bind(L_ok);
618 __ block_comment("} verify_i2ce ");
619 #endif
620 }
621
622 // Cut-out for having no stack args.
623 int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
624 if (comp_args_on_stack) {
625 __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
626 __ andr(sp, rscratch1, -16);
627 }
628
629 // Will jump to the compiled code just as if compiled code was doing it.
630 // Pre-load the register-jump target early, to schedule it better.
631 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
632
633 #if INCLUDE_JVMCI
634 if (EnableJVMCI) {
635 // check if this call should be routed towards a specific entry point
636 __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
637 Label no_alternative_target;
638 __ cbz(rscratch2, no_alternative_target);
639 __ mov(rscratch1, rscratch2);
640 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
641 __ bind(no_alternative_target);
642 }
643 #endif // INCLUDE_JVMCI
644
645 // Now generate the shuffle code.
646 for (int i = 0; i < total_args_passed; i++) {
647 if (sig_bt[i] == T_VOID) {
648 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
649 continue;
650 }
651
652 // Pick up 0, 1 or 2 words from SP+offset.
653
654 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
655 "scrambled load targets?");
656 // Load in argument order going down.
657 int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
658 // Point to interpreter value (vs. tag)
659 int next_off = ld_off - Interpreter::stackElementSize;
660 //
661 //
662 //
663 VMReg r_1 = regs[i].first();
664 VMReg r_2 = regs[i].second();
665 if (!r_1->is_valid()) {
666 assert(!r_2->is_valid(), "");
667 continue;
668 }
669 if (r_1->is_stack()) {
670 // Convert stack slot to an SP offset (+ wordSize to account for return address )
671 int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
672 if (!r_2->is_valid()) {
673 // sign extend???
674 __ ldrsw(rscratch2, Address(esp, ld_off));
675 __ str(rscratch2, Address(sp, st_off));
676 } else {
677 //
678 // We are using two optoregs. This can be either T_OBJECT,
679 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
680 // two slots but only uses one for thr T_LONG or T_DOUBLE case
681 // So we must adjust where to pick up the data to match the
682 // interpreter.
683 //
684 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
685 // are accessed as negative so LSW is at LOW address
686
687 // ld_off is MSW so get LSW
688 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
689 next_off : ld_off;
690 __ ldr(rscratch2, Address(esp, offset));
691 // st_off is LSW (i.e. reg.first())
692 __ str(rscratch2, Address(sp, st_off));
693 }
694 } else if (r_1->is_Register()) { // Register argument
695 Register r = r_1->as_Register();
696 if (r_2->is_valid()) {
697 //
698 // We are using two VMRegs. This can be either T_OBJECT,
699 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
700 // two slots but only uses one for thr T_LONG or T_DOUBLE case
701 // So we must adjust where to pick up the data to match the
702 // interpreter.
703
704 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
705 next_off : ld_off;
706
707 // this can be a misaligned move
708 __ ldr(r, Address(esp, offset));
709 } else {
710 // sign extend and use a full word?
711 __ ldrw(r, Address(esp, ld_off));
712 }
713 } else {
714 if (!r_2->is_valid()) {
715 __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
716 } else {
717 __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
718 }
719 }
720 }
721
722 // 6243940 We might end up in handle_wrong_method if
723 // the callee is deoptimized as we race thru here. If that
724 // happens we don't want to take a safepoint because the
725 // caller frame will look interpreted and arguments are now
726 // "compiled" so it is much better to make this transition
727 // invisible to the stack walking code. Unfortunately if
728 // we try and find the callee by normal means a safepoint
729 // is possible. So we stash the desired callee in the thread
730 // and the vm will find there should this case occur.
731
732 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
733
734 __ br(rscratch1);
735 }
736
737 // ---------------------------------------------------------------
738 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
739 int total_args_passed,
740 int comp_args_on_stack,
741 const BasicType *sig_bt,
742 const VMRegPair *regs,
743 AdapterFingerPrint* fingerprint) {
744 address i2c_entry = __ pc();
745
746 gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
747
748 address c2i_unverified_entry = __ pc();
749 Label skip_fixup;
750
751 Label ok;
752
753 Register holder = rscratch2;
754 Register receiver = j_rarg0;
755 Register tmp = r10; // A call-clobbered register not used for arg passing
756
757 // -------------------------------------------------------------------------
758 // Generate a C2I adapter. On entry we know rmethod holds the Method* during calls
759 // to the interpreter. The args start out packed in the compiled layout. They
760 // need to be unpacked into the interpreter layout. This will almost always
761 // require some stack space. We grow the current (compiled) stack, then repack
762 // the args. We finally end in a jump to the generic interpreter entry point.
763 // On exit from the interpreter, the interpreter will restore our SP (lest the
764 // compiled code, which relys solely on SP and not FP, get sick).
765
766 {
767 __ block_comment("c2i_unverified_entry {");
768 __ load_klass(rscratch1, receiver);
769 __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
770 __ cmp(rscratch1, tmp);
771 __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
772 __ br(Assembler::EQ, ok);
773 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
774
775 __ bind(ok);
776 // Method might have been compiled since the call site was patched to
777 // interpreted; if that is the case treat it as a miss so we can get
778 // the call site corrected.
779 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
780 __ cbz(rscratch1, skip_fixup);
781 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
782 __ block_comment("} c2i_unverified_entry");
783 }
784
785 address c2i_entry = __ pc();
786
787 // Class initialization barrier for static methods
788 address c2i_no_clinit_check_entry = NULL;
789 if (VM_Version::supports_fast_class_init_checks()) {
790 Label L_skip_barrier;
791
792 { // Bypass the barrier for non-static methods
793 __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
794 __ andsw(zr, rscratch1, JVM_ACC_STATIC);
795 __ br(Assembler::EQ, L_skip_barrier); // non-static
796 }
797
798 __ load_method_holder(rscratch2, rmethod);
799 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
800 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
801
802 __ bind(L_skip_barrier);
803 c2i_no_clinit_check_entry = __ pc();
804 }
805
806 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
807 bs->c2i_entry_barrier(masm);
808
809 gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
810
811 __ flush();
812 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
813 }
814
815 static int c_calling_convention_priv(const BasicType *sig_bt,
816 VMRegPair *regs,
817 VMRegPair *regs2,
818 int total_args_passed) {
819 assert(regs2 == NULL, "not needed on AArch64");
820
821 // We return the amount of VMRegImpl stack slots we need to reserve for all
822 // the arguments NOT counting out_preserve_stack_slots.
823
824 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
825 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
826 };
827 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
828 c_farg0, c_farg1, c_farg2, c_farg3,
829 c_farg4, c_farg5, c_farg6, c_farg7
830 };
831
832 uint int_args = 0;
833 uint fp_args = 0;
834 uint stk_args = 0; // inc by 2 each time
835
836 for (int i = 0; i < total_args_passed; i++) {
837 switch (sig_bt[i]) {
838 case T_BOOLEAN:
839 case T_CHAR:
840 case T_BYTE:
841 case T_SHORT:
842 case T_INT:
843 if (int_args < Argument::n_int_register_parameters_c) {
844 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
845 } else {
846 #ifdef __APPLE__
847 // Less-than word types are stored one after another.
848 // The code is unable to handle this so bailout.
849 return -1;
850 #endif
851 regs[i].set1(VMRegImpl::stack2reg(stk_args));
852 stk_args += 2;
853 }
854 break;
855 case T_LONG:
856 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
857 // fall through
858 case T_OBJECT:
859 case T_ARRAY:
860 case T_ADDRESS:
861 case T_METADATA:
862 if (int_args < Argument::n_int_register_parameters_c) {
863 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
864 } else {
865 regs[i].set2(VMRegImpl::stack2reg(stk_args));
866 stk_args += 2;
867 }
868 break;
869 case T_FLOAT:
870 if (fp_args < Argument::n_float_register_parameters_c) {
871 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
872 } else {
873 #ifdef __APPLE__
874 // Less-than word types are stored one after another.
875 // The code is unable to handle this so bailout.
876 return -1;
877 #endif
878 regs[i].set1(VMRegImpl::stack2reg(stk_args));
879 stk_args += 2;
880 }
881 break;
882 case T_DOUBLE:
883 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
884 if (fp_args < Argument::n_float_register_parameters_c) {
885 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
886 } else {
887 regs[i].set2(VMRegImpl::stack2reg(stk_args));
888 stk_args += 2;
889 }
890 break;
891 case T_VOID: // Halves of longs and doubles
892 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
893 regs[i].set_bad();
894 break;
895 default:
896 ShouldNotReachHere();
897 break;
898 }
899 }
900
901 return stk_args;
902 }
903
904 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
905 uint num_bits,
906 uint total_args_passed) {
907 Unimplemented();
908 return 0;
909 }
910
911 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
912 VMRegPair *regs,
913 VMRegPair *regs2,
914 int total_args_passed)
915 {
916 int result = c_calling_convention_priv(sig_bt, regs, regs2, total_args_passed);
917 guarantee(result >= 0, "Unsupported arguments configuration");
918 return result;
919 }
920
921 // On 64 bit we will store integer like items to the stack as
922 // 64 bits items (Aarch64 abi) even though java would only store
923 // 32bits for a parameter. On 32bit it will simply be 32 bits
924 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
925 static void move32_64(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
926 if (src.first()->is_stack()) {
927 if (dst.first()->is_stack()) {
928 // stack to stack
929 __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
930 __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
931 } else {
932 // stack to reg
933 __ ldrsw(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
934 }
935 } else if (dst.first()->is_stack()) {
936 // reg to stack
937 // Do we really have to sign extend???
938 // __ movslq(src.first()->as_Register(), src.first()->as_Register());
939 __ str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
940 } else {
941 if (dst.first() != src.first()) {
942 __ sxtw(dst.first()->as_Register(), src.first()->as_Register());
943 }
944 }
945 }
946
947 // An oop arg. Must pass a handle not the oop itself
948 static void object_move(MacroAssembler* masm,
949 OopMap* map,
950 int oop_handle_offset,
951 int framesize_in_slots,
952 VMRegPair src,
953 VMRegPair dst,
954 bool is_receiver,
955 int* receiver_offset) {
956
957 // must pass a handle. First figure out the location we use as a handle
958
959 Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register();
960
961 // See if oop is NULL if it is we need no handle
962
963 if (src.first()->is_stack()) {
964
965 // Oop is already on the stack as an argument
966 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
967 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
968 if (is_receiver) {
969 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
970 }
971
972 __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
973 __ lea(rHandle, Address(rfp, reg2offset_in(src.first())));
974 // conditionally move a NULL
975 __ cmp(rscratch1, zr);
976 __ csel(rHandle, zr, rHandle, Assembler::EQ);
977 } else {
978
979 // Oop is in an a register we must store it to the space we reserve
980 // on the stack for oop_handles and pass a handle if oop is non-NULL
981
982 const Register rOop = src.first()->as_Register();
983 int oop_slot;
984 if (rOop == j_rarg0)
985 oop_slot = 0;
986 else if (rOop == j_rarg1)
987 oop_slot = 1;
988 else if (rOop == j_rarg2)
989 oop_slot = 2;
990 else if (rOop == j_rarg3)
991 oop_slot = 3;
992 else if (rOop == j_rarg4)
993 oop_slot = 4;
994 else if (rOop == j_rarg5)
995 oop_slot = 5;
996 else if (rOop == j_rarg6)
997 oop_slot = 6;
998 else {
999 assert(rOop == j_rarg7, "wrong register");
1000 oop_slot = 7;
1001 }
1002
1003 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
1004 int offset = oop_slot*VMRegImpl::stack_slot_size;
1005
1006 map->set_oop(VMRegImpl::stack2reg(oop_slot));
1007 // Store oop in handle area, may be NULL
1008 __ str(rOop, Address(sp, offset));
1009 if (is_receiver) {
1010 *receiver_offset = offset;
1011 }
1012
1013 __ cmp(rOop, zr);
1014 __ lea(rHandle, Address(sp, offset));
1015 // conditionally move a NULL
1016 __ csel(rHandle, zr, rHandle, Assembler::EQ);
1017 }
1018
1019 // If arg is on the stack then place it otherwise it is already in correct reg.
1020 if (dst.first()->is_stack()) {
1021 __ str(rHandle, Address(sp, reg2offset_out(dst.first())));
1022 }
1023 }
1024
1025 // A float arg may have to do float reg int reg conversion
1026 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1027 assert(src.first()->is_stack() && dst.first()->is_stack() ||
1028 src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
1029 if (src.first()->is_stack()) {
1030 if (dst.first()->is_stack()) {
1031 __ ldrw(rscratch1, Address(rfp, reg2offset_in(src.first())));
1032 __ strw(rscratch1, Address(sp, reg2offset_out(dst.first())));
1033 } else {
1034 ShouldNotReachHere();
1035 }
1036 } else if (src.first() != dst.first()) {
1037 if (src.is_single_phys_reg() && dst.is_single_phys_reg())
1038 __ fmovs(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1039 else
1040 ShouldNotReachHere();
1041 }
1042 }
1043
1044 // A long move
1045 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1046 if (src.first()->is_stack()) {
1047 if (dst.first()->is_stack()) {
1048 // stack to stack
1049 __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1050 __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
1051 } else {
1052 // stack to reg
1053 __ ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
1054 }
1055 } else if (dst.first()->is_stack()) {
1056 // reg to stack
1057 // Do we really have to sign extend???
1058 // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1059 __ str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
1060 } else {
1061 if (dst.first() != src.first()) {
1062 __ mov(dst.first()->as_Register(), src.first()->as_Register());
1063 }
1064 }
1065 }
1066
1067
1068 // A double move
1069 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1070 assert(src.first()->is_stack() && dst.first()->is_stack() ||
1071 src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
1072 if (src.first()->is_stack()) {
1073 if (dst.first()->is_stack()) {
1074 __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1075 __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
1076 } else {
1077 ShouldNotReachHere();
1078 }
1079 } else if (src.first() != dst.first()) {
1080 if (src.is_single_phys_reg() && dst.is_single_phys_reg())
1081 __ fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1082 else
1083 ShouldNotReachHere();
1084 }
1085 }
1086
1087
1088 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1089 // We always ignore the frame_slots arg and just use the space just below frame pointer
1090 // which by this time is free to use
1091 switch (ret_type) {
1092 case T_FLOAT:
1093 __ strs(v0, Address(rfp, -wordSize));
1094 break;
1095 case T_DOUBLE:
1096 __ strd(v0, Address(rfp, -wordSize));
1097 break;
1098 case T_VOID: break;
1099 default: {
1100 __ str(r0, Address(rfp, -wordSize));
1101 }
1102 }
1103 }
1104
1105 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1106 // We always ignore the frame_slots arg and just use the space just below frame pointer
1107 // which by this time is free to use
1108 switch (ret_type) {
1109 case T_FLOAT:
1110 __ ldrs(v0, Address(rfp, -wordSize));
1111 break;
1112 case T_DOUBLE:
1113 __ ldrd(v0, Address(rfp, -wordSize));
1114 break;
1115 case T_VOID: break;
1116 default: {
1117 __ ldr(r0, Address(rfp, -wordSize));
1118 }
1119 }
1120 }
1121 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1122 RegSet x;
1123 for ( int i = first_arg ; i < arg_count ; i++ ) {
1124 if (args[i].first()->is_Register()) {
1125 x = x + args[i].first()->as_Register();
1126 } else if (args[i].first()->is_FloatRegister()) {
1127 __ strd(args[i].first()->as_FloatRegister(), Address(__ pre(sp, -2 * wordSize)));
1128 }
1129 }
1130 __ push(x, sp);
1131 }
1132
1133 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1134 RegSet x;
1135 for ( int i = first_arg ; i < arg_count ; i++ ) {
1136 if (args[i].first()->is_Register()) {
1137 x = x + args[i].first()->as_Register();
1138 } else {
1139 ;
1140 }
1141 }
1142 __ pop(x, sp);
1143 for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
1144 if (args[i].first()->is_Register()) {
1145 ;
1146 } else if (args[i].first()->is_FloatRegister()) {
1147 __ ldrd(args[i].first()->as_FloatRegister(), Address(__ post(sp, 2 * wordSize)));
1148 }
1149 }
1150 }
1151
1152 static void rt_call(MacroAssembler* masm, address dest) {
1153 CodeBlob *cb = CodeCache::find_blob(dest);
1154 if (cb) {
1155 __ far_call(RuntimeAddress(dest));
1156 } else {
1157 __ lea(rscratch1, RuntimeAddress(dest));
1158 __ blr(rscratch1);
1159 }
1160 }
1161
1162 static void verify_oop_args(MacroAssembler* masm,
1163 const methodHandle& method,
1164 const BasicType* sig_bt,
1165 const VMRegPair* regs) {
1166 Register temp_reg = r19; // not part of any compiled calling seq
1167 if (VerifyOops) {
1168 for (int i = 0; i < method->size_of_parameters(); i++) {
1169 if (sig_bt[i] == T_OBJECT ||
1170 sig_bt[i] == T_ARRAY) {
1171 VMReg r = regs[i].first();
1172 assert(r->is_valid(), "bad oop arg");
1173 if (r->is_stack()) {
1174 __ ldr(temp_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1175 __ verify_oop(temp_reg);
1176 } else {
1177 __ verify_oop(r->as_Register());
1178 }
1179 }
1180 }
1181 }
1182 }
1183
1184 static void gen_special_dispatch(MacroAssembler* masm,
1185 const methodHandle& method,
1186 const BasicType* sig_bt,
1187 const VMRegPair* regs) {
1188 verify_oop_args(masm, method, sig_bt, regs);
1189 vmIntrinsics::ID iid = method->intrinsic_id();
1190
1191 // Now write the args into the outgoing interpreter space
1192 bool has_receiver = false;
1193 Register receiver_reg = noreg;
1194 int member_arg_pos = -1;
1195 Register member_reg = noreg;
1196 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1197 if (ref_kind != 0) {
1198 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
1199 member_reg = r19; // known to be free at this point
1200 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1201 } else if (iid == vmIntrinsics::_invokeBasic || iid == vmIntrinsics::_linkToNative) {
1202 has_receiver = true;
1203 } else {
1204 fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
1205 }
1206
1207 if (member_reg != noreg) {
1208 // Load the member_arg into register, if necessary.
1209 SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1210 VMReg r = regs[member_arg_pos].first();
1211 if (r->is_stack()) {
1212 __ ldr(member_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1213 } else {
1214 // no data motion is needed
1215 member_reg = r->as_Register();
1216 }
1217 }
1218
1219 if (has_receiver) {
1220 // Make sure the receiver is loaded into a register.
1221 assert(method->size_of_parameters() > 0, "oob");
1222 assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1223 VMReg r = regs[0].first();
1224 assert(r->is_valid(), "bad receiver arg");
1225 if (r->is_stack()) {
1226 // Porting note: This assumes that compiled calling conventions always
1227 // pass the receiver oop in a register. If this is not true on some
1228 // platform, pick a temp and load the receiver from stack.
1229 fatal("receiver always in a register");
1230 receiver_reg = r2; // known to be free at this point
1231 __ ldr(receiver_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1232 } else {
1233 // no data motion is needed
1234 receiver_reg = r->as_Register();
1235 }
1236 }
1237
1238 // Figure out which address we are really jumping to:
1239 MethodHandles::generate_method_handle_dispatch(masm, iid,
1240 receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1241 }
1242
1243 // ---------------------------------------------------------------------------
1244 // Generate a native wrapper for a given method. The method takes arguments
1245 // in the Java compiled code convention, marshals them to the native
1246 // convention (handlizes oops, etc), transitions to native, makes the call,
1247 // returns to java state (possibly blocking), unhandlizes any result and
1248 // returns.
1249 //
1250 // Critical native functions are a shorthand for the use of
1251 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1252 // functions. The wrapper is expected to unpack the arguments before
1253 // passing them to the callee. Critical native functions leave the state _in_Java,
1254 // since they block out GC.
1255 // Some other parts of JNI setup are skipped like the tear down of the JNI handle
1256 // block and the check for pending exceptions it's impossible for them
1257 // to be thrown.
1258 //
1259 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1260 const methodHandle& method,
1261 int compile_id,
1262 BasicType* in_sig_bt,
1263 VMRegPair* in_regs,
1264 BasicType ret_type) {
1265 if (method->is_method_handle_intrinsic()) {
1266 vmIntrinsics::ID iid = method->intrinsic_id();
1267 intptr_t start = (intptr_t)__ pc();
1268 int vep_offset = ((intptr_t)__ pc()) - start;
1269
1270 // First instruction must be a nop as it may need to be patched on deoptimisation
1271 __ nop();
1272 gen_special_dispatch(masm,
1273 method,
1274 in_sig_bt,
1275 in_regs);
1276 int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
1277 __ flush();
1278 int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
1279 return nmethod::new_native_nmethod(method,
1280 compile_id,
1281 masm->code(),
1282 vep_offset,
1283 frame_complete,
1284 stack_slots / VMRegImpl::slots_per_word,
1285 in_ByteSize(-1),
1286 in_ByteSize(-1),
1287 (OopMapSet*)NULL);
1288 }
1289 address native_func = method->native_function();
1290 assert(native_func != NULL, "must have function");
1291
1292 // An OopMap for lock (and class if static)
1293 OopMapSet *oop_maps = new OopMapSet();
1294 intptr_t start = (intptr_t)__ pc();
1295
1296 // We have received a description of where all the java arg are located
1297 // on entry to the wrapper. We need to convert these args to where
1298 // the jni function will expect them. To figure out where they go
1299 // we convert the java signature to a C signature by inserting
1300 // the hidden arguments as arg[0] and possibly arg[1] (static method)
1301
1302 const int total_in_args = method->size_of_parameters();
1303 int total_c_args = total_in_args + (method->is_static() ? 2 : 1);
1304
1305 BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1306 VMRegPair* out_regs = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1307 BasicType* in_elem_bt = NULL;
1308
1309 int argc = 0;
1310 out_sig_bt[argc++] = T_ADDRESS;
1311 if (method->is_static()) {
1312 out_sig_bt[argc++] = T_OBJECT;
1313 }
1314
1315 for (int i = 0; i < total_in_args ; i++ ) {
1316 out_sig_bt[argc++] = in_sig_bt[i];
1317 }
1318
1319 // Now figure out where the args must be stored and how much stack space
1320 // they require.
1321 int out_arg_slots;
1322 out_arg_slots = c_calling_convention_priv(out_sig_bt, out_regs, NULL, total_c_args);
1323
1324 if (out_arg_slots < 0) {
1325 return NULL;
1326 }
1327
1328 // Compute framesize for the wrapper. We need to handlize all oops in
1329 // incoming registers
1330
1331 // Calculate the total number of stack slots we will need.
1332
1333 // First count the abi requirement plus all of the outgoing args
1334 int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1335
1336 // Now the space for the inbound oop handle area
1337 int total_save_slots = 8 * VMRegImpl::slots_per_word; // 8 arguments passed in registers
1338
1339 int oop_handle_offset = stack_slots;
1340 stack_slots += total_save_slots;
1341
1342 // Now any space we need for handlizing a klass if static method
1343
1344 int klass_slot_offset = 0;
1345 int klass_offset = -1;
1346 int lock_slot_offset = 0;
1347 bool is_static = false;
1348
1349 if (method->is_static()) {
1350 klass_slot_offset = stack_slots;
1351 stack_slots += VMRegImpl::slots_per_word;
1352 klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
1353 is_static = true;
1354 }
1355
1356 // Plus a lock if needed
1357
1358 if (method->is_synchronized()) {
1359 lock_slot_offset = stack_slots;
1360 stack_slots += VMRegImpl::slots_per_word;
1361 }
1362
1363 // Now a place (+2) to save return values or temp during shuffling
1364 // + 4 for return address (which we own) and saved rfp
1365 stack_slots += 6;
1366
1367 // Ok The space we have allocated will look like:
1368 //
1369 //
1370 // FP-> | |
1371 // |---------------------|
1372 // | 2 slots for moves |
1373 // |---------------------|
1374 // | lock box (if sync) |
1375 // |---------------------| <- lock_slot_offset
1376 // | klass (if static) |
1377 // |---------------------| <- klass_slot_offset
1378 // | oopHandle area |
1379 // |---------------------| <- oop_handle_offset (8 java arg registers)
1380 // | outbound memory |
1381 // | based arguments |
1382 // | |
1383 // |---------------------|
1384 // | |
1385 // SP-> | out_preserved_slots |
1386 //
1387 //
1388
1389
1390 // Now compute actual number of stack words we need rounding to make
1391 // stack properly aligned.
1392 stack_slots = align_up(stack_slots, StackAlignmentInSlots);
1393
1394 int stack_size = stack_slots * VMRegImpl::stack_slot_size;
1395
1396 // First thing make an ic check to see if we should even be here
1397
1398 // We are free to use all registers as temps without saving them and
1399 // restoring them except rfp. rfp is the only callee save register
1400 // as far as the interpreter and the compiler(s) are concerned.
1401
1402
1403 const Register ic_reg = rscratch2;
1404 const Register receiver = j_rarg0;
1405
1406 Label hit;
1407 Label exception_pending;
1408
1409 assert_different_registers(ic_reg, receiver, rscratch1);
1410 __ verify_oop(receiver);
1411 __ cmp_klass(receiver, ic_reg, rscratch1);
1412 __ br(Assembler::EQ, hit);
1413
1414 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1415
1416 // Verified entry point must be aligned
1417 __ align(8);
1418
1419 __ bind(hit);
1420
1421 int vep_offset = ((intptr_t)__ pc()) - start;
1422
1423 // If we have to make this method not-entrant we'll overwrite its
1424 // first instruction with a jump. For this action to be legal we
1425 // must ensure that this first instruction is a B, BL, NOP, BKPT,
1426 // SVC, HVC, or SMC. Make it a NOP.
1427 __ nop();
1428
1429 if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
1430 Label L_skip_barrier;
1431 __ mov_metadata(rscratch2, method->method_holder()); // InstanceKlass*
1432 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1433 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1434
1435 __ bind(L_skip_barrier);
1436 }
1437
1438 // Generate stack overflow check
1439 __ bang_stack_with_offset(checked_cast<int>(StackOverflow::stack_shadow_zone_size()));
1440
1441 // Generate a new frame for the wrapper.
1442 __ enter();
1443 // -2 because return address is already present and so is saved rfp
1444 __ sub(sp, sp, stack_size - 2*wordSize);
1445
1446 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1447 bs->nmethod_entry_barrier(masm);
1448
1449 // Frame is now completed as far as size and linkage.
1450 int frame_complete = ((intptr_t)__ pc()) - start;
1451
1452 // We use r20 as the oop handle for the receiver/klass
1453 // It is callee save so it survives the call to native
1454
1455 const Register oop_handle_reg = r20;
1456
1457 //
1458 // We immediately shuffle the arguments so that any vm call we have to
1459 // make from here on out (sync slow path, jvmti, etc.) we will have
1460 // captured the oops from our caller and have a valid oopMap for
1461 // them.
1462
1463 // -----------------
1464 // The Grand Shuffle
1465
1466 // The Java calling convention is either equal (linux) or denser (win64) than the
1467 // c calling convention. However the because of the jni_env argument the c calling
1468 // convention always has at least one more (and two for static) arguments than Java.
1469 // Therefore if we move the args from java -> c backwards then we will never have
1470 // a register->register conflict and we don't have to build a dependency graph
1471 // and figure out how to break any cycles.
1472 //
1473
1474 // Record esp-based slot for receiver on stack for non-static methods
1475 int receiver_offset = -1;
1476
1477 // This is a trick. We double the stack slots so we can claim
1478 // the oops in the caller's frame. Since we are sure to have
1479 // more args than the caller doubling is enough to make
1480 // sure we can capture all the incoming oop args from the
1481 // caller.
1482 //
1483 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1484
1485 // Mark location of rfp (someday)
1486 // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rfp));
1487
1488
1489 int float_args = 0;
1490 int int_args = 0;
1491
1492 #ifdef ASSERT
1493 bool reg_destroyed[RegisterImpl::number_of_registers];
1494 bool freg_destroyed[FloatRegisterImpl::number_of_registers];
1495 for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
1496 reg_destroyed[r] = false;
1497 }
1498 for ( int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++ ) {
1499 freg_destroyed[f] = false;
1500 }
1501
1502 #endif /* ASSERT */
1503
1504 // For JNI natives the incoming and outgoing registers are offset upwards.
1505 GrowableArray<int> arg_order(2 * total_in_args);
1506 VMRegPair tmp_vmreg;
1507 tmp_vmreg.set2(r19->as_VMReg());
1508
1509 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1510 arg_order.push(i);
1511 arg_order.push(c_arg);
1512 }
1513
1514 int temploc = -1;
1515 for (int ai = 0; ai < arg_order.length(); ai += 2) {
1516 int i = arg_order.at(ai);
1517 int c_arg = arg_order.at(ai + 1);
1518 __ block_comment(err_msg("move %d -> %d", i, c_arg));
1519 assert(c_arg != -1 && i != -1, "wrong order");
1520 #ifdef ASSERT
1521 if (in_regs[i].first()->is_Register()) {
1522 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1523 } else if (in_regs[i].first()->is_FloatRegister()) {
1524 assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1525 }
1526 if (out_regs[c_arg].first()->is_Register()) {
1527 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1528 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1529 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1530 }
1531 #endif /* ASSERT */
1532 switch (in_sig_bt[i]) {
1533 case T_ARRAY:
1534 case T_OBJECT:
1535 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1536 ((i == 0) && (!is_static)),
1537 &receiver_offset);
1538 int_args++;
1539 break;
1540 case T_VOID:
1541 break;
1542
1543 case T_FLOAT:
1544 float_move(masm, in_regs[i], out_regs[c_arg]);
1545 float_args++;
1546 break;
1547
1548 case T_DOUBLE:
1549 assert( i + 1 < total_in_args &&
1550 in_sig_bt[i + 1] == T_VOID &&
1551 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1552 double_move(masm, in_regs[i], out_regs[c_arg]);
1553 float_args++;
1554 break;
1555
1556 case T_LONG :
1557 long_move(masm, in_regs[i], out_regs[c_arg]);
1558 int_args++;
1559 break;
1560
1561 case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
1562
1563 default:
1564 move32_64(masm, in_regs[i], out_regs[c_arg]);
1565 int_args++;
1566 }
1567 }
1568
1569 // point c_arg at the first arg that is already loaded in case we
1570 // need to spill before we call out
1571 int c_arg = total_c_args - total_in_args;
1572
1573 // Pre-load a static method's oop into c_rarg1.
1574 if (method->is_static()) {
1575
1576 // load oop into a register
1577 __ movoop(c_rarg1,
1578 JNIHandles::make_local(method->method_holder()->java_mirror()),
1579 /*immediate*/true);
1580
1581 // Now handlize the static class mirror it's known not-null.
1582 __ str(c_rarg1, Address(sp, klass_offset));
1583 map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
1584
1585 // Now get the handle
1586 __ lea(c_rarg1, Address(sp, klass_offset));
1587 // and protect the arg if we must spill
1588 c_arg--;
1589 }
1590
1591 // Change state to native (we save the return address in the thread, since it might not
1592 // be pushed on the stack when we do a stack traversal).
1593 // We use the same pc/oopMap repeatedly when we call out
1594
1595 Label native_return;
1596 __ set_last_Java_frame(sp, noreg, native_return, rscratch1);
1597
1598 Label dtrace_method_entry, dtrace_method_entry_done;
1599 {
1600 uint64_t offset;
1601 __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
1602 __ ldrb(rscratch1, Address(rscratch1, offset));
1603 __ cbnzw(rscratch1, dtrace_method_entry);
1604 __ bind(dtrace_method_entry_done);
1605 }
1606
1607 // RedefineClasses() tracing support for obsolete method entry
1608 if (log_is_enabled(Trace, redefine, class, obsolete)) {
1609 // protect the args we've loaded
1610 save_args(masm, total_c_args, c_arg, out_regs);
1611 __ mov_metadata(c_rarg1, method());
1612 __ call_VM_leaf(
1613 CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1614 rthread, c_rarg1);
1615 restore_args(masm, total_c_args, c_arg, out_regs);
1616 }
1617
1618 // Lock a synchronized method
1619
1620 // Register definitions used by locking and unlocking
1621
1622 const Register swap_reg = r0;
1623 const Register obj_reg = r19; // Will contain the oop
1624 const Register lock_reg = r13; // Address of compiler lock object (BasicLock)
1625 const Register old_hdr = r13; // value of old header at unlock time
1626 const Register tmp = lr;
1627
1628 Label slow_path_lock;
1629 Label lock_done;
1630
1631 if (method->is_synchronized()) {
1632
1633 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1634
1635 // Get the handle (the 2nd argument)
1636 __ mov(oop_handle_reg, c_rarg1);
1637
1638 // Get address of the box
1639
1640 __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1641
1642 // Load the oop from the handle
1643 __ ldr(obj_reg, Address(oop_handle_reg, 0));
1644
1645 // Load (object->mark() | 1) into swap_reg %r0
1646 __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1647 __ orr(swap_reg, rscratch1, 1);
1648
1649 // Save (object->mark() | 1) into BasicLock's displaced header
1650 __ str(swap_reg, Address(lock_reg, mark_word_offset));
1651
1652 // src -> dest iff dest == r0 else r0 <- dest
1653 { Label here;
1654 __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
1655 }
1656
1657 // Hmm should this move to the slow path code area???
1658
1659 // Test if the oopMark is an obvious stack pointer, i.e.,
1660 // 1) (mark & 3) == 0, and
1661 // 2) sp <= mark < mark + os::pagesize()
1662 // These 3 tests can be done by evaluating the following
1663 // expression: ((mark - sp) & (3 - os::vm_page_size())),
1664 // assuming both stack pointer and pagesize have their
1665 // least significant 2 bits clear.
1666 // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1667
1668 __ sub(swap_reg, sp, swap_reg);
1669 __ neg(swap_reg, swap_reg);
1670 __ ands(swap_reg, swap_reg, 3 - os::vm_page_size());
1671
1672 // Save the test result, for recursive case, the result is zero
1673 __ str(swap_reg, Address(lock_reg, mark_word_offset));
1674 __ br(Assembler::NE, slow_path_lock);
1675
1676 // Slow path will re-enter here
1677
1678 __ bind(lock_done);
1679 }
1680
1681
1682 // Finally just about ready to make the JNI call
1683
1684 // get JNIEnv* which is first argument to native
1685 __ lea(c_rarg0, Address(rthread, in_bytes(JavaThread::jni_environment_offset())));
1686
1687 // Now set thread in native
1688 __ mov(rscratch1, _thread_in_native);
1689 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1690 __ stlrw(rscratch1, rscratch2);
1691
1692 rt_call(masm, native_func);
1693
1694 __ bind(native_return);
1695
1696 intptr_t return_pc = (intptr_t) __ pc();
1697 oop_maps->add_gc_map(return_pc - start, map);
1698
1699 // Unpack native results.
1700 switch (ret_type) {
1701 case T_BOOLEAN: __ c2bool(r0); break;
1702 case T_CHAR : __ ubfx(r0, r0, 0, 16); break;
1703 case T_BYTE : __ sbfx(r0, r0, 0, 8); break;
1704 case T_SHORT : __ sbfx(r0, r0, 0, 16); break;
1705 case T_INT : __ sbfx(r0, r0, 0, 32); break;
1706 case T_DOUBLE :
1707 case T_FLOAT :
1708 // Result is in v0 we'll save as needed
1709 break;
1710 case T_ARRAY: // Really a handle
1711 case T_OBJECT: // Really a handle
1712 break; // can't de-handlize until after safepoint check
1713 case T_VOID: break;
1714 case T_LONG: break;
1715 default : ShouldNotReachHere();
1716 }
1717
1718 Label safepoint_in_progress, safepoint_in_progress_done;
1719 Label after_transition;
1720
1721 // Switch thread to "native transition" state before reading the synchronization state.
1722 // This additional state is necessary because reading and testing the synchronization
1723 // state is not atomic w.r.t. GC, as this scenario demonstrates:
1724 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
1725 // VM thread changes sync state to synchronizing and suspends threads for GC.
1726 // Thread A is resumed to finish this native method, but doesn't block here since it
1727 // didn't see any synchronization is progress, and escapes.
1728 __ mov(rscratch1, _thread_in_native_trans);
1729
1730 __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
1731
1732 // Force this write out before the read below
1733 __ dmb(Assembler::ISH);
1734
1735 __ verify_sve_vector_length();
1736
1737 // Check for safepoint operation in progress and/or pending suspend requests.
1738 {
1739 // We need an acquire here to ensure that any subsequent load of the
1740 // global SafepointSynchronize::_state flag is ordered after this load
1741 // of the thread-local polling word. We don't want this poll to
1742 // return false (i.e. not safepointing) and a later poll of the global
1743 // SafepointSynchronize::_state spuriously to return true.
1744 //
1745 // This is to avoid a race when we're in a native->Java transition
1746 // racing the code which wakes up from a safepoint.
1747
1748 __ safepoint_poll(safepoint_in_progress, true /* at_return */, true /* acquire */, false /* in_nmethod */);
1749 __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
1750 __ cbnzw(rscratch1, safepoint_in_progress);
1751 __ bind(safepoint_in_progress_done);
1752 }
1753
1754 // change thread state
1755 __ mov(rscratch1, _thread_in_Java);
1756 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1757 __ stlrw(rscratch1, rscratch2);
1758 __ bind(after_transition);
1759
1760 Label reguard;
1761 Label reguard_done;
1762 __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
1763 __ cmpw(rscratch1, StackOverflow::stack_guard_yellow_reserved_disabled);
1764 __ br(Assembler::EQ, reguard);
1765 __ bind(reguard_done);
1766
1767 // native result if any is live
1768
1769 // Unlock
1770 Label unlock_done;
1771 Label slow_path_unlock;
1772 if (method->is_synchronized()) {
1773
1774 // Get locked oop from the handle we passed to jni
1775 __ ldr(obj_reg, Address(oop_handle_reg, 0));
1776
1777 Label done;
1778 // Simple recursive lock?
1779
1780 __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1781 __ cbz(rscratch1, done);
1782
1783 // Must save r0 if if it is live now because cmpxchg must use it
1784 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
1785 save_native_result(masm, ret_type, stack_slots);
1786 }
1787
1788
1789 // get address of the stack lock
1790 __ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1791 // get old displaced header
1792 __ ldr(old_hdr, Address(r0, 0));
1793
1794 // Atomic swap old header if oop still contains the stack lock
1795 Label succeed;
1796 __ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, succeed, &slow_path_unlock);
1797 __ bind(succeed);
1798
1799 // slow path re-enters here
1800 __ bind(unlock_done);
1801 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
1802 restore_native_result(masm, ret_type, stack_slots);
1803 }
1804
1805 __ bind(done);
1806 }
1807
1808 Label dtrace_method_exit, dtrace_method_exit_done;
1809 {
1810 uint64_t offset;
1811 __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
1812 __ ldrb(rscratch1, Address(rscratch1, offset));
1813 __ cbnzw(rscratch1, dtrace_method_exit);
1814 __ bind(dtrace_method_exit_done);
1815 }
1816
1817 __ reset_last_Java_frame(false);
1818
1819 // Unbox oop result, e.g. JNIHandles::resolve result.
1820 if (is_reference_type(ret_type)) {
1821 __ resolve_jobject(r0, rthread, rscratch2);
1822 }
1823
1824 if (CheckJNICalls) {
1825 // clear_pending_jni_exception_check
1826 __ str(zr, Address(rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1827 }
1828
1829 // reset handle block
1830 __ ldr(r2, Address(rthread, JavaThread::active_handles_offset()));
1831 __ str(zr, Address(r2, JNIHandleBlock::top_offset_in_bytes()));
1832
1833 __ leave();
1834
1835 // Any exception pending?
1836 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1837 __ cbnz(rscratch1, exception_pending);
1838
1839 // We're done
1840 __ ret(lr);
1841
1842 // Unexpected paths are out of line and go here
1843
1844 // forward the exception
1845 __ bind(exception_pending);
1846
1847 // and forward the exception
1848 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1849
1850 // Slow path locking & unlocking
1851 if (method->is_synchronized()) {
1852
1853 __ block_comment("Slow path lock {");
1854 __ bind(slow_path_lock);
1855
1856 // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
1857 // args are (oop obj, BasicLock* lock, JavaThread* thread)
1858
1859 // protect the args we've loaded
1860 save_args(masm, total_c_args, c_arg, out_regs);
1861
1862 __ mov(c_rarg0, obj_reg);
1863 __ mov(c_rarg1, lock_reg);
1864 __ mov(c_rarg2, rthread);
1865
1866 // Not a leaf but we have last_Java_frame setup as we want
1867 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
1868 restore_args(masm, total_c_args, c_arg, out_regs);
1869
1870 #ifdef ASSERT
1871 { Label L;
1872 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1873 __ cbz(rscratch1, L);
1874 __ stop("no pending exception allowed on exit from monitorenter");
1875 __ bind(L);
1876 }
1877 #endif
1878 __ b(lock_done);
1879
1880 __ block_comment("} Slow path lock");
1881
1882 __ block_comment("Slow path unlock {");
1883 __ bind(slow_path_unlock);
1884
1885 // If we haven't already saved the native result we must save it now as xmm registers
1886 // are still exposed.
1887
1888 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
1889 save_native_result(masm, ret_type, stack_slots);
1890 }
1891
1892 __ mov(c_rarg2, rthread);
1893 __ lea(c_rarg1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1894 __ mov(c_rarg0, obj_reg);
1895
1896 // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
1897 // NOTE that obj_reg == r19 currently
1898 __ ldr(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1899 __ str(zr, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1900
1901 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
1902
1903 #ifdef ASSERT
1904 {
1905 Label L;
1906 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1907 __ cbz(rscratch1, L);
1908 __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
1909 __ bind(L);
1910 }
1911 #endif /* ASSERT */
1912
1913 __ str(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1914
1915 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
1916 restore_native_result(masm, ret_type, stack_slots);
1917 }
1918 __ b(unlock_done);
1919
1920 __ block_comment("} Slow path unlock");
1921
1922 } // synchronized
1923
1924 // SLOW PATH Reguard the stack if needed
1925
1926 __ bind(reguard);
1927 save_native_result(masm, ret_type, stack_slots);
1928 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1929 restore_native_result(masm, ret_type, stack_slots);
1930 // and continue
1931 __ b(reguard_done);
1932
1933 // SLOW PATH safepoint
1934 {
1935 __ block_comment("safepoint {");
1936 __ bind(safepoint_in_progress);
1937
1938 // Don't use call_VM as it will see a possible pending exception and forward it
1939 // and never return here preventing us from clearing _last_native_pc down below.
1940 //
1941 save_native_result(masm, ret_type, stack_slots);
1942 __ mov(c_rarg0, rthread);
1943 #ifndef PRODUCT
1944 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
1945 #endif
1946 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
1947 __ blr(rscratch1);
1948
1949 // Restore any method result value
1950 restore_native_result(masm, ret_type, stack_slots);
1951
1952 __ b(safepoint_in_progress_done);
1953 __ block_comment("} safepoint");
1954 }
1955
1956 // SLOW PATH dtrace support
1957 {
1958 __ block_comment("dtrace entry {");
1959 __ bind(dtrace_method_entry);
1960
1961 // We have all of the arguments setup at this point. We must not touch any register
1962 // argument registers at this point (what if we save/restore them there are no oop?
1963
1964 save_args(masm, total_c_args, c_arg, out_regs);
1965 __ mov_metadata(c_rarg1, method());
1966 __ call_VM_leaf(
1967 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
1968 rthread, c_rarg1);
1969 restore_args(masm, total_c_args, c_arg, out_regs);
1970 __ b(dtrace_method_entry_done);
1971 __ block_comment("} dtrace entry");
1972 }
1973
1974 {
1975 __ block_comment("dtrace exit {");
1976 __ bind(dtrace_method_exit);
1977 save_native_result(masm, ret_type, stack_slots);
1978 __ mov_metadata(c_rarg1, method());
1979 __ call_VM_leaf(
1980 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
1981 rthread, c_rarg1);
1982 restore_native_result(masm, ret_type, stack_slots);
1983 __ b(dtrace_method_exit_done);
1984 __ block_comment("} dtrace exit");
1985 }
1986
1987
1988 __ flush();
1989
1990 nmethod *nm = nmethod::new_native_nmethod(method,
1991 compile_id,
1992 masm->code(),
1993 vep_offset,
1994 frame_complete,
1995 stack_slots / VMRegImpl::slots_per_word,
1996 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
1997 in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
1998 oop_maps);
1999
2000 return nm;
2001 }
2002
2003 // this function returns the adjust size (in number of words) to a c2i adapter
2004 // activation for use during deoptimization
2005 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2006 assert(callee_locals >= callee_parameters,
2007 "test and remove; got more parms than locals");
2008 if (callee_locals < callee_parameters)
2009 return 0; // No adjustment for negative locals
2010 int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2011 // diff is counted in stack words
2012 return align_up(diff, 2);
2013 }
2014
2015
2016 //------------------------------generate_deopt_blob----------------------------
2017 void SharedRuntime::generate_deopt_blob() {
2018 // Allocate space for the code
2019 ResourceMark rm;
2020 // Setup code generation tools
2021 int pad = 0;
2022 #if INCLUDE_JVMCI
2023 if (EnableJVMCI) {
2024 pad += 512; // Increase the buffer size when compiling for JVMCI
2025 }
2026 #endif
2027 CodeBuffer buffer("deopt_blob", 2048+pad, 1024);
2028 MacroAssembler* masm = new MacroAssembler(&buffer);
2029 int frame_size_in_words;
2030 OopMap* map = NULL;
2031 OopMapSet *oop_maps = new OopMapSet();
2032 RegisterSaver reg_save(COMPILER2_OR_JVMCI != 0);
2033
2034 // -------------
2035 // This code enters when returning to a de-optimized nmethod. A return
2036 // address has been pushed on the the stack, and return values are in
2037 // registers.
2038 // If we are doing a normal deopt then we were called from the patched
2039 // nmethod from the point we returned to the nmethod. So the return
2040 // address on the stack is wrong by NativeCall::instruction_size
2041 // We will adjust the value so it looks like we have the original return
2042 // address on the stack (like when we eagerly deoptimized).
2043 // In the case of an exception pending when deoptimizing, we enter
2044 // with a return address on the stack that points after the call we patched
2045 // into the exception handler. We have the following register state from,
2046 // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2047 // r0: exception oop
2048 // r19: exception handler
2049 // r3: throwing pc
2050 // So in this case we simply jam r3 into the useless return address and
2051 // the stack looks just like we want.
2052 //
2053 // At this point we need to de-opt. We save the argument return
2054 // registers. We call the first C routine, fetch_unroll_info(). This
2055 // routine captures the return values and returns a structure which
2056 // describes the current frame size and the sizes of all replacement frames.
2057 // The current frame is compiled code and may contain many inlined
2058 // functions, each with their own JVM state. We pop the current frame, then
2059 // push all the new frames. Then we call the C routine unpack_frames() to
2060 // populate these frames. Finally unpack_frames() returns us the new target
2061 // address. Notice that callee-save registers are BLOWN here; they have
2062 // already been captured in the vframeArray at the time the return PC was
2063 // patched.
2064 address start = __ pc();
2065 Label cont;
2066
2067 // Prolog for non exception case!
2068
2069 // Save everything in sight.
2070 map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2071
2072 // Normal deoptimization. Save exec mode for unpack_frames.
2073 __ movw(rcpool, Deoptimization::Unpack_deopt); // callee-saved
2074 __ b(cont);
2075
2076 int reexecute_offset = __ pc() - start;
2077 #if INCLUDE_JVMCI && !defined(COMPILER1)
2078 if (EnableJVMCI && UseJVMCICompiler) {
2079 // JVMCI does not use this kind of deoptimization
2080 __ should_not_reach_here();
2081 }
2082 #endif
2083
2084 // Reexecute case
2085 // return address is the pc describes what bci to do re-execute at
2086
2087 // No need to update map as each call to save_live_registers will produce identical oopmap
2088 (void) reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2089
2090 __ movw(rcpool, Deoptimization::Unpack_reexecute); // callee-saved
2091 __ b(cont);
2092
2093 #if INCLUDE_JVMCI
2094 Label after_fetch_unroll_info_call;
2095 int implicit_exception_uncommon_trap_offset = 0;
2096 int uncommon_trap_offset = 0;
2097
2098 if (EnableJVMCI) {
2099 implicit_exception_uncommon_trap_offset = __ pc() - start;
2100
2101 __ ldr(lr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2102 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2103
2104 uncommon_trap_offset = __ pc() - start;
2105
2106 // Save everything in sight.
2107 reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2108 // fetch_unroll_info needs to call last_java_frame()
2109 Label retaddr;
2110 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2111
2112 __ ldrw(c_rarg1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2113 __ movw(rscratch1, -1);
2114 __ strw(rscratch1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2115
2116 __ movw(rcpool, (int32_t)Deoptimization::Unpack_reexecute);
2117 __ mov(c_rarg0, rthread);
2118 __ movw(c_rarg2, rcpool); // exec mode
2119 __ lea(rscratch1,
2120 RuntimeAddress(CAST_FROM_FN_PTR(address,
2121 Deoptimization::uncommon_trap)));
2122 __ blr(rscratch1);
2123 __ bind(retaddr);
2124 oop_maps->add_gc_map( __ pc()-start, map->deep_copy());
2125
2126 __ reset_last_Java_frame(false);
2127
2128 __ b(after_fetch_unroll_info_call);
2129 } // EnableJVMCI
2130 #endif // INCLUDE_JVMCI
2131
2132 int exception_offset = __ pc() - start;
2133
2134 // Prolog for exception case
2135
2136 // all registers are dead at this entry point, except for r0, and
2137 // r3 which contain the exception oop and exception pc
2138 // respectively. Set them in TLS and fall thru to the
2139 // unpack_with_exception_in_tls entry point.
2140
2141 __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2142 __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2143
2144 int exception_in_tls_offset = __ pc() - start;
2145
2146 // new implementation because exception oop is now passed in JavaThread
2147
2148 // Prolog for exception case
2149 // All registers must be preserved because they might be used by LinearScan
2150 // Exceptiop oop and throwing PC are passed in JavaThread
2151 // tos: stack at point of call to method that threw the exception (i.e. only
2152 // args are on the stack, no return address)
2153
2154 // The return address pushed by save_live_registers will be patched
2155 // later with the throwing pc. The correct value is not available
2156 // now because loading it from memory would destroy registers.
2157
2158 // NB: The SP at this point must be the SP of the method that is
2159 // being deoptimized. Deoptimization assumes that the frame created
2160 // here by save_live_registers is immediately below the method's SP.
2161 // This is a somewhat fragile mechanism.
2162
2163 // Save everything in sight.
2164 map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2165
2166 // Now it is safe to overwrite any register
2167
2168 // Deopt during an exception. Save exec mode for unpack_frames.
2169 __ mov(rcpool, Deoptimization::Unpack_exception); // callee-saved
2170
2171 // load throwing pc from JavaThread and patch it as the return address
2172 // of the current frame. Then clear the field in JavaThread
2173
2174 __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2175 __ str(r3, Address(rfp, wordSize));
2176 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2177
2178 #ifdef ASSERT
2179 // verify that there is really an exception oop in JavaThread
2180 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2181 __ verify_oop(r0);
2182
2183 // verify that there is no pending exception
2184 Label no_pending_exception;
2185 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2186 __ cbz(rscratch1, no_pending_exception);
2187 __ stop("must not have pending exception here");
2188 __ bind(no_pending_exception);
2189 #endif
2190
2191 __ bind(cont);
2192
2193 // Call C code. Need thread and this frame, but NOT official VM entry
2194 // crud. We cannot block on this call, no GC can happen.
2195 //
2196 // UnrollBlock* fetch_unroll_info(JavaThread* thread)
2197
2198 // fetch_unroll_info needs to call last_java_frame().
2199
2200 Label retaddr;
2201 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2202 #ifdef ASSERT0
2203 { Label L;
2204 __ ldr(rscratch1, Address(rthread,
2205 JavaThread::last_Java_fp_offset()));
2206 __ cbz(rscratch1, L);
2207 __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2208 __ bind(L);
2209 }
2210 #endif // ASSERT
2211 __ mov(c_rarg0, rthread);
2212 __ mov(c_rarg1, rcpool);
2213 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
2214 __ blr(rscratch1);
2215 __ bind(retaddr);
2216
2217 // Need to have an oopmap that tells fetch_unroll_info where to
2218 // find any register it might need.
2219 oop_maps->add_gc_map(__ pc() - start, map);
2220
2221 __ reset_last_Java_frame(false);
2222
2223 #if INCLUDE_JVMCI
2224 if (EnableJVMCI) {
2225 __ bind(after_fetch_unroll_info_call);
2226 }
2227 #endif
2228
2229 // Load UnrollBlock* into r5
2230 __ mov(r5, r0);
2231
2232 __ ldrw(rcpool, Address(r5, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2233 Label noException;
2234 __ cmpw(rcpool, Deoptimization::Unpack_exception); // Was exception pending?
2235 __ br(Assembler::NE, noException);
2236 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2237 // QQQ this is useless it was NULL above
2238 __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2239 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
2240 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2241
2242 __ verify_oop(r0);
2243
2244 // Overwrite the result registers with the exception results.
2245 __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2246 // I think this is useless
2247 // __ str(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2248
2249 __ bind(noException);
2250
2251 // Only register save data is on the stack.
2252 // Now restore the result registers. Everything else is either dead
2253 // or captured in the vframeArray.
2254
2255 // Restore fp result register
2256 __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2257 // Restore integer result register
2258 __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2259
2260 // Pop all of the register save area off the stack
2261 __ add(sp, sp, frame_size_in_words * wordSize);
2262
2263 // All of the register save area has been popped of the stack. Only the
2264 // return address remains.
2265
2266 // Pop all the frames we must move/replace.
2267 //
2268 // Frame picture (youngest to oldest)
2269 // 1: self-frame (no frame link)
2270 // 2: deopting frame (no frame link)
2271 // 3: caller of deopting frame (could be compiled/interpreted).
2272 //
2273 // Note: by leaving the return address of self-frame on the stack
2274 // and using the size of frame 2 to adjust the stack
2275 // when we are done the return to frame 3 will still be on the stack.
2276
2277 // Pop deoptimized frame
2278 __ ldrw(r2, Address(r5, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2279 __ sub(r2, r2, 2 * wordSize);
2280 __ add(sp, sp, r2);
2281 __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2282 // LR should now be the return address to the caller (3)
2283
2284 #ifdef ASSERT
2285 // Compilers generate code that bang the stack by as much as the
2286 // interpreter would need. So this stack banging should never
2287 // trigger a fault. Verify that it does not on non product builds.
2288 __ ldrw(r19, Address(r5, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2289 __ bang_stack_size(r19, r2);
2290 #endif
2291 // Load address of array of frame pcs into r2
2292 __ ldr(r2, Address(r5, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2293
2294 // Trash the old pc
2295 // __ addptr(sp, wordSize); FIXME ????
2296
2297 // Load address of array of frame sizes into r4
2298 __ ldr(r4, Address(r5, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2299
2300 // Load counter into r3
2301 __ ldrw(r3, Address(r5, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2302
2303 // Now adjust the caller's stack to make up for the extra locals
2304 // but record the original sp so that we can save it in the skeletal interpreter
2305 // frame and the stack walking of interpreter_sender will get the unextended sp
2306 // value and not the "real" sp value.
2307
2308 const Register sender_sp = r6;
2309
2310 __ mov(sender_sp, sp);
2311 __ ldrw(r19, Address(r5,
2312 Deoptimization::UnrollBlock::
2313 caller_adjustment_offset_in_bytes()));
2314 __ sub(sp, sp, r19);
2315
2316 // Push interpreter frames in a loop
2317 __ mov(rscratch1, (uint64_t)0xDEADDEAD); // Make a recognizable pattern
2318 __ mov(rscratch2, rscratch1);
2319 Label loop;
2320 __ bind(loop);
2321 __ ldr(r19, Address(__ post(r4, wordSize))); // Load frame size
2322 __ sub(r19, r19, 2*wordSize); // We'll push pc and fp by hand
2323 __ ldr(lr, Address(__ post(r2, wordSize))); // Load pc
2324 __ enter(); // Save old & set new fp
2325 __ sub(sp, sp, r19); // Prolog
2326 // This value is corrected by layout_activation_impl
2327 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2328 __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2329 __ mov(sender_sp, sp); // Pass sender_sp to next frame
2330 __ sub(r3, r3, 1); // Decrement counter
2331 __ cbnz(r3, loop);
2332
2333 // Re-push self-frame
2334 __ ldr(lr, Address(r2));
2335 __ enter();
2336
2337 // Allocate a full sized register save area. We subtract 2 because
2338 // enter() just pushed 2 words
2339 __ sub(sp, sp, (frame_size_in_words - 2) * wordSize);
2340
2341 // Restore frame locals after moving the frame
2342 __ strd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2343 __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2344
2345 // Call C code. Need thread but NOT official VM entry
2346 // crud. We cannot block on this call, no GC can happen. Call should
2347 // restore return values to their stack-slots with the new SP.
2348 //
2349 // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
2350
2351 // Use rfp because the frames look interpreted now
2352 // Don't need the precise return PC here, just precise enough to point into this code blob.
2353 address the_pc = __ pc();
2354 __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2355
2356 __ mov(c_rarg0, rthread);
2357 __ movw(c_rarg1, rcpool); // second arg: exec_mode
2358 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2359 __ blr(rscratch1);
2360
2361 // Set an oopmap for the call site
2362 // Use the same PC we used for the last java frame
2363 oop_maps->add_gc_map(the_pc - start,
2364 new OopMap( frame_size_in_words, 0 ));
2365
2366 // Clear fp AND pc
2367 __ reset_last_Java_frame(true);
2368
2369 // Collect return values
2370 __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2371 __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2372 // I think this is useless (throwing pc?)
2373 // __ ldr(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2374
2375 // Pop self-frame.
2376 __ leave(); // Epilog
2377
2378 // Jump to interpreter
2379 __ ret(lr);
2380
2381 // Make sure all code is generated
2382 masm->flush();
2383
2384 _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
2385 _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2386 #if INCLUDE_JVMCI
2387 if (EnableJVMCI) {
2388 _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
2389 _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
2390 }
2391 #endif
2392 }
2393
2394 // Number of stack slots between incoming argument block and the start of
2395 // a new frame. The PROLOG must add this many slots to the stack. The
2396 // EPILOG must remove this many slots. aarch64 needs two slots for
2397 // return address and fp.
2398 // TODO think this is correct but check
2399 uint SharedRuntime::in_preserve_stack_slots() {
2400 return 4;
2401 }
2402
2403 uint SharedRuntime::out_preserve_stack_slots() {
2404 return 0;
2405 }
2406
2407 #ifdef COMPILER2
2408 //------------------------------generate_uncommon_trap_blob--------------------
2409 void SharedRuntime::generate_uncommon_trap_blob() {
2410 // Allocate space for the code
2411 ResourceMark rm;
2412 // Setup code generation tools
2413 CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
2414 MacroAssembler* masm = new MacroAssembler(&buffer);
2415
2416 assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2417
2418 address start = __ pc();
2419
2420 // Push self-frame. We get here with a return address in LR
2421 // and sp should be 16 byte aligned
2422 // push rfp and retaddr by hand
2423 __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2424 // we don't expect an arg reg save area
2425 #ifndef PRODUCT
2426 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2427 #endif
2428 // compiler left unloaded_class_index in j_rarg0 move to where the
2429 // runtime expects it.
2430 if (c_rarg1 != j_rarg0) {
2431 __ movw(c_rarg1, j_rarg0);
2432 }
2433
2434 // we need to set the past SP to the stack pointer of the stub frame
2435 // and the pc to the address where this runtime call will return
2436 // although actually any pc in this code blob will do).
2437 Label retaddr;
2438 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2439
2440 // Call C code. Need thread but NOT official VM entry
2441 // crud. We cannot block on this call, no GC can happen. Call should
2442 // capture callee-saved registers as well as return values.
2443 // Thread is in rdi already.
2444 //
2445 // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index);
2446 //
2447 // n.b. 2 gp args, 0 fp args, integral return type
2448
2449 __ mov(c_rarg0, rthread);
2450 __ movw(c_rarg2, (unsigned)Deoptimization::Unpack_uncommon_trap);
2451 __ lea(rscratch1,
2452 RuntimeAddress(CAST_FROM_FN_PTR(address,
2453 Deoptimization::uncommon_trap)));
2454 __ blr(rscratch1);
2455 __ bind(retaddr);
2456
2457 // Set an oopmap for the call site
2458 OopMapSet* oop_maps = new OopMapSet();
2459 OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0);
2460
2461 // location of rfp is known implicitly by the frame sender code
2462
2463 oop_maps->add_gc_map(__ pc() - start, map);
2464
2465 __ reset_last_Java_frame(false);
2466
2467 // move UnrollBlock* into r4
2468 __ mov(r4, r0);
2469
2470 #ifdef ASSERT
2471 { Label L;
2472 __ ldrw(rscratch1, Address(r4, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2473 __ cmpw(rscratch1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2474 __ br(Assembler::EQ, L);
2475 __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2476 __ bind(L);
2477 }
2478 #endif
2479
2480 // Pop all the frames we must move/replace.
2481 //
2482 // Frame picture (youngest to oldest)
2483 // 1: self-frame (no frame link)
2484 // 2: deopting frame (no frame link)
2485 // 3: caller of deopting frame (could be compiled/interpreted).
2486
2487 // Pop self-frame. We have no frame, and must rely only on r0 and sp.
2488 __ add(sp, sp, (SimpleRuntimeFrame::framesize) << LogBytesPerInt); // Epilog!
2489
2490 // Pop deoptimized frame (int)
2491 __ ldrw(r2, Address(r4,
2492 Deoptimization::UnrollBlock::
2493 size_of_deoptimized_frame_offset_in_bytes()));
2494 __ sub(r2, r2, 2 * wordSize);
2495 __ add(sp, sp, r2);
2496 __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2497 // LR should now be the return address to the caller (3) frame
2498
2499 #ifdef ASSERT
2500 // Compilers generate code that bang the stack by as much as the
2501 // interpreter would need. So this stack banging should never
2502 // trigger a fault. Verify that it does not on non product builds.
2503 __ ldrw(r1, Address(r4,
2504 Deoptimization::UnrollBlock::
2505 total_frame_sizes_offset_in_bytes()));
2506 __ bang_stack_size(r1, r2);
2507 #endif
2508
2509 // Load address of array of frame pcs into r2 (address*)
2510 __ ldr(r2, Address(r4,
2511 Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2512
2513 // Load address of array of frame sizes into r5 (intptr_t*)
2514 __ ldr(r5, Address(r4,
2515 Deoptimization::UnrollBlock::
2516 frame_sizes_offset_in_bytes()));
2517
2518 // Counter
2519 __ ldrw(r3, Address(r4,
2520 Deoptimization::UnrollBlock::
2521 number_of_frames_offset_in_bytes())); // (int)
2522
2523 // Now adjust the caller's stack to make up for the extra locals but
2524 // record the original sp so that we can save it in the skeletal
2525 // interpreter frame and the stack walking of interpreter_sender
2526 // will get the unextended sp value and not the "real" sp value.
2527
2528 const Register sender_sp = r8;
2529
2530 __ mov(sender_sp, sp);
2531 __ ldrw(r1, Address(r4,
2532 Deoptimization::UnrollBlock::
2533 caller_adjustment_offset_in_bytes())); // (int)
2534 __ sub(sp, sp, r1);
2535
2536 // Push interpreter frames in a loop
2537 Label loop;
2538 __ bind(loop);
2539 __ ldr(r1, Address(r5, 0)); // Load frame size
2540 __ sub(r1, r1, 2 * wordSize); // We'll push pc and rfp by hand
2541 __ ldr(lr, Address(r2, 0)); // Save return address
2542 __ enter(); // and old rfp & set new rfp
2543 __ sub(sp, sp, r1); // Prolog
2544 __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2545 // This value is corrected by layout_activation_impl
2546 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2547 __ mov(sender_sp, sp); // Pass sender_sp to next frame
2548 __ add(r5, r5, wordSize); // Bump array pointer (sizes)
2549 __ add(r2, r2, wordSize); // Bump array pointer (pcs)
2550 __ subsw(r3, r3, 1); // Decrement counter
2551 __ br(Assembler::GT, loop);
2552 __ ldr(lr, Address(r2, 0)); // save final return address
2553 // Re-push self-frame
2554 __ enter(); // & old rfp & set new rfp
2555
2556 // Use rfp because the frames look interpreted now
2557 // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
2558 // Don't need the precise return PC here, just precise enough to point into this code blob.
2559 address the_pc = __ pc();
2560 __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2561
2562 // Call C code. Need thread but NOT official VM entry
2563 // crud. We cannot block on this call, no GC can happen. Call should
2564 // restore return values to their stack-slots with the new SP.
2565 // Thread is in rdi already.
2566 //
2567 // BasicType unpack_frames(JavaThread* thread, int exec_mode);
2568 //
2569 // n.b. 2 gp args, 0 fp args, integral return type
2570
2571 // sp should already be aligned
2572 __ mov(c_rarg0, rthread);
2573 __ movw(c_rarg1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2574 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2575 __ blr(rscratch1);
2576
2577 // Set an oopmap for the call site
2578 // Use the same PC we used for the last java frame
2579 oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
2580
2581 // Clear fp AND pc
2582 __ reset_last_Java_frame(true);
2583
2584 // Pop self-frame.
2585 __ leave(); // Epilog
2586
2587 // Jump to interpreter
2588 __ ret(lr);
2589
2590 // Make sure all code is generated
2591 masm->flush();
2592
2593 _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, oop_maps,
2594 SimpleRuntimeFrame::framesize >> 1);
2595 }
2596 #endif // COMPILER2
2597
2598
2599 //------------------------------generate_handler_blob------
2600 //
2601 // Generate a special Compile2Runtime blob that saves all registers,
2602 // and setup oopmap.
2603 //
2604 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
2605 ResourceMark rm;
2606 OopMapSet *oop_maps = new OopMapSet();
2607 OopMap* map;
2608
2609 // Allocate space for the code. Setup code generation tools.
2610 CodeBuffer buffer("handler_blob", 2048, 1024);
2611 MacroAssembler* masm = new MacroAssembler(&buffer);
2612
2613 address start = __ pc();
2614 address call_pc = NULL;
2615 int frame_size_in_words;
2616 bool cause_return = (poll_type == POLL_AT_RETURN);
2617 RegisterSaver reg_save(poll_type == POLL_AT_VECTOR_LOOP /* save_vectors */);
2618
2619 // Save Integer and Float registers.
2620 map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2621
2622 // The following is basically a call_VM. However, we need the precise
2623 // address of the call in order to generate an oopmap. Hence, we do all the
2624 // work outselves.
2625
2626 Label retaddr;
2627 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2628
2629 // The return address must always be correct so that frame constructor never
2630 // sees an invalid pc.
2631
2632 if (!cause_return) {
2633 // overwrite the return address pushed by save_live_registers
2634 // Additionally, r20 is a callee-saved register so we can look at
2635 // it later to determine if someone changed the return address for
2636 // us!
2637 __ ldr(r20, Address(rthread, JavaThread::saved_exception_pc_offset()));
2638 __ str(r20, Address(rfp, wordSize));
2639 }
2640
2641 // Do the call
2642 __ mov(c_rarg0, rthread);
2643 __ lea(rscratch1, RuntimeAddress(call_ptr));
2644 __ blr(rscratch1);
2645 __ bind(retaddr);
2646
2647 // Set an oopmap for the call site. This oopmap will map all
2648 // oop-registers and debug-info registers as callee-saved. This
2649 // will allow deoptimization at this safepoint to find all possible
2650 // debug-info recordings, as well as let GC find all oops.
2651
2652 oop_maps->add_gc_map( __ pc() - start, map);
2653
2654 Label noException;
2655
2656 __ reset_last_Java_frame(false);
2657
2658 __ membar(Assembler::LoadLoad | Assembler::LoadStore);
2659
2660 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2661 __ cbz(rscratch1, noException);
2662
2663 // Exception pending
2664
2665 reg_save.restore_live_registers(masm);
2666
2667 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2668
2669 // No exception case
2670 __ bind(noException);
2671
2672 Label no_adjust, bail;
2673 if (!cause_return) {
2674 // If our stashed return pc was modified by the runtime we avoid touching it
2675 __ ldr(rscratch1, Address(rfp, wordSize));
2676 __ cmp(r20, rscratch1);
2677 __ br(Assembler::NE, no_adjust);
2678
2679 #ifdef ASSERT
2680 // Verify the correct encoding of the poll we're about to skip.
2681 // See NativeInstruction::is_ldrw_to_zr()
2682 __ ldrw(rscratch1, Address(r20));
2683 __ ubfx(rscratch2, rscratch1, 22, 10);
2684 __ cmpw(rscratch2, 0b1011100101);
2685 __ br(Assembler::NE, bail);
2686 __ ubfx(rscratch2, rscratch1, 0, 5);
2687 __ cmpw(rscratch2, 0b11111);
2688 __ br(Assembler::NE, bail);
2689 #endif
2690 // Adjust return pc forward to step over the safepoint poll instruction
2691 __ add(r20, r20, NativeInstruction::instruction_size);
2692 __ str(r20, Address(rfp, wordSize));
2693 }
2694
2695 __ bind(no_adjust);
2696 // Normal exit, restore registers and exit.
2697 reg_save.restore_live_registers(masm);
2698
2699 __ ret(lr);
2700
2701 #ifdef ASSERT
2702 __ bind(bail);
2703 __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
2704 #endif
2705
2706 // Make sure all code is generated
2707 masm->flush();
2708
2709 // Fill-out other meta info
2710 return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
2711 }
2712
2713 //
2714 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
2715 //
2716 // Generate a stub that calls into vm to find out the proper destination
2717 // of a java call. All the argument registers are live at this point
2718 // but since this is generic code we don't know what they are and the caller
2719 // must do any gc of the args.
2720 //
2721 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
2722 assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
2723
2724 // allocate space for the code
2725 ResourceMark rm;
2726
2727 CodeBuffer buffer(name, 1000, 512);
2728 MacroAssembler* masm = new MacroAssembler(&buffer);
2729
2730 int frame_size_in_words;
2731 RegisterSaver reg_save(false /* save_vectors */);
2732
2733 OopMapSet *oop_maps = new OopMapSet();
2734 OopMap* map = NULL;
2735
2736 int start = __ offset();
2737
2738 map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2739
2740 int frame_complete = __ offset();
2741
2742 {
2743 Label retaddr;
2744 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2745
2746 __ mov(c_rarg0, rthread);
2747 __ lea(rscratch1, RuntimeAddress(destination));
2748
2749 __ blr(rscratch1);
2750 __ bind(retaddr);
2751 }
2752
2753 // Set an oopmap for the call site.
2754 // We need this not only for callee-saved registers, but also for volatile
2755 // registers that the compiler might be keeping live across a safepoint.
2756
2757 oop_maps->add_gc_map( __ offset() - start, map);
2758
2759 // r0 contains the address we are going to jump to assuming no exception got installed
2760
2761 // clear last_Java_sp
2762 __ reset_last_Java_frame(false);
2763 // check for pending exceptions
2764 Label pending;
2765 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2766 __ cbnz(rscratch1, pending);
2767
2768 // get the returned Method*
2769 __ get_vm_result_2(rmethod, rthread);
2770 __ str(rmethod, Address(sp, reg_save.reg_offset_in_bytes(rmethod)));
2771
2772 // r0 is where we want to jump, overwrite rscratch1 which is saved and scratch
2773 __ str(r0, Address(sp, reg_save.rscratch1_offset_in_bytes()));
2774 reg_save.restore_live_registers(masm);
2775
2776 // We are back the the original state on entry and ready to go.
2777
2778 __ br(rscratch1);
2779
2780 // Pending exception after the safepoint
2781
2782 __ bind(pending);
2783
2784 reg_save.restore_live_registers(masm);
2785
2786 // exception pending => remove activation and forward to exception handler
2787
2788 __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
2789
2790 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
2791 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2792
2793 // -------------
2794 // make sure all code is generated
2795 masm->flush();
2796
2797 // return the blob
2798 // frame_size_words or bytes??
2799 return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
2800 }
2801
2802 #ifdef COMPILER2
2803 // This is here instead of runtime_x86_64.cpp because it uses SimpleRuntimeFrame
2804 //
2805 //------------------------------generate_exception_blob---------------------------
2806 // creates exception blob at the end
2807 // Using exception blob, this code is jumped from a compiled method.
2808 // (see emit_exception_handler in x86_64.ad file)
2809 //
2810 // Given an exception pc at a call we call into the runtime for the
2811 // handler in this method. This handler might merely restore state
2812 // (i.e. callee save registers) unwind the frame and jump to the
2813 // exception handler for the nmethod if there is no Java level handler
2814 // for the nmethod.
2815 //
2816 // This code is entered with a jmp.
2817 //
2818 // Arguments:
2819 // r0: exception oop
2820 // r3: exception pc
2821 //
2822 // Results:
2823 // r0: exception oop
2824 // r3: exception pc in caller or ???
2825 // destination: exception handler of caller
2826 //
2827 // Note: the exception pc MUST be at a call (precise debug information)
2828 // Registers r0, r3, r2, r4, r5, r8-r11 are not callee saved.
2829 //
2830
2831 void OptoRuntime::generate_exception_blob() {
2832 assert(!OptoRuntime::is_callee_saved_register(R3_num), "");
2833 assert(!OptoRuntime::is_callee_saved_register(R0_num), "");
2834 assert(!OptoRuntime::is_callee_saved_register(R2_num), "");
2835
2836 assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2837
2838 // Allocate space for the code
2839 ResourceMark rm;
2840 // Setup code generation tools
2841 CodeBuffer buffer("exception_blob", 2048, 1024);
2842 MacroAssembler* masm = new MacroAssembler(&buffer);
2843
2844 // TODO check various assumptions made here
2845 //
2846 // make sure we do so before running this
2847
2848 address start = __ pc();
2849
2850 // push rfp and retaddr by hand
2851 // Exception pc is 'return address' for stack walker
2852 __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2853 // there are no callee save registers and we don't expect an
2854 // arg reg save area
2855 #ifndef PRODUCT
2856 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2857 #endif
2858 // Store exception in Thread object. We cannot pass any arguments to the
2859 // handle_exception call, since we do not want to make any assumption
2860 // about the size of the frame where the exception happened in.
2861 __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2862 __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2863
2864 // This call does all the hard work. It checks if an exception handler
2865 // exists in the method.
2866 // If so, it returns the handler address.
2867 // If not, it prepares for stack-unwinding, restoring the callee-save
2868 // registers of the frame being removed.
2869 //
2870 // address OptoRuntime::handle_exception_C(JavaThread* thread)
2871 //
2872 // n.b. 1 gp arg, 0 fp args, integral return type
2873
2874 // the stack should always be aligned
2875 address the_pc = __ pc();
2876 __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
2877 __ mov(c_rarg0, rthread);
2878 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
2879 __ blr(rscratch1);
2880 // handle_exception_C is a special VM call which does not require an explicit
2881 // instruction sync afterwards.
2882
2883 // May jump to SVE compiled code
2884 __ reinitialize_ptrue();
2885
2886 // Set an oopmap for the call site. This oopmap will only be used if we
2887 // are unwinding the stack. Hence, all locations will be dead.
2888 // Callee-saved registers will be the same as the frame above (i.e.,
2889 // handle_exception_stub), since they were restored when we got the
2890 // exception.
2891
2892 OopMapSet* oop_maps = new OopMapSet();
2893
2894 oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
2895
2896 __ reset_last_Java_frame(false);
2897
2898 // Restore callee-saved registers
2899
2900 // rfp is an implicitly saved callee saved register (i.e. the calling
2901 // convention will save restore it in prolog/epilog) Other than that
2902 // there are no callee save registers now that adapter frames are gone.
2903 // and we dont' expect an arg reg save area
2904 __ ldp(rfp, r3, Address(__ post(sp, 2 * wordSize)));
2905
2906 // r0: exception handler
2907
2908 // We have a handler in r0 (could be deopt blob).
2909 __ mov(r8, r0);
2910
2911 // Get the exception oop
2912 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2913 // Get the exception pc in case we are deoptimized
2914 __ ldr(r4, Address(rthread, JavaThread::exception_pc_offset()));
2915 #ifdef ASSERT
2916 __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
2917 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2918 #endif
2919 // Clear the exception oop so GC no longer processes it as a root.
2920 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
2921
2922 // r0: exception oop
2923 // r8: exception handler
2924 // r4: exception pc
2925 // Jump to handler
2926
2927 __ br(r8);
2928
2929 // Make sure all code is generated
2930 masm->flush();
2931
2932 // Set exception blob
2933 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
2934 }
2935
2936 // ---------------------------------------------------------------
2937
2938 class NativeInvokerGenerator : public StubCodeGenerator {
2939 address _call_target;
2940 int _shadow_space_bytes;
2941
2942 const GrowableArray<VMReg>& _input_registers;
2943 const GrowableArray<VMReg>& _output_registers;
2944
2945 int _frame_complete;
2946 int _framesize;
2947 OopMapSet* _oop_maps;
2948 public:
2949 NativeInvokerGenerator(CodeBuffer* buffer,
2950 address call_target,
2951 int shadow_space_bytes,
2952 const GrowableArray<VMReg>& input_registers,
2953 const GrowableArray<VMReg>& output_registers)
2954 : StubCodeGenerator(buffer, PrintMethodHandleStubs),
2955 _call_target(call_target),
2956 _shadow_space_bytes(shadow_space_bytes),
2957 _input_registers(input_registers),
2958 _output_registers(output_registers),
2959 _frame_complete(0),
2960 _framesize(0),
2961 _oop_maps(NULL) {
2962 assert(_output_registers.length() <= 1
2963 || (_output_registers.length() == 2 && !_output_registers.at(1)->is_valid()), "no multi-reg returns");
2964 }
2965
2966 void generate();
2967
2968 int spill_size_in_bytes() const {
2969 if (_output_registers.length() == 0) {
2970 return 0;
2971 }
2972 VMReg reg = _output_registers.at(0);
2973 assert(reg->is_reg(), "must be a register");
2974 if (reg->is_Register()) {
2975 return 8;
2976 } else if (reg->is_FloatRegister()) {
2977 bool use_sve = Matcher::supports_scalable_vector();
2978 if (use_sve) {
2979 return Matcher::scalable_vector_reg_size(T_BYTE);
2980 }
2981 return 16;
2982 } else {
2983 ShouldNotReachHere();
2984 }
2985 return 0;
2986 }
2987
2988 void spill_output_registers() {
2989 if (_output_registers.length() == 0) {
2990 return;
2991 }
2992 VMReg reg = _output_registers.at(0);
2993 assert(reg->is_reg(), "must be a register");
2994 MacroAssembler* masm = _masm;
2995 if (reg->is_Register()) {
2996 __ spill(reg->as_Register(), true, 0);
2997 } else if (reg->is_FloatRegister()) {
2998 bool use_sve = Matcher::supports_scalable_vector();
2999 if (use_sve) {
3000 __ spill_sve_vector(reg->as_FloatRegister(), 0, Matcher::scalable_vector_reg_size(T_BYTE));
3001 } else {
3002 __ spill(reg->as_FloatRegister(), __ Q, 0);
3003 }
3004 } else {
3005 ShouldNotReachHere();
3006 }
3007 }
3008
3009 void fill_output_registers() {
3010 if (_output_registers.length() == 0) {
3011 return;
3012 }
3013 VMReg reg = _output_registers.at(0);
3014 assert(reg->is_reg(), "must be a register");
3015 MacroAssembler* masm = _masm;
3016 if (reg->is_Register()) {
3017 __ unspill(reg->as_Register(), true, 0);
3018 } else if (reg->is_FloatRegister()) {
3019 bool use_sve = Matcher::supports_scalable_vector();
3020 if (use_sve) {
3021 __ unspill_sve_vector(reg->as_FloatRegister(), 0, Matcher::scalable_vector_reg_size(T_BYTE));
3022 } else {
3023 __ unspill(reg->as_FloatRegister(), __ Q, 0);
3024 }
3025 } else {
3026 ShouldNotReachHere();
3027 }
3028 }
3029
3030 int frame_complete() const {
3031 return _frame_complete;
3032 }
3033
3034 int framesize() const {
3035 return (_framesize >> (LogBytesPerWord - LogBytesPerInt));
3036 }
3037
3038 OopMapSet* oop_maps() const {
3039 return _oop_maps;
3040 }
3041
3042 private:
3043 #ifdef ASSERT
3044 bool target_uses_register(VMReg reg) {
3045 return _input_registers.contains(reg) || _output_registers.contains(reg);
3046 }
3047 #endif
3048 };
3049
3050 static const int native_invoker_code_size = 1024;
3051
3052 RuntimeStub* SharedRuntime::make_native_invoker(address call_target,
3053 int shadow_space_bytes,
3054 const GrowableArray<VMReg>& input_registers,
3055 const GrowableArray<VMReg>& output_registers) {
3056 int locs_size = 64;
3057 CodeBuffer code("nep_invoker_blob", native_invoker_code_size, locs_size);
3058 NativeInvokerGenerator g(&code, call_target, shadow_space_bytes, input_registers, output_registers);
3059 g.generate();
3060 code.log_section_sizes("nep_invoker_blob");
3061
3062 RuntimeStub* stub =
3063 RuntimeStub::new_runtime_stub("nep_invoker_blob",
3064 &code,
3065 g.frame_complete(),
3066 g.framesize(),
3067 g.oop_maps(), false);
3068 return stub;
3069 }
3070
3071 void NativeInvokerGenerator::generate() {
3072 assert(!(target_uses_register(rscratch1->as_VMReg())
3073 || target_uses_register(rscratch2->as_VMReg())
3074 || target_uses_register(rthread->as_VMReg())),
3075 "Register conflict");
3076
3077 enum layout {
3078 rbp_off,
3079 rbp_off2,
3080 return_off,
3081 return_off2,
3082 framesize // inclusive of return address
3083 };
3084
3085 assert(_shadow_space_bytes == 0, "not expecting shadow space on AArch64");
3086 _framesize = align_up(framesize + (spill_size_in_bytes() >> LogBytesPerInt), 4);
3087 assert(is_even(_framesize/2), "sp not 16-byte aligned");
3088
3089 _oop_maps = new OopMapSet();
3090 MacroAssembler* masm = _masm;
3091
3092 address start = __ pc();
3093
3094 __ enter();
3095
3096 // lr and fp are already in place
3097 __ sub(sp, rfp, ((unsigned)_framesize-4) << LogBytesPerInt); // prolog
3098
3099 _frame_complete = __ pc() - start;
3100
3101 address the_pc = __ pc();
3102 __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
3103 OopMap* map = new OopMap(_framesize, 0);
3104 _oop_maps->add_gc_map(the_pc - start, map);
3105
3106 // State transition
3107 __ mov(rscratch1, _thread_in_native);
3108 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
3109 __ stlrw(rscratch1, rscratch2);
3110
3111 rt_call(masm, _call_target);
3112
3113 __ mov(rscratch1, _thread_in_native_trans);
3114 __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
3115
3116 // Force this write out before the read below
3117 __ membar(Assembler::LoadLoad | Assembler::LoadStore |
3118 Assembler::StoreLoad | Assembler::StoreStore);
3119
3120 __ verify_sve_vector_length();
3121
3122 Label L_after_safepoint_poll;
3123 Label L_safepoint_poll_slow_path;
3124
3125 __ safepoint_poll(L_safepoint_poll_slow_path, true /* at_return */, true /* acquire */, false /* in_nmethod */);
3126
3127 __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
3128 __ cbnzw(rscratch1, L_safepoint_poll_slow_path);
3129
3130 __ bind(L_after_safepoint_poll);
3131
3132 // change thread state
3133 __ mov(rscratch1, _thread_in_Java);
3134 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
3135 __ stlrw(rscratch1, rscratch2);
3136
3137 __ block_comment("reguard stack check");
3138 Label L_reguard;
3139 Label L_after_reguard;
3140 __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
3141 __ cmpw(rscratch1, StackOverflow::stack_guard_yellow_reserved_disabled);
3142 __ br(Assembler::EQ, L_reguard);
3143 __ bind(L_after_reguard);
3144
3145 __ reset_last_Java_frame(true);
3146
3147 __ leave(); // required for proper stackwalking of RuntimeStub frame
3148 __ ret(lr);
3149
3150 //////////////////////////////////////////////////////////////////////////////
3151
3152 __ block_comment("{ L_safepoint_poll_slow_path");
3153 __ bind(L_safepoint_poll_slow_path);
3154
3155 // Need to save the native result registers around any runtime calls.
3156 spill_output_registers();
3157
3158 __ mov(c_rarg0, rthread);
3159 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
3160 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
3161 __ blr(rscratch1);
3162
3163 fill_output_registers();
3164
3165 __ b(L_after_safepoint_poll);
3166 __ block_comment("} L_safepoint_poll_slow_path");
3167
3168 //////////////////////////////////////////////////////////////////////////////
3169
3170 __ block_comment("{ L_reguard");
3171 __ bind(L_reguard);
3172
3173 spill_output_registers();
3174
3175 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3176
3177 fill_output_registers();
3178
3179 __ b(L_after_reguard);
3180
3181 __ block_comment("} L_reguard");
3182
3183 //////////////////////////////////////////////////////////////////////////////
3184
3185 __ flush();
3186 }
3187 #endif // COMPILER2
--- EOF ---