1 /*
2 * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Defs.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "compiler/disassembler.hpp"
33 #include "compiler/oopMap.hpp"
34 #include "gc/shared/cardTable.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/tlab_globals.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "memory/universe.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "prims/jvmtiExport.hpp"
43 #include "register_aarch64.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/signature.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "runtime/vframe.hpp"
48 #include "runtime/vframeArray.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_aarch64.inline.hpp"
51
52
53 // Implementation of StubAssembler
54
55 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
56 // setup registers
57 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
58 assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
59 assert(args_size >= 0, "illegal args_size");
60 bool align_stack = false;
61
62 mov(c_rarg0, rthread);
63 set_num_rt_args(0); // Nothing on stack
64
65 Label retaddr;
66 set_last_Java_frame(sp, rfp, retaddr, rscratch1);
67
68 // do the call
69 lea(rscratch1, RuntimeAddress(entry));
70 blr(rscratch1);
71 bind(retaddr);
72 int call_offset = offset();
73 // verify callee-saved register
74 #ifdef ASSERT
75 push(r0, sp);
76 { Label L;
77 get_thread(r0);
78 cmp(rthread, r0);
79 br(Assembler::EQ, L);
80 stop("StubAssembler::call_RT: rthread not callee saved?");
81 bind(L);
82 }
83 pop(r0, sp);
84 #endif
85 reset_last_Java_frame(true);
86
87 // check for pending exceptions
88 { Label L;
89 // check for pending exceptions (java_thread is set upon return)
90 ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
91 cbz(rscratch1, L);
92 // exception pending => remove activation and forward to exception handler
93 // make sure that the vm_results are cleared
94 if (oop_result1->is_valid()) {
95 str(zr, Address(rthread, JavaThread::vm_result_offset()));
96 }
97 if (metadata_result->is_valid()) {
98 str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
99 }
100 if (frame_size() == no_frame_size) {
101 leave();
102 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
103 } else if (_stub_id == Runtime1::forward_exception_id) {
104 should_not_reach_here();
105 } else {
106 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
107 }
108 bind(L);
109 }
110 // get oop results if there are any and reset the values in the thread
111 if (oop_result1->is_valid()) {
112 get_vm_result(oop_result1, rthread);
113 }
114 if (metadata_result->is_valid()) {
115 get_vm_result_2(metadata_result, rthread);
116 }
117 return call_offset;
118 }
119
120
121 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
122 mov(c_rarg1, arg1);
123 return call_RT(oop_result1, metadata_result, entry, 1);
124 }
125
126
127 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
128 if (c_rarg1 == arg2) {
129 if (c_rarg2 == arg1) {
130 mov(rscratch1, arg1);
131 mov(arg1, arg2);
132 mov(arg2, rscratch1);
133 } else {
134 mov(c_rarg2, arg2);
135 mov(c_rarg1, arg1);
136 }
137 } else {
138 mov(c_rarg1, arg1);
139 mov(c_rarg2, arg2);
140 }
141 return call_RT(oop_result1, metadata_result, entry, 2);
142 }
143
144
145 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
146 // if there is any conflict use the stack
147 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
148 arg2 == c_rarg1 || arg2 == c_rarg3 ||
149 arg3 == c_rarg1 || arg3 == c_rarg2) {
150 stp(arg3, arg2, Address(pre(sp, -2 * wordSize)));
151 stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
152 ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
153 ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
154 } else {
155 mov(c_rarg1, arg1);
156 mov(c_rarg2, arg2);
157 mov(c_rarg3, arg3);
158 }
159 return call_RT(oop_result1, metadata_result, entry, 3);
160 }
161
162 enum return_state_t {
163 does_not_return, requires_return
164 };
165
166
167 // Implementation of StubFrame
168
169 class StubFrame: public StackObj {
170 private:
171 StubAssembler* _sasm;
172 bool _return_state;
173 bool _use_pop_on_epilogue;
174
175 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments,
176 return_state_t return_state, bool use_pop_on_epilogue);
177
178 public:
179 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilogue);
180 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state);
181 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
182 ~StubFrame();
183
184 void load_argument(int offset_in_words, Register reg);
185 };;
186
187 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
188 set_info(name, must_gc_arguments);
189 enter();
190 }
191
192 void StubAssembler::epilogue(bool use_pop) {
193 // use_pop when this frame may have been frozen on one carrier
194 // thread then thawed on another carrier thread, rendering the fp
195 // register invalid. We must restore the previous FP because it is
196 // used as a call-saved scratch register by compiled code.
197 if (use_pop) {
198 ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
199 } else {
200 leave();
201 }
202 ret(lr);
203 }
204
205 #define __ _sasm->
206
207 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments,
208 return_state_t return_state, bool use_pop_on_epilogue)
209 : _sasm(sasm), _return_state(return_state), _use_pop_on_epilogue(use_pop_on_epilogue) {
210 __ prologue(name, must_gc_arguments);
211 }
212
213 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments,
214 bool use_pop_on_epilogue) :
215 StubFrame(sasm, name, must_gc_arguments, requires_return, use_pop_on_epilogue) {}
216
217 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments,
218 return_state_t return_state) :
219 StubFrame(sasm, name, must_gc_arguments, return_state, /*use_pop_on_epilogue*/false) {}
220
221 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) :
222 StubFrame(sasm, name, must_gc_arguments, requires_return, /*use_pop_on_epilogue*/false) {}
223
224 // load parameters that were stored with LIR_Assembler::store_parameter
225 // Note: offsets for store_parameter and load_argument must match
226 void StubFrame::load_argument(int offset_in_words, Register reg) {
227 __ load_parameter(offset_in_words, reg);
228 }
229
230 StubFrame::~StubFrame() {
231 __ epilogue(_use_pop_on_epilogue);
232 }
233
234 #undef __
235
236
237 // Implementation of Runtime1
238
239 #define __ sasm->
240
241 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
242
243 // Stack layout for saving/restoring all the registers needed during a runtime
244 // call (this includes deoptimization)
245 // Note: note that users of this frame may well have arguments to some runtime
246 // while these values are on the stack. These positions neglect those arguments
247 // but the code in save_live_registers will take the argument count into
248 // account.
249 //
250
251 enum reg_save_layout {
252 reg_save_frame_size = 32 /* float */ + 32 /* integer */
253 };
254
255 // Save off registers which might be killed by calls into the runtime.
256 // Tries to smart of about FP registers. In particular we separate
257 // saving and describing the FPU registers for deoptimization since we
258 // have to save the FPU registers twice if we describe them. The
259 // deopt blob is the only thing which needs to describe FPU registers.
260 // In all other cases it should be sufficient to simply save their
261 // current value.
262
263 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
264 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
265 static int reg_save_size_in_words;
266 static int frame_size_in_bytes = -1;
267
268 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
269 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
270 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
271 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
272 OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
273
274 for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
275 Register r = as_Register(i);
276 if (r == rthread || (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding())) {
277 int sp_offset = cpu_reg_save_offsets[i];
278 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
279 r->as_VMReg());
280 }
281 }
282
283 if (save_fpu_registers) {
284 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
285 FloatRegister r = as_FloatRegister(i);
286 {
287 int sp_offset = fpu_reg_save_offsets[i];
288 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
289 r->as_VMReg());
290 }
291 }
292 }
293 return oop_map;
294 }
295
296 static OopMap* save_live_registers(StubAssembler* sasm,
297 bool save_fpu_registers = true) {
298 __ block_comment("save_live_registers");
299
300 __ push(RegSet::range(r0, r29), sp); // integer registers except lr & sp
301
302 if (save_fpu_registers) {
303 for (int i = 31; i>= 0; i -= 4) {
304 __ sub(sp, sp, 4 * wordSize); // no pre-increment for st1. Emulate it without modifying other registers
305 __ st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
306 as_FloatRegister(i), __ T1D, Address(sp));
307 }
308 } else {
309 __ add(sp, sp, -32 * wordSize);
310 }
311
312 return generate_oop_map(sasm, save_fpu_registers);
313 }
314
315 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
316 if (restore_fpu_registers) {
317 for (int i = 0; i < 32; i += 4)
318 __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
319 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
320 } else {
321 __ add(sp, sp, 32 * wordSize);
322 }
323
324 __ pop(RegSet::range(r0, r29), sp);
325 }
326
327 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true) {
328
329 if (restore_fpu_registers) {
330 for (int i = 0; i < 32; i += 4)
331 __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
332 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
333 } else {
334 __ add(sp, sp, 32 * wordSize);
335 }
336
337 __ ldp(zr, r1, Address(__ post(sp, 16)));
338 __ pop(RegSet::range(r2, r29), sp);
339 }
340
341
342
343 void Runtime1::initialize_pd() {
344 int i;
345 int sp_offset = 0;
346
347 // all float registers are saved explicitly
348 assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
349 for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
350 fpu_reg_save_offsets[i] = sp_offset;
351 sp_offset += 2; // SP offsets are in halfwords
352 }
353
354 for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
355 Register r = as_Register(i);
356 cpu_reg_save_offsets[i] = sp_offset;
357 sp_offset += 2; // SP offsets are in halfwords
358 }
359 }
360
361 // return: offset in 64-bit words.
362 uint Runtime1::runtime_blob_current_thread_offset(frame f) {
363 CodeBlob* cb = f.cb();
364 assert(cb == Runtime1::blob_for(Runtime1::monitorenter_id) ||
365 cb == Runtime1::blob_for(Runtime1::monitorenter_nofpu_id), "must be");
366 assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame");
367 int offset = cpu_reg_save_offsets[rthread->encoding()];
368 return offset / 2; // SP offsets are in halfwords
369 }
370
371 // target: the entry point of the method that creates and posts the exception oop
372 // has_argument: true if the exception needs arguments (passed in rscratch1 and rscratch2)
373
374 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
375 // make a frame and preserve the caller's caller-save registers
376 OopMap* oop_map = save_live_registers(sasm);
377 int call_offset;
378 if (!has_argument) {
379 call_offset = __ call_RT(noreg, noreg, target);
380 } else {
381 __ mov(c_rarg1, rscratch1);
382 __ mov(c_rarg2, rscratch2);
383 call_offset = __ call_RT(noreg, noreg, target);
384 }
385 OopMapSet* oop_maps = new OopMapSet();
386 oop_maps->add_gc_map(call_offset, oop_map);
387 return oop_maps;
388 }
389
390
391 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
392 __ block_comment("generate_handle_exception");
393
394 // incoming parameters
395 const Register exception_oop = r0;
396 const Register exception_pc = r3;
397 // other registers used in this stub
398
399 // Save registers, if required.
400 OopMapSet* oop_maps = new OopMapSet();
401 OopMap* oop_map = nullptr;
402 switch (id) {
403 case forward_exception_id:
404 // We're handling an exception in the context of a compiled frame.
405 // The registers have been saved in the standard places. Perform
406 // an exception lookup in the caller and dispatch to the handler
407 // if found. Otherwise unwind and dispatch to the callers
408 // exception handler.
409 oop_map = generate_oop_map(sasm, 1 /*thread*/);
410
411 // load and clear pending exception oop into r0
412 __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
413 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
414
415 // load issuing PC (the return address for this stub) into r3
416 __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
417 __ authenticate_return_address(exception_pc);
418
419 // make sure that the vm_results are cleared (may be unnecessary)
420 __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
421 __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
422 break;
423 case handle_exception_nofpu_id:
424 case handle_exception_id:
425 // At this point all registers MAY be live.
426 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
427 break;
428 case handle_exception_from_callee_id: {
429 // At this point all registers except exception oop (r0) and
430 // exception pc (lr) are dead.
431 const int frame_size = 2 /*fp, return address*/;
432 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
433 sasm->set_frame_size(frame_size);
434 break;
435 }
436 default: ShouldNotReachHere();
437 }
438
439 // verify that only r0 and r3 are valid at this time
440 __ invalidate_registers(false, true, true, false, true, true);
441 // verify that r0 contains a valid exception
442 __ verify_not_null_oop(exception_oop);
443
444 #ifdef ASSERT
445 // check that fields in JavaThread for exception oop and issuing pc are
446 // empty before writing to them
447 Label oop_empty;
448 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
449 __ cbz(rscratch1, oop_empty);
450 __ stop("exception oop already set");
451 __ bind(oop_empty);
452
453 Label pc_empty;
454 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
455 __ cbz(rscratch1, pc_empty);
456 __ stop("exception pc already set");
457 __ bind(pc_empty);
458 #endif
459
460 // save exception oop and issuing pc into JavaThread
461 // (exception handler will load it from here)
462 __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
463 __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
464
465 // patch throwing pc into return address (has bci & oop map)
466 __ protect_return_address(exception_pc);
467 __ str(exception_pc, Address(rfp, 1*BytesPerWord));
468
469 // compute the exception handler.
470 // the exception oop and the throwing pc are read from the fields in JavaThread
471 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
472 oop_maps->add_gc_map(call_offset, oop_map);
473
474 // r0: handler address
475 // will be the deopt blob if nmethod was deoptimized while we looked up
476 // handler regardless of whether handler existed in the nmethod.
477
478 // only r0 is valid at this time, all other registers have been destroyed by the runtime call
479 __ invalidate_registers(false, true, true, true, true, true);
480
481 // patch the return address, this stub will directly return to the exception handler
482 __ protect_return_address(r0);
483 __ str(r0, Address(rfp, 1*BytesPerWord));
484
485 switch (id) {
486 case forward_exception_id:
487 case handle_exception_nofpu_id:
488 case handle_exception_id:
489 // Restore the registers that were saved at the beginning.
490 restore_live_registers(sasm, id != handle_exception_nofpu_id);
491 break;
492 case handle_exception_from_callee_id:
493 break;
494 default: ShouldNotReachHere();
495 }
496
497 return oop_maps;
498 }
499
500
501 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
502 // incoming parameters
503 const Register exception_oop = r0;
504 // callee-saved copy of exception_oop during runtime call
505 const Register exception_oop_callee_saved = r19;
506 // other registers used in this stub
507 const Register exception_pc = r3;
508 const Register handler_addr = r1;
509
510 if (AbortVMOnException) {
511 __ mov(rscratch1, exception_oop);
512 __ enter();
513 save_live_registers(sasm);
514 __ call_VM_leaf(CAST_FROM_FN_PTR(address, check_abort_on_vm_exception), rscratch1);
515 restore_live_registers(sasm);
516 __ leave();
517 }
518
519 // verify that only r0, is valid at this time
520 __ invalidate_registers(false, true, true, true, true, true);
521
522 #ifdef ASSERT
523 // check that fields in JavaThread for exception oop and issuing pc are empty
524 Label oop_empty;
525 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
526 __ cbz(rscratch1, oop_empty);
527 __ stop("exception oop must be empty");
528 __ bind(oop_empty);
529
530 Label pc_empty;
531 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
532 __ cbz(rscratch1, pc_empty);
533 __ stop("exception pc must be empty");
534 __ bind(pc_empty);
535 #endif
536
537 // Save our return address because
538 // exception_handler_for_return_address will destroy it. We also
539 // save exception_oop
540 __ mov(r3, lr);
541 __ protect_return_address();
542 __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
543
544 // search the exception handler address of the caller (using the return address)
545 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, r3);
546 // r0: exception handler address of the caller
547
548 // Only R0 is valid at this time; all other registers have been
549 // destroyed by the call.
550 __ invalidate_registers(false, true, true, true, false, true);
551
552 // move result of call into correct register
553 __ mov(handler_addr, r0);
554
555 // get throwing pc (= return address).
556 // lr has been destroyed by the call
557 __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
558 __ authenticate_return_address();
559 __ mov(r3, lr);
560
561 __ verify_not_null_oop(exception_oop);
562
563 // continue at exception handler (return address removed)
564 // note: do *not* remove arguments when unwinding the
565 // activation since the caller assumes having
566 // all arguments on the stack when entering the
567 // runtime to determine the exception handler
568 // (GC happens at call site with arguments!)
569 // r0: exception oop
570 // r3: throwing pc
571 // r1: exception handler
572 __ br(handler_addr);
573 }
574
575
576
577 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
578 // use the maximum number of runtime-arguments here because it is difficult to
579 // distinguish each RT-Call.
580 // Note: This number affects also the RT-Call in generate_handle_exception because
581 // the oop-map is shared for all calls.
582 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
583 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
584
585 OopMap* oop_map = save_live_registers(sasm);
586
587 __ mov(c_rarg0, rthread);
588 Label retaddr;
589 __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
590 // do the call
591 __ lea(rscratch1, RuntimeAddress(target));
592 __ blr(rscratch1);
593 __ bind(retaddr);
594 OopMapSet* oop_maps = new OopMapSet();
595 oop_maps->add_gc_map(__ offset(), oop_map);
596 // verify callee-saved register
597 #ifdef ASSERT
598 { Label L;
599 __ get_thread(rscratch1);
600 __ cmp(rthread, rscratch1);
601 __ br(Assembler::EQ, L);
602 __ stop("StubAssembler::call_RT: rthread not callee saved?");
603 __ bind(L);
604 }
605 #endif
606
607 __ reset_last_Java_frame(true);
608
609 #ifdef ASSERT
610 // check that fields in JavaThread for exception oop and issuing pc are empty
611 Label oop_empty;
612 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
613 __ cbz(rscratch1, oop_empty);
614 __ stop("exception oop must be empty");
615 __ bind(oop_empty);
616
617 Label pc_empty;
618 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
619 __ cbz(rscratch1, pc_empty);
620 __ stop("exception pc must be empty");
621 __ bind(pc_empty);
622 #endif
623
624 // Runtime will return true if the nmethod has been deoptimized, this is the
625 // expected scenario and anything else is an error. Note that we maintain a
626 // check on the result purely as a defensive measure.
627 Label no_deopt;
628 __ cbz(r0, no_deopt); // Have we deoptimized?
629
630 // Perform a re-execute. The proper return address is already on the stack,
631 // we just need to restore registers, pop all of our frame but the return
632 // address and jump to the deopt blob.
633 restore_live_registers(sasm);
634 __ leave();
635 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
636
637 __ bind(no_deopt);
638 __ stop("deopt not performed");
639
640 return oop_maps;
641 }
642
643
644 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
645
646 const Register exception_oop = r0;
647 const Register exception_pc = r3;
648
649 // for better readability
650 const bool must_gc_arguments = true;
651 const bool dont_gc_arguments = false;
652
653 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
654 bool save_fpu_registers = true;
655
656 // stub code & info for the different stubs
657 OopMapSet* oop_maps = nullptr;
658 OopMap* oop_map = nullptr;
659 switch (id) {
660 {
661 case forward_exception_id:
662 {
663 oop_maps = generate_handle_exception(id, sasm);
664 __ leave();
665 __ ret(lr);
666 }
667 break;
668
669 case throw_div0_exception_id:
670 { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return);
671 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
672 }
673 break;
674
675 case throw_null_pointer_exception_id:
676 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return);
677 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
678 }
679 break;
680
681 case new_instance_id:
682 case fast_new_instance_id:
683 case fast_new_instance_init_check_id:
684 {
685 Register klass = r3; // Incoming
686 Register obj = r0; // Result
687
688 if (id == new_instance_id) {
689 __ set_info("new_instance", dont_gc_arguments);
690 } else if (id == fast_new_instance_id) {
691 __ set_info("fast new_instance", dont_gc_arguments);
692 } else {
693 assert(id == fast_new_instance_init_check_id, "bad StubID");
694 __ set_info("fast new_instance init check", dont_gc_arguments);
695 }
696
697 __ enter();
698 OopMap* map = save_live_registers(sasm);
699 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
700 oop_maps = new OopMapSet();
701 oop_maps->add_gc_map(call_offset, map);
702 restore_live_registers_except_r0(sasm);
703 __ verify_oop(obj);
704 __ leave();
705 __ ret(lr);
706
707 // r0,: new instance
708 }
709
710 break;
711
712 case counter_overflow_id:
713 {
714 Register bci = r0, method = r1;
715 __ enter();
716 OopMap* map = save_live_registers(sasm);
717 // Retrieve bci
718 __ ldrw(bci, Address(rfp, 2*BytesPerWord));
719 // And a pointer to the Method*
720 __ ldr(method, Address(rfp, 3*BytesPerWord));
721 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
722 oop_maps = new OopMapSet();
723 oop_maps->add_gc_map(call_offset, map);
724 restore_live_registers(sasm);
725 __ leave();
726 __ ret(lr);
727 }
728 break;
729
730 case new_type_array_id:
731 case new_object_array_id:
732 {
733 Register length = r19; // Incoming
734 Register klass = r3; // Incoming
735 Register obj = r0; // Result
736
737 if (id == new_type_array_id) {
738 __ set_info("new_type_array", dont_gc_arguments);
739 } else {
740 __ set_info("new_object_array", dont_gc_arguments);
741 }
742
743 #ifdef ASSERT
744 // assert object type is really an array of the proper kind
745 {
746 Label ok;
747 Register t0 = obj;
748 __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
749 __ asrw(t0, t0, Klass::_lh_array_tag_shift);
750 int tag = ((id == new_type_array_id)
751 ? Klass::_lh_array_tag_type_value
752 : Klass::_lh_array_tag_obj_value);
753 __ mov(rscratch1, tag);
754 __ cmpw(t0, rscratch1);
755 __ br(Assembler::EQ, ok);
756 __ stop("assert(is an array klass)");
757 __ should_not_reach_here();
758 __ bind(ok);
759 }
760 #endif // ASSERT
761
762 __ enter();
763 OopMap* map = save_live_registers(sasm);
764 int call_offset;
765 if (id == new_type_array_id) {
766 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
767 } else {
768 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
769 }
770
771 oop_maps = new OopMapSet();
772 oop_maps->add_gc_map(call_offset, map);
773 restore_live_registers_except_r0(sasm);
774
775 __ verify_oop(obj);
776 __ leave();
777 __ ret(lr);
778
779 // r0: new array
780 }
781 break;
782
783 case new_multi_array_id:
784 { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
785 // r0,: klass
786 // r19,: rank
787 // r2: address of 1st dimension
788 OopMap* map = save_live_registers(sasm);
789 __ mov(c_rarg1, r0);
790 __ mov(c_rarg3, r2);
791 __ mov(c_rarg2, r19);
792 int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
793
794 oop_maps = new OopMapSet();
795 oop_maps->add_gc_map(call_offset, map);
796 restore_live_registers_except_r0(sasm);
797
798 // r0,: new multi array
799 __ verify_oop(r0);
800 }
801 break;
802
803 case register_finalizer_id:
804 {
805 __ set_info("register_finalizer", dont_gc_arguments);
806
807 // This is called via call_runtime so the arguments
808 // will be place in C abi locations
809
810 __ verify_oop(c_rarg0);
811
812 // load the klass and check the has finalizer flag
813 Label register_finalizer;
814 Register t = r5;
815 __ load_klass(t, r0);
816 __ ldrw(t, Address(t, Klass::access_flags_offset()));
817 __ tbnz(t, exact_log2(JVM_ACC_HAS_FINALIZER), register_finalizer);
818 __ ret(lr);
819
820 __ bind(register_finalizer);
821 __ enter();
822 OopMap* oop_map = save_live_registers(sasm);
823 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
824 oop_maps = new OopMapSet();
825 oop_maps->add_gc_map(call_offset, oop_map);
826
827 // Now restore all the live registers
828 restore_live_registers(sasm);
829
830 __ leave();
831 __ ret(lr);
832 }
833 break;
834
835 case throw_class_cast_exception_id:
836 { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return);
837 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
838 }
839 break;
840
841 case throw_incompatible_class_change_error_id:
842 { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return);
843 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
844 }
845 break;
846
847 case slow_subtype_check_id:
848 {
849 // Typical calling sequence:
850 // __ push(klass_RInfo); // object klass or other subclass
851 // __ push(sup_k_RInfo); // array element klass or other superclass
852 // __ bl(slow_subtype_check);
853 // Note that the subclass is pushed first, and is therefore deepest.
854 enum layout {
855 r0_off, r0_off_hi,
856 r2_off, r2_off_hi,
857 r4_off, r4_off_hi,
858 r5_off, r5_off_hi,
859 sup_k_off, sup_k_off_hi,
860 klass_off, klass_off_hi,
861 framesize,
862 result_off = sup_k_off
863 };
864
865 __ set_info("slow_subtype_check", dont_gc_arguments);
866 __ push(RegSet::of(r0, r2, r4, r5), sp);
867
868 // This is called by pushing args and not with C abi
869 // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
870 // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
871
872 __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
873
874 Label miss;
875 __ check_klass_subtype_slow_path(r4, r0, r2, r5, nullptr, &miss);
876
877 // fallthrough on success:
878 __ mov(rscratch1, 1);
879 __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
880 __ pop(RegSet::of(r0, r2, r4, r5), sp);
881 __ ret(lr);
882
883 __ bind(miss);
884 __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
885 __ pop(RegSet::of(r0, r2, r4, r5), sp);
886 __ ret(lr);
887 }
888 break;
889
890 case monitorenter_nofpu_id:
891 save_fpu_registers = false;
892 // fall through
893 case monitorenter_id:
894 {
895 StubFrame f(sasm, "monitorenter", dont_gc_arguments, /*use_pop_on_epilogue*/true);
896 OopMap* map = save_live_registers(sasm, save_fpu_registers);
897
898 // Called with store_parameter and not C abi
899
900 f.load_argument(1, r0); // r0,: object
901 f.load_argument(0, r1); // r1,: lock address
902
903 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
904
905 oop_maps = new OopMapSet();
906 oop_maps->add_gc_map(call_offset, map);
907 restore_live_registers(sasm, save_fpu_registers);
908 }
909 break;
910
911 case monitorexit_nofpu_id:
912 save_fpu_registers = false;
913 // fall through
914 case monitorexit_id:
915 {
916 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
917 OopMap* map = save_live_registers(sasm, save_fpu_registers);
918
919 // Called with store_parameter and not C abi
920
921 f.load_argument(0, r0); // r0,: lock address
922
923 // note: really a leaf routine but must setup last java sp
924 // => use call_RT for now (speed can be improved by
925 // doing last java sp setup manually)
926 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
927
928 oop_maps = new OopMapSet();
929 oop_maps->add_gc_map(call_offset, map);
930 restore_live_registers(sasm, save_fpu_registers);
931 }
932 break;
933
934 case deoptimize_id:
935 {
936 StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return);
937 OopMap* oop_map = save_live_registers(sasm);
938 f.load_argument(0, c_rarg1);
939 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
940
941 oop_maps = new OopMapSet();
942 oop_maps->add_gc_map(call_offset, oop_map);
943 restore_live_registers(sasm);
944 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
945 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
946 __ leave();
947 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
948 }
949 break;
950
951 case throw_range_check_failed_id:
952 { StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return);
953 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
954 }
955 break;
956
957 case unwind_exception_id:
958 { __ set_info("unwind_exception", dont_gc_arguments);
959 // note: no stubframe since we are about to leave the current
960 // activation and we are calling a leaf VM function only.
961 generate_unwind_exception(sasm);
962 }
963 break;
964
965 case access_field_patching_id:
966 { StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return);
967 // we should set up register map
968 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
969 }
970 break;
971
972 case load_klass_patching_id:
973 { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return);
974 // we should set up register map
975 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
976 }
977 break;
978
979 case load_mirror_patching_id:
980 { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return);
981 // we should set up register map
982 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
983 }
984 break;
985
986 case load_appendix_patching_id:
987 { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return);
988 // we should set up register map
989 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
990 }
991 break;
992
993 case handle_exception_nofpu_id:
994 case handle_exception_id:
995 { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
996 oop_maps = generate_handle_exception(id, sasm);
997 }
998 break;
999
1000 case handle_exception_from_callee_id:
1001 { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1002 oop_maps = generate_handle_exception(id, sasm);
1003 }
1004 break;
1005
1006 case throw_index_exception_id:
1007 { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return);
1008 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1009 }
1010 break;
1011
1012 case throw_array_store_exception_id:
1013 { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return);
1014 // tos + 0: link
1015 // + 1: return address
1016 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1017 }
1018 break;
1019
1020 case predicate_failed_trap_id:
1021 {
1022 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return);
1023
1024 OopMap* map = save_live_registers(sasm);
1025
1026 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1027 oop_maps = new OopMapSet();
1028 oop_maps->add_gc_map(call_offset, map);
1029 restore_live_registers(sasm);
1030 __ leave();
1031 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1032 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1033
1034 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1035 }
1036 break;
1037
1038 case dtrace_object_alloc_id:
1039 { // c_rarg0: object
1040 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1041 save_live_registers(sasm);
1042
1043 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), c_rarg0);
1044
1045 restore_live_registers(sasm);
1046 }
1047 break;
1048
1049 default:
1050 { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return);
1051 __ mov(r0, (int)id);
1052 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1053 }
1054 break;
1055 }
1056 }
1057 return oop_maps;
1058 }
1059
1060 #undef __
1061
1062 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }