1 /*
2 * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Defs.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "compiler/disassembler.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/shared/cardTable.hpp"
36 #include "gc/shared/cardTableBarrierSet.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "memory/universe.hpp"
40 #include "nativeInst_riscv.hpp"
41 #include "oops/compiledICHolder.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "prims/jvmtiExport.hpp"
44 #include "register_riscv.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/signature.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/vframe.hpp"
49 #include "runtime/vframeArray.hpp"
50 #include "utilities/powerOfTwo.hpp"
51 #include "vmreg_riscv.inline.hpp"
52
53
54 // Implementation of StubAssembler
55
56 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, int args_size) {
57 // setup registers
58 assert(!(oop_result->is_valid() || metadata_result->is_valid()) || oop_result != metadata_result,
59 "registers must be different");
60 assert(oop_result != xthread && metadata_result != xthread, "registers must be different");
61 assert(args_size >= 0, "illegal args_size");
62 bool align_stack = false;
63
64 mv(c_rarg0, xthread);
65 set_num_rt_args(0); // Nothing on stack
66
67 Label retaddr;
68 set_last_Java_frame(sp, fp, retaddr, t0);
69
70 // do the call
71 int32_t off = 0;
72 la_patchable(t0, RuntimeAddress(entry), off);
73 jalr(x1, t0, off);
74 bind(retaddr);
75 int call_offset = offset();
76 // verify callee-saved register
77 #ifdef ASSERT
78 push_reg(x10, sp);
79 { Label L;
80 get_thread(x10);
81 beq(xthread, x10, L);
82 stop("StubAssembler::call_RT: xthread not callee saved?");
83 bind(L);
84 }
85 pop_reg(x10, sp);
86 #endif
87 reset_last_Java_frame(true);
88
89 // check for pending exceptions
90 { Label L;
91 // check for pending exceptions (java_thread is set upon return)
92 ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
93 beqz(t0, L);
94 // exception pending => remove activation and forward to exception handler
95 // make sure that the vm_results are cleared
96 if (oop_result->is_valid()) {
97 sd(zr, Address(xthread, JavaThread::vm_result_offset()));
98 }
99 if (metadata_result->is_valid()) {
100 sd(zr, Address(xthread, JavaThread::vm_result_2_offset()));
101 }
102 if (frame_size() == no_frame_size) {
103 leave();
104 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
105 } else if (_stub_id == Runtime1::forward_exception_id) {
106 should_not_reach_here();
107 } else {
108 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
109 }
110 bind(L);
111 }
112 // get oop results if there are any and reset the values in the thread
113 if (oop_result->is_valid()) {
114 get_vm_result(oop_result, xthread);
115 }
116 if (metadata_result->is_valid()) {
117 get_vm_result_2(metadata_result, xthread);
118 }
119 return call_offset;
120 }
121
122 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1) {
123 mv(c_rarg1, arg1);
124 return call_RT(oop_result, metadata_result, entry, 1);
125 }
126
127 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2) {
128 const int arg_num = 2;
129 if (c_rarg1 == arg2) {
130 if (c_rarg2 == arg1) {
131 xorr(arg1, arg1, arg2);
132 xorr(arg2, arg1, arg2);
133 xorr(arg1, arg1, arg2);
134 } else {
135 mv(c_rarg2, arg2);
136 mv(c_rarg1, arg1);
137 }
138 } else {
139 mv(c_rarg1, arg1);
140 mv(c_rarg2, arg2);
141 }
142 return call_RT(oop_result, metadata_result, entry, arg_num);
143 }
144
145 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
146 const int arg_num = 3;
147 // if there is any conflict use the stack
148 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
149 arg2 == c_rarg1 || arg2 == c_rarg3 ||
150 arg3 == c_rarg1 || arg3 == c_rarg2) {
151 const int arg1_sp_offset = 0;
152 const int arg2_sp_offset = 1;
153 const int arg3_sp_offset = 2;
154 addi(sp, sp, -(arg_num + 1) * wordSize);
155 sd(arg1, Address(sp, arg1_sp_offset * wordSize));
156 sd(arg2, Address(sp, arg2_sp_offset * wordSize));
157 sd(arg3, Address(sp, arg3_sp_offset * wordSize));
158
159 ld(c_rarg1, Address(sp, arg1_sp_offset * wordSize));
160 ld(c_rarg2, Address(sp, arg2_sp_offset * wordSize));
161 ld(c_rarg3, Address(sp, arg3_sp_offset * wordSize));
162 addi(sp, sp, (arg_num + 1) * wordSize);
163 } else {
164 mv(c_rarg1, arg1);
165 mv(c_rarg2, arg2);
166 mv(c_rarg3, arg3);
167 }
168 return call_RT(oop_result, metadata_result, entry, arg_num);
169 }
170
171 enum return_state_t {
172 does_not_return, requires_return
173 };
174
175 // Implementation of StubFrame
176
177 class StubFrame: public StackObj {
178 private:
179 StubAssembler* _sasm;
180 bool _return_state;
181
182 public:
183 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return);
184 void load_argument(int offset_in_words, Register reg);
185
186 ~StubFrame();
187 };;
188
189 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
190 set_info(name, must_gc_arguments);
191 enter();
192 }
193
194 void StubAssembler::epilogue() {
195 leave();
196 ret();
197 }
198
199 #define __ _sasm->
200
201 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) {
202 _sasm = sasm;
203 _return_state = return_state;
204 __ prologue(name, must_gc_arguments);
205 }
206
207 // load parameters that were stored with LIR_Assembler::store_parameter
208 // Note: offsets for store_parameter and load_argument must match
209 void StubFrame::load_argument(int offset_in_words, Register reg) {
210 __ load_parameter(offset_in_words, reg);
211 }
212
213
214 StubFrame::~StubFrame() {
215 if (_return_state == requires_return) {
216 __ epilogue();
217 } else {
218 __ should_not_reach_here();
219 }
220 _sasm = NULL;
221 }
222
223 #undef __
224
225
226 // Implementation of Runtime1
227
228 #define __ sasm->
229
230 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
231
232 // Stack layout for saving/restoring all the registers needed during a runtime
233 // call (this includes deoptimization)
234 // Note: note that users of this frame may well have arguments to some runtime
235 // while these values are on the stack. These positions neglect those arguments
236 // but the code in save_live_registers will take the argument count into
237 // account.
238 //
239
240 enum reg_save_layout {
241 reg_save_frame_size = 32 /* float */ + 30 /* integer excluding x3, x4 */
242 };
243
244 // Save off registers which might be killed by calls into the runtime.
245 // Tries to smart of about FPU registers. In particular we separate
246 // saving and describing the FPU registers for deoptimization since we
247 // have to save the FPU registers twice if we describe them. The
248 // deopt blob is the only thing which needs to describe FPU registers.
249 // In all other cases it should be sufficient to simply save their
250 // current value.
251
252 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
253 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
254
255 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
256 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
257 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
258 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
259 OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
260 assert_cond(oop_map != NULL);
261
262 // caller save registers only, see FrameMap::initialize
263 // in c1_FrameMap_riscv.cpp for detail.
264 const static Register caller_save_cpu_regs[FrameMap::max_nof_caller_save_cpu_regs] = {
265 x7, x10, x11, x12, x13, x14, x15, x16, x17, x28, x29, x30, x31
266 };
267
268 for (int i = 0; i < FrameMap::max_nof_caller_save_cpu_regs; i++) {
269 Register r = caller_save_cpu_regs[i];
270 int sp_offset = cpu_reg_save_offsets[r->encoding()];
271 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
272 r->as_VMReg());
273 }
274
275 // fpu_regs
276 if (save_fpu_registers) {
277 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
278 FloatRegister r = as_FloatRegister(i);
279 int sp_offset = fpu_reg_save_offsets[i];
280 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
281 r->as_VMReg());
282 }
283 }
284 return oop_map;
285 }
286
287 static OopMap* save_live_registers(StubAssembler* sasm,
288 bool save_fpu_registers = true) {
289 __ block_comment("save_live_registers");
290
291 // if the number of pushed regs is odd, one slot will be reserved for alignment
292 __ push_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4)
293
294 if (save_fpu_registers) {
295 // float registers
296 __ addi(sp, sp, -(FrameMap::nof_fpu_regs * wordSize));
297 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
298 __ fsd(as_FloatRegister(i), Address(sp, i * wordSize));
299 }
300 } else {
301 // we define reg_save_layout = 62 as the fixed frame size,
302 // we should also sub 32 * wordSize to sp when save_fpu_registers == false
303 __ addi(sp, sp, -32 * wordSize);
304 }
305
306 return generate_oop_map(sasm, save_fpu_registers);
307 }
308
309 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
310 if (restore_fpu_registers) {
311 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
312 __ fld(as_FloatRegister(i), Address(sp, i * wordSize));
313 }
314 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize);
315 } else {
316 // we define reg_save_layout = 64 as the fixed frame size,
317 // we should also add 32 * wordSize to sp when save_fpu_registers == false
318 __ addi(sp, sp, 32 * wordSize);
319 }
320
321 // if the number of popped regs is odd, the reserved slot for alignment will be removed
322 __ pop_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4)
323 }
324
325 static void restore_live_registers_except_r10(StubAssembler* sasm, bool restore_fpu_registers = true) {
326 if (restore_fpu_registers) {
327 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
328 __ fld(as_FloatRegister(i), Address(sp, i * wordSize));
329 }
330 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize);
331 } else {
332 // we define reg_save_layout = 64 as the fixed frame size,
333 // we should also add 32 * wordSize to sp when save_fpu_registers == false
334 __ addi(sp, sp, 32 * wordSize);
335 }
336
337 // pop integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) & x10
338 // there is one reserved slot for alignment on the stack in save_live_registers().
339 __ pop_reg(RegSet::range(x5, x9), sp); // pop x5 ~ x9 with the reserved slot for alignment
340 __ pop_reg(RegSet::range(x11, x31), sp); // pop x11 ~ x31; x10 will be automatically skipped here
341 }
342
343 void Runtime1::initialize_pd() {
344 int i = 0;
345 int sp_offset = 0;
346 const int step = 2; // SP offsets are in halfwords
347
348 // all float registers are saved explicitly
349 for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
350 fpu_reg_save_offsets[i] = sp_offset;
351 sp_offset += step;
352 }
353
354 // a slot reserved for stack 16-byte alignment, see MacroAssembler::push_reg
355 sp_offset += step;
356 // we save x5 ~ x31, except x0 ~ x4: loop starts from x5
357 for (i = 5; i < FrameMap::nof_cpu_regs; i++) {
358 cpu_reg_save_offsets[i] = sp_offset;
359 sp_offset += step;
360 }
361 }
362
363 // target: the entry point of the method that creates and posts the exception oop
364 // has_argument: true if the exception needs arguments (passed in t0 and t1)
365
366 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
367 // make a frame and preserve the caller's caller-save registers
368 OopMap* oop_map = save_live_registers(sasm);
369 assert_cond(oop_map != NULL);
370 int call_offset = 0;
371 if (!has_argument) {
372 call_offset = __ call_RT(noreg, noreg, target);
373 } else {
374 __ mv(c_rarg1, t0);
375 __ mv(c_rarg2, t1);
376 call_offset = __ call_RT(noreg, noreg, target);
377 }
378 OopMapSet* oop_maps = new OopMapSet();
379 assert_cond(oop_maps != NULL);
380 oop_maps->add_gc_map(call_offset, oop_map);
381
382 return oop_maps;
383 }
384
385 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
386 __ block_comment("generate_handle_exception");
387
388 // incoming parameters
389 const Register exception_oop = x10;
390 const Register exception_pc = x13;
391
392 OopMapSet* oop_maps = new OopMapSet();
393 assert_cond(oop_maps != NULL);
394 OopMap* oop_map = NULL;
395
396 switch (id) {
397 case forward_exception_id:
398 // We're handling an exception in the context of a compiled frame.
399 // The registers have been saved in the standard places. Perform
400 // an exception lookup in the caller and dispatch to the handler
401 // if found. Otherwise unwind and dispatch to the callers
402 // exception handler.
403 oop_map = generate_oop_map(sasm, 1 /* thread */);
404
405 // load and clear pending exception oop into x10
406 __ ld(exception_oop, Address(xthread, Thread::pending_exception_offset()));
407 __ sd(zr, Address(xthread, Thread::pending_exception_offset()));
408
409 // load issuing PC (the return address for this stub) into x13
410 __ ld(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord));
411
412 // make sure that the vm_results are cleared (may be unnecessary)
413 __ sd(zr, Address(xthread, JavaThread::vm_result_offset()));
414 __ sd(zr, Address(xthread, JavaThread::vm_result_2_offset()));
415 break;
416 case handle_exception_nofpu_id:
417 case handle_exception_id:
418 // At this point all registers MAY be live.
419 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
420 break;
421 case handle_exception_from_callee_id: {
422 // At this point all registers except exception oop (x10) and
423 // exception pc (ra) are dead.
424 const int frame_size = 2 /* fp, return address */;
425 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
426 sasm->set_frame_size(frame_size);
427 break;
428 }
429 default: ShouldNotReachHere();
430 }
431
432 // verify that only x10 and x13 are valid at this time
433 __ invalidate_registers(false, true, true, false, true, true);
434 // verify that x10 contains a valid exception
435 __ verify_not_null_oop(exception_oop);
436
437 #ifdef ASSERT
438 // check that fields in JavaThread for exception oop and issuing pc are
439 // empty before writing to them
440 Label oop_empty;
441 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset()));
442 __ beqz(t0, oop_empty);
443 __ stop("exception oop already set");
444 __ bind(oop_empty);
445
446 Label pc_empty;
447 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
448 __ beqz(t0, pc_empty);
449 __ stop("exception pc already set");
450 __ bind(pc_empty);
451 #endif
452
453 // save exception oop and issuing pc into JavaThread
454 // (exception handler will load it from here)
455 __ sd(exception_oop, Address(xthread, JavaThread::exception_oop_offset()));
456 __ sd(exception_pc, Address(xthread, JavaThread::exception_pc_offset()));
457
458 // patch throwing pc into return address (has bci & oop map)
459 __ sd(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord));
460
461 // compute the exception handler.
462 // the exception oop and the throwing pc are read from the fields in JavaThread
463 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
464 guarantee(oop_map != NULL, "NULL oop_map!");
465 oop_maps->add_gc_map(call_offset, oop_map);
466
467 // x10: handler address
468 // will be the deopt blob if nmethod was deoptimized while we looked up
469 // handler regardless of whether handler existed in the nmethod.
470
471 // only x10 is valid at this time, all other registers have been destroyed by the runtime call
472 __ invalidate_registers(false, true, true, true, true, true);
473
474 // patch the return address, this stub will directly return to the exception handler
475 __ sd(x10, Address(fp, frame::return_addr_offset * BytesPerWord));
476
477 switch (id) {
478 case forward_exception_id:
479 case handle_exception_nofpu_id:
480 case handle_exception_id:
481 // Restore the registers that were saved at the beginning.
482 restore_live_registers(sasm, id != handle_exception_nofpu_id);
483 break;
484 case handle_exception_from_callee_id:
485 break;
486 default: ShouldNotReachHere();
487 }
488
489 return oop_maps;
490 }
491
492
493 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
494 // incoming parameters
495 const Register exception_oop = x10;
496 // other registers used in this stub
497 const Register handler_addr = x11;
498
499 // verify that only x10, is valid at this time
500 __ invalidate_registers(false, true, true, true, true, true);
501
502 #ifdef ASSERT
503 // check that fields in JavaThread for exception oop and issuing pc are empty
504 Label oop_empty;
505 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset()));
506 __ beqz(t0, oop_empty);
507 __ stop("exception oop must be empty");
508 __ bind(oop_empty);
509
510 Label pc_empty;
511 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
512 __ beqz(t0, pc_empty);
513 __ stop("exception pc must be empty");
514 __ bind(pc_empty);
515 #endif
516
517 // Save our return address because
518 // exception_handler_for_return_address will destroy it. We also
519 // save exception_oop
520 __ addi(sp, sp, -2 * wordSize);
521 __ sd(exception_oop, Address(sp, wordSize));
522 __ sd(ra, Address(sp));
523
524 // search the exception handler address of the caller (using the return address)
525 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, ra);
526 // x10: exception handler address of the caller
527
528 // Only x10 is valid at this time; all other registers have been
529 // destroyed by the call.
530 __ invalidate_registers(false, true, true, true, false, true);
531
532 // move result of call into correct register
533 __ mv(handler_addr, x10);
534
535 // get throwing pc (= return address).
536 // ra has been destroyed by the call
537 __ ld(ra, Address(sp));
538 __ ld(exception_oop, Address(sp, wordSize));
539 __ addi(sp, sp, 2 * wordSize);
540 __ mv(x13, ra);
541
542 __ verify_not_null_oop(exception_oop);
543
544 // continue at exception handler (return address removed)
545 // note: do *not* remove arguments when unwinding the
546 // activation since the caller assumes having
547 // all arguments on the stack when entering the
548 // runtime to determine the exception handler
549 // (GC happens at call site with arguments!)
550 // x10: exception oop
551 // x13: throwing pc
552 // x11: exception handler
553 __ jr(handler_addr);
554 }
555
556 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
557 // use the maximum number of runtime-arguments here because it is difficult to
558 // distinguish each RT-Call.
559 // Note: This number affects also the RT-Call in generate_handle_exception because
560 // the oop-map is shared for all calls.
561 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
562 assert(deopt_blob != NULL, "deoptimization blob must have been created");
563
564 OopMap* oop_map = save_live_registers(sasm);
565 assert_cond(oop_map != NULL);
566
567 __ mv(c_rarg0, xthread);
568 Label retaddr;
569 __ set_last_Java_frame(sp, fp, retaddr, t0);
570 // do the call
571 int32_t off = 0;
572 __ la_patchable(t0, RuntimeAddress(target), off);
573 __ jalr(x1, t0, off);
574 __ bind(retaddr);
575 OopMapSet* oop_maps = new OopMapSet();
576 assert_cond(oop_maps != NULL);
577 oop_maps->add_gc_map(__ offset(), oop_map);
578 // verify callee-saved register
579 #ifdef ASSERT
580 { Label L;
581 __ get_thread(t0);
582 __ beq(xthread, t0, L);
583 __ stop("StubAssembler::call_RT: xthread not callee saved?");
584 __ bind(L);
585 }
586 #endif
587 __ reset_last_Java_frame(true);
588
589 #ifdef ASSERT
590 // Check that fields in JavaThread for exception oop and issuing pc are empty
591 Label oop_empty;
592 __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
593 __ beqz(t0, oop_empty);
594 __ stop("exception oop must be empty");
595 __ bind(oop_empty);
596
597 Label pc_empty;
598 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
599 __ beqz(t0, pc_empty);
600 __ stop("exception pc must be empty");
601 __ bind(pc_empty);
602 #endif
603
604 // Runtime will return true if the nmethod has been deoptimized, this is the
605 // expected scenario and anything else is an error. Note that we maintain a
606 // check on the result purely as a defensive measure.
607 Label no_deopt;
608 __ beqz(x10, no_deopt); // Have we deoptimized?
609
610 // Perform a re-execute. The proper return address is already on the stack,
611 // we just need to restore registers, pop all of our frames but the return
612 // address and jump to the deopt blob.
613
614 restore_live_registers(sasm);
615 __ leave();
616 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
617
618 __ bind(no_deopt);
619 __ stop("deopt not performed");
620
621 return oop_maps;
622 }
623
624 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
625 // for better readability
626 const bool dont_gc_arguments = false;
627
628 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
629 bool save_fpu_registers = true;
630
631 // stub code & info for the different stubs
632 OopMapSet* oop_maps = NULL;
633 switch (id) {
634 {
635 case forward_exception_id:
636 {
637 oop_maps = generate_handle_exception(id, sasm);
638 __ leave();
639 __ ret();
640 }
641 break;
642
643 case throw_div0_exception_id:
644 {
645 StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return);
646 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
647 }
648 break;
649
650 case throw_null_pointer_exception_id:
651 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return);
652 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
653 }
654 break;
655
656 case new_instance_id:
657 case fast_new_instance_id:
658 case fast_new_instance_init_check_id:
659 {
660 Register klass = x13; // Incoming
661 Register obj = x10; // Result
662
663 if (id == new_instance_id) {
664 __ set_info("new_instance", dont_gc_arguments);
665 } else if (id == fast_new_instance_id) {
666 __ set_info("fast new_instance", dont_gc_arguments);
667 } else {
668 assert(id == fast_new_instance_init_check_id, "bad StubID");
669 __ set_info("fast new_instance init check", dont_gc_arguments);
670 }
671
672 // If TLAB is disabled, see if there is support for inlining contiguous
673 // allocations.
674 // Otherwise, just go to the slow path.
675 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
676 !UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
677 Label slow_path;
678 Register obj_size = x12;
679 Register tmp1 = x9;
680 Register tmp2 = x14;
681 assert_different_registers(klass, obj, obj_size, tmp1, tmp2);
682
683 const int sp_offset = 2;
684 const int x9_offset = 1;
685 const int zr_offset = 0;
686 __ addi(sp, sp, -(sp_offset * wordSize));
687 __ sd(x9, Address(sp, x9_offset * wordSize));
688 __ sd(zr, Address(sp, zr_offset * wordSize));
689
690 if (id == fast_new_instance_init_check_id) {
691 // make sure the klass is initialized
692 __ lbu(t0, Address(klass, InstanceKlass::init_state_offset()));
693 __ mv(t1, InstanceKlass::fully_initialized);
694 __ bne(t0, t1, slow_path);
695 }
696
697 #ifdef ASSERT
698 // assert object can be fast path allocated
699 {
700 Label ok, not_ok;
701 __ lw(obj_size, Address(klass, Klass::layout_helper_offset()));
702 // make sure it's an instance. For instances, layout helper is a positive number.
703 // For arrays, layout helper is a negative number
704 __ blez(obj_size, not_ok);
705 __ andi(t0, obj_size, Klass::_lh_instance_slow_path_bit);
706 __ beqz(t0, ok);
707 __ bind(not_ok);
708 __ stop("assert(can be fast path allocated)");
709 __ should_not_reach_here();
710 __ bind(ok);
711 }
712 #endif // ASSERT
713
714 // get the instance size
715 __ lwu(obj_size, Address(klass, Klass::layout_helper_offset()));
716
717 __ eden_allocate(obj, obj_size, 0, tmp1, slow_path);
718
719 __ initialize_object(obj, klass, obj_size, 0, tmp1, tmp2, /* is_tlab_allocated */ false);
720 __ verify_oop(obj);
721 __ ld(x9, Address(sp, x9_offset * wordSize));
722 __ ld(zr, Address(sp, zr_offset * wordSize));
723 __ addi(sp, sp, sp_offset * wordSize);
724 __ ret();
725
726 __ bind(slow_path);
727 __ ld(x9, Address(sp, x9_offset * wordSize));
728 __ ld(zr, Address(sp, zr_offset * wordSize));
729 __ addi(sp, sp, sp_offset * wordSize);
730 }
731
732 __ enter();
733 OopMap* map = save_live_registers(sasm);
734 assert_cond(map != NULL);
735 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
736 oop_maps = new OopMapSet();
737 assert_cond(oop_maps != NULL);
738 oop_maps->add_gc_map(call_offset, map);
739 restore_live_registers_except_r10(sasm);
740 __ verify_oop(obj);
741 __ leave();
742 __ ret();
743
744 // x10: new instance
745 }
746
747 break;
748
749 case counter_overflow_id:
750 {
751 Register bci = x10;
752 Register method = x11;
753 __ enter();
754 OopMap* map = save_live_registers(sasm);
755 assert_cond(map != NULL);
756
757 const int bci_off = 0;
758 const int method_off = 1;
759 // Retrieve bci
760 __ lw(bci, Address(fp, bci_off * BytesPerWord));
761 // And a pointer to the Method*
762 __ ld(method, Address(fp, method_off * BytesPerWord));
763 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
764 oop_maps = new OopMapSet();
765 assert_cond(oop_maps != NULL);
766 oop_maps->add_gc_map(call_offset, map);
767 restore_live_registers(sasm);
768 __ leave();
769 __ ret();
770 }
771 break;
772
773 case new_type_array_id:
774 case new_object_array_id:
775 {
776 Register length = x9; // Incoming
777 Register klass = x13; // Incoming
778 Register obj = x10; // Result
779
780 if (id == new_type_array_id) {
781 __ set_info("new_type_array", dont_gc_arguments);
782 } else {
783 __ set_info("new_object_array", dont_gc_arguments);
784 }
785
786 #ifdef ASSERT
787 // assert object type is really an array of the proper kind
788 {
789 Label ok;
790 Register tmp = obj;
791 __ lwu(tmp, Address(klass, Klass::layout_helper_offset()));
792 __ sraiw(tmp, tmp, Klass::_lh_array_tag_shift);
793 int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value);
794 __ mv(t0, tag);
795 __ beq(t0, tmp, ok);
796 __ stop("assert(is an array klass)");
797 __ should_not_reach_here();
798 __ bind(ok);
799 }
800 #endif // ASSERT
801
802 // If TLAB is disabled, see if there is support for inlining contiguous
803 // allocations.
804 // Otherwise, just go to the slow path.
805 if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
806 Register arr_size = x14;
807 Register tmp1 = x12;
808 Register tmp2 = x15;
809 Label slow_path;
810 assert_different_registers(length, klass, obj, arr_size, tmp1, tmp2);
811
812 // check that array length is small enough for fast path.
813 __ mv(t0, C1_MacroAssembler::max_array_allocation_length);
814 __ bgtu(length, t0, slow_path);
815
816 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
817 __ lwu(tmp1, Address(klass, Klass::layout_helper_offset()));
818 __ andi(t0, tmp1, 0x1f);
819 __ sll(arr_size, length, t0);
820 int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1);
821 int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width;
822 __ slli(tmp1, tmp1, XLEN - lh_header_size_msb);
823 __ srli(tmp1, tmp1, XLEN - lh_header_size_width);
824 __ add(arr_size, arr_size, tmp1);
825 __ addi(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
826 __ andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
827
828 __ eden_allocate(obj, arr_size, 0, tmp1, slow_path); // preserves arr_size
829
830 __ initialize_header(obj, klass, length, tmp1, tmp2);
831 __ lbu(tmp1, Address(klass,
832 in_bytes(Klass::layout_helper_offset()) +
833 (Klass::_lh_header_size_shift / BitsPerByte)));
834 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
835 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
836 __ andi(tmp1, tmp1, Klass::_lh_header_size_mask);
837 __ sub(arr_size, arr_size, tmp1); // body length
838 __ add(tmp1, tmp1, obj); // body start
839 __ initialize_body(tmp1, arr_size, 0, tmp2);
840 __ membar(MacroAssembler::StoreStore);
841 __ verify_oop(obj);
842
843 __ ret();
844
845 __ bind(slow_path);
846 }
847
848 __ enter();
849 OopMap* map = save_live_registers(sasm);
850 assert_cond(map != NULL);
851 int call_offset = 0;
852 if (id == new_type_array_id) {
853 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
854 } else {
855 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
856 }
857
858 oop_maps = new OopMapSet();
859 assert_cond(oop_maps != NULL);
860 oop_maps->add_gc_map(call_offset, map);
861 restore_live_registers_except_r10(sasm);
862
863 __ verify_oop(obj);
864 __ leave();
865 __ ret();
866
867 // x10: new array
868 }
869 break;
870
871 case new_multi_array_id:
872 {
873 StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
874 // x10: klass
875 // x9: rank
876 // x12: address of 1st dimension
877 OopMap* map = save_live_registers(sasm);
878 assert_cond(map != NULL);
879 __ mv(c_rarg1, x10);
880 __ mv(c_rarg3, x12);
881 __ mv(c_rarg2, x9);
882 int call_offset = __ call_RT(x10, noreg, CAST_FROM_FN_PTR(address, new_multi_array), x11, x12, x13);
883
884 oop_maps = new OopMapSet();
885 assert_cond(oop_maps != NULL);
886 oop_maps->add_gc_map(call_offset, map);
887 restore_live_registers_except_r10(sasm);
888
889 // x10: new multi array
890 __ verify_oop(x10);
891 }
892 break;
893
894 case register_finalizer_id:
895 {
896 __ set_info("register_finalizer", dont_gc_arguments);
897
898 // This is called via call_runtime so the arguments
899 // will be place in C abi locations
900 __ verify_oop(c_rarg0);
901
902 // load the klass and check the has finalizer flag
903 Label register_finalizer;
904 Register t = x15;
905 __ load_klass(t, x10);
906 __ lwu(t, Address(t, Klass::access_flags_offset()));
907 __ andi(t0, t, JVM_ACC_HAS_FINALIZER);
908 __ bnez(t0, register_finalizer);
909 __ ret();
910
911 __ bind(register_finalizer);
912 __ enter();
913 OopMap* oop_map = save_live_registers(sasm);
914 assert_cond(oop_map != NULL);
915 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), x10);
916 oop_maps = new OopMapSet();
917 assert_cond(oop_maps != NULL);
918 oop_maps->add_gc_map(call_offset, oop_map);
919
920 // Now restore all the live registers
921 restore_live_registers(sasm);
922
923 __ leave();
924 __ ret();
925 }
926 break;
927
928 case throw_class_cast_exception_id:
929 {
930 StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return);
931 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
932 }
933 break;
934
935 case throw_incompatible_class_change_error_id:
936 {
937 StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return);
938 oop_maps = generate_exception_throw(sasm,
939 CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
940 }
941 break;
942
943 case slow_subtype_check_id:
944 {
945 // Typical calling sequence:
946 // push klass_RInfo (object klass or other subclass)
947 // push sup_k_RInfo (array element klass or other superclass)
948 // jump to slow_subtype_check
949 // Note that the subclass is pushed first, and is therefore deepest.
950 enum layout {
951 x10_off, x10_off_hi,
952 x12_off, x12_off_hi,
953 x14_off, x14_off_hi,
954 x15_off, x15_off_hi,
955 sup_k_off, sup_k_off_hi,
956 klass_off, klass_off_hi,
957 framesize,
958 result_off = sup_k_off
959 };
960
961 __ set_info("slow_subtype_check", dont_gc_arguments);
962 __ push_reg(RegSet::of(x10, x12, x14, x15), sp);
963
964 __ ld(x14, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // sub klass
965 __ ld(x10, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // super klass
966
967 Label miss;
968 __ check_klass_subtype_slow_path(x14, x10, x12, x15, NULL, &miss);
969
970 // fallthrough on success:
971 __ li(t0, 1);
972 __ sd(t0, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
973 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp);
974 __ ret();
975
976 __ bind(miss);
977 __ sd(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
978 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp);
979 __ ret();
980 }
981 break;
982
983 case monitorenter_nofpu_id:
984 save_fpu_registers = false;
985 // fall through
986 case monitorenter_id:
987 {
988 StubFrame f(sasm, "monitorenter", dont_gc_arguments);
989 OopMap* map = save_live_registers(sasm, save_fpu_registers);
990 assert_cond(map != NULL);
991
992 // Called with store_parameter and not C abi
993 f.load_argument(1, x10); // x10: object
994 f.load_argument(0, x11); // x11: lock address
995
996 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), x10, x11);
997
998 oop_maps = new OopMapSet();
999 assert_cond(oop_maps != NULL);
1000 oop_maps->add_gc_map(call_offset, map);
1001 restore_live_registers(sasm, save_fpu_registers);
1002 }
1003 break;
1004
1005 case monitorexit_nofpu_id:
1006 save_fpu_registers = false;
1007 // fall through
1008 case monitorexit_id:
1009 {
1010 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1011 OopMap* map = save_live_registers(sasm, save_fpu_registers);
1012 assert_cond(map != NULL);
1013
1014 // Called with store_parameter and not C abi
1015 f.load_argument(0, x10); // x10: lock address
1016
1017 // note: really a leaf routine but must setup last java sp
1018 // => use call_RT for now (speed can be improved by
1019 // doing last java sp setup manually)
1020 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), x10);
1021
1022 oop_maps = new OopMapSet();
1023 assert_cond(oop_maps != NULL);
1024 oop_maps->add_gc_map(call_offset, map);
1025 restore_live_registers(sasm, save_fpu_registers);
1026 }
1027 break;
1028
1029 case deoptimize_id:
1030 {
1031 StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return);
1032 OopMap* oop_map = save_live_registers(sasm);
1033 assert_cond(oop_map != NULL);
1034 f.load_argument(0, c_rarg1);
1035 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
1036
1037 oop_maps = new OopMapSet();
1038 assert_cond(oop_maps != NULL);
1039 oop_maps->add_gc_map(call_offset, oop_map);
1040 restore_live_registers(sasm);
1041 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1042 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1043 __ leave();
1044 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1045 }
1046 break;
1047
1048 case throw_range_check_failed_id:
1049 {
1050 StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return);
1051 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1052 }
1053 break;
1054
1055 case unwind_exception_id:
1056 {
1057 __ set_info("unwind_exception", dont_gc_arguments);
1058 // note: no stubframe since we are about to leave the current
1059 // activation and we are calling a leaf VM function only.
1060 generate_unwind_exception(sasm);
1061 }
1062 break;
1063
1064 case access_field_patching_id:
1065 {
1066 StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return);
1067 // we should set up register map
1068 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1069 }
1070 break;
1071
1072 case load_klass_patching_id:
1073 {
1074 StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return);
1075 // we should set up register map
1076 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1077 }
1078 break;
1079
1080 case load_mirror_patching_id:
1081 {
1082 StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return);
1083 // we should set up register map
1084 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1085 }
1086 break;
1087
1088 case load_appendix_patching_id:
1089 {
1090 StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return);
1091 // we should set up register map
1092 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1093 }
1094 break;
1095
1096 case handle_exception_nofpu_id:
1097 case handle_exception_id:
1098 {
1099 StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1100 oop_maps = generate_handle_exception(id, sasm);
1101 }
1102 break;
1103
1104 case handle_exception_from_callee_id:
1105 {
1106 StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1107 oop_maps = generate_handle_exception(id, sasm);
1108 }
1109 break;
1110
1111 case throw_index_exception_id:
1112 {
1113 StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return);
1114 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1115 }
1116 break;
1117
1118 case throw_array_store_exception_id:
1119 {
1120 StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return);
1121 // tos + 0: link
1122 // + 1: return address
1123 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1124 }
1125 break;
1126
1127 case predicate_failed_trap_id:
1128 {
1129 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return);
1130
1131 OopMap* map = save_live_registers(sasm);
1132 assert_cond(map != NULL);
1133
1134 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1135 oop_maps = new OopMapSet();
1136 assert_cond(oop_maps != NULL);
1137 oop_maps->add_gc_map(call_offset, map);
1138 restore_live_registers(sasm);
1139 __ leave();
1140 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1141 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1142
1143 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1144 }
1145 break;
1146
1147 case dtrace_object_alloc_id:
1148 { // c_rarg0: object
1149 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1150 save_live_registers(sasm);
1151
1152 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0);
1153
1154 restore_live_registers(sasm);
1155 }
1156 break;
1157
1158 default:
1159 {
1160 StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return);
1161 __ li(x10, (int) id);
1162 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), x10);
1163 __ should_not_reach_here();
1164 }
1165 break;
1166 }
1167 }
1168 return oop_maps;
1169 }
1170
1171 #undef __
1172
1173 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }