1 /*
2 * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Defs.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "compiler/disassembler.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/shared/cardTable.hpp"
36 #include "gc/shared/cardTableBarrierSet.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/universe.hpp"
39 #include "nativeInst_riscv.hpp"
40 #include "oops/compiledICHolder.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "prims/jvmtiExport.hpp"
43 #include "register_riscv.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/signature.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "runtime/vframe.hpp"
48 #include "runtime/vframeArray.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_riscv.inline.hpp"
51
52
53 // Implementation of StubAssembler
54
55 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, int args_size) {
56 // setup registers
57 assert(!(oop_result->is_valid() || metadata_result->is_valid()) || oop_result != metadata_result,
58 "registers must be different");
59 assert(oop_result != xthread && metadata_result != xthread, "registers must be different");
60 assert(args_size >= 0, "illegal args_size");
61 bool align_stack = false;
62
63 mv(c_rarg0, xthread);
64 set_num_rt_args(0); // Nothing on stack
65
66 Label retaddr;
67 set_last_Java_frame(sp, fp, retaddr, t0);
68
69 // do the call
70 int32_t off = 0;
71 la_patchable(t0, RuntimeAddress(entry), off);
72 jalr(x1, t0, off);
73 bind(retaddr);
74 int call_offset = offset();
75 // verify callee-saved register
76 #ifdef ASSERT
77 push_reg(x10, sp);
78 { Label L;
79 get_thread(x10);
80 beq(xthread, x10, L);
81 stop("StubAssembler::call_RT: xthread not callee saved?");
82 bind(L);
83 }
84 pop_reg(x10, sp);
85 #endif
86 reset_last_Java_frame(true);
87
88 // check for pending exceptions
89 { Label L;
90 // check for pending exceptions (java_thread is set upon return)
91 ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
92 beqz(t0, L);
93 // exception pending => remove activation and forward to exception handler
94 // make sure that the vm_results are cleared
95 if (oop_result->is_valid()) {
96 sd(zr, Address(xthread, JavaThread::vm_result_offset()));
97 }
98 if (metadata_result->is_valid()) {
99 sd(zr, Address(xthread, JavaThread::vm_result_2_offset()));
100 }
101 if (frame_size() == no_frame_size) {
102 leave();
103 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
104 } else if (_stub_id == Runtime1::forward_exception_id) {
105 should_not_reach_here();
106 } else {
107 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
108 }
109 bind(L);
110 }
111 // get oop results if there are any and reset the values in the thread
112 if (oop_result->is_valid()) {
113 get_vm_result(oop_result, xthread);
114 }
115 if (metadata_result->is_valid()) {
116 get_vm_result_2(metadata_result, xthread);
117 }
118 return call_offset;
119 }
120
121 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1) {
122 mv(c_rarg1, arg1);
123 return call_RT(oop_result, metadata_result, entry, 1);
124 }
125
126 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2) {
127 const int arg_num = 2;
128 if (c_rarg1 == arg2) {
129 if (c_rarg2 == arg1) {
130 xorr(arg1, arg1, arg2);
131 xorr(arg2, arg1, arg2);
132 xorr(arg1, arg1, arg2);
133 } else {
134 mv(c_rarg2, arg2);
135 mv(c_rarg1, arg1);
136 }
137 } else {
138 mv(c_rarg1, arg1);
139 mv(c_rarg2, arg2);
140 }
141 return call_RT(oop_result, metadata_result, entry, arg_num);
142 }
143
144 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
145 const int arg_num = 3;
146 // if there is any conflict use the stack
147 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
148 arg2 == c_rarg1 || arg2 == c_rarg3 ||
149 arg3 == c_rarg1 || arg3 == c_rarg2) {
150 const int arg1_sp_offset = 0;
151 const int arg2_sp_offset = 1;
152 const int arg3_sp_offset = 2;
153 addi(sp, sp, -(arg_num + 1) * wordSize);
154 sd(arg1, Address(sp, arg1_sp_offset * wordSize));
155 sd(arg2, Address(sp, arg2_sp_offset * wordSize));
156 sd(arg3, Address(sp, arg3_sp_offset * wordSize));
157
158 ld(c_rarg1, Address(sp, arg1_sp_offset * wordSize));
159 ld(c_rarg2, Address(sp, arg2_sp_offset * wordSize));
160 ld(c_rarg3, Address(sp, arg3_sp_offset * wordSize));
161 addi(sp, sp, (arg_num + 1) * wordSize);
162 } else {
163 mv(c_rarg1, arg1);
164 mv(c_rarg2, arg2);
165 mv(c_rarg3, arg3);
166 }
167 return call_RT(oop_result, metadata_result, entry, arg_num);
168 }
169
170 enum return_state_t {
171 does_not_return, requires_return
172 };
173
174 // Implementation of StubFrame
175
176 class StubFrame: public StackObj {
177 private:
178 StubAssembler* _sasm;
179 bool _return_state;
180
181 public:
182 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return);
183 void load_argument(int offset_in_words, Register reg);
184
185 ~StubFrame();
186 };;
187
188 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
189 set_info(name, must_gc_arguments);
190 enter();
191 }
192
193 void StubAssembler::epilogue() {
194 leave();
195 ret();
196 }
197
198 #define __ _sasm->
199
200 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) {
201 _sasm = sasm;
202 _return_state = return_state;
203 __ prologue(name, must_gc_arguments);
204 }
205
206 // load parameters that were stored with LIR_Assembler::store_parameter
207 // Note: offsets for store_parameter and load_argument must match
208 void StubFrame::load_argument(int offset_in_words, Register reg) {
209 __ load_parameter(offset_in_words, reg);
210 }
211
212
213 StubFrame::~StubFrame() {
214 if (_return_state == requires_return) {
215 __ epilogue();
216 } else {
217 __ should_not_reach_here();
218 }
219 _sasm = NULL;
220 }
221
222 #undef __
223
224
225 // Implementation of Runtime1
226
227 #define __ sasm->
228
229 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
230
231 // Stack layout for saving/restoring all the registers needed during a runtime
232 // call (this includes deoptimization)
233 // Note: note that users of this frame may well have arguments to some runtime
234 // while these values are on the stack. These positions neglect those arguments
235 // but the code in save_live_registers will take the argument count into
236 // account.
237 //
238
239 enum reg_save_layout {
240 reg_save_frame_size = 32 /* float */ + 32 /* integer */
241 };
242
243 // Save off registers which might be killed by calls into the runtime.
244 // Tries to smart of about FP registers. In particular we separate
245 // saving and describing the FPU registers for deoptimization since we
246 // have to save the FPU registers twice if we describe them. The
247 // deopt blob is the only thing which needs to describe FPU registers.
248 // In all other cases it should be sufficient to simply save their
249 // current value.
250
251 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
252 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
253
254 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
255 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
256 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
257 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
258 OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
259 assert_cond(oop_map != NULL);
260
261 // cpu_regs, caller save registers only, see FrameMap::initialize
262 // in c1_FrameMap_riscv64.cpp for detail.
263 const static Register caller_save_cpu_regs[FrameMap::max_nof_caller_save_cpu_regs] = {x3, x7, x10, x11, x12,
264 x13, x14, x15, x16, x17,
265 x28, x29, x30, x31};
266 for (int i = 0; i < FrameMap::max_nof_caller_save_cpu_regs; i++) {
267 Register r = caller_save_cpu_regs[i];
268 int sp_offset = cpu_reg_save_offsets[r->encoding()];
269 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
270 r->as_VMReg());
271 }
272
273 // fpu_regs
274 if (save_fpu_registers) {
275 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
276 FloatRegister r = as_FloatRegister(i);
277 int sp_offset = fpu_reg_save_offsets[i];
278 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
279 r->as_VMReg());
280 }
281 }
282 return oop_map;
283 }
284
285 static OopMap* save_live_registers(StubAssembler* sasm,
286 bool save_fpu_registers = true) {
287 __ block_comment("save_live_registers");
288
289 // if the number of pushed regs is odd, zr will be added
290 __ push_reg(RegSet::range(x3, x31), sp); // integer registers except ra(x1) & sp(x2)
291
292 if (save_fpu_registers) {
293 // float registers
294 __ addi(sp, sp, -(FrameMap::nof_fpu_regs * wordSize));
295 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
296 __ fsd(as_FloatRegister(i), Address(sp, i * wordSize));
297 }
298 } else {
299 // we define reg_save_layout = 64 as the fixed frame size,
300 // we should also sub 32 * wordSize to sp when save_fpu_registers == false
301 __ addi(sp, sp, -32 * wordSize);
302 }
303
304 return generate_oop_map(sasm, save_fpu_registers);
305 }
306
307 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
308 if (restore_fpu_registers) {
309 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
310 __ fld(as_FloatRegister(i), Address(sp, i * wordSize));
311 }
312 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize);
313 } else {
314 // we define reg_save_layout = 64 as the fixed frame size,
315 // we should also add 32 * wordSize to sp when save_fpu_registers == false
316 __ addi(sp, sp, 32 * wordSize);
317 }
318
319 // if the number of popped regs is odd, zr will be added
320 __ pop_reg(RegSet::range(x3, x31), sp); // integer registers except ra(x1) & sp(x2)
321 }
322
323 static void restore_live_registers_except_r10(StubAssembler* sasm, bool restore_fpu_registers = true) {
324 if (restore_fpu_registers) {
325 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
326 __ fld(as_FloatRegister(i), Address(sp, i * wordSize));
327 }
328 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize);
329 } else {
330 // we define reg_save_layout = 64 as the fixed frame size,
331 // we should also add 32 * wordSize to sp when save_fpu_registers == false
332 __ addi(sp, sp, 32 * wordSize);
333 }
334
335 // if the number of popped regs is odd, zr will be added
336 // integer registers except ra(x1) & sp(x2) & x10
337 __ pop_reg(RegSet::range(x3, x9), sp); // pop zr, x3 ~ x9
338 __ pop_reg(RegSet::range(x11, x31), sp); // pop x10 ~ x31, x10 will be loaded to zr
339 }
340
341 void Runtime1::initialize_pd() {
342 int i = 0;
343 int sp_offset = 0;
344 const int step = 2; // SP offsets are in halfwords
345
346 // all float registers are saved explicitly
347 for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
348 fpu_reg_save_offsets[i] = sp_offset;
349 sp_offset += step;
350 }
351
352 // we save x0, x3 ~ x31, except x1, x2
353 cpu_reg_save_offsets[0] = sp_offset;
354 sp_offset += step;
355 // 3: loop starts from x3
356 for (i = 3; i < FrameMap::nof_cpu_regs; i++) {
357 cpu_reg_save_offsets[i] = sp_offset;
358 sp_offset += step;
359 }
360 }
361
362 // target: the entry point of the method that creates and posts the exception oop
363 // has_argument: true if the exception needs arguments (passed in t0 and t1)
364
365 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
366 // make a frame and preserve the caller's caller-save registers
367 OopMap* oop_map = save_live_registers(sasm);
368 assert_cond(oop_map != NULL);
369 int call_offset = 0;
370 if (!has_argument) {
371 call_offset = __ call_RT(noreg, noreg, target);
372 } else {
373 __ mv(c_rarg1, t0);
374 __ mv(c_rarg2, t1);
375 call_offset = __ call_RT(noreg, noreg, target);
376 }
377 OopMapSet* oop_maps = new OopMapSet();
378 assert_cond(oop_maps != NULL);
379 oop_maps->add_gc_map(call_offset, oop_map);
380
381 return oop_maps;
382 }
383
384 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
385 __ block_comment("generate_handle_exception");
386
387 // incoming parameters
388 const Register exception_oop = x10;
389 const Register exception_pc = x13;
390
391 OopMapSet* oop_maps = new OopMapSet();
392 assert_cond(oop_maps != NULL);
393 OopMap* oop_map = NULL;
394
395 switch (id) {
396 case forward_exception_id:
397 // We're handling an exception in the context of a compiled frame.
398 // The registers have been saved in the standard places. Perform
399 // an exception lookup in the caller and dispatch to the handler
400 // if found. Otherwise unwind and dispatch to the callers
401 // exception handler.
402 oop_map = generate_oop_map(sasm, 1 /* thread */);
403
404 // load and clear pending exception oop into x10
405 __ ld(exception_oop, Address(xthread, Thread::pending_exception_offset()));
406 __ sd(zr, Address(xthread, Thread::pending_exception_offset()));
407
408 // load issuing PC (the return address for this stub) into x13
409 __ ld(exception_pc, Address(fp, 1 * BytesPerWord));
410
411 // make sure that the vm_results are cleared (may be unnecessary)
412 __ sd(zr, Address(xthread, JavaThread::vm_result_offset()));
413 __ sd(zr, Address(xthread, JavaThread::vm_result_2_offset()));
414 break;
415 case handle_exception_nofpu_id:
416 case handle_exception_id:
417 // At this point all registers MAY be live.
418 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
419 break;
420 case handle_exception_from_callee_id: {
421 // At this point all registers except exception oop (x10) and
422 // exception pc (lr) are dead.
423 const int frame_size = 2 /* fp, return address */;
424 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
425 sasm->set_frame_size(frame_size);
426 break;
427 }
428 default:
429 ShouldNotReachHere();
430 break;
431 }
432
433 // verify that only x10 and x13 are valid at this time
434 __ invalidate_registers(false, true, true, false, true, true);
435 // verify that x10 contains a valid exception
436 __ verify_not_null_oop(exception_oop);
437
438 #ifdef ASSERT
439 // check that fields in JavaThread for exception oop and issuing pc are
440 // empty before writing to them
441 Label oop_empty;
442 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset()));
443 __ beqz(t0, oop_empty);
444 __ stop("exception oop already set");
445 __ bind(oop_empty);
446
447 Label pc_empty;
448 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
449 __ beqz(t0, pc_empty);
450 __ stop("exception pc already set");
451 __ bind(pc_empty);
452 #endif
453
454 // save exception oop and issuing pc into JavaThread
455 // (exception handler will load it from here)
456 __ sd(exception_oop, Address(xthread, JavaThread::exception_oop_offset()));
457 __ sd(exception_pc, Address(xthread, JavaThread::exception_pc_offset()));
458
459 // patch throwing pc into return address (has bci & oop map)
460 __ sd(exception_pc, Address(fp, 1 * BytesPerWord));
461
462 // compute the exception handler.
463 // the exception oop and the throwing pc are read from the fields in JavaThread
464 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
465 guarantee(oop_map != NULL, "NULL oop_map!");
466 oop_maps->add_gc_map(call_offset, oop_map);
467
468 // x10: handler address
469 // will be the deopt blob if nmethod was deoptimized while we looked up
470 // handler regardless of whether handler existed in the nmethod.
471
472 // only x10 is valid at this time, all other registers have been destroyed by the runtime call
473 __ invalidate_registers(false, true, true, true, true, true);
474
475 // patch the return address, this stub will directly return to the exception handler
476 __ sd(x10, Address(fp, 1 * BytesPerWord));
477
478 switch (id) {
479 case forward_exception_id:
480 case handle_exception_nofpu_id:
481 case handle_exception_id:
482 // Restore the registers that were saved at the beginning.
483 restore_live_registers(sasm, id != handle_exception_nofpu_id);
484 break;
485 case handle_exception_from_callee_id:
486 break;
487 default: ShouldNotReachHere();
488 }
489
490 return oop_maps;
491 }
492
493
494 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
495 // incoming parameters
496 const Register exception_oop = x10;
497 // other registers used in this stub
498 const Register handler_addr = x11;
499
500 // verify that only x10, is valid at this time
501 __ invalidate_registers(false, true, true, true, true, true);
502
503 #ifdef ASSERT
504 // check that fields in JavaThread for exception oop and issuing pc are empty
505 Label oop_empty;
506 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset()));
507 __ beqz(t0, oop_empty);
508 __ stop("exception oop must be empty");
509 __ bind(oop_empty);
510
511 Label pc_empty;
512 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
513 __ beqz(t0, pc_empty);
514 __ stop("exception pc must be empty");
515 __ bind(pc_empty);
516 #endif
517
518 // Save our return address because
519 // exception_handler_for_return_address will destroy it. We also
520 // save exception_oop
521 __ addi(sp, sp, -2 * wordSize);
522 __ sd(exception_oop, Address(sp, wordSize));
523 __ sd(lr, Address(sp));
524
525 // search the exception handler address of the caller (using the return address)
526 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, lr);
527 // x10: exception handler address of the caller
528
529 // Only x10 is valid at this time; all other registers have been
530 // destroyed by the call.
531 __ invalidate_registers(false, true, true, true, false, true);
532
533 // move result of call into correct register
534 __ mv(handler_addr, x10);
535
536 // get throwing pc (= return address).
537 // lr has been destroyed by the call
538 __ ld(lr, Address(sp));
539 __ ld(exception_oop, Address(sp, wordSize));
540 __ addi(sp, sp, 2 * wordSize);
541 __ mv(x13, lr);
542
543 __ verify_not_null_oop(exception_oop);
544
545 // continue at exception handler (return address removed)
546 // note: do *not* remove arguments when unwinding the
547 // activation since the caller assumes having
548 // all arguments on the stack when entering the
549 // runtime to determine the exception handler
550 // (GC happens at call site with arguments!)
551 // x10: exception oop
552 // x13: throwing pc
553 // x11: exception handler
554 __ jr(handler_addr);
555 }
556
557 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
558 // use the maximum number of runtime-arguments here because it is difficult to
559 // distinguish each RT-Call.
560 // Note: This number affects also the RT-Call in generate_handle_exception because
561 // the oop-map is shared for all calls.
562 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
563 assert(deopt_blob != NULL, "deoptimization blob must have been created");
564
565 OopMap* oop_map = save_live_registers(sasm);
566 assert_cond(oop_map != NULL);
567
568 __ mv(c_rarg0, xthread);
569 Label retaddr;
570 __ set_last_Java_frame(sp, fp, retaddr, t0);
571 // do the call
572 int32_t off = 0;
573 __ la_patchable(t0, RuntimeAddress(target), off);
574 __ jalr(x1, t0, off);
575 __ bind(retaddr);
576 OopMapSet* oop_maps = new OopMapSet();
577 assert_cond(oop_maps != NULL);
578 oop_maps->add_gc_map(__ offset(), oop_map);
579 // verify callee-saved register
580 #ifdef ASSERT
581 { Label L;
582 __ get_thread(t0);
583 __ beq(xthread, t0, L);
584 __ stop("StubAssembler::call_RT: xthread not callee saved?");
585 __ bind(L);
586 }
587 #endif
588 __ reset_last_Java_frame(true);
589
590 #ifdef ASSERT
591 // Check that fields in JavaThread for exception oop and issuing pc are empty
592 Label oop_empty;
593 __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
594 __ beqz(t0, oop_empty);
595 __ stop("exception oop must be empty");
596 __ bind(oop_empty);
597
598 Label pc_empty;
599 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset()));
600 __ beqz(t0, pc_empty);
601 __ stop("exception pc must be empty");
602 __ bind(pc_empty);
603 #endif
604
605 // Runtime will return true if the nmethod has been deoptimized, this is the
606 // expected scenario and anything else is an error. Note that we maintain a
607 // check on the result purely as a defensive measure.
608 Label no_deopt;
609 __ beqz(x10, no_deopt); // Have we deoptimized?
610
611 // Perform a re-execute. The proper return address is already on the stack,
612 // we just need to restore registers, pop all of our frames but the return
613 // address and jump to the deopt blob.
614
615 restore_live_registers(sasm);
616 __ leave();
617 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
618
619 __ bind(no_deopt);
620 __ stop("deopt not performed");
621
622 return oop_maps;
623 }
624
625 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
626 // for better readability
627 const bool dont_gc_arguments = false;
628
629 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
630 bool save_fpu_registers = true;
631
632 // stub code & info for the different stubs
633 OopMapSet* oop_maps = NULL;
634 switch (id) {
635 {
636 case forward_exception_id:
637 {
638 oop_maps = generate_handle_exception(id, sasm);
639 __ leave();
640 __ ret();
641 }
642 break;
643
644 case throw_div0_exception_id:
645 {
646 StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return);
647 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
648 }
649 break;
650
651 case throw_null_pointer_exception_id:
652 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return);
653 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
654 }
655 break;
656
657 case new_instance_id:
658 case fast_new_instance_id:
659 case fast_new_instance_init_check_id:
660 {
661 Register klass = x13; // Incoming
662 Register obj = x10; // Result
663
664 if (id == new_instance_id) {
665 __ set_info("new_instance", dont_gc_arguments);
666 } else if (id == fast_new_instance_id) {
667 __ set_info("fast new_instance", dont_gc_arguments);
668 } else {
669 assert(id == fast_new_instance_init_check_id, "bad StubID");
670 __ set_info("fast new_instance init check", dont_gc_arguments);
671 }
672
673 // If TLAB is disabled, see if there is support for inlining contiguous
674 // allocations.
675 // Otherwise, just go to the slow path.
676 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
677 !UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
678 Label slow_path;
679 Register obj_size = x12;
680 Register tmp1 = x9;
681 Register tmp2 = x14;
682 assert_different_registers(klass, obj, obj_size, tmp1, tmp2);
683
684 const int sp_offset = 2;
685 const int x9_offset = 1;
686 const int zr_offset = 0;
687 __ addi(sp, sp, -(sp_offset * wordSize));
688 __ sd(x9, Address(sp, x9_offset * wordSize));
689 __ sd(zr, Address(sp, zr_offset * wordSize));
690
691 if (id == fast_new_instance_init_check_id) {
692 // make sure the klass is initialized
693 __ lbu(t0, Address(klass, InstanceKlass::init_state_offset()));
694 __ mv(t1, InstanceKlass::fully_initialized);
695 __ bne(t0, t1, slow_path);
696 }
697
698 #ifdef ASSERT
699 // assert object can be fast path allocated
700 {
701 Label ok, not_ok;
702 __ lw(obj_size, Address(klass, Klass::layout_helper_offset()));
703 // make sure it's an instance. For instances, layout helper is a positive number.
704 // For arrays, layout helper is a negative number
705 __ blez(obj_size, not_ok);
706 __ andi(t0, obj_size, Klass::_lh_instance_slow_path_bit);
707 __ beqz(t0, ok);
708 __ bind(not_ok);
709 __ stop("assert(can be fast path allocated)");
710 __ should_not_reach_here();
711 __ bind(ok);
712 }
713 #endif // ASSERT
714
715 // get the instance size
716 __ lwu(obj_size, Address(klass, Klass::layout_helper_offset()));
717
718 __ eden_allocate(obj, obj_size, 0, tmp1, slow_path);
719
720 __ initialize_object(obj, klass, obj_size, 0, tmp1, tmp2, /* is_tlab_allocated */ false);
721 __ verify_oop(obj);
722 __ ld(x9, Address(sp, x9_offset * wordSize));
723 __ ld(zr, Address(sp, zr_offset * wordSize));
724 __ addi(sp, sp, sp_offset * wordSize);
725 __ ret();
726
727 __ bind(slow_path);
728 __ ld(x9, Address(sp, x9_offset * wordSize));
729 __ ld(zr, Address(sp, zr_offset * wordSize));
730 __ addi(sp, sp, sp_offset * wordSize);
731 }
732
733 __ enter();
734 OopMap* map = save_live_registers(sasm);
735 assert_cond(map != NULL);
736 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
737 oop_maps = new OopMapSet();
738 assert_cond(oop_maps != NULL);
739 oop_maps->add_gc_map(call_offset, map);
740 restore_live_registers_except_r10(sasm);
741 __ verify_oop(obj);
742 __ leave();
743 __ ret();
744
745 // x10: new instance
746 }
747
748 break;
749
750 case counter_overflow_id:
751 {
752 Register bci = x10;
753 Register method = x11;
754 __ enter();
755 OopMap* map = save_live_registers(sasm);
756 assert_cond(map != NULL);
757
758 const int bci_off = 2;
759 const int method_off = 3;
760 // Retrieve bci
761 __ lw(bci, Address(fp, bci_off * BytesPerWord));
762 // And a pointer to the Method*
763 __ ld(method, Address(fp, method_off * BytesPerWord));
764 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
765 oop_maps = new OopMapSet();
766 assert_cond(oop_maps != NULL);
767 oop_maps->add_gc_map(call_offset, map);
768 restore_live_registers(sasm);
769 __ leave();
770 __ ret();
771 }
772 break;
773
774 case new_type_array_id:
775 case new_object_array_id:
776 {
777 Register length = x9; // Incoming
778 Register klass = x13; // Incoming
779 Register obj = x10; // Result
780
781 if (id == new_type_array_id) {
782 __ set_info("new_type_array", dont_gc_arguments);
783 } else {
784 __ set_info("new_object_array", dont_gc_arguments);
785 }
786
787 #ifdef ASSERT
788 // assert object type is really an array of the proper kind
789 {
790 Label ok;
791 Register tmp = obj;
792 __ lwu(tmp, Address(klass, Klass::layout_helper_offset()));
793 __ sraiw(tmp, tmp, Klass::_lh_array_tag_shift);
794 int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value);
795 __ mv(t0, tag);
796 __ beq(t0, tmp, ok);
797 __ stop("assert(is an array klass)");
798 __ should_not_reach_here();
799 __ bind(ok);
800 }
801 #endif // ASSERT
802
803 // If TLAB is disabled, see if there is support for inlining contiguous
804 // allocations.
805 // Otherwise, just go to the slow path.
806 if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
807 Register arr_size = x14;
808 Register tmp1 = x12;
809 Register tmp2 = x15;
810 Label slow_path;
811 assert_different_registers(length, klass, obj, arr_size, tmp1, tmp2);
812
813 // check that array length is small enough for fast path.
814 __ mv(t0, C1_MacroAssembler::max_array_allocation_length);
815 __ bgtu(length, t0, slow_path);
816
817 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
818 __ lwu(tmp1, Address(klass, Klass::layout_helper_offset()));
819 __ andi(t0, tmp1, 0x1f);
820 __ sll(arr_size, length, t0);
821 int lh_header_size_width = log2i_exact(Klass::_lh_header_size_mask + 1);
822 int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width;
823 __ slli(tmp1, tmp1, registerSize - lh_header_size_msb);
824 __ srli(tmp1, tmp1, registerSize - lh_header_size_width);
825 __ add(arr_size, arr_size, tmp1);
826 __ addi(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
827 __ andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
828
829 __ eden_allocate(obj, arr_size, 0, tmp1, slow_path); // preserves arr_size
830
831 __ initialize_header(obj, klass, length, tmp1, tmp2);
832 __ lbu(tmp1, Address(klass,
833 in_bytes(Klass::layout_helper_offset()) +
834 (Klass::_lh_header_size_shift / BitsPerByte)));
835 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
836 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
837 __ andi(tmp1, tmp1, Klass::_lh_header_size_mask);
838 __ sub(arr_size, arr_size, tmp1); // body length
839 __ add(tmp1, tmp1, obj); // body start
840 __ initialize_body(tmp1, arr_size, 0, tmp2);
841 __ membar(MacroAssembler::StoreStore);
842 __ verify_oop(obj);
843
844 __ ret();
845
846 __ bind(slow_path);
847 }
848
849 __ enter();
850 OopMap* map = save_live_registers(sasm);
851 assert_cond(map != NULL);
852 int call_offset = 0;
853 if (id == new_type_array_id) {
854 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
855 } else {
856 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
857 }
858
859 oop_maps = new OopMapSet();
860 assert_cond(oop_maps != NULL);
861 oop_maps->add_gc_map(call_offset, map);
862 restore_live_registers_except_r10(sasm);
863
864 __ verify_oop(obj);
865 __ leave();
866 __ ret();
867
868 // x10: new array
869 }
870 break;
871
872 case new_multi_array_id:
873 {
874 StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
875 // x10: klass
876 // x9: rank
877 // x12: address of 1st dimension
878 OopMap* map = save_live_registers(sasm);
879 assert_cond(map != NULL);
880 __ mv(c_rarg1, x10);
881 __ mv(c_rarg3, x12);
882 __ mv(c_rarg2, x9);
883 int call_offset = __ call_RT(x10, noreg, CAST_FROM_FN_PTR(address, new_multi_array), x11, x12, x13);
884
885 oop_maps = new OopMapSet();
886 assert_cond(oop_maps != NULL);
887 oop_maps->add_gc_map(call_offset, map);
888 restore_live_registers_except_r10(sasm);
889
890 // x10: new multi array
891 __ verify_oop(x10);
892 }
893 break;
894
895 case register_finalizer_id:
896 {
897 __ set_info("register_finalizer", dont_gc_arguments);
898
899 // This is called via call_runtime so the arguments
900 // will be place in C abi locations
901 __ verify_oop(c_rarg0);
902
903 // load the klass and check the has finalizer flag
904 Label register_finalizer;
905 Register t = x15;
906 __ load_klass(t, x10);
907 __ lwu(t, Address(t, Klass::access_flags_offset()));
908 __ andi(t0, t, JVM_ACC_HAS_FINALIZER);
909 __ bnez(t0, register_finalizer);
910 __ ret();
911
912 __ bind(register_finalizer);
913 __ enter();
914 OopMap* oop_map = save_live_registers(sasm);
915 assert_cond(oop_map != NULL);
916 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), x10);
917 oop_maps = new OopMapSet();
918 assert_cond(oop_maps != NULL);
919 oop_maps->add_gc_map(call_offset, oop_map);
920
921 // Now restore all the live registers
922 restore_live_registers(sasm);
923
924 __ leave();
925 __ ret();
926 }
927 break;
928
929 case throw_class_cast_exception_id:
930 {
931 StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return);
932 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
933 }
934 break;
935
936 case throw_incompatible_class_change_error_id:
937 {
938 StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return);
939 oop_maps = generate_exception_throw(sasm,
940 CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
941 }
942 break;
943
944 case slow_subtype_check_id:
945 {
946 // Typical calling sequence:
947 // push klass_RInfo (object klass or other subclass)
948 // push sup_k_RInfo (array element klass or other superclass)
949 // jump to slow_subtype_check
950 // Note that the subclass is pushed first, and is therefore deepest.
951 enum layout {
952 x10_off, x10_off_hi,
953 x12_off, x12_off_hi,
954 x14_off, x14_off_hi,
955 x15_off, x15_off_hi,
956 sup_k_off, sup_k_off_hi,
957 klass_off, klass_off_hi,
958 framesize,
959 result_off = sup_k_off
960 };
961
962 __ set_info("slow_subtype_check", dont_gc_arguments);
963 __ push_reg(RegSet::of(x10, x12, x14, x15), sp);
964
965 __ ld(x14, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // sub klass
966 __ ld(x10, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // super klass
967
968 Label miss;
969 __ check_klass_subtype_slow_path(x14, x10, x12, x15, NULL, &miss);
970
971 // fallthrough on success:
972 __ li(t0, 1);
973 __ sd(t0, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
974 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp);
975 __ ret();
976
977 __ bind(miss);
978 __ sd(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
979 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp);
980 __ ret();
981 }
982 break;
983
984 case monitorenter_nofpu_id:
985 save_fpu_registers = false;
986 // fall through
987 case monitorenter_id:
988 {
989 StubFrame f(sasm, "monitorenter", dont_gc_arguments);
990 OopMap* map = save_live_registers(sasm, save_fpu_registers);
991 assert_cond(map != NULL);
992
993 // Called with store_parameter and not C abi
994 f.load_argument(1, x10); // x10: object
995 f.load_argument(0, x11); // x11: lock address
996
997 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), x10, x11);
998
999 oop_maps = new OopMapSet();
1000 assert_cond(oop_maps != NULL);
1001 oop_maps->add_gc_map(call_offset, map);
1002 restore_live_registers(sasm, save_fpu_registers);
1003 }
1004 break;
1005
1006 case monitorexit_nofpu_id:
1007 save_fpu_registers = false;
1008 // fall through
1009 case monitorexit_id:
1010 {
1011 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1012 OopMap* map = save_live_registers(sasm, save_fpu_registers);
1013 assert_cond(map != NULL);
1014
1015 // Called with store_parameter and not C abi
1016 f.load_argument(0, x10); // x10: lock address
1017
1018 // note: really a leaf routine but must setup last java sp
1019 // => use call_RT for now (speed can be improved by
1020 // doing last java sp setup manually)
1021 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), x10);
1022
1023 oop_maps = new OopMapSet();
1024 assert_cond(oop_maps != NULL);
1025 oop_maps->add_gc_map(call_offset, map);
1026 restore_live_registers(sasm, save_fpu_registers);
1027 }
1028 break;
1029
1030 case deoptimize_id:
1031 {
1032 StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return);
1033 OopMap* oop_map = save_live_registers(sasm);
1034 assert_cond(oop_map != NULL);
1035 f.load_argument(0, c_rarg1);
1036 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
1037
1038 oop_maps = new OopMapSet();
1039 assert_cond(oop_maps != NULL);
1040 oop_maps->add_gc_map(call_offset, oop_map);
1041 restore_live_registers(sasm);
1042 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1043 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1044 __ leave();
1045 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1046 }
1047 break;
1048
1049 case throw_range_check_failed_id:
1050 {
1051 StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return);
1052 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1053 }
1054 break;
1055
1056 case unwind_exception_id:
1057 {
1058 __ set_info("unwind_exception", dont_gc_arguments);
1059 // note: no stubframe since we are about to leave the current
1060 // activation and we are calling a leaf VM function only.
1061 generate_unwind_exception(sasm);
1062 }
1063 break;
1064
1065 case access_field_patching_id:
1066 {
1067 StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return);
1068 // we should set up register map
1069 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1070 }
1071 break;
1072
1073 case load_klass_patching_id:
1074 {
1075 StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return);
1076 // we should set up register map
1077 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1078 }
1079 break;
1080
1081 case load_mirror_patching_id:
1082 {
1083 StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return);
1084 // we should set up register map
1085 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1086 }
1087 break;
1088
1089 case load_appendix_patching_id:
1090 {
1091 StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return);
1092 // we should set up register map
1093 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1094 }
1095 break;
1096
1097 case handle_exception_nofpu_id:
1098 case handle_exception_id:
1099 {
1100 StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1101 oop_maps = generate_handle_exception(id, sasm);
1102 }
1103 break;
1104
1105 case handle_exception_from_callee_id:
1106 {
1107 StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1108 oop_maps = generate_handle_exception(id, sasm);
1109 }
1110 break;
1111
1112 case throw_index_exception_id:
1113 {
1114 StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return);
1115 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1116 }
1117 break;
1118
1119 case throw_array_store_exception_id:
1120 {
1121 StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return);
1122 // tos + 0: link
1123 // + 1: return address
1124 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1125 }
1126 break;
1127
1128 case predicate_failed_trap_id:
1129 {
1130 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return);
1131
1132 OopMap* map = save_live_registers(sasm);
1133 assert_cond(map != NULL);
1134
1135 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1136 oop_maps = new OopMapSet();
1137 assert_cond(oop_maps != NULL);
1138 oop_maps->add_gc_map(call_offset, map);
1139 restore_live_registers(sasm);
1140 __ leave();
1141 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1142 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1143
1144 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1145 }
1146 break;
1147
1148 case dtrace_object_alloc_id:
1149 { // c_rarg0: object
1150 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1151 save_live_registers(sasm);
1152
1153 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0);
1154
1155 restore_live_registers(sasm);
1156 }
1157 break;
1158
1159 default:
1160 {
1161 StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return);
1162 __ li(x10, (int) id);
1163 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), x10);
1164 __ should_not_reach_here();
1165 }
1166 break;
1167 }
1168 }
1169 return oop_maps;
1170 }
1171
1172 #undef __
1173
1174 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }