1 /*
2 * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "c1/c1_Defs.hpp"
27 #include "c1/c1_FrameMap.hpp"
28 #include "c1/c1_MacroAssembler.hpp"
29 #include "c1/c1_Runtime1.hpp"
30 #include "ci/ciUtilities.hpp"
31 #include "compiler/compilerDefinitions.inline.hpp"
32 #include "compiler/oopMap.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/cardTableBarrierSet.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/universe.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "register_x86.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/signature.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/vframeArray.hpp"
47 #include "utilities/macros.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50 // Implementation of StubAssembler
51
52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
53 // setup registers
54 const Register thread = r15_thread;
55 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
56 assert(oop_result1 != thread && metadata_result != thread, "registers must be different");
57 assert(args_size >= 0, "illegal args_size");
58 bool align_stack = false;
59
60 // At a method handle call, the stack may not be properly aligned
61 // when returning with an exception.
62 align_stack = (stub_id() == (int)C1StubId::handle_exception_from_callee_id);
63
64 mov(c_rarg0, thread);
65 set_num_rt_args(0); // Nothing on stack
66
67 int call_offset = -1;
68 if (!align_stack) {
69 set_last_Java_frame(noreg, rbp, nullptr, rscratch1);
70 } else {
71 address the_pc = pc();
72 call_offset = offset();
73 set_last_Java_frame(noreg, rbp, the_pc, rscratch1);
74 andptr(rsp, -(StackAlignmentInBytes)); // Align stack
75 }
76
77 // do the call
78 call(RuntimeAddress(entry));
79 if (!align_stack) {
80 call_offset = offset();
81 }
82 // verify callee-saved register
83 #ifdef ASSERT
84 guarantee(thread != rax, "change this code");
85 push(rax);
86 { Label L;
87 get_thread_slow(rax);
88 cmpptr(thread, rax);
89 jcc(Assembler::equal, L);
90 int3();
91 stop("StubAssembler::call_RT: rdi not callee saved?");
92 bind(L);
93 }
94 pop(rax);
95 #endif
96 reset_last_Java_frame(true);
97
98 // check for pending exceptions
99 { Label L;
100 cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
101 jcc(Assembler::equal, L);
102 // exception pending => remove activation and forward to exception handler
103 movptr(rax, Address(thread, Thread::pending_exception_offset()));
104 // make sure that the vm_results are cleared
105 if (oop_result1->is_valid()) {
106 movptr(Address(thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
107 }
108 if (metadata_result->is_valid()) {
109 movptr(Address(thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
110 }
111 if (frame_size() == no_frame_size) {
112 leave();
113 jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
114 } else if (_stub_id == (int)C1StubId::forward_exception_id) {
115 should_not_reach_here();
116 } else {
117 jump(RuntimeAddress(Runtime1::entry_for(C1StubId::forward_exception_id)));
118 }
119 bind(L);
120 }
121 // get oop results if there are any and reset the values in the thread
122 if (oop_result1->is_valid()) {
123 get_vm_result_oop(oop_result1);
124 }
125 if (metadata_result->is_valid()) {
126 get_vm_result_metadata(metadata_result);
127 }
128
129 assert(call_offset >= 0, "Should be set");
130 return call_offset;
131 }
132
133
134 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
135 mov(c_rarg1, arg1);
136 return call_RT(oop_result1, metadata_result, entry, 1);
137 }
138
139
140 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
141 if (c_rarg1 == arg2) {
142 if (c_rarg2 == arg1) {
143 xchgq(arg1, arg2);
144 } else {
145 mov(c_rarg2, arg2);
146 mov(c_rarg1, arg1);
147 }
148 } else {
149 mov(c_rarg1, arg1);
150 mov(c_rarg2, arg2);
151 }
152 return call_RT(oop_result1, metadata_result, entry, 2);
153 }
154
155
156 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
157 // if there is any conflict use the stack
158 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
159 arg2 == c_rarg1 || arg2 == c_rarg3 ||
160 arg3 == c_rarg1 || arg3 == c_rarg2) {
161 push(arg3);
162 push(arg2);
163 push(arg1);
164 pop(c_rarg1);
165 pop(c_rarg2);
166 pop(c_rarg3);
167 } else {
168 mov(c_rarg1, arg1);
169 mov(c_rarg2, arg2);
170 mov(c_rarg3, arg3);
171 }
172 return call_RT(oop_result1, metadata_result, entry, 3);
173 }
174
175
176 // Implementation of StubFrame
177
178 class StubFrame: public StackObj {
179 private:
180 StubAssembler* _sasm;
181 bool _use_pop_on_epilog;
182
183 public:
184 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog = false);
185 void load_argument(int offset_in_words, Register reg);
186
187 ~StubFrame();
188 };
189
190 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
191 set_info(name, must_gc_arguments);
192 enter();
193 }
194
195 void StubAssembler::epilogue(bool use_pop) {
196 // Avoid using a leave instruction when this frame may
197 // have been frozen, since the current value of rbp
198 // restored from the stub would be invalid. We still
199 // must restore the rbp value saved on enter though.
200 use_pop ? pop(rbp) : leave();
201 ret(0);
202 }
203
204 #define __ _sasm->
205
206 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog) {
207 _sasm = sasm;
208 _use_pop_on_epilog = use_pop_on_epilog;
209 __ prologue(name, must_gc_arguments);
210 }
211
212 // load parameters that were stored with LIR_Assembler::store_parameter
213 // Note: offsets for store_parameter and load_argument must match
214 void StubFrame::load_argument(int offset_in_words, Register reg) {
215 __ load_parameter(offset_in_words, reg);
216 }
217
218
219 StubFrame::~StubFrame() {
220 __ epilogue(_use_pop_on_epilog);
221 }
222
223 #undef __
224
225
226 // Implementation of Runtime1
227
228 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
229 const int xmm_regs_as_doubles_size_in_slots = FrameMap::nof_xmm_regs * 2;
230
231 // Stack layout for saving/restoring all the registers needed during a runtime
232 // call (this includes deoptimization)
233 // Note: note that users of this frame may well have arguments to some runtime
234 // while these values are on the stack. These positions neglect those arguments
235 // but the code in save_live_registers will take the argument count into
236 // account.
237 //
238 #define SLOT2(x) x,
239 #define SLOT_PER_WORD 2
240
241 enum reg_save_layout {
242 // 64bit needs to keep stack 16 byte aligned. So we add some alignment dummies to make that
243 // happen and will assert if the stack size we create is misaligned
244 align_dummy_0, align_dummy_1,
245 #ifdef _WIN64
246 // Windows always allocates space for it's argument registers (see
247 // frame::arg_reg_save_area_bytes).
248 arg_reg_save_1, arg_reg_save_1H, // 0, 4
249 arg_reg_save_2, arg_reg_save_2H, // 8, 12
250 arg_reg_save_3, arg_reg_save_3H, // 16, 20
251 arg_reg_save_4, arg_reg_save_4H, // 24, 28
252 #endif // _WIN64
253 xmm_regs_as_doubles_off, // 32
254 float_regs_as_doubles_off = xmm_regs_as_doubles_off + xmm_regs_as_doubles_size_in_slots, // 160
255 fpu_state_off = float_regs_as_doubles_off + float_regs_as_doubles_size_in_slots, // 224
256 // fpu_state_end_off is exclusive
257 fpu_state_end_off = fpu_state_off + (FPUStateSizeInWords / SLOT_PER_WORD), // 352
258 marker = fpu_state_end_off, SLOT2(markerH) // 352, 356
259 extra_space_offset, // 360
260 r15_off = extra_space_offset, r15H_off, // 360, 364
261 r14_off, r14H_off, // 368, 372
262 r13_off, r13H_off, // 376, 380
263 r12_off, r12H_off, // 384, 388
264 r11_off, r11H_off, // 392, 396
265 r10_off, r10H_off, // 400, 404
266 r9_off, r9H_off, // 408, 412
267 r8_off, r8H_off, // 416, 420
268 rdi_off, rdiH_off, // 424, 428
269 rsi_off, SLOT2(rsiH_off) // 432, 436
270 rbp_off, SLOT2(rbpH_off) // 440, 444
271 rsp_off, SLOT2(rspH_off) // 448, 452
272 rbx_off, SLOT2(rbxH_off) // 456, 460
273 rdx_off, SLOT2(rdxH_off) // 464, 468
274 rcx_off, SLOT2(rcxH_off) // 472, 476
275 rax_off, SLOT2(raxH_off) // 480, 484
276 saved_rbp_off, SLOT2(saved_rbpH_off) // 488, 492
277 return_off, SLOT2(returnH_off) // 496, 500
278 reg_save_frame_size // As noted: neglects any parameters to runtime // 504
279 };
280
281 // Save off registers which might be killed by calls into the runtime.
282 // Tries to smart of about FP registers. In particular we separate
283 // saving and describing the FPU registers for deoptimization since we
284 // have to save the FPU registers twice if we describe them and on P4
285 // saving FPU registers which don't contain anything appears
286 // expensive. The deopt blob is the only thing which needs to
287 // describe FPU registers. In all other cases it should be sufficient
288 // to simply save their current value.
289 //
290 static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
291 bool save_fpu_registers = true) {
292
293 // In 64bit all the args are in regs so there are no additional stack slots
294 num_rt_args = 0;
295 assert((reg_save_frame_size * VMRegImpl::stack_slot_size) % 16 == 0, "must be 16 byte aligned");
296 int frame_size_in_slots = reg_save_frame_size + num_rt_args; // args + thread
297 sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word);
298
299 // record saved value locations in an OopMap
300 // locations are offsets from sp after runtime call; num_rt_args is number of arguments in call, including thread
301 OopMap* map = new OopMap(frame_size_in_slots, 0);
302 map->set_callee_saved(VMRegImpl::stack2reg(rax_off + num_rt_args), rax->as_VMReg());
303 map->set_callee_saved(VMRegImpl::stack2reg(rcx_off + num_rt_args), rcx->as_VMReg());
304 map->set_callee_saved(VMRegImpl::stack2reg(rdx_off + num_rt_args), rdx->as_VMReg());
305 map->set_callee_saved(VMRegImpl::stack2reg(rbx_off + num_rt_args), rbx->as_VMReg());
306 map->set_callee_saved(VMRegImpl::stack2reg(rsi_off + num_rt_args), rsi->as_VMReg());
307 map->set_callee_saved(VMRegImpl::stack2reg(rdi_off + num_rt_args), rdi->as_VMReg());
308 map->set_callee_saved(VMRegImpl::stack2reg(r8_off + num_rt_args), r8->as_VMReg());
309 map->set_callee_saved(VMRegImpl::stack2reg(r9_off + num_rt_args), r9->as_VMReg());
310 map->set_callee_saved(VMRegImpl::stack2reg(r10_off + num_rt_args), r10->as_VMReg());
311 map->set_callee_saved(VMRegImpl::stack2reg(r11_off + num_rt_args), r11->as_VMReg());
312 map->set_callee_saved(VMRegImpl::stack2reg(r12_off + num_rt_args), r12->as_VMReg());
313 map->set_callee_saved(VMRegImpl::stack2reg(r13_off + num_rt_args), r13->as_VMReg());
314 map->set_callee_saved(VMRegImpl::stack2reg(r14_off + num_rt_args), r14->as_VMReg());
315 map->set_callee_saved(VMRegImpl::stack2reg(r15_off + num_rt_args), r15->as_VMReg());
316
317 // This is stupid but needed.
318 map->set_callee_saved(VMRegImpl::stack2reg(raxH_off + num_rt_args), rax->as_VMReg()->next());
319 map->set_callee_saved(VMRegImpl::stack2reg(rcxH_off + num_rt_args), rcx->as_VMReg()->next());
320 map->set_callee_saved(VMRegImpl::stack2reg(rdxH_off + num_rt_args), rdx->as_VMReg()->next());
321 map->set_callee_saved(VMRegImpl::stack2reg(rbxH_off + num_rt_args), rbx->as_VMReg()->next());
322 map->set_callee_saved(VMRegImpl::stack2reg(rsiH_off + num_rt_args), rsi->as_VMReg()->next());
323 map->set_callee_saved(VMRegImpl::stack2reg(rdiH_off + num_rt_args), rdi->as_VMReg()->next());
324
325 map->set_callee_saved(VMRegImpl::stack2reg(r8H_off + num_rt_args), r8->as_VMReg()->next());
326 map->set_callee_saved(VMRegImpl::stack2reg(r9H_off + num_rt_args), r9->as_VMReg()->next());
327 map->set_callee_saved(VMRegImpl::stack2reg(r10H_off + num_rt_args), r10->as_VMReg()->next());
328 map->set_callee_saved(VMRegImpl::stack2reg(r11H_off + num_rt_args), r11->as_VMReg()->next());
329 map->set_callee_saved(VMRegImpl::stack2reg(r12H_off + num_rt_args), r12->as_VMReg()->next());
330 map->set_callee_saved(VMRegImpl::stack2reg(r13H_off + num_rt_args), r13->as_VMReg()->next());
331 map->set_callee_saved(VMRegImpl::stack2reg(r14H_off + num_rt_args), r14->as_VMReg()->next());
332 map->set_callee_saved(VMRegImpl::stack2reg(r15H_off + num_rt_args), r15->as_VMReg()->next());
333
334 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
335
336 if (save_fpu_registers) {
337 int xmm_off = xmm_regs_as_doubles_off;
338 for (int n = 0; n < FrameMap::nof_xmm_regs; n++) {
339 if (n < xmm_bypass_limit) {
340 VMReg xmm_name_0 = as_XMMRegister(n)->as_VMReg();
341 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + num_rt_args), xmm_name_0);
342 // %%% This is really a waste but we'll keep things as they were for now
343 if (true) {
344 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + 1 + num_rt_args), xmm_name_0->next());
345 }
346 }
347 xmm_off += 2;
348 }
349 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
350 }
351
352 return map;
353 }
354
355 #define __ this->
356
357 void C1_MacroAssembler::save_live_registers_no_oop_map(bool save_fpu_registers) {
358 __ block_comment("save_live_registers");
359
360 // Push CPU state in multiple of 16 bytes
361 __ save_legacy_gprs();
362
363 __ subptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
364
365 #ifdef ASSERT
366 __ movptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
367 #endif
368
369 if (save_fpu_registers) {
370 // save XMM registers
371 // XMM registers can contain float or double values, but this is not known here,
372 // so always save them as doubles.
373 // note that float values are _not_ converted automatically, so for float values
374 // the second word contains only garbage data.
375 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
376 int offset = 0;
377 for (int n = 0; n < xmm_bypass_limit; n++) {
378 XMMRegister xmm_name = as_XMMRegister(n);
379 __ movdbl(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
380 offset += 8;
381 }
382 }
383 }
384
385 #undef __
386 #define __ sasm->
387
388 static void restore_fpu(C1_MacroAssembler* sasm, bool restore_fpu_registers) {
389 if (restore_fpu_registers) {
390 // restore XMM registers
391 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
392 int offset = 0;
393 for (int n = 0; n < xmm_bypass_limit; n++) {
394 XMMRegister xmm_name = as_XMMRegister(n);
395 __ movdbl(xmm_name, Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset));
396 offset += 8;
397 }
398 }
399
400 #ifdef ASSERT
401 {
402 Label ok;
403 __ cmpptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
404 __ jcc(Assembler::equal, ok);
405 __ stop("bad offsets in frame");
406 __ bind(ok);
407 }
408 #endif // ASSERT
409
410 __ addptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
411 }
412
413 #undef __
414 #define __ this->
415
416 void C1_MacroAssembler::restore_live_registers(bool restore_fpu_registers) {
417 __ block_comment("restore_live_registers");
418
419 restore_fpu(this, restore_fpu_registers);
420 __ restore_legacy_gprs();
421 }
422
423
424 void C1_MacroAssembler::restore_live_registers_except_rax(bool restore_fpu_registers) {
425 __ block_comment("restore_live_registers_except_rax");
426
427 restore_fpu(this, restore_fpu_registers);
428
429 __ movptr(r15, Address(rsp, 0));
430 __ movptr(r14, Address(rsp, wordSize));
431 __ movptr(r13, Address(rsp, 2 * wordSize));
432 __ movptr(r12, Address(rsp, 3 * wordSize));
433 __ movptr(r11, Address(rsp, 4 * wordSize));
434 __ movptr(r10, Address(rsp, 5 * wordSize));
435 __ movptr(r9, Address(rsp, 6 * wordSize));
436 __ movptr(r8, Address(rsp, 7 * wordSize));
437 __ movptr(rdi, Address(rsp, 8 * wordSize));
438 __ movptr(rsi, Address(rsp, 9 * wordSize));
439 __ movptr(rbp, Address(rsp, 10 * wordSize));
440 // skip rsp
441 __ movptr(rbx, Address(rsp, 12 * wordSize));
442 __ movptr(rdx, Address(rsp, 13 * wordSize));
443 __ movptr(rcx, Address(rsp, 14 * wordSize));
444
445 __ addptr(rsp, 16 * wordSize);
446 }
447
448 #undef __
449 #define __ sasm->
450
451 static OopMap* save_live_registers(StubAssembler* sasm, int num_rt_args,
452 bool save_fpu_registers = true) {
453 __ save_live_registers_no_oop_map(save_fpu_registers);
454 return generate_oop_map(sasm, num_rt_args, save_fpu_registers);
455 }
456
457 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
458 __ restore_live_registers(restore_fpu_registers);
459 }
460
461 static void restore_live_registers_except_rax(StubAssembler* sasm, bool restore_fpu_registers = true) {
462 sasm->restore_live_registers_except_rax(restore_fpu_registers);
463 }
464
465
466 void Runtime1::initialize_pd() {
467 // nothing to do
468 }
469
470 // return: offset in 64-bit words.
471 uint Runtime1::runtime_blob_current_thread_offset(frame f) {
472 return r15_off / 2; // rsp offsets are in halfwords
473 }
474
475 // Target: the entry point of the method that creates and posts the exception oop.
476 // has_argument: true if the exception needs arguments (passed on the stack because
477 // registers must be preserved).
478 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
479 // Preserve all registers.
480 int num_rt_args = has_argument ? (2 + 1) : 1;
481 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
482
483 // Now all registers are saved and can be used freely.
484 // Verify that no old value is used accidentally.
485 __ invalidate_registers(true, true, true, true, true, true);
486
487 // Registers used by this stub.
488 const Register temp_reg = rbx;
489
490 // Load arguments for exception that are passed as arguments into the stub.
491 if (has_argument) {
492 __ movptr(c_rarg1, Address(rbp, 2*BytesPerWord));
493 __ movptr(c_rarg2, Address(rbp, 3*BytesPerWord));
494 }
495 int call_offset = __ call_RT(noreg, noreg, target, num_rt_args - 1);
496
497 OopMapSet* oop_maps = new OopMapSet();
498 oop_maps->add_gc_map(call_offset, oop_map);
499
500 __ stop("should not reach here");
501
502 return oop_maps;
503 }
504
505
506 OopMapSet* Runtime1::generate_handle_exception(C1StubId id, StubAssembler *sasm) {
507 __ block_comment("generate_handle_exception");
508
509 // incoming parameters
510 const Register exception_oop = rax;
511 const Register exception_pc = rdx;
512 // other registers used in this stub
513 const Register thread = r15_thread;
514
515 // Save registers, if required.
516 OopMapSet* oop_maps = new OopMapSet();
517 OopMap* oop_map = nullptr;
518 switch (id) {
519 case C1StubId::forward_exception_id:
520 // We're handling an exception in the context of a compiled frame.
521 // The registers have been saved in the standard places. Perform
522 // an exception lookup in the caller and dispatch to the handler
523 // if found. Otherwise unwind and dispatch to the callers
524 // exception handler.
525 oop_map = generate_oop_map(sasm, 1 /*thread*/);
526
527 // load and clear pending exception oop into RAX
528 __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
529 __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
530
531 // load issuing PC (the return address for this stub) into rdx
532 __ movptr(exception_pc, Address(rbp, 1*BytesPerWord));
533
534 // make sure that the vm_results are cleared (may be unnecessary)
535 __ movptr(Address(thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
536 __ movptr(Address(thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
537 break;
538 case C1StubId::handle_exception_nofpu_id:
539 case C1StubId::handle_exception_id:
540 // At this point all registers MAY be live.
541 oop_map = save_live_registers(sasm, 1 /*thread*/, id != C1StubId::handle_exception_nofpu_id);
542 break;
543 case C1StubId::handle_exception_from_callee_id: {
544 // At this point all registers except exception oop (RAX) and
545 // exception pc (RDX) are dead.
546 const int frame_size = 2 /*BP, return address*/ WIN64_ONLY(+ frame::arg_reg_save_area_bytes / BytesPerWord);
547 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
548 sasm->set_frame_size(frame_size);
549 WIN64_ONLY(__ subq(rsp, frame::arg_reg_save_area_bytes));
550 break;
551 }
552 default: ShouldNotReachHere();
553 }
554
555 // verify that only rax, and rdx is valid at this time
556 __ invalidate_registers(false, true, true, false, true, true);
557 // verify that rax, contains a valid exception
558 __ verify_not_null_oop(exception_oop);
559
560 #ifdef ASSERT
561 // check that fields in JavaThread for exception oop and issuing pc are
562 // empty before writing to them
563 Label oop_empty;
564 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
565 __ jcc(Assembler::equal, oop_empty);
566 __ stop("exception oop already set");
567 __ bind(oop_empty);
568
569 Label pc_empty;
570 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), 0);
571 __ jcc(Assembler::equal, pc_empty);
572 __ stop("exception pc already set");
573 __ bind(pc_empty);
574 #endif
575
576 // save exception oop and issuing pc into JavaThread
577 // (exception handler will load it from here)
578 __ movptr(Address(thread, JavaThread::exception_oop_offset()), exception_oop);
579 __ movptr(Address(thread, JavaThread::exception_pc_offset()), exception_pc);
580
581 // patch throwing pc into return address (has bci & oop map)
582 __ movptr(Address(rbp, 1*BytesPerWord), exception_pc);
583
584 // compute the exception handler.
585 // the exception oop and the throwing pc are read from the fields in JavaThread
586 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
587 oop_maps->add_gc_map(call_offset, oop_map);
588
589 // rax: handler address
590 // will be the deopt blob if nmethod was deoptimized while we looked up
591 // handler regardless of whether handler existed in the nmethod.
592
593 // only rax, is valid at this time, all other registers have been destroyed by the runtime call
594 __ invalidate_registers(false, true, true, true, true, true);
595
596 // patch the return address, this stub will directly return to the exception handler
597 __ movptr(Address(rbp, 1*BytesPerWord), rax);
598
599 switch (id) {
600 case C1StubId::forward_exception_id:
601 case C1StubId::handle_exception_nofpu_id:
602 case C1StubId::handle_exception_id:
603 // Restore the registers that were saved at the beginning.
604 restore_live_registers(sasm, id != C1StubId::handle_exception_nofpu_id);
605 break;
606 case C1StubId::handle_exception_from_callee_id:
607 // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP
608 // since we do a leave anyway.
609
610 // Pop the return address.
611 __ leave();
612 __ pop(rcx);
613 __ jmp(rcx); // jump to exception handler
614 break;
615 default: ShouldNotReachHere();
616 }
617
618 return oop_maps;
619 }
620
621
622 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
623 // incoming parameters
624 const Register exception_oop = rax;
625 // callee-saved copy of exception_oop during runtime call
626 const Register exception_oop_callee_saved = r14;
627 // other registers used in this stub
628 const Register exception_pc = rdx;
629 const Register handler_addr = rbx;
630 const Register thread = r15_thread;
631
632 if (AbortVMOnException) {
633 __ enter();
634 save_live_registers(sasm, 2);
635 __ call_VM_leaf(CAST_FROM_FN_PTR(address, check_abort_on_vm_exception), rax);
636 restore_live_registers(sasm);
637 __ leave();
638 }
639
640 // verify that only rax, is valid at this time
641 __ invalidate_registers(false, true, true, true, true, true);
642
643 #ifdef ASSERT
644 // check that fields in JavaThread for exception oop and issuing pc are empty
645 Label oop_empty;
646 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), 0);
647 __ jcc(Assembler::equal, oop_empty);
648 __ stop("exception oop must be empty");
649 __ bind(oop_empty);
650
651 Label pc_empty;
652 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), 0);
653 __ jcc(Assembler::equal, pc_empty);
654 __ stop("exception pc must be empty");
655 __ bind(pc_empty);
656 #endif
657
658 // save exception_oop in callee-saved register to preserve it during runtime calls
659 __ verify_not_null_oop(exception_oop);
660 __ movptr(exception_oop_callee_saved, exception_oop);
661
662 // Get return address (is on top of stack after leave).
663 __ movptr(exception_pc, Address(rsp, 0));
664
665 // search the exception handler address of the caller (using the return address)
666 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
667 // rax: exception handler address of the caller
668
669 // Only RAX and RSI are valid at this time, all other registers have been destroyed by the call.
670 __ invalidate_registers(false, true, true, true, false, true);
671
672 // move result of call into correct register
673 __ movptr(handler_addr, rax);
674
675 // Restore exception oop to RAX (required convention of exception handler).
676 __ movptr(exception_oop, exception_oop_callee_saved);
677
678 // verify that there is really a valid exception in rax
679 __ verify_not_null_oop(exception_oop);
680
681 // get throwing pc (= return address).
682 // rdx has been destroyed by the call, so it must be set again
683 // the pop is also necessary to simulate the effect of a ret(0)
684 __ pop(exception_pc);
685
686 // continue at exception handler (return address removed)
687 // note: do *not* remove arguments when unwinding the
688 // activation since the caller assumes having
689 // all arguments on the stack when entering the
690 // runtime to determine the exception handler
691 // (GC happens at call site with arguments!)
692 // rax: exception oop
693 // rdx: throwing pc
694 // rbx: exception handler
695 __ jmp(handler_addr);
696 }
697
698
699 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
700 // use the maximum number of runtime-arguments here because it is difficult to
701 // distinguish each RT-Call.
702 // Note: This number affects also the RT-Call in generate_handle_exception because
703 // the oop-map is shared for all calls.
704 const int num_rt_args = 2; // thread + dummy
705
706 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
707 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
708
709 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
710
711 const Register thread = r15_thread;
712 // No need to worry about dummy
713 __ mov(c_rarg0, thread);
714 __ set_last_Java_frame(noreg, rbp, nullptr, rscratch1);
715 // do the call
716 __ call(RuntimeAddress(target));
717 OopMapSet* oop_maps = new OopMapSet();
718 oop_maps->add_gc_map(__ offset(), oop_map);
719 // verify callee-saved register
720 #ifdef ASSERT
721 guarantee(thread != rax, "change this code");
722 __ push(rax);
723 { Label L;
724 __ get_thread_slow(rax);
725 __ cmpptr(thread, rax);
726 __ jcc(Assembler::equal, L);
727 __ stop("StubAssembler::call_RT: rdi/r15 not callee saved?");
728 __ bind(L);
729 }
730 __ pop(rax);
731 #endif
732 __ reset_last_Java_frame(true);
733
734 // check for pending exceptions
735 { Label L;
736 __ cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
737 __ jcc(Assembler::equal, L);
738 // exception pending => remove activation and forward to exception handler
739
740 __ testptr(rax, rax); // have we deoptimized?
741 __ jump_cc(Assembler::equal,
742 RuntimeAddress(Runtime1::entry_for(C1StubId::forward_exception_id)));
743
744 // the deopt blob expects exceptions in the special fields of
745 // JavaThread, so copy and clear pending exception.
746
747 // load and clear pending exception
748 __ movptr(rax, Address(thread, Thread::pending_exception_offset()));
749 __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
750
751 // check that there is really a valid exception
752 __ verify_not_null_oop(rax);
753
754 // load throwing pc: this is the return address of the stub
755 __ movptr(rdx, Address(rsp, return_off * VMRegImpl::stack_slot_size));
756
757 #ifdef ASSERT
758 // check that fields in JavaThread for exception oop and issuing pc are empty
759 Label oop_empty;
760 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
761 __ jcc(Assembler::equal, oop_empty);
762 __ stop("exception oop must be empty");
763 __ bind(oop_empty);
764
765 Label pc_empty;
766 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
767 __ jcc(Assembler::equal, pc_empty);
768 __ stop("exception pc must be empty");
769 __ bind(pc_empty);
770 #endif
771
772 // store exception oop and throwing pc to JavaThread
773 __ movptr(Address(thread, JavaThread::exception_oop_offset()), rax);
774 __ movptr(Address(thread, JavaThread::exception_pc_offset()), rdx);
775
776 restore_live_registers(sasm);
777
778 __ leave();
779 __ addptr(rsp, BytesPerWord); // remove return address from stack
780
781 // Forward the exception directly to deopt blob. We can blow no
782 // registers and must leave throwing pc on the stack. A patch may
783 // have values live in registers so the entry point with the
784 // exception in tls.
785 __ jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
786
787 __ bind(L);
788 }
789
790
791 // Runtime will return true if the nmethod has been deoptimized during
792 // the patching process. In that case we must do a deopt reexecute instead.
793
794 Label cont;
795
796 __ testptr(rax, rax); // have we deoptimized?
797 __ jcc(Assembler::equal, cont); // no
798
799 // Will reexecute. Proper return address is already on the stack we just restore
800 // registers, pop all of our frame but the return address and jump to the deopt blob
801 restore_live_registers(sasm);
802 __ leave();
803 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
804
805 __ bind(cont);
806 restore_live_registers(sasm);
807 __ leave();
808 __ ret(0);
809
810 return oop_maps;
811 }
812
813
814 OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) {
815
816 // for better readability
817 const bool must_gc_arguments = true;
818 const bool dont_gc_arguments = false;
819
820 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
821 bool save_fpu_registers = true;
822
823 // stub code & info for the different stubs
824 OopMapSet* oop_maps = nullptr;
825 switch (id) {
826 case C1StubId::forward_exception_id:
827 {
828 oop_maps = generate_handle_exception(id, sasm);
829 __ leave();
830 __ ret(0);
831 }
832 break;
833
834 case C1StubId::new_instance_id:
835 case C1StubId::fast_new_instance_id:
836 case C1StubId::fast_new_instance_init_check_id:
837 {
838 Register klass = rdx; // Incoming
839 Register obj = rax; // Result
840
841 if (id == C1StubId::new_instance_id) {
842 __ set_info("new_instance", dont_gc_arguments);
843 } else if (id == C1StubId::fast_new_instance_id) {
844 __ set_info("fast new_instance", dont_gc_arguments);
845 } else {
846 assert(id == C1StubId::fast_new_instance_init_check_id, "bad C1StubId");
847 __ set_info("fast new_instance init check", dont_gc_arguments);
848 }
849
850 __ enter();
851 OopMap* map = save_live_registers(sasm, 2);
852 int call_offset;
853 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
854 oop_maps = new OopMapSet();
855 oop_maps->add_gc_map(call_offset, map);
856 restore_live_registers_except_rax(sasm);
857 __ verify_oop(obj);
858 __ leave();
859 __ ret(0);
860
861 // rax,: new instance
862 }
863
864 break;
865
866 case C1StubId::counter_overflow_id:
867 {
868 Register bci = rax, method = rbx;
869 __ enter();
870 OopMap* map = save_live_registers(sasm, 3);
871 // Retrieve bci
872 __ movl(bci, Address(rbp, 2*BytesPerWord));
873 // And a pointer to the Method*
874 __ movptr(method, Address(rbp, 3*BytesPerWord));
875 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
876 oop_maps = new OopMapSet();
877 oop_maps->add_gc_map(call_offset, map);
878 restore_live_registers(sasm);
879 __ leave();
880 __ ret(0);
881 }
882 break;
883
884 case C1StubId::new_type_array_id:
885 case C1StubId::new_object_array_id:
886 case C1StubId::new_null_free_array_id:
887 {
888 Register length = rbx; // Incoming
889 Register klass = rdx; // Incoming
890 Register obj = rax; // Result
891
892 if (id == C1StubId::new_type_array_id) {
893 __ set_info("new_type_array", dont_gc_arguments);
894 } else if (id == C1StubId::new_object_array_id) {
895 __ set_info("new_object_array", dont_gc_arguments);
896 } else {
897 __ set_info("new_null_free_array", dont_gc_arguments);
898 }
899
900 #ifdef ASSERT
901 // assert object type is really an array of the proper kind
902 {
903 Label ok;
904 Register t0 = obj;
905 __ movl(t0, Address(klass, Klass::layout_helper_offset()));
906 __ sarl(t0, Klass::_lh_array_tag_shift);
907 switch (id) {
908 case C1StubId::new_type_array_id:
909 __ cmpl(t0, Klass::_lh_array_tag_type_value);
910 __ jcc(Assembler::equal, ok);
911 __ stop("assert(is a type array klass)");
912 break;
913 case C1StubId::new_object_array_id:
914 __ cmpl(t0, Klass::_lh_array_tag_obj_value); // new "[Ljava/lang/Object;"
915 __ jcc(Assembler::equal, ok);
916 __ cmpl(t0, Klass::_lh_array_tag_vt_value); // new "[LVT;"
917 __ jcc(Assembler::equal, ok);
918 __ stop("assert(is an object or inline type array klass)");
919 break;
920 case C1StubId::new_null_free_array_id:
921 __ cmpl(t0, Klass::_lh_array_tag_vt_value); // the array can be a flat array.
922 __ jcc(Assembler::equal, ok);
923 __ cmpl(t0, Klass::_lh_array_tag_obj_value); // the array cannot be a flat array (due to InlineArrayElementMaxFlatSize, etc)
924 __ jcc(Assembler::equal, ok);
925 __ stop("assert(is an object or inline type array klass)");
926 break;
927 default: ShouldNotReachHere();
928 }
929 __ should_not_reach_here();
930 __ bind(ok);
931 }
932 #endif // ASSERT
933
934 __ enter();
935 OopMap* map = save_live_registers(sasm, 3);
936 int call_offset;
937 if (id == C1StubId::new_type_array_id) {
938 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
939 } else if (id == C1StubId::new_object_array_id) {
940 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
941 } else {
942 assert(id == C1StubId::new_null_free_array_id, "must be");
943 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_null_free_array), klass, length);
944 }
945
946 oop_maps = new OopMapSet();
947 oop_maps->add_gc_map(call_offset, map);
948 restore_live_registers_except_rax(sasm);
949
950 __ verify_oop(obj);
951 __ leave();
952 __ ret(0);
953
954 // rax,: new array
955 }
956 break;
957
958 case C1StubId::new_multi_array_id:
959 { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
960 // rax,: klass
961 // rbx,: rank
962 // rcx: address of 1st dimension
963 OopMap* map = save_live_registers(sasm, 4);
964 int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, new_multi_array), rax, rbx, rcx);
965
966 oop_maps = new OopMapSet();
967 oop_maps->add_gc_map(call_offset, map);
968 restore_live_registers_except_rax(sasm);
969
970 // rax,: new multi array
971 __ verify_oop(rax);
972 }
973 break;
974
975 case C1StubId::load_flat_array_id:
976 {
977 StubFrame f(sasm, "load_flat_array", dont_gc_arguments);
978 OopMap* map = save_live_registers(sasm, 3);
979
980 // Called with store_parameter and not C abi
981
982 f.load_argument(1, rax); // rax,: array
983 f.load_argument(0, rbx); // rbx,: index
984 int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, load_flat_array), rax, rbx);
985
986 oop_maps = new OopMapSet();
987 oop_maps->add_gc_map(call_offset, map);
988 restore_live_registers_except_rax(sasm);
989
990 // rax,: loaded element at array[index]
991 __ verify_oop(rax);
992 }
993 break;
994
995 case C1StubId::store_flat_array_id:
996 {
997 StubFrame f(sasm, "store_flat_array", dont_gc_arguments);
998 OopMap* map = save_live_registers(sasm, 4);
999
1000 // Called with store_parameter and not C abi
1001
1002 f.load_argument(2, rax); // rax,: array
1003 f.load_argument(1, rbx); // rbx,: index
1004 f.load_argument(0, rcx); // rcx,: value
1005 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, store_flat_array), rax, rbx, rcx);
1006
1007 oop_maps = new OopMapSet();
1008 oop_maps->add_gc_map(call_offset, map);
1009 restore_live_registers_except_rax(sasm);
1010 }
1011 break;
1012
1013 case C1StubId::substitutability_check_id:
1014 {
1015 StubFrame f(sasm, "substitutability_check", dont_gc_arguments);
1016 OopMap* map = save_live_registers(sasm, 3);
1017
1018 // Called with store_parameter and not C abi
1019
1020 f.load_argument(1, rax); // rax,: left
1021 f.load_argument(0, rbx); // rbx,: right
1022 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, substitutability_check), rax, rbx);
1023
1024 oop_maps = new OopMapSet();
1025 oop_maps->add_gc_map(call_offset, map);
1026 restore_live_registers_except_rax(sasm);
1027
1028 // rax,: are the two operands substitutable
1029 }
1030 break;
1031
1032
1033 case C1StubId::buffer_inline_args_id:
1034 case C1StubId::buffer_inline_args_no_receiver_id:
1035 {
1036 const char* name = (id == C1StubId::buffer_inline_args_id) ?
1037 "buffer_inline_args" : "buffer_inline_args_no_receiver";
1038 StubFrame f(sasm, name, dont_gc_arguments);
1039 OopMap* map = save_live_registers(sasm, 2);
1040 Register method = rbx;
1041 address entry = (id == C1StubId::buffer_inline_args_id) ?
1042 CAST_FROM_FN_PTR(address, buffer_inline_args) :
1043 CAST_FROM_FN_PTR(address, buffer_inline_args_no_receiver);
1044 int call_offset = __ call_RT(rax, noreg, entry, method);
1045 oop_maps = new OopMapSet();
1046 oop_maps->add_gc_map(call_offset, map);
1047 restore_live_registers_except_rax(sasm);
1048 __ verify_oop(rax); // rax: an array of buffered value objects
1049 }
1050 break;
1051
1052 case C1StubId::register_finalizer_id:
1053 {
1054 __ set_info("register_finalizer", dont_gc_arguments);
1055
1056 // This is called via call_runtime so the arguments
1057 // will be place in C abi locations
1058
1059 __ verify_oop(c_rarg0);
1060 __ mov(rax, c_rarg0);
1061
1062 // load the klass and check the has finalizer flag
1063 Label register_finalizer;
1064 Register t = rsi;
1065 __ load_klass(t, rax, rscratch1);
1066 __ testb(Address(t, Klass::misc_flags_offset()), KlassFlags::_misc_has_finalizer);
1067 __ jcc(Assembler::notZero, register_finalizer);
1068 __ ret(0);
1069
1070 __ bind(register_finalizer);
1071 __ enter();
1072 OopMap* oop_map = save_live_registers(sasm, 2 /*num_rt_args */);
1073 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), rax);
1074 oop_maps = new OopMapSet();
1075 oop_maps->add_gc_map(call_offset, oop_map);
1076
1077 // Now restore all the live registers
1078 restore_live_registers(sasm);
1079
1080 __ leave();
1081 __ ret(0);
1082 }
1083 break;
1084
1085 case C1StubId::throw_range_check_failed_id:
1086 { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1087 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1088 }
1089 break;
1090
1091 case C1StubId::throw_index_exception_id:
1092 { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1093 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1094 }
1095 break;
1096
1097 case C1StubId::throw_div0_exception_id:
1098 { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
1099 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
1100 }
1101 break;
1102
1103 case C1StubId::throw_null_pointer_exception_id:
1104 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
1105 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
1106 }
1107 break;
1108
1109 case C1StubId::handle_exception_nofpu_id:
1110 case C1StubId::handle_exception_id:
1111 { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1112 oop_maps = generate_handle_exception(id, sasm);
1113 }
1114 break;
1115
1116 case C1StubId::handle_exception_from_callee_id:
1117 { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1118 oop_maps = generate_handle_exception(id, sasm);
1119 }
1120 break;
1121
1122 case C1StubId::unwind_exception_id:
1123 { __ set_info("unwind_exception", dont_gc_arguments);
1124 // note: no stubframe since we are about to leave the current
1125 // activation and we are calling a leaf VM function only.
1126 generate_unwind_exception(sasm);
1127 }
1128 break;
1129
1130 case C1StubId::throw_array_store_exception_id:
1131 { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1132 // tos + 0: link
1133 // + 1: return address
1134 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1135 }
1136 break;
1137
1138 case C1StubId::throw_class_cast_exception_id:
1139 { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
1140 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
1141 }
1142 break;
1143
1144 case C1StubId::throw_incompatible_class_change_error_id:
1145 { StubFrame f(sasm, "throw_incompatible_class_change_error", dont_gc_arguments);
1146 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
1147 }
1148 break;
1149
1150 case C1StubId::throw_illegal_monitor_state_exception_id:
1151 { StubFrame f(sasm, "throw_illegal_monitor_state_exception", dont_gc_arguments);
1152 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_illegal_monitor_state_exception), false);
1153 }
1154 break;
1155
1156 case C1StubId::throw_identity_exception_id:
1157 { StubFrame f(sasm, "throw_identity_exception", dont_gc_arguments);
1158 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_identity_exception), true);
1159 }
1160 break;
1161
1162 case C1StubId::slow_subtype_check_id:
1163 {
1164 // Typical calling sequence:
1165 // __ push(klass_RInfo); // object klass or other subclass
1166 // __ push(sup_k_RInfo); // array element klass or other superclass
1167 // __ call(slow_subtype_check);
1168 // Note that the subclass is pushed first, and is therefore deepest.
1169 // Previous versions of this code reversed the names 'sub' and 'super'.
1170 // This was operationally harmless but made the code unreadable.
1171 enum layout {
1172 rax_off, SLOT2(raxH_off)
1173 rcx_off, SLOT2(rcxH_off)
1174 rsi_off, SLOT2(rsiH_off)
1175 rdi_off, SLOT2(rdiH_off)
1176 // saved_rbp_off, SLOT2(saved_rbpH_off)
1177 return_off, SLOT2(returnH_off)
1178 sup_k_off, SLOT2(sup_kH_off)
1179 klass_off, SLOT2(superH_off)
1180 framesize,
1181 result_off = klass_off // deepest argument is also the return value
1182 };
1183
1184 __ set_info("slow_subtype_check", dont_gc_arguments);
1185 __ push(rdi);
1186 __ push(rsi);
1187 __ push(rcx);
1188 __ push(rax);
1189
1190 // This is called by pushing args and not with C abi
1191 __ movptr(rsi, Address(rsp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1192 __ movptr(rax, Address(rsp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1193
1194 Label miss;
1195 __ check_klass_subtype_slow_path(rsi, rax, rcx, rdi, nullptr, &miss);
1196
1197 // fallthrough on success:
1198 __ movptr(Address(rsp, (result_off) * VMRegImpl::stack_slot_size), 1); // result
1199 __ pop(rax);
1200 __ pop(rcx);
1201 __ pop(rsi);
1202 __ pop(rdi);
1203 __ ret(0);
1204
1205 __ bind(miss);
1206 __ movptr(Address(rsp, (result_off) * VMRegImpl::stack_slot_size), NULL_WORD); // result
1207 __ pop(rax);
1208 __ pop(rcx);
1209 __ pop(rsi);
1210 __ pop(rdi);
1211 __ ret(0);
1212 }
1213 break;
1214
1215 case C1StubId::is_instance_of_id:
1216 {
1217 // Mirror: c_rarg0 (Windows: rcx, SysV: rdi)
1218 // Object: c_rarg1 (Windows: rdx, SysV: rsi)
1219 // ObjClass: r9
1220 // Temps: rcx, r8, r10, r11
1221 // Result: rax
1222
1223 Register klass = r9, obj = c_rarg1, result = rax;
1224 Register temp0 = rcx, temp1 = r8, temp2 = r10, temp3 = r11;
1225
1226 // Get the Klass* into r9. c_rarg0 is now dead.
1227 __ movptr(klass, Address(c_rarg0, java_lang_Class::klass_offset()));
1228
1229 Label done, is_secondary, same;
1230
1231 __ xorq(result, result);
1232 __ testq(klass, klass);
1233 __ jcc(Assembler::equal, done); // Klass is null
1234
1235 __ testq(obj, obj);
1236 __ jcc(Assembler::equal, done); // obj is null
1237
1238 __ movl(temp0, Address(klass, in_bytes(Klass::super_check_offset_offset())));
1239 __ cmpl(temp0, in_bytes(Klass::secondary_super_cache_offset()));
1240 __ jcc(Assembler::equal, is_secondary); // Klass is a secondary superclass
1241
1242 // Klass is a concrete class
1243 __ load_klass(temp2, obj, /*tmp*/temp1);
1244 __ cmpptr(klass, Address(temp2, temp0));
1245 __ setcc(Assembler::equal, result);
1246 __ ret(0);
1247
1248 __ bind(is_secondary);
1249
1250 __ load_klass(obj, obj, /*tmp*/temp1);
1251
1252 // This is necessary because I am never in my own secondary_super list.
1253 __ cmpptr(obj, klass);
1254 __ jcc(Assembler::equal, same);
1255
1256 __ lookup_secondary_supers_table_var(obj, klass,
1257 /*temps*/temp0, temp1, temp2, temp3,
1258 result);
1259 __ testq(result, result);
1260
1261 __ bind(same);
1262 __ setcc(Assembler::equal, result);
1263
1264 __ bind(done);
1265 __ ret(0);
1266 }
1267 break;
1268
1269 case C1StubId::monitorenter_nofpu_id:
1270 save_fpu_registers = false;
1271 // fall through
1272 case C1StubId::monitorenter_id:
1273 {
1274 StubFrame f(sasm, "monitorenter", dont_gc_arguments, true /* use_pop_on_epilog */);
1275 OopMap* map = save_live_registers(sasm, 3, save_fpu_registers);
1276
1277 // Called with store_parameter and not C abi
1278
1279 f.load_argument(1, rax); // rax,: object
1280 f.load_argument(0, rbx); // rbx,: lock address
1281
1282 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), rax, rbx);
1283
1284 oop_maps = new OopMapSet();
1285 oop_maps->add_gc_map(call_offset, map);
1286 restore_live_registers(sasm, save_fpu_registers);
1287 }
1288 break;
1289
1290 case C1StubId::monitorexit_nofpu_id:
1291 save_fpu_registers = false;
1292 // fall through
1293 case C1StubId::monitorexit_id:
1294 {
1295 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1296 OopMap* map = save_live_registers(sasm, 2, save_fpu_registers);
1297
1298 // Called with store_parameter and not C abi
1299
1300 f.load_argument(0, rax); // rax,: lock address
1301
1302 // note: really a leaf routine but must setup last java sp
1303 // => use call_RT for now (speed can be improved by
1304 // doing last java sp setup manually)
1305 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax);
1306
1307 oop_maps = new OopMapSet();
1308 oop_maps->add_gc_map(call_offset, map);
1309 restore_live_registers(sasm, save_fpu_registers);
1310 }
1311 break;
1312
1313 case C1StubId::deoptimize_id:
1314 {
1315 StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1316 const int num_rt_args = 2; // thread, trap_request
1317 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
1318 f.load_argument(0, rax);
1319 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), rax);
1320 oop_maps = new OopMapSet();
1321 oop_maps->add_gc_map(call_offset, oop_map);
1322 restore_live_registers(sasm);
1323 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1324 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1325 __ leave();
1326 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1327 }
1328 break;
1329
1330 case C1StubId::access_field_patching_id:
1331 { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1332 // we should set up register map
1333 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1334 }
1335 break;
1336
1337 case C1StubId::load_klass_patching_id:
1338 { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1339 // we should set up register map
1340 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1341 }
1342 break;
1343
1344 case C1StubId::load_mirror_patching_id:
1345 { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1346 // we should set up register map
1347 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1348 }
1349 break;
1350
1351 case C1StubId::load_appendix_patching_id:
1352 { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1353 // we should set up register map
1354 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1355 }
1356 break;
1357
1358 case C1StubId::dtrace_object_alloc_id:
1359 { // rax,: object
1360 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1361 // we can't gc here so skip the oopmap but make sure that all
1362 // the live registers get saved.
1363 save_live_registers(sasm, 1);
1364
1365 __ mov(c_rarg0, rax);
1366 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc))));
1367
1368 restore_live_registers(sasm);
1369 }
1370 break;
1371
1372 case C1StubId::fpu2long_stub_id:
1373 {
1374 Label done;
1375 __ cvttsd2siq(rax, Address(rsp, wordSize));
1376 __ cmp64(rax, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
1377 __ jccb(Assembler::notEqual, done);
1378 __ movq(rax, Address(rsp, wordSize));
1379 __ subptr(rsp, 8);
1380 __ movq(Address(rsp, 0), rax);
1381 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
1382 __ pop(rax);
1383 __ bind(done);
1384 __ ret(0);
1385 }
1386 break;
1387
1388 case C1StubId::predicate_failed_trap_id:
1389 {
1390 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1391
1392 OopMap* map = save_live_registers(sasm, 1);
1393
1394 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1395 oop_maps = new OopMapSet();
1396 oop_maps->add_gc_map(call_offset, map);
1397 restore_live_registers(sasm);
1398 __ leave();
1399 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1400 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1401
1402 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1403 }
1404 break;
1405
1406 default:
1407 { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1408 __ movptr(rax, (int)id);
1409 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
1410 __ should_not_reach_here();
1411 }
1412 break;
1413 }
1414 return oop_maps;
1415 }
1416
1417 #undef __
1418
1419 const char *Runtime1::pd_name_for_address(address entry) {
1420 return "<unknown function>";
1421 }