1 /*
2 * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "c1/c1_Defs.hpp"
27 #include "c1/c1_FrameMap.hpp"
28 #include "c1/c1_MacroAssembler.hpp"
29 #include "c1/c1_Runtime1.hpp"
30 #include "ci/ciUtilities.hpp"
31 #include "compiler/compilerDefinitions.inline.hpp"
32 #include "compiler/oopMap.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/cardTableBarrierSet.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/universe.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "register_x86.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/signature.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/vframeArray.hpp"
47 #include "utilities/macros.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50 // Implementation of StubAssembler
51
52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
53 // setup registers
54 const Register thread = r15_thread;
55 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
56 assert(oop_result1 != thread && metadata_result != thread, "registers must be different");
57 assert(args_size >= 0, "illegal args_size");
58 bool align_stack = false;
59
60 // At a method handle call, the stack may not be properly aligned
61 // when returning with an exception.
62 align_stack = (stub_id() == (int)StubId::c1_handle_exception_from_callee_id);
63
64 mov(c_rarg0, thread);
65 set_num_rt_args(0); // Nothing on stack
66
67 int call_offset = -1;
68 if (!align_stack) {
69 set_last_Java_frame(noreg, rbp, nullptr, rscratch1);
70 } else {
71 address the_pc = pc();
72 call_offset = offset();
73 set_last_Java_frame(noreg, rbp, the_pc, rscratch1);
74 andptr(rsp, -(StackAlignmentInBytes)); // Align stack
75 }
76
77 // do the call
78 call(RuntimeAddress(entry));
79 if (!align_stack) {
80 call_offset = offset();
81 }
82 // verify callee-saved register
83 #ifdef ASSERT
84 guarantee(thread != rax, "change this code");
85 push(rax);
86 { Label L;
87 get_thread_slow(rax);
88 cmpptr(thread, rax);
89 jcc(Assembler::equal, L);
90 int3();
91 stop("StubAssembler::call_RT: rdi not callee saved?");
92 bind(L);
93 }
94 pop(rax);
95 #endif
96 reset_last_Java_frame(true);
97
98 // check for pending exceptions
99 { Label L;
100 cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
101 jcc(Assembler::equal, L);
102 // exception pending => remove activation and forward to exception handler
103 movptr(rax, Address(thread, Thread::pending_exception_offset()));
104 // make sure that the vm_results are cleared
105 if (oop_result1->is_valid()) {
106 movptr(Address(thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
107 }
108 if (metadata_result->is_valid()) {
109 movptr(Address(thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
110 }
111 if (frame_size() == no_frame_size) {
112 leave();
113 jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
114 } else if (_stub_id == (int)StubId::c1_forward_exception_id) {
115 should_not_reach_here();
116 } else {
117 jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_forward_exception_id)));
118 }
119 bind(L);
120 }
121 // get oop results if there are any and reset the values in the thread
122 if (oop_result1->is_valid()) {
123 get_vm_result_oop(oop_result1);
124 }
125 if (metadata_result->is_valid()) {
126 get_vm_result_metadata(metadata_result);
127 }
128
129 assert(call_offset >= 0, "Should be set");
130 return call_offset;
131 }
132
133
134 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
135 mov(c_rarg1, arg1);
136 return call_RT(oop_result1, metadata_result, entry, 1);
137 }
138
139
140 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
141 if (c_rarg1 == arg2) {
142 if (c_rarg2 == arg1) {
143 xchgq(arg1, arg2);
144 } else {
145 mov(c_rarg2, arg2);
146 mov(c_rarg1, arg1);
147 }
148 } else {
149 mov(c_rarg1, arg1);
150 mov(c_rarg2, arg2);
151 }
152 return call_RT(oop_result1, metadata_result, entry, 2);
153 }
154
155
156 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
157 // if there is any conflict use the stack
158 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
159 arg2 == c_rarg1 || arg2 == c_rarg3 ||
160 arg3 == c_rarg1 || arg3 == c_rarg2) {
161 push(arg3);
162 push(arg2);
163 push(arg1);
164 pop(c_rarg1);
165 pop(c_rarg2);
166 pop(c_rarg3);
167 } else {
168 mov(c_rarg1, arg1);
169 mov(c_rarg2, arg2);
170 mov(c_rarg3, arg3);
171 }
172 return call_RT(oop_result1, metadata_result, entry, 3);
173 }
174
175
176 // Implementation of StubFrame
177
178 class StubFrame: public StackObj {
179 private:
180 StubAssembler* _sasm;
181 bool _use_pop_on_epilog;
182
183 public:
184 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog = false);
185 void load_argument(int offset_in_words, Register reg);
186
187 ~StubFrame();
188 };
189
190 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
191 set_info(name, must_gc_arguments);
192 enter();
193 }
194
195 void StubAssembler::epilogue(bool use_pop) {
196 // Avoid using a leave instruction when this frame may
197 // have been frozen, since the current value of rbp
198 // restored from the stub would be invalid. We still
199 // must restore the rbp value saved on enter though.
200 use_pop ? pop(rbp) : leave();
201 ret(0);
202 }
203
204 #define __ _sasm->
205
206 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog) {
207 _sasm = sasm;
208 _use_pop_on_epilog = use_pop_on_epilog;
209 __ prologue(name, must_gc_arguments);
210 }
211
212 // load parameters that were stored with LIR_Assembler::store_parameter
213 // Note: offsets for store_parameter and load_argument must match
214 void StubFrame::load_argument(int offset_in_words, Register reg) {
215 __ load_parameter(offset_in_words, reg);
216 }
217
218
219 StubFrame::~StubFrame() {
220 __ epilogue(_use_pop_on_epilog);
221 }
222
223 #undef __
224
225
226 // Implementation of Runtime1
227
228 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
229 const int xmm_regs_as_doubles_size_in_slots = FrameMap::nof_xmm_regs * 2;
230
231 // Stack layout for saving/restoring all the registers needed during a runtime
232 // call (this includes deoptimization)
233 // Note: note that users of this frame may well have arguments to some runtime
234 // while these values are on the stack. These positions neglect those arguments
235 // but the code in save_live_registers will take the argument count into
236 // account.
237 //
238 #define SLOT2(x) x,
239 #define SLOT_PER_WORD 2
240
241 enum reg_save_layout {
242 // 64bit needs to keep stack 16 byte aligned. So we add some alignment dummies to make that
243 // happen and will assert if the stack size we create is misaligned
244 align_dummy_0, align_dummy_1,
245 #ifdef _WIN64
246 // Windows always allocates space for it's argument registers (see
247 // frame::arg_reg_save_area_bytes).
248 arg_reg_save_1, arg_reg_save_1H, // 0, 4
249 arg_reg_save_2, arg_reg_save_2H, // 8, 12
250 arg_reg_save_3, arg_reg_save_3H, // 16, 20
251 arg_reg_save_4, arg_reg_save_4H, // 24, 28
252 #endif // _WIN64
253 xmm_regs_as_doubles_off, // 32
254 float_regs_as_doubles_off = xmm_regs_as_doubles_off + xmm_regs_as_doubles_size_in_slots, // 160
255 fpu_state_off = float_regs_as_doubles_off + float_regs_as_doubles_size_in_slots, // 224
256 // fpu_state_end_off is exclusive
257 fpu_state_end_off = fpu_state_off + (FPUStateSizeInWords / SLOT_PER_WORD), // 352
258 marker = fpu_state_end_off, SLOT2(markerH) // 352, 356
259 extra_space_offset, // 360
260 r15_off = extra_space_offset, r15H_off, // 360, 364
261 r14_off, r14H_off, // 368, 372
262 r13_off, r13H_off, // 376, 380
263 r12_off, r12H_off, // 384, 388
264 r11_off, r11H_off, // 392, 396
265 r10_off, r10H_off, // 400, 404
266 r9_off, r9H_off, // 408, 412
267 r8_off, r8H_off, // 416, 420
268 rdi_off, rdiH_off, // 424, 428
269 rsi_off, SLOT2(rsiH_off) // 432, 436
270 rbp_off, SLOT2(rbpH_off) // 440, 444
271 rsp_off, SLOT2(rspH_off) // 448, 452
272 rbx_off, SLOT2(rbxH_off) // 456, 460
273 rdx_off, SLOT2(rdxH_off) // 464, 468
274 rcx_off, SLOT2(rcxH_off) // 472, 476
275 rax_off, SLOT2(raxH_off) // 480, 484
276 saved_rbp_off, SLOT2(saved_rbpH_off) // 488, 492
277 return_off, SLOT2(returnH_off) // 496, 500
278 reg_save_frame_size // As noted: neglects any parameters to runtime // 504
279 };
280
281 // Save off registers which might be killed by calls into the runtime.
282 // Tries to smart of about FP registers. In particular we separate
283 // saving and describing the FPU registers for deoptimization since we
284 // have to save the FPU registers twice if we describe them and on P4
285 // saving FPU registers which don't contain anything appears
286 // expensive. The deopt blob is the only thing which needs to
287 // describe FPU registers. In all other cases it should be sufficient
288 // to simply save their current value.
289 //
290 static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
291 bool save_fpu_registers = true) {
292
293 // In 64bit all the args are in regs so there are no additional stack slots
294 num_rt_args = 0;
295 assert((reg_save_frame_size * VMRegImpl::stack_slot_size) % 16 == 0, "must be 16 byte aligned");
296 int frame_size_in_slots = reg_save_frame_size + num_rt_args; // args + thread
297 sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word);
298
299 // record saved value locations in an OopMap
300 // locations are offsets from sp after runtime call; num_rt_args is number of arguments in call, including thread
301 OopMap* map = new OopMap(frame_size_in_slots, 0);
302 map->set_callee_saved(VMRegImpl::stack2reg(rax_off + num_rt_args), rax->as_VMReg());
303 map->set_callee_saved(VMRegImpl::stack2reg(rcx_off + num_rt_args), rcx->as_VMReg());
304 map->set_callee_saved(VMRegImpl::stack2reg(rdx_off + num_rt_args), rdx->as_VMReg());
305 map->set_callee_saved(VMRegImpl::stack2reg(rbx_off + num_rt_args), rbx->as_VMReg());
306 map->set_callee_saved(VMRegImpl::stack2reg(rsi_off + num_rt_args), rsi->as_VMReg());
307 map->set_callee_saved(VMRegImpl::stack2reg(rdi_off + num_rt_args), rdi->as_VMReg());
308 map->set_callee_saved(VMRegImpl::stack2reg(r8_off + num_rt_args), r8->as_VMReg());
309 map->set_callee_saved(VMRegImpl::stack2reg(r9_off + num_rt_args), r9->as_VMReg());
310 map->set_callee_saved(VMRegImpl::stack2reg(r10_off + num_rt_args), r10->as_VMReg());
311 map->set_callee_saved(VMRegImpl::stack2reg(r11_off + num_rt_args), r11->as_VMReg());
312 map->set_callee_saved(VMRegImpl::stack2reg(r12_off + num_rt_args), r12->as_VMReg());
313 map->set_callee_saved(VMRegImpl::stack2reg(r13_off + num_rt_args), r13->as_VMReg());
314 map->set_callee_saved(VMRegImpl::stack2reg(r14_off + num_rt_args), r14->as_VMReg());
315 map->set_callee_saved(VMRegImpl::stack2reg(r15_off + num_rt_args), r15->as_VMReg());
316
317 // This is stupid but needed.
318 map->set_callee_saved(VMRegImpl::stack2reg(raxH_off + num_rt_args), rax->as_VMReg()->next());
319 map->set_callee_saved(VMRegImpl::stack2reg(rcxH_off + num_rt_args), rcx->as_VMReg()->next());
320 map->set_callee_saved(VMRegImpl::stack2reg(rdxH_off + num_rt_args), rdx->as_VMReg()->next());
321 map->set_callee_saved(VMRegImpl::stack2reg(rbxH_off + num_rt_args), rbx->as_VMReg()->next());
322 map->set_callee_saved(VMRegImpl::stack2reg(rsiH_off + num_rt_args), rsi->as_VMReg()->next());
323 map->set_callee_saved(VMRegImpl::stack2reg(rdiH_off + num_rt_args), rdi->as_VMReg()->next());
324
325 map->set_callee_saved(VMRegImpl::stack2reg(r8H_off + num_rt_args), r8->as_VMReg()->next());
326 map->set_callee_saved(VMRegImpl::stack2reg(r9H_off + num_rt_args), r9->as_VMReg()->next());
327 map->set_callee_saved(VMRegImpl::stack2reg(r10H_off + num_rt_args), r10->as_VMReg()->next());
328 map->set_callee_saved(VMRegImpl::stack2reg(r11H_off + num_rt_args), r11->as_VMReg()->next());
329 map->set_callee_saved(VMRegImpl::stack2reg(r12H_off + num_rt_args), r12->as_VMReg()->next());
330 map->set_callee_saved(VMRegImpl::stack2reg(r13H_off + num_rt_args), r13->as_VMReg()->next());
331 map->set_callee_saved(VMRegImpl::stack2reg(r14H_off + num_rt_args), r14->as_VMReg()->next());
332 map->set_callee_saved(VMRegImpl::stack2reg(r15H_off + num_rt_args), r15->as_VMReg()->next());
333
334 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
335
336 if (save_fpu_registers) {
337 int xmm_off = xmm_regs_as_doubles_off;
338 for (int n = 0; n < FrameMap::nof_xmm_regs; n++) {
339 if (n < xmm_bypass_limit) {
340 VMReg xmm_name_0 = as_XMMRegister(n)->as_VMReg();
341 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + num_rt_args), xmm_name_0);
342 // %%% This is really a waste but we'll keep things as they were for now
343 if (true) {
344 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + 1 + num_rt_args), xmm_name_0->next());
345 }
346 }
347 xmm_off += 2;
348 }
349 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
350 }
351
352 return map;
353 }
354
355 #define __ this->
356
357 void C1_MacroAssembler::save_live_registers_no_oop_map(bool save_fpu_registers) {
358 __ block_comment("save_live_registers");
359
360 // Push CPU state in multiple of 16 bytes
361 __ save_legacy_gprs();
362
363 __ subptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
364
365 #ifdef ASSERT
366 __ movptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
367 #endif
368
369 if (save_fpu_registers) {
370 // save XMM registers
371 // XMM registers can contain float or double values, but this is not known here,
372 // so always save them as doubles.
373 // note that float values are _not_ converted automatically, so for float values
374 // the second word contains only garbage data.
375 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
376 int offset = 0;
377 for (int n = 0; n < xmm_bypass_limit; n++) {
378 XMMRegister xmm_name = as_XMMRegister(n);
379 __ movdbl(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
380 offset += 8;
381 }
382 }
383 }
384
385 #undef __
386 #define __ sasm->
387
388 static void restore_fpu(C1_MacroAssembler* sasm, bool restore_fpu_registers) {
389 if (restore_fpu_registers) {
390 // restore XMM registers
391 int xmm_bypass_limit = FrameMap::get_num_caller_save_xmms();
392 int offset = 0;
393 for (int n = 0; n < xmm_bypass_limit; n++) {
394 XMMRegister xmm_name = as_XMMRegister(n);
395 __ movdbl(xmm_name, Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset));
396 offset += 8;
397 }
398 }
399
400 #ifdef ASSERT
401 {
402 Label ok;
403 __ cmpptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
404 __ jcc(Assembler::equal, ok);
405 __ stop("bad offsets in frame");
406 __ bind(ok);
407 }
408 #endif // ASSERT
409
410 __ addptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
411 }
412
413 #undef __
414 #define __ this->
415
416 void C1_MacroAssembler::restore_live_registers(bool restore_fpu_registers) {
417 __ block_comment("restore_live_registers");
418
419 restore_fpu(this, restore_fpu_registers);
420 __ restore_legacy_gprs();
421 }
422
423
424 void C1_MacroAssembler::restore_live_registers_except_rax(bool restore_fpu_registers) {
425 __ block_comment("restore_live_registers_except_rax");
426
427 restore_fpu(this, restore_fpu_registers);
428
429 __ movptr(r15, Address(rsp, 0));
430 __ movptr(r14, Address(rsp, wordSize));
431 __ movptr(r13, Address(rsp, 2 * wordSize));
432 __ movptr(r12, Address(rsp, 3 * wordSize));
433 __ movptr(r11, Address(rsp, 4 * wordSize));
434 __ movptr(r10, Address(rsp, 5 * wordSize));
435 __ movptr(r9, Address(rsp, 6 * wordSize));
436 __ movptr(r8, Address(rsp, 7 * wordSize));
437 __ movptr(rdi, Address(rsp, 8 * wordSize));
438 __ movptr(rsi, Address(rsp, 9 * wordSize));
439 __ movptr(rbp, Address(rsp, 10 * wordSize));
440 // skip rsp
441 __ movptr(rbx, Address(rsp, 12 * wordSize));
442 __ movptr(rdx, Address(rsp, 13 * wordSize));
443 __ movptr(rcx, Address(rsp, 14 * wordSize));
444
445 __ addptr(rsp, 16 * wordSize);
446 }
447
448 #undef __
449 #define __ sasm->
450
451 static OopMap* save_live_registers(StubAssembler* sasm, int num_rt_args,
452 bool save_fpu_registers = true) {
453 __ save_live_registers_no_oop_map(save_fpu_registers);
454 return generate_oop_map(sasm, num_rt_args, save_fpu_registers);
455 }
456
457 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
458 __ restore_live_registers(restore_fpu_registers);
459 }
460
461 static void restore_live_registers_except_rax(StubAssembler* sasm, bool restore_fpu_registers = true) {
462 sasm->restore_live_registers_except_rax(restore_fpu_registers);
463 }
464
465
466 void Runtime1::initialize_pd() {
467 // nothing to do
468 }
469
470 // return: offset in 64-bit words.
471 uint Runtime1::runtime_blob_current_thread_offset(frame f) {
472 return r15_off / 2; // rsp offsets are in halfwords
473 }
474
475 // Target: the entry point of the method that creates and posts the exception oop.
476 // has_argument: true if the exception needs arguments (passed on the stack because
477 // registers must be preserved).
478 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
479 // Preserve all registers.
480 int num_rt_args = has_argument ? (2 + 1) : 1;
481 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
482
483 // Now all registers are saved and can be used freely.
484 // Verify that no old value is used accidentally.
485 __ invalidate_registers(true, true, true, true, true, true);
486
487 // Registers used by this stub.
488 const Register temp_reg = rbx;
489
490 // Load arguments for exception that are passed as arguments into the stub.
491 if (has_argument) {
492 __ movptr(c_rarg1, Address(rbp, 2*BytesPerWord));
493 __ movptr(c_rarg2, Address(rbp, 3*BytesPerWord));
494 }
495 int call_offset = __ call_RT(noreg, noreg, target, num_rt_args - 1);
496
497 OopMapSet* oop_maps = new OopMapSet();
498 oop_maps->add_gc_map(call_offset, oop_map);
499
500 __ stop("should not reach here");
501
502 return oop_maps;
503 }
504
505
506 OopMapSet* Runtime1::generate_handle_exception(StubId id, StubAssembler *sasm) {
507 __ block_comment("generate_handle_exception");
508
509 // incoming parameters
510 const Register exception_oop = rax;
511 const Register exception_pc = rdx;
512 // other registers used in this stub
513 const Register thread = r15_thread;
514
515 // Save registers, if required.
516 OopMapSet* oop_maps = new OopMapSet();
517 OopMap* oop_map = nullptr;
518 switch (id) {
519 case StubId::c1_forward_exception_id:
520 // We're handling an exception in the context of a compiled frame.
521 // The registers have been saved in the standard places. Perform
522 // an exception lookup in the caller and dispatch to the handler
523 // if found. Otherwise unwind and dispatch to the callers
524 // exception handler.
525 oop_map = generate_oop_map(sasm, 1 /*thread*/);
526
527 // load and clear pending exception oop into RAX
528 __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
529 __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
530
531 // load issuing PC (the return address for this stub) into rdx
532 __ movptr(exception_pc, Address(rbp, 1*BytesPerWord));
533
534 // make sure that the vm_results are cleared (may be unnecessary)
535 __ movptr(Address(thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
536 __ movptr(Address(thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
537 break;
538 case StubId::c1_handle_exception_nofpu_id:
539 case StubId::c1_handle_exception_id:
540 // At this point all registers MAY be live.
541 oop_map = save_live_registers(sasm, 1 /*thread*/, id != StubId::c1_handle_exception_nofpu_id);
542 break;
543 case StubId::c1_handle_exception_from_callee_id: {
544 // At this point all registers except exception oop (RAX) and
545 // exception pc (RDX) are dead.
546 const int frame_size = 2 /*BP, return address*/ WIN64_ONLY(+ frame::arg_reg_save_area_bytes / BytesPerWord);
547 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
548 sasm->set_frame_size(frame_size);
549 WIN64_ONLY(__ subq(rsp, frame::arg_reg_save_area_bytes));
550 break;
551 }
552 default: ShouldNotReachHere();
553 }
554
555 // verify that only rax, and rdx is valid at this time
556 __ invalidate_registers(false, true, true, false, true, true);
557 // verify that rax, contains a valid exception
558 __ verify_not_null_oop(exception_oop);
559
560 #ifdef ASSERT
561 // check that fields in JavaThread for exception oop and issuing pc are
562 // empty before writing to them
563 Label oop_empty;
564 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
565 __ jcc(Assembler::equal, oop_empty);
566 __ stop("exception oop already set");
567 __ bind(oop_empty);
568
569 Label pc_empty;
570 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), 0);
571 __ jcc(Assembler::equal, pc_empty);
572 __ stop("exception pc already set");
573 __ bind(pc_empty);
574 #endif
575
576 // save exception oop and issuing pc into JavaThread
577 // (exception handler will load it from here)
578 __ movptr(Address(thread, JavaThread::exception_oop_offset()), exception_oop);
579 __ movptr(Address(thread, JavaThread::exception_pc_offset()), exception_pc);
580
581 // patch throwing pc into return address (has bci & oop map)
582 __ movptr(Address(rbp, 1*BytesPerWord), exception_pc);
583
584 // compute the exception handler.
585 // the exception oop and the throwing pc are read from the fields in JavaThread
586 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
587 oop_maps->add_gc_map(call_offset, oop_map);
588
589 // rax: handler address
590 // will be the deopt blob if nmethod was deoptimized while we looked up
591 // handler regardless of whether handler existed in the nmethod.
592
593 // only rax, is valid at this time, all other registers have been destroyed by the runtime call
594 __ invalidate_registers(false, true, true, true, true, true);
595
596 // patch the return address, this stub will directly return to the exception handler
597 __ movptr(Address(rbp, 1*BytesPerWord), rax);
598
599 switch (id) {
600 case StubId::c1_forward_exception_id:
601 case StubId::c1_handle_exception_nofpu_id:
602 case StubId::c1_handle_exception_id:
603 // Restore the registers that were saved at the beginning.
604 restore_live_registers(sasm, id != StubId::c1_handle_exception_nofpu_id);
605 break;
606 case StubId::c1_handle_exception_from_callee_id:
607 // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP
608 // since we do a leave anyway.
609
610 // Pop the return address.
611 __ leave();
612 __ pop(rcx);
613 __ jmp(rcx); // jump to exception handler
614 break;
615 default: ShouldNotReachHere();
616 }
617
618 return oop_maps;
619 }
620
621
622 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
623 // incoming parameters
624 const Register exception_oop = rax;
625 // callee-saved copy of exception_oop during runtime call
626 const Register exception_oop_callee_saved = r14;
627 // other registers used in this stub
628 const Register exception_pc = rdx;
629 const Register handler_addr = rbx;
630 const Register thread = r15_thread;
631
632 if (AbortVMOnException) {
633 __ enter();
634 save_live_registers(sasm, 2);
635 __ call_VM_leaf(CAST_FROM_FN_PTR(address, check_abort_on_vm_exception), rax);
636 restore_live_registers(sasm);
637 __ leave();
638 }
639
640 // verify that only rax, is valid at this time
641 __ invalidate_registers(false, true, true, true, true, true);
642
643 #ifdef ASSERT
644 // check that fields in JavaThread for exception oop and issuing pc are empty
645 Label oop_empty;
646 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), 0);
647 __ jcc(Assembler::equal, oop_empty);
648 __ stop("exception oop must be empty");
649 __ bind(oop_empty);
650
651 Label pc_empty;
652 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), 0);
653 __ jcc(Assembler::equal, pc_empty);
654 __ stop("exception pc must be empty");
655 __ bind(pc_empty);
656 #endif
657
658 // save exception_oop in callee-saved register to preserve it during runtime calls
659 __ verify_not_null_oop(exception_oop);
660 __ movptr(exception_oop_callee_saved, exception_oop);
661
662 // Get return address (is on top of stack after leave).
663 __ movptr(exception_pc, Address(rsp, 0));
664
665 // search the exception handler address of the caller (using the return address)
666 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
667 // rax: exception handler address of the caller
668
669 // Only RAX and RSI are valid at this time, all other registers have been destroyed by the call.
670 __ invalidate_registers(false, true, true, true, false, true);
671
672 // move result of call into correct register
673 __ movptr(handler_addr, rax);
674
675 // Restore exception oop to RAX (required convention of exception handler).
676 __ movptr(exception_oop, exception_oop_callee_saved);
677
678 // verify that there is really a valid exception in rax
679 __ verify_not_null_oop(exception_oop);
680
681 // get throwing pc (= return address).
682 // rdx has been destroyed by the call, so it must be set again
683 // the pop is also necessary to simulate the effect of a ret(0)
684 __ pop(exception_pc);
685
686 // continue at exception handler (return address removed)
687 // note: do *not* remove arguments when unwinding the
688 // activation since the caller assumes having
689 // all arguments on the stack when entering the
690 // runtime to determine the exception handler
691 // (GC happens at call site with arguments!)
692 // rax: exception oop
693 // rdx: throwing pc
694 // rbx: exception handler
695 __ jmp(handler_addr);
696 }
697
698
699 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
700 // use the maximum number of runtime-arguments here because it is difficult to
701 // distinguish each RT-Call.
702 // Note: This number affects also the RT-Call in generate_handle_exception because
703 // the oop-map is shared for all calls.
704 const int num_rt_args = 2; // thread + dummy
705
706 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
707 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
708
709 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
710
711 const Register thread = r15_thread;
712 // No need to worry about dummy
713 __ mov(c_rarg0, thread);
714 __ set_last_Java_frame(noreg, rbp, nullptr, rscratch1);
715 // do the call
716 __ call(RuntimeAddress(target));
717 OopMapSet* oop_maps = new OopMapSet();
718 oop_maps->add_gc_map(__ offset(), oop_map);
719 // verify callee-saved register
720 #ifdef ASSERT
721 guarantee(thread != rax, "change this code");
722 __ push_ppx(rax);
723 { Label L;
724 __ get_thread_slow(rax);
725 __ cmpptr(thread, rax);
726 __ jcc(Assembler::equal, L);
727 __ stop("StubAssembler::call_RT: rdi/r15 not callee saved?");
728 __ bind(L);
729 }
730 __ pop_ppx(rax);
731 #endif
732 __ reset_last_Java_frame(true);
733
734 // check for pending exceptions
735 { Label L;
736 __ cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
737 __ jcc(Assembler::equal, L);
738 // exception pending => remove activation and forward to exception handler
739
740 __ testptr(rax, rax); // have we deoptimized?
741 __ jump_cc(Assembler::equal,
742 RuntimeAddress(Runtime1::entry_for(StubId::c1_forward_exception_id)));
743
744 // the deopt blob expects exceptions in the special fields of
745 // JavaThread, so copy and clear pending exception.
746
747 // load and clear pending exception
748 __ movptr(rax, Address(thread, Thread::pending_exception_offset()));
749 __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
750
751 // check that there is really a valid exception
752 __ verify_not_null_oop(rax);
753
754 // load throwing pc: this is the return address of the stub
755 __ movptr(rdx, Address(rsp, return_off * VMRegImpl::stack_slot_size));
756
757 #ifdef ASSERT
758 // check that fields in JavaThread for exception oop and issuing pc are empty
759 Label oop_empty;
760 __ cmpptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
761 __ jcc(Assembler::equal, oop_empty);
762 __ stop("exception oop must be empty");
763 __ bind(oop_empty);
764
765 Label pc_empty;
766 __ cmpptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
767 __ jcc(Assembler::equal, pc_empty);
768 __ stop("exception pc must be empty");
769 __ bind(pc_empty);
770 #endif
771
772 // store exception oop and throwing pc to JavaThread
773 __ movptr(Address(thread, JavaThread::exception_oop_offset()), rax);
774 __ movptr(Address(thread, JavaThread::exception_pc_offset()), rdx);
775
776 restore_live_registers(sasm);
777
778 __ leave();
779 __ addptr(rsp, BytesPerWord); // remove return address from stack
780
781 // Forward the exception directly to deopt blob. We can blow no
782 // registers and must leave throwing pc on the stack. A patch may
783 // have values live in registers so the entry point with the
784 // exception in tls.
785 __ jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
786
787 __ bind(L);
788 }
789
790
791 // Runtime will return true if the nmethod has been deoptimized during
792 // the patching process. In that case we must do a deopt reexecute instead.
793
794 Label cont;
795
796 __ testptr(rax, rax); // have we deoptimized?
797 __ jcc(Assembler::equal, cont); // no
798
799 // Will reexecute. Proper return address is already on the stack we just restore
800 // registers, pop all of our frame but the return address and jump to the deopt blob
801 restore_live_registers(sasm);
802 __ leave();
803 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
804
805 __ bind(cont);
806 restore_live_registers(sasm);
807 __ leave();
808 __ ret(0);
809
810 return oop_maps;
811 }
812
813
814 OopMapSet* Runtime1::generate_code_for(StubId id, StubAssembler* sasm) {
815
816 // for better readability
817 const bool must_gc_arguments = true;
818 const bool dont_gc_arguments = false;
819
820 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
821 bool save_fpu_registers = true;
822
823 // stub code & info for the different stubs
824 OopMapSet* oop_maps = nullptr;
825 switch (id) {
826 case StubId::c1_forward_exception_id:
827 {
828 oop_maps = generate_handle_exception(id, sasm);
829 __ leave();
830 __ ret(0);
831 }
832 break;
833
834 case StubId::c1_new_instance_id:
835 case StubId::c1_fast_new_instance_id:
836 case StubId::c1_fast_new_instance_init_check_id:
837 {
838 Register klass = rdx; // Incoming
839 Register obj = rax; // Result
840
841 if (id == StubId::c1_new_instance_id) {
842 __ set_info("new_instance", dont_gc_arguments);
843 } else if (id == StubId::c1_fast_new_instance_id) {
844 __ set_info("fast new_instance", dont_gc_arguments);
845 } else {
846 assert(id == StubId::c1_fast_new_instance_init_check_id, "bad StubId");
847 __ set_info("fast new_instance init check", dont_gc_arguments);
848 }
849
850 __ enter();
851 OopMap* map = save_live_registers(sasm, 2);
852 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
853 oop_maps = new OopMapSet();
854 oop_maps->add_gc_map(call_offset, map);
855 restore_live_registers_except_rax(sasm);
856 __ verify_oop(obj);
857 __ leave();
858 __ ret(0);
859
860 // rax,: new instance
861 }
862
863 break;
864
865 case StubId::c1_counter_overflow_id:
866 {
867 Register bci = rax, method = rbx;
868 __ enter();
869 OopMap* map = save_live_registers(sasm, 3);
870 // Retrieve bci
871 __ movl(bci, Address(rbp, 2*BytesPerWord));
872 // And a pointer to the Method*
873 __ movptr(method, Address(rbp, 3*BytesPerWord));
874 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
875 oop_maps = new OopMapSet();
876 oop_maps->add_gc_map(call_offset, map);
877 restore_live_registers(sasm);
878 __ leave();
879 __ ret(0);
880 }
881 break;
882
883 case StubId::c1_new_type_array_id:
884 case StubId::c1_new_object_array_id:
885 case StubId::c1_new_null_free_array_id:
886 {
887 Register length = rbx; // Incoming
888 Register klass = rdx; // Incoming
889 Register obj = rax; // Result
890
891 if (id == StubId::c1_new_type_array_id) {
892 __ set_info("new_type_array", dont_gc_arguments);
893 } else if (id == StubId::c1_new_object_array_id) {
894 __ set_info("new_object_array", dont_gc_arguments);
895 } else {
896 __ set_info("new_null_free_array", dont_gc_arguments);
897 }
898
899 #ifdef ASSERT
900 // assert object type is really an array of the proper kind
901 {
902 Label ok;
903 Register t0 = obj;
904 __ movl(t0, Address(klass, Klass::layout_helper_offset()));
905 __ sarl(t0, Klass::_lh_array_tag_shift);
906 switch (id) {
907 case StubId::c1_new_type_array_id:
908 __ cmpl(t0, Klass::_lh_array_tag_type_value);
909 __ jcc(Assembler::equal, ok);
910 __ stop("assert(is a type array klass)");
911 break;
912 case StubId::c1_new_object_array_id:
913 __ cmpl(t0, (Klass::_lh_array_tag_ref_value)); // new "[Ljava/lang/Object;"
914 __ jcc(Assembler::equal, ok);
915 __ cmpl(t0, Klass::_lh_array_tag_flat_value); // new "[LVT;"
916 __ jcc(Assembler::equal, ok);
917 __ stop("assert(is an object or inline type array klass)");
918 break;
919 case StubId::c1_new_null_free_array_id:
920 __ cmpl(t0, Klass::_lh_array_tag_flat_value); // the array can be a flat array.
921 __ jcc(Assembler::equal, ok);
922 __ cmpl(t0, (Klass::_lh_array_tag_ref_value)); // the array cannot be a flat array (due to InlineArrayElementMaxFlatSize, etc)
923 __ jcc(Assembler::equal, ok);
924 __ stop("assert(is an object or inline type array klass)");
925 break;
926 default: ShouldNotReachHere();
927 }
928 __ should_not_reach_here();
929 __ bind(ok);
930 }
931 #endif // ASSERT
932
933 __ enter();
934 OopMap* map = save_live_registers(sasm, 3);
935 int call_offset;
936 if (id == StubId::c1_new_type_array_id) {
937 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
938 } else if (id == StubId::c1_new_object_array_id) {
939 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
940 } else {
941 assert(id == StubId::c1_new_null_free_array_id, "must be");
942 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_null_free_array), klass, length);
943 }
944
945 oop_maps = new OopMapSet();
946 oop_maps->add_gc_map(call_offset, map);
947 restore_live_registers_except_rax(sasm);
948
949 __ verify_oop(obj);
950 __ leave();
951 __ ret(0);
952
953 // rax,: new array
954 }
955 break;
956
957 case StubId::c1_new_multi_array_id:
958 { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
959 // rax,: klass
960 // rbx,: rank
961 // rcx: address of 1st dimension
962 OopMap* map = save_live_registers(sasm, 4);
963 int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, new_multi_array), rax, rbx, rcx);
964
965 oop_maps = new OopMapSet();
966 oop_maps->add_gc_map(call_offset, map);
967 restore_live_registers_except_rax(sasm);
968
969 // rax,: new multi array
970 __ verify_oop(rax);
971 }
972 break;
973
974 case StubId::c1_load_flat_array_id:
975 {
976 StubFrame f(sasm, "load_flat_array", dont_gc_arguments);
977 OopMap* map = save_live_registers(sasm, 3);
978
979 // Called with store_parameter and not C abi
980
981 f.load_argument(1, rax); // rax,: array
982 f.load_argument(0, rbx); // rbx,: index
983 int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, load_flat_array), rax, rbx);
984
985 oop_maps = new OopMapSet();
986 oop_maps->add_gc_map(call_offset, map);
987 restore_live_registers_except_rax(sasm);
988
989 // rax,: loaded element at array[index]
990 __ verify_oop(rax);
991 }
992 break;
993
994 case StubId::c1_store_flat_array_id:
995 {
996 StubFrame f(sasm, "store_flat_array", dont_gc_arguments);
997 OopMap* map = save_live_registers(sasm, 4);
998
999 // Called with store_parameter and not C abi
1000
1001 f.load_argument(2, rax); // rax,: array
1002 f.load_argument(1, rbx); // rbx,: index
1003 f.load_argument(0, rcx); // rcx,: value
1004 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, store_flat_array), rax, rbx, rcx);
1005
1006 oop_maps = new OopMapSet();
1007 oop_maps->add_gc_map(call_offset, map);
1008 restore_live_registers_except_rax(sasm);
1009 }
1010 break;
1011
1012 case StubId::c1_substitutability_check_id:
1013 {
1014 StubFrame f(sasm, "substitutability_check", dont_gc_arguments);
1015 OopMap* map = save_live_registers(sasm, 3);
1016
1017 // Called with store_parameter and not C abi
1018
1019 f.load_argument(1, rax); // rax,: left
1020 f.load_argument(0, rbx); // rbx,: right
1021 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, substitutability_check), rax, rbx);
1022
1023 oop_maps = new OopMapSet();
1024 oop_maps->add_gc_map(call_offset, map);
1025 restore_live_registers_except_rax(sasm);
1026
1027 // rax,: are the two operands substitutable
1028 }
1029 break;
1030
1031
1032 case StubId::c1_buffer_inline_args_id:
1033 case StubId::c1_buffer_inline_args_no_receiver_id:
1034 {
1035 const char* name = (id == StubId::c1_buffer_inline_args_id) ?
1036 "buffer_inline_args" : "buffer_inline_args_no_receiver";
1037 StubFrame f(sasm, name, dont_gc_arguments);
1038 OopMap* map = save_live_registers(sasm, 2);
1039 Register method = rbx;
1040 address entry = (id == StubId::c1_buffer_inline_args_id) ?
1041 CAST_FROM_FN_PTR(address, buffer_inline_args) :
1042 CAST_FROM_FN_PTR(address, buffer_inline_args_no_receiver);
1043 int call_offset = __ call_RT(rax, noreg, entry, method);
1044 oop_maps = new OopMapSet();
1045 oop_maps->add_gc_map(call_offset, map);
1046 restore_live_registers_except_rax(sasm);
1047 __ verify_oop(rax); // rax: an array of buffered value objects
1048 }
1049 break;
1050
1051 case StubId::c1_register_finalizer_id:
1052 {
1053 __ set_info("register_finalizer", dont_gc_arguments);
1054
1055 // This is called via call_runtime so the arguments
1056 // will be place in C abi locations
1057
1058 __ verify_oop(c_rarg0);
1059 __ mov(rax, c_rarg0);
1060
1061 // load the klass and check the has finalizer flag
1062 Label register_finalizer;
1063 Register t = rsi;
1064 __ load_klass(t, rax, rscratch1);
1065 __ testb(Address(t, Klass::misc_flags_offset()), KlassFlags::_misc_has_finalizer);
1066 __ jcc(Assembler::notZero, register_finalizer);
1067 __ ret(0);
1068
1069 __ bind(register_finalizer);
1070 __ enter();
1071 OopMap* oop_map = save_live_registers(sasm, 2 /*num_rt_args */);
1072 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), rax);
1073 oop_maps = new OopMapSet();
1074 oop_maps->add_gc_map(call_offset, oop_map);
1075
1076 // Now restore all the live registers
1077 restore_live_registers(sasm);
1078
1079 __ leave();
1080 __ ret(0);
1081 }
1082 break;
1083
1084 case StubId::c1_throw_range_check_failed_id:
1085 { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1086 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1087 }
1088 break;
1089
1090 case StubId::c1_throw_index_exception_id:
1091 { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1092 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1093 }
1094 break;
1095
1096 case StubId::c1_throw_div0_exception_id:
1097 { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
1098 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
1099 }
1100 break;
1101
1102 case StubId::c1_throw_null_pointer_exception_id:
1103 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
1104 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
1105 }
1106 break;
1107
1108 case StubId::c1_handle_exception_nofpu_id:
1109 case StubId::c1_handle_exception_id:
1110 { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1111 oop_maps = generate_handle_exception(id, sasm);
1112 }
1113 break;
1114
1115 case StubId::c1_handle_exception_from_callee_id:
1116 { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1117 oop_maps = generate_handle_exception(id, sasm);
1118 }
1119 break;
1120
1121 case StubId::c1_unwind_exception_id:
1122 { __ set_info("unwind_exception", dont_gc_arguments);
1123 // note: no stubframe since we are about to leave the current
1124 // activation and we are calling a leaf VM function only.
1125 generate_unwind_exception(sasm);
1126 }
1127 break;
1128
1129 case StubId::c1_throw_array_store_exception_id:
1130 { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1131 // tos + 0: link
1132 // + 1: return address
1133 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1134 }
1135 break;
1136
1137 case StubId::c1_throw_class_cast_exception_id:
1138 { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
1139 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
1140 }
1141 break;
1142
1143 case StubId::c1_throw_incompatible_class_change_error_id:
1144 { StubFrame f(sasm, "throw_incompatible_class_change_error", dont_gc_arguments);
1145 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
1146 }
1147 break;
1148
1149 case StubId::c1_throw_illegal_monitor_state_exception_id:
1150 { StubFrame f(sasm, "throw_illegal_monitor_state_exception", dont_gc_arguments);
1151 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_illegal_monitor_state_exception), false);
1152 }
1153 break;
1154
1155 case StubId::c1_throw_identity_exception_id:
1156 { StubFrame f(sasm, "throw_identity_exception", dont_gc_arguments);
1157 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_identity_exception), true);
1158 }
1159 break;
1160
1161 case StubId::c1_slow_subtype_check_id:
1162 {
1163 // Typical calling sequence:
1164 // __ push(klass_RInfo); // object klass or other subclass
1165 // __ push(sup_k_RInfo); // array element klass or other superclass
1166 // __ call(slow_subtype_check);
1167 // Note that the subclass is pushed first, and is therefore deepest.
1168 // Previous versions of this code reversed the names 'sub' and 'super'.
1169 // This was operationally harmless but made the code unreadable.
1170 enum layout {
1171 rax_off, SLOT2(raxH_off)
1172 rcx_off, SLOT2(rcxH_off)
1173 rsi_off, SLOT2(rsiH_off)
1174 rdi_off, SLOT2(rdiH_off)
1175 // saved_rbp_off, SLOT2(saved_rbpH_off)
1176 return_off, SLOT2(returnH_off)
1177 sup_k_off, SLOT2(sup_kH_off)
1178 klass_off, SLOT2(superH_off)
1179 framesize,
1180 result_off = klass_off // deepest argument is also the return value
1181 };
1182
1183 __ set_info("slow_subtype_check", dont_gc_arguments);
1184 __ push_ppx(rdi);
1185 __ push_ppx(rsi);
1186 __ push_ppx(rcx);
1187 __ push_ppx(rax);
1188
1189 // This is called by pushing args and not with C abi
1190 __ movptr(rsi, Address(rsp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1191 __ movptr(rax, Address(rsp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1192
1193 Label miss;
1194 __ check_klass_subtype_slow_path(rsi, rax, rcx, rdi, nullptr, &miss);
1195
1196 // fallthrough on success:
1197 __ movptr(Address(rsp, (result_off) * VMRegImpl::stack_slot_size), 1); // result
1198 __ pop_ppx(rax);
1199 __ pop_ppx(rcx);
1200 __ pop_ppx(rsi);
1201 __ pop_ppx(rdi);
1202 __ ret(0);
1203
1204 __ bind(miss);
1205 __ movptr(Address(rsp, (result_off) * VMRegImpl::stack_slot_size), NULL_WORD); // result
1206 __ pop(rax);
1207 __ pop(rcx);
1208 __ pop(rsi);
1209 __ pop(rdi);
1210 __ ret(0);
1211 }
1212 break;
1213
1214 case StubId::c1_is_instance_of_id:
1215 {
1216 // Mirror: c_rarg0 (Windows: rcx, SysV: rdi)
1217 // Object: c_rarg1 (Windows: rdx, SysV: rsi)
1218 // ObjClass: r9
1219 // Temps: rcx, r8, r10, r11
1220 // Result: rax
1221
1222 Register klass = r9, obj = c_rarg1, result = rax;
1223 Register temp0 = rcx, temp1 = r8, temp2 = r10, temp3 = r11;
1224
1225 // Get the Klass* into r9. c_rarg0 is now dead.
1226 __ movptr(klass, Address(c_rarg0, java_lang_Class::klass_offset()));
1227
1228 Label done, is_secondary, same;
1229
1230 __ xorq(result, result);
1231 __ testq(klass, klass);
1232 __ jcc(Assembler::equal, done); // Klass is null
1233
1234 __ testq(obj, obj);
1235 __ jcc(Assembler::equal, done); // obj is null
1236
1237 __ movl(temp0, Address(klass, in_bytes(Klass::super_check_offset_offset())));
1238 __ cmpl(temp0, in_bytes(Klass::secondary_super_cache_offset()));
1239 __ jcc(Assembler::equal, is_secondary); // Klass is a secondary superclass
1240
1241 // Klass is a concrete class
1242 __ load_klass(temp2, obj, /*tmp*/temp1);
1243 __ cmpptr(klass, Address(temp2, temp0));
1244 __ setcc(Assembler::equal, result);
1245 __ ret(0);
1246
1247 __ bind(is_secondary);
1248
1249 __ load_klass(obj, obj, /*tmp*/temp1);
1250
1251 // This is necessary because I am never in my own secondary_super list.
1252 __ cmpptr(obj, klass);
1253 __ jcc(Assembler::equal, same);
1254
1255 __ lookup_secondary_supers_table_var(obj, klass,
1256 /*temps*/temp0, temp1, temp2, temp3,
1257 result);
1258 __ testq(result, result);
1259
1260 __ bind(same);
1261 __ setcc(Assembler::equal, result);
1262
1263 __ bind(done);
1264 __ ret(0);
1265 }
1266 break;
1267
1268 case StubId::c1_monitorenter_nofpu_id:
1269 save_fpu_registers = false;
1270 // fall through
1271 case StubId::c1_monitorenter_id:
1272 {
1273 StubFrame f(sasm, "monitorenter", dont_gc_arguments, true /* use_pop_on_epilog */);
1274 OopMap* map = save_live_registers(sasm, 3, save_fpu_registers);
1275
1276 // Called with store_parameter and not C abi
1277
1278 f.load_argument(1, rax); // rax,: object
1279 f.load_argument(0, rbx); // rbx,: lock address
1280
1281 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), rax, rbx);
1282
1283 oop_maps = new OopMapSet();
1284 oop_maps->add_gc_map(call_offset, map);
1285 restore_live_registers(sasm, save_fpu_registers);
1286 }
1287 break;
1288
1289 case StubId::c1_monitorexit_nofpu_id:
1290 save_fpu_registers = false;
1291 // fall through
1292 case StubId::c1_monitorexit_id:
1293 {
1294 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1295 OopMap* map = save_live_registers(sasm, 2, save_fpu_registers);
1296
1297 // Called with store_parameter and not C abi
1298
1299 f.load_argument(0, rax); // rax,: lock address
1300
1301 // note: really a leaf routine but must setup last java sp
1302 // => use call_RT for now (speed can be improved by
1303 // doing last java sp setup manually)
1304 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax);
1305
1306 oop_maps = new OopMapSet();
1307 oop_maps->add_gc_map(call_offset, map);
1308 restore_live_registers(sasm, save_fpu_registers);
1309 }
1310 break;
1311
1312 case StubId::c1_deoptimize_id:
1313 {
1314 StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1315 const int num_rt_args = 2; // thread, trap_request
1316 OopMap* oop_map = save_live_registers(sasm, num_rt_args);
1317 f.load_argument(0, rax);
1318 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), rax);
1319 oop_maps = new OopMapSet();
1320 oop_maps->add_gc_map(call_offset, oop_map);
1321 restore_live_registers(sasm);
1322 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1323 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1324 __ leave();
1325 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1326 }
1327 break;
1328
1329 case StubId::c1_access_field_patching_id:
1330 { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1331 // we should set up register map
1332 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1333 }
1334 break;
1335
1336 case StubId::c1_load_klass_patching_id:
1337 { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1338 // we should set up register map
1339 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1340 }
1341 break;
1342
1343 case StubId::c1_load_mirror_patching_id:
1344 { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1345 // we should set up register map
1346 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1347 }
1348 break;
1349
1350 case StubId::c1_load_appendix_patching_id:
1351 { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1352 // we should set up register map
1353 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1354 }
1355 break;
1356
1357 case StubId::c1_dtrace_object_alloc_id:
1358 { // rax,: object
1359 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1360 // we can't gc here so skip the oopmap but make sure that all
1361 // the live registers get saved.
1362 save_live_registers(sasm, 1);
1363
1364 __ mov(c_rarg0, rax);
1365 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc))));
1366
1367 restore_live_registers(sasm);
1368 }
1369 break;
1370
1371 case StubId::c1_fpu2long_stub_id:
1372 {
1373 Label done;
1374 __ cvttsd2siq(rax, Address(rsp, wordSize));
1375 __ cmp64(rax, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
1376 __ jccb(Assembler::notEqual, done);
1377 __ movq(rax, Address(rsp, wordSize));
1378 __ subptr(rsp, 8);
1379 __ movq(Address(rsp, 0), rax);
1380 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
1381 __ pop(rax);
1382 __ bind(done);
1383 __ ret(0);
1384 }
1385 break;
1386
1387 case StubId::c1_predicate_failed_trap_id:
1388 {
1389 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1390
1391 OopMap* map = save_live_registers(sasm, 1);
1392
1393 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1394 oop_maps = new OopMapSet();
1395 oop_maps->add_gc_map(call_offset, map);
1396 restore_live_registers(sasm);
1397 __ leave();
1398 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1399 assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1400
1401 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1402 }
1403 break;
1404
1405 default:
1406 { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1407 __ movptr(rax, (int)id);
1408 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
1409 __ should_not_reach_here();
1410 }
1411 break;
1412 }
1413 return oop_maps;
1414 }
1415
1416 #undef __
1417
1418 const char *Runtime1::pd_name_for_address(address entry) {
1419 return "<unknown function>";
1420 }