1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 1999, 2011, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Defs.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "compiler/disassembler.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "nativeInst_aarch64.hpp"
36 #include "oops/compiledICHolder.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "prims/jvmtiExport.hpp"
39 #include "register_aarch64.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/signature.hpp"
42 #include "runtime/vframe.hpp"
43 #include "runtime/vframeArray.hpp"
44 #include "vmreg_aarch64.inline.hpp"
45 #if INCLUDE_ALL_GCS
46 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
47 #endif
48
49
50 // Implementation of StubAssembler
51
52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
53 // setup registers
54 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
55 assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
56 assert(args_size >= 0, "illegal args_size");
57 bool align_stack = false;
58
59 mov(c_rarg0, rthread);
60 set_num_rt_args(0); // Nothing on stack
61
62 Label retaddr;
63 set_last_Java_frame(sp, rfp, retaddr, rscratch1);
64
65 // do the call
66 lea(rscratch1, RuntimeAddress(entry));
67 blr(rscratch1);
68 bind(retaddr);
69 int call_offset = offset();
70 // verify callee-saved register
71 #ifdef ASSERT
72 push(r0, sp);
73 { Label L;
74 get_thread(r0);
75 cmp(rthread, r0);
76 br(Assembler::EQ, L);
77 stop("StubAssembler::call_RT: rthread not callee saved?");
78 bind(L);
79 }
80 pop(r0, sp);
81 #endif
82 reset_last_Java_frame(true);
83 maybe_isb();
84
85 // check for pending exceptions
86 { Label L;
87 // check for pending exceptions (java_thread is set upon return)
88 ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
89 cbz(rscratch1, L);
90 // exception pending => remove activation and forward to exception handler
91 // make sure that the vm_results are cleared
92 if (oop_result1->is_valid()) {
93 str(zr, Address(rthread, JavaThread::vm_result_offset()));
94 }
95 if (metadata_result->is_valid()) {
96 str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
97 }
98 if (frame_size() == no_frame_size) {
99 leave();
100 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
101 } else if (_stub_id == Runtime1::forward_exception_id) {
102 should_not_reach_here();
103 } else {
104 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
105 }
106 bind(L);
107 }
108 // get oop results if there are any and reset the values in the thread
109 if (oop_result1->is_valid()) {
110 get_vm_result(oop_result1, rthread);
111 }
112 if (metadata_result->is_valid()) {
113 get_vm_result_2(metadata_result, rthread);
114 }
115 return call_offset;
116 }
117
118
119 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
120 mov(c_rarg1, arg1);
121 return call_RT(oop_result1, metadata_result, entry, 1);
122 }
123
124
125 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
126 if (c_rarg1 == arg2) {
127 if (c_rarg2 == arg1) {
128 mov(rscratch1, arg1);
129 mov(arg1, arg2);
130 mov(arg2, rscratch1);
131 } else {
132 mov(c_rarg2, arg2);
133 mov(c_rarg1, arg1);
134 }
135 } else {
136 mov(c_rarg1, arg1);
137 mov(c_rarg2, arg2);
138 }
139 return call_RT(oop_result1, metadata_result, entry, 2);
140 }
141
142
143 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
144 // if there is any conflict use the stack
145 if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
146 arg2 == c_rarg1 || arg2 == c_rarg3 ||
147 arg3 == c_rarg1 || arg3 == c_rarg2) {
148 stp(arg3, arg2, Address(pre(sp, -2 * wordSize)));
149 stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
150 ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
151 ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
152 } else {
153 mov(c_rarg1, arg1);
154 mov(c_rarg2, arg2);
155 mov(c_rarg3, arg3);
156 }
157 return call_RT(oop_result1, metadata_result, entry, 3);
158 }
159
160 // Implementation of StubFrame
161
162 class StubFrame: public StackObj {
163 private:
164 StubAssembler* _sasm;
165
166 public:
167 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
168 void load_argument(int offset_in_words, Register reg);
169
170 ~StubFrame();
171 };;
172
173
174 #define __ _sasm->
175
176 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
177 _sasm = sasm;
178 __ set_info(name, must_gc_arguments);
179 __ enter();
180 }
181
182 // load parameters that were stored with LIR_Assembler::store_parameter
183 // Note: offsets for store_parameter and load_argument must match
184 void StubFrame::load_argument(int offset_in_words, Register reg) {
185 // rbp, + 0: link
186 // + 1: return address
187 // + 2: argument with offset 0
188 // + 3: argument with offset 1
189 // + 4: ...
190
191 __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));
192 }
193
194
195 StubFrame::~StubFrame() {
196 __ leave();
197 __ ret(lr);
198 }
199
200 #undef __
201
202
203 // Implementation of Runtime1
204
205 #define __ sasm->
206
207 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
208
209 // Stack layout for saving/restoring all the registers needed during a runtime
210 // call (this includes deoptimization)
211 // Note: note that users of this frame may well have arguments to some runtime
212 // while these values are on the stack. These positions neglect those arguments
213 // but the code in save_live_registers will take the argument count into
214 // account.
215 //
216
217 enum reg_save_layout {
218 reg_save_frame_size = 32 /* float */ + 32 /* integer */
219 };
220
221 // Save off registers which might be killed by calls into the runtime.
222 // Tries to smart of about FP registers. In particular we separate
223 // saving and describing the FPU registers for deoptimization since we
224 // have to save the FPU registers twice if we describe them. The
225 // deopt blob is the only thing which needs to describe FPU registers.
226 // In all other cases it should be sufficient to simply save their
227 // current value.
228
229 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
230 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
231 static int reg_save_size_in_words;
232 static int frame_size_in_bytes = -1;
233
234 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
235 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
236 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
237 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
238 OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
239
240 for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
241 Register r = as_Register(i);
242 if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
243 int sp_offset = cpu_reg_save_offsets[i];
244 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
245 r->as_VMReg());
246 }
247 }
248
249 if (save_fpu_registers) {
250 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
251 FloatRegister r = as_FloatRegister(i);
252 {
253 int sp_offset = fpu_reg_save_offsets[i];
254 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
255 r->as_VMReg());
256 }
257 }
258 }
259 return oop_map;
260 }
261
262 static OopMap* save_live_registers(StubAssembler* sasm,
263 bool save_fpu_registers = true) {
264 __ block_comment("save_live_registers");
265
266 __ push(RegSet::range(r0, r29), sp); // integer registers except lr & sp
267
268 if (save_fpu_registers) {
269 for (int i = 30; i >= 0; i -= 2)
270 __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),
271 Address(__ pre(sp, -2 * wordSize)));
272 } else {
273 __ add(sp, sp, -32 * wordSize);
274 }
275
276 return generate_oop_map(sasm, save_fpu_registers);
277 }
278
279 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
280 if (restore_fpu_registers) {
281 for (int i = 0; i < 32; i += 2)
282 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
283 Address(__ post(sp, 2 * wordSize)));
284 } else {
285 __ add(sp, sp, 32 * wordSize);
286 }
287
288 __ pop(RegSet::range(r0, r29), sp);
289 }
290
291 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true) {
292
293 if (restore_fpu_registers) {
294 for (int i = 0; i < 32; i += 2)
295 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
296 Address(__ post(sp, 2 * wordSize)));
297 } else {
298 __ add(sp, sp, 32 * wordSize);
299 }
300
301 __ ldp(zr, r1, Address(__ post(sp, 16)));
302 __ pop(RegSet::range(r2, r29), sp);
303 }
304
305
306
307 void Runtime1::initialize_pd() {
308 int i;
309 int sp_offset = 0;
310
311 // all float registers are saved explicitly
312 assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
313 for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
314 fpu_reg_save_offsets[i] = sp_offset;
315 sp_offset += 2; // SP offsets are in halfwords
316 }
317
318 for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
319 Register r = as_Register(i);
320 cpu_reg_save_offsets[i] = sp_offset;
321 sp_offset += 2; // SP offsets are in halfwords
322 }
323 }
324
325
326 // target: the entry point of the method that creates and posts the exception oop
327 // has_argument: true if the exception needs an argument (passed in rscratch1)
328
329 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
330 // make a frame and preserve the caller's caller-save registers
331 OopMap* oop_map = save_live_registers(sasm);
332 int call_offset;
333 if (!has_argument) {
334 call_offset = __ call_RT(noreg, noreg, target);
335 } else {
336 call_offset = __ call_RT(noreg, noreg, target, rscratch1);
337 }
338 OopMapSet* oop_maps = new OopMapSet();
339 oop_maps->add_gc_map(call_offset, oop_map);
340
341 __ should_not_reach_here();
342 return oop_maps;
343 }
344
345
346 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
347 __ block_comment("generate_handle_exception");
348
349 // incoming parameters
350 const Register exception_oop = r0;
351 const Register exception_pc = r3;
352 // other registers used in this stub
353
354 // Save registers, if required.
355 OopMapSet* oop_maps = new OopMapSet();
356 OopMap* oop_map = NULL;
357 switch (id) {
358 case forward_exception_id:
359 // We're handling an exception in the context of a compiled frame.
360 // The registers have been saved in the standard places. Perform
361 // an exception lookup in the caller and dispatch to the handler
362 // if found. Otherwise unwind and dispatch to the callers
363 // exception handler.
364 oop_map = generate_oop_map(sasm, 1 /*thread*/);
365
366 // load and clear pending exception oop into r0
367 __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
368 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
369
370 // load issuing PC (the return address for this stub) into r3
371 __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
372
373 // make sure that the vm_results are cleared (may be unnecessary)
374 __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
375 __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
376 break;
377 case handle_exception_nofpu_id:
378 case handle_exception_id:
379 // At this point all registers MAY be live.
380 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
381 break;
382 case handle_exception_from_callee_id: {
383 // At this point all registers except exception oop (r0) and
384 // exception pc (lr) are dead.
385 const int frame_size = 2 /*fp, return address*/;
386 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
387 sasm->set_frame_size(frame_size);
388 break;
389 }
390 default:
391 __ should_not_reach_here();
392 break;
393 }
394
395 // verify that only r0 and r3 are valid at this time
396 __ invalidate_registers(false, true, true, false, true, true);
397 // verify that r0 contains a valid exception
398 __ verify_not_null_oop(exception_oop);
399
400 #ifdef ASSERT
401 // check that fields in JavaThread for exception oop and issuing pc are
402 // empty before writing to them
403 Label oop_empty;
404 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
405 __ cbz(rscratch1, oop_empty);
406 __ stop("exception oop already set");
407 __ bind(oop_empty);
408
409 Label pc_empty;
410 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
411 __ cbz(rscratch1, pc_empty);
412 __ stop("exception pc already set");
413 __ bind(pc_empty);
414 #endif
415
416 // save exception oop and issuing pc into JavaThread
417 // (exception handler will load it from here)
418 __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
419 __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
420
421 // patch throwing pc into return address (has bci & oop map)
422 __ str(exception_pc, Address(rfp, 1*BytesPerWord));
423
424 // compute the exception handler.
425 // the exception oop and the throwing pc are read from the fields in JavaThread
426 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
427 oop_maps->add_gc_map(call_offset, oop_map);
428
429 // r0: handler address
430 // will be the deopt blob if nmethod was deoptimized while we looked up
431 // handler regardless of whether handler existed in the nmethod.
432
433 // only r0 is valid at this time, all other registers have been destroyed by the runtime call
434 __ invalidate_registers(false, true, true, true, true, true);
435
436 // patch the return address, this stub will directly return to the exception handler
437 __ str(r0, Address(rfp, 1*BytesPerWord));
438
439 switch (id) {
440 case forward_exception_id:
441 case handle_exception_nofpu_id:
442 case handle_exception_id:
443 // Restore the registers that were saved at the beginning.
444 restore_live_registers(sasm, id != handle_exception_nofpu_id);
445 break;
446 case handle_exception_from_callee_id:
447 // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP
448 // since we do a leave anyway.
449
450 // Pop the return address since we are possibly changing SP (restoring from BP).
451 __ leave();
452
453 // Restore SP from FP if the exception PC is a method handle call site.
454 {
455 Label nope;
456 __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
457 __ cbzw(rscratch1, nope);
458 __ mov(sp, rfp);
459 __ bind(nope);
460 }
461
462 __ ret(lr); // jump to exception handler
463 break;
464 default: ShouldNotReachHere();
465 }
466
467 return oop_maps;
468 }
469
470
471 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
472 // incoming parameters
473 const Register exception_oop = r0;
474 // callee-saved copy of exception_oop during runtime call
475 const Register exception_oop_callee_saved = r19;
476 // other registers used in this stub
477 const Register exception_pc = r3;
478 const Register handler_addr = r1;
479
480 // verify that only r0, is valid at this time
481 __ invalidate_registers(false, true, true, true, true, true);
482
483 #ifdef ASSERT
484 // check that fields in JavaThread for exception oop and issuing pc are empty
485 Label oop_empty;
486 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
487 __ cbz(rscratch1, oop_empty);
488 __ stop("exception oop must be empty");
489 __ bind(oop_empty);
490
491 Label pc_empty;
492 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
493 __ cbz(rscratch1, pc_empty);
494 __ stop("exception pc must be empty");
495 __ bind(pc_empty);
496 #endif
497
498 // Save our return address because
499 // exception_handler_for_return_address will destroy it. We also
500 // save exception_oop
501 __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
502
503 // search the exception handler address of the caller (using the return address)
504 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
505 // r0: exception handler address of the caller
506
507 // Only R0 is valid at this time; all other registers have been
508 // destroyed by the call.
509 __ invalidate_registers(false, true, true, true, false, true);
510
511 // move result of call into correct register
512 __ mov(handler_addr, r0);
513
514 // get throwing pc (= return address).
515 // lr has been destroyed by the call
516 __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
517 __ mov(r3, lr);
518
519 __ verify_not_null_oop(exception_oop);
520
521 {
522 Label foo;
523 __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
524 __ cbzw(rscratch1, foo);
525 __ mov(sp, rfp);
526 __ bind(foo);
527 }
528
529 // continue at exception handler (return address removed)
530 // note: do *not* remove arguments when unwinding the
531 // activation since the caller assumes having
532 // all arguments on the stack when entering the
533 // runtime to determine the exception handler
534 // (GC happens at call site with arguments!)
535 // r0: exception oop
536 // r3: throwing pc
537 // r1: exception handler
538 __ br(handler_addr);
539 }
540
541
542
543 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
544 // use the maximum number of runtime-arguments here because it is difficult to
545 // distinguish each RT-Call.
546 // Note: This number affects also the RT-Call in generate_handle_exception because
547 // the oop-map is shared for all calls.
548 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
549 assert(deopt_blob != NULL, "deoptimization blob must have been created");
550
551 OopMap* oop_map = save_live_registers(sasm);
552
553 __ mov(c_rarg0, rthread);
554 Label retaddr;
555 __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
556 // do the call
557 __ lea(rscratch1, RuntimeAddress(target));
558 __ blr(rscratch1);
559 __ bind(retaddr);
560 OopMapSet* oop_maps = new OopMapSet();
561 oop_maps->add_gc_map(__ offset(), oop_map);
562 // verify callee-saved register
563 #ifdef ASSERT
564 { Label L;
565 __ get_thread(rscratch1);
566 __ cmp(rthread, rscratch1);
567 __ br(Assembler::EQ, L);
568 __ stop("StubAssembler::call_RT: rthread not callee saved?");
569 __ bind(L);
570 }
571 #endif
572 __ reset_last_Java_frame(true);
573 __ maybe_isb();
574
575 // check for pending exceptions
576 { Label L;
577 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
578 __ cbz(rscratch1, L);
579 // exception pending => remove activation and forward to exception handler
580
581 { Label L1;
582 __ cbnz(r0, L1); // have we deoptimized?
583 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
584 __ bind(L1);
585 }
586
587 // the deopt blob expects exceptions in the special fields of
588 // JavaThread, so copy and clear pending exception.
589
590 // load and clear pending exception
591 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
592 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
593
594 // check that there is really a valid exception
595 __ verify_not_null_oop(r0);
596
597 // load throwing pc: this is the return address of the stub
598 __ mov(r3, lr);
599
600 #ifdef ASSERT
601 // check that fields in JavaThread for exception oop and issuing pc are empty
602 Label oop_empty;
603 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
604 __ cbz(rscratch1, oop_empty);
605 __ stop("exception oop must be empty");
606 __ bind(oop_empty);
607
608 Label pc_empty;
609 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
610 __ cbz(rscratch1, pc_empty);
611 __ stop("exception pc must be empty");
612 __ bind(pc_empty);
613 #endif
614
615 // store exception oop and throwing pc to JavaThread
616 __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
617 __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
618
619 restore_live_registers(sasm);
620
621 __ leave();
622
623 // Forward the exception directly to deopt blob. We can blow no
624 // registers and must leave throwing pc on the stack. A patch may
625 // have values live in registers so the entry point with the
626 // exception in tls.
627 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
628
629 __ bind(L);
630 }
631
632
633 // Runtime will return true if the nmethod has been deoptimized during
634 // the patching process. In that case we must do a deopt reexecute instead.
635
636 Label reexecuteEntry, cont;
637
638 __ cbz(r0, cont); // have we deoptimized?
639
640 // Will reexecute. Proper return address is already on the stack we just restore
641 // registers, pop all of our frame but the return address and jump to the deopt blob
642 restore_live_registers(sasm);
643 __ leave();
644 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
645
646 __ bind(cont);
647 restore_live_registers(sasm);
648 __ leave();
649 __ ret(lr);
650
651 return oop_maps;
652 }
653
654
655 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
656
657 const Register exception_oop = r0;
658 const Register exception_pc = r3;
659
660 // for better readability
661 const bool must_gc_arguments = true;
662 const bool dont_gc_arguments = false;
663
664 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
665 bool save_fpu_registers = true;
666
667 // stub code & info for the different stubs
668 OopMapSet* oop_maps = NULL;
669 OopMap* oop_map = NULL;
670 switch (id) {
671 {
672 case forward_exception_id:
673 {
674 oop_maps = generate_handle_exception(id, sasm);
675 __ leave();
676 __ ret(lr);
677 }
678 break;
679
680 case throw_div0_exception_id:
681 { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
682 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
683 }
684 break;
685
686 case throw_null_pointer_exception_id:
687 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
688 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
689 }
690 break;
691
692 case new_instance_id:
693 case fast_new_instance_id:
694 case fast_new_instance_init_check_id:
695 {
696 Register klass = r3; // Incoming
697 Register obj = r0; // Result
698
699 if (id == new_instance_id) {
700 __ set_info("new_instance", dont_gc_arguments);
701 } else if (id == fast_new_instance_id) {
702 __ set_info("fast new_instance", dont_gc_arguments);
703 } else {
704 assert(id == fast_new_instance_init_check_id, "bad StubID");
705 __ set_info("fast new_instance init check", dont_gc_arguments);
706 }
707
708 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
709 UseTLAB && FastTLABRefill) {
710 Label slow_path;
711 Register obj_size = r2;
712 Register t1 = r19;
713 Register t2 = r4;
714 assert_different_registers(klass, obj, obj_size, t1, t2);
715
716 __ stp(r5, r19, Address(__ pre(sp, -2 * wordSize)));
717
718 if (id == fast_new_instance_init_check_id) {
719 // make sure the klass is initialized
720 __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
721 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
722 __ br(Assembler::NE, slow_path);
723 }
724
725 #ifdef ASSERT
726 // assert object can be fast path allocated
727 {
728 Label ok, not_ok;
729 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
730 __ cmp(obj_size, 0u);
731 __ br(Assembler::LE, not_ok); // make sure it's an instance (LH > 0)
732 __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
733 __ br(Assembler::EQ, ok);
734 __ bind(not_ok);
735 __ stop("assert(can be fast path allocated)");
736 __ should_not_reach_here();
737 __ bind(ok);
738 }
739 #endif // ASSERT
740
741 // if we got here then the TLAB allocation failed, so try
742 // refilling the TLAB or allocating directly from eden.
743 Label retry_tlab, try_eden;
744 __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy r3 (klass), returns r5
745
746 __ bind(retry_tlab);
747
748 // get the instance size (size is postive so movl is fine for 64bit)
749 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
750
751 __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
752
753 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
754 __ verify_oop(obj);
755 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
756 __ ret(lr);
757
758 __ bind(try_eden);
759 // get the instance size (size is postive so movl is fine for 64bit)
760 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
761
762 __ eden_allocate(obj, obj_size, 0, t1, slow_path);
763 __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1);
764
765 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
766 __ verify_oop(obj);
767 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
768 __ ret(lr);
769
770 __ bind(slow_path);
771 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
772 }
773
774 __ enter();
775 OopMap* map = save_live_registers(sasm);
776 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
777 oop_maps = new OopMapSet();
778 oop_maps->add_gc_map(call_offset, map);
779 restore_live_registers_except_r0(sasm);
780 __ verify_oop(obj);
781 __ leave();
782 __ ret(lr);
783
784 // r0,: new instance
785 }
786
787 break;
788
789 case counter_overflow_id:
790 {
791 Register bci = r0, method = r1;
792 __ enter();
793 OopMap* map = save_live_registers(sasm);
794 // Retrieve bci
795 __ ldrw(bci, Address(rfp, 2*BytesPerWord));
796 // And a pointer to the Method*
797 __ ldr(method, Address(rfp, 3*BytesPerWord));
798 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
799 oop_maps = new OopMapSet();
800 oop_maps->add_gc_map(call_offset, map);
801 restore_live_registers(sasm);
802 __ leave();
803 __ ret(lr);
804 }
805 break;
806
807 case new_type_array_id:
808 case new_object_array_id:
809 {
810 Register length = r19; // Incoming
811 Register klass = r3; // Incoming
812 Register obj = r0; // Result
813
814 if (id == new_type_array_id) {
815 __ set_info("new_type_array", dont_gc_arguments);
816 } else {
817 __ set_info("new_object_array", dont_gc_arguments);
818 }
819
820 #ifdef ASSERT
821 // assert object type is really an array of the proper kind
822 {
823 Label ok;
824 Register t0 = obj;
825 __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
826 __ asrw(t0, t0, Klass::_lh_array_tag_shift);
827 int tag = ((id == new_type_array_id)
828 ? Klass::_lh_array_tag_type_value
829 : Klass::_lh_array_tag_obj_value);
830 __ mov(rscratch1, tag);
831 __ cmpw(t0, rscratch1);
832 __ br(Assembler::EQ, ok);
833 __ stop("assert(is an array klass)");
834 __ should_not_reach_here();
835 __ bind(ok);
836 }
837 #endif // ASSERT
838
839 if (UseTLAB && FastTLABRefill) {
840 Register arr_size = r4;
841 Register t1 = r2;
842 Register t2 = r5;
843 Label slow_path;
844 assert_different_registers(length, klass, obj, arr_size, t1, t2);
845
846 // check that array length is small enough for fast path.
847 __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
848 __ cmpw(length, rscratch1);
849 __ br(Assembler::HI, slow_path);
850
851 // if we got here then the TLAB allocation failed, so try
852 // refilling the TLAB or allocating directly from eden.
853 Label retry_tlab, try_eden;
854 const Register thread =
855 __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves r19 & r3, returns rthread
856
857 __ bind(retry_tlab);
858
859 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
860 // since size is positive ldrw does right thing on 64bit
861 __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
862 __ lslvw(arr_size, length, t1);
863 __ ubfx(t1, t1, Klass::_lh_header_size_shift,
864 exact_log2(Klass::_lh_header_size_mask + 1));
865 __ add(arr_size, arr_size, t1);
866 __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
867 __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
868
869 __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size
870
871 __ initialize_header(obj, klass, length, t1, t2);
872 __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
873 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
874 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
875 __ andr(t1, t1, Klass::_lh_header_size_mask);
876 __ sub(arr_size, arr_size, t1); // body length
877 __ add(t1, t1, obj); // body start
878 __ initialize_body(t1, arr_size, 0, t2);
879 __ membar(Assembler::StoreStore);
880 __ verify_oop(obj);
881
882 __ ret(lr);
883
884 __ bind(try_eden);
885 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
886 // since size is positive ldrw does right thing on 64bit
887 __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
888 // since size is postive movw does right thing on 64bit
889 __ movw(arr_size, length);
890 __ lslvw(arr_size, length, t1);
891 __ ubfx(t1, t1, Klass::_lh_header_size_shift,
892 exact_log2(Klass::_lh_header_size_mask + 1));
893 __ add(arr_size, arr_size, t1);
894 __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
895 __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
896
897 __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size
898 __ incr_allocated_bytes(thread, arr_size, 0, rscratch1);
899
900 __ initialize_header(obj, klass, length, t1, t2);
901 __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
902 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
903 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
904 __ andr(t1, t1, Klass::_lh_header_size_mask);
905 __ sub(arr_size, arr_size, t1); // body length
906 __ add(t1, t1, obj); // body start
907 __ initialize_body(t1, arr_size, 0, t2);
908 __ membar(Assembler::StoreStore);
909 __ verify_oop(obj);
910
911 __ ret(lr);
912
913 __ bind(slow_path);
914 }
915
916 __ enter();
917 OopMap* map = save_live_registers(sasm);
918 int call_offset;
919 if (id == new_type_array_id) {
920 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
921 } else {
922 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
923 }
924
925 oop_maps = new OopMapSet();
926 oop_maps->add_gc_map(call_offset, map);
927 restore_live_registers_except_r0(sasm);
928
929 __ verify_oop(obj);
930 __ leave();
931 __ ret(lr);
932
933 // r0: new array
934 }
935 break;
936
937 case new_multi_array_id:
938 { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
939 // r0,: klass
940 // r19,: rank
941 // r2: address of 1st dimension
942 OopMap* map = save_live_registers(sasm);
943 __ mov(c_rarg1, r0);
944 __ mov(c_rarg3, r2);
945 __ mov(c_rarg2, r19);
946 int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
947
948 oop_maps = new OopMapSet();
949 oop_maps->add_gc_map(call_offset, map);
950 restore_live_registers_except_r0(sasm);
951
952 // r0,: new multi array
953 __ verify_oop(r0);
954 }
955 break;
956
957 case register_finalizer_id:
958 {
959 __ set_info("register_finalizer", dont_gc_arguments);
960
961 // This is called via call_runtime so the arguments
962 // will be place in C abi locations
963
964 __ verify_oop(c_rarg0);
965
966 // load the klass and check the has finalizer flag
967 Label register_finalizer;
968 Register t = r5;
969 __ load_klass(t, r0);
970 __ ldrw(t, Address(t, Klass::access_flags_offset()));
971 __ tst(t, JVM_ACC_HAS_FINALIZER);
972 __ br(Assembler::NE, register_finalizer);
973 __ ret(lr);
974
975 __ bind(register_finalizer);
976 __ enter();
977 OopMap* oop_map = save_live_registers(sasm);
978 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
979 oop_maps = new OopMapSet();
980 oop_maps->add_gc_map(call_offset, oop_map);
981
982 // Now restore all the live registers
983 restore_live_registers(sasm);
984
985 __ leave();
986 __ ret(lr);
987 }
988 break;
989
990 case throw_class_cast_exception_id:
991 { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
992 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
993 }
994 break;
995
996 case throw_incompatible_class_change_error_id:
997 { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
998 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
999 }
1000 break;
1001
1002 case slow_subtype_check_id:
1003 {
1004 // Typical calling sequence:
1005 // __ push(klass_RInfo); // object klass or other subclass
1006 // __ push(sup_k_RInfo); // array element klass or other superclass
1007 // __ bl(slow_subtype_check);
1008 // Note that the subclass is pushed first, and is therefore deepest.
1009 enum layout {
1010 r0_off, r0_off_hi,
1011 r2_off, r2_off_hi,
1012 r4_off, r4_off_hi,
1013 r5_off, r5_off_hi,
1014 sup_k_off, sup_k_off_hi,
1015 klass_off, klass_off_hi,
1016 framesize,
1017 result_off = sup_k_off
1018 };
1019
1020 __ set_info("slow_subtype_check", dont_gc_arguments);
1021 __ push(RegSet::of(r0, r2, r4, r5), sp);
1022
1023 // This is called by pushing args and not with C abi
1024 // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1025 // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1026
1027 __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
1028
1029 Label miss;
1030 __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
1031
1032 // fallthrough on success:
1033 __ mov(rscratch1, 1);
1034 __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1035 __ pop(RegSet::of(r0, r2, r4, r5), sp);
1036 __ ret(lr);
1037
1038 __ bind(miss);
1039 __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1040 __ pop(RegSet::of(r0, r2, r4, r5), sp);
1041 __ ret(lr);
1042 }
1043 break;
1044
1045 case monitorenter_nofpu_id:
1046 save_fpu_registers = false;
1047 // fall through
1048 case monitorenter_id:
1049 {
1050 StubFrame f(sasm, "monitorenter", dont_gc_arguments);
1051 OopMap* map = save_live_registers(sasm, save_fpu_registers);
1052
1053 // Called with store_parameter and not C abi
1054
1055 f.load_argument(1, r0); // r0,: object
1056 f.load_argument(0, r1); // r1,: lock address
1057
1058 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
1059
1060 oop_maps = new OopMapSet();
1061 oop_maps->add_gc_map(call_offset, map);
1062 restore_live_registers(sasm, save_fpu_registers);
1063 }
1064 break;
1065
1066 case monitorexit_nofpu_id:
1067 save_fpu_registers = false;
1068 // fall through
1069 case monitorexit_id:
1070 {
1071 StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1072 OopMap* map = save_live_registers(sasm, save_fpu_registers);
1073
1074 // Called with store_parameter and not C abi
1075
1076 f.load_argument(0, r0); // r0,: lock address
1077
1078 // note: really a leaf routine but must setup last java sp
1079 // => use call_RT for now (speed can be improved by
1080 // doing last java sp setup manually)
1081 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1082
1083 oop_maps = new OopMapSet();
1084 oop_maps->add_gc_map(call_offset, map);
1085 restore_live_registers(sasm, save_fpu_registers);
1086 }
1087 break;
1088
1089 case deoptimize_id:
1090 {
1091 StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1092 OopMap* oop_map = save_live_registers(sasm);
1093 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize));
1094 oop_maps = new OopMapSet();
1095 oop_maps->add_gc_map(call_offset, oop_map);
1096 restore_live_registers(sasm);
1097 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1098 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1099 __ leave();
1100 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1101 }
1102 break;
1103
1104 case throw_range_check_failed_id:
1105 { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1106 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1107 }
1108 break;
1109
1110 case unwind_exception_id:
1111 { __ set_info("unwind_exception", dont_gc_arguments);
1112 // note: no stubframe since we are about to leave the current
1113 // activation and we are calling a leaf VM function only.
1114 generate_unwind_exception(sasm);
1115 }
1116 break;
1117
1118 case access_field_patching_id:
1119 { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1120 // we should set up register map
1121 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1122 }
1123 break;
1124
1125 case load_klass_patching_id:
1126 { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1127 // we should set up register map
1128 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1129 }
1130 break;
1131
1132 case load_mirror_patching_id:
1133 { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1134 // we should set up register map
1135 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1136 }
1137 break;
1138
1139 case load_appendix_patching_id:
1140 { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1141 // we should set up register map
1142 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1143 }
1144 break;
1145
1146 case handle_exception_nofpu_id:
1147 case handle_exception_id:
1148 { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1149 oop_maps = generate_handle_exception(id, sasm);
1150 }
1151 break;
1152
1153 case handle_exception_from_callee_id:
1154 { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1155 oop_maps = generate_handle_exception(id, sasm);
1156 }
1157 break;
1158
1159 case throw_index_exception_id:
1160 { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1161 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1162 }
1163 break;
1164
1165 case throw_array_store_exception_id:
1166 { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1167 // tos + 0: link
1168 // + 1: return address
1169 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1170 }
1171 break;
1172
1173 #if INCLUDE_ALL_GCS
1174
1175 case g1_pre_barrier_slow_id:
1176 {
1177 StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1178 // arg0 : previous value of memory
1179
1180 BarrierSet* bs = Universe::heap()->barrier_set();
1181 if (bs->kind() != BarrierSet::G1SATBCTLogging) {
1182 __ mov(r0, (int)id);
1183 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1184 __ should_not_reach_here();
1185 break;
1186 }
1187
1188 const Register pre_val = r0;
1189 const Register thread = rthread;
1190 const Register tmp = rscratch1;
1191
1192 Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1193 PtrQueue::byte_offset_of_active()));
1194
1195 Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1196 PtrQueue::byte_offset_of_index()));
1197 Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1198 PtrQueue::byte_offset_of_buf()));
1199
1200 Label done;
1201 Label runtime;
1202
1203 // Can we store original value in the thread's buffer?
1204 __ ldr(tmp, queue_index);
1205 __ cbz(tmp, runtime);
1206
1207 __ sub(tmp, tmp, wordSize);
1208 __ str(tmp, queue_index);
1209 __ ldr(rscratch2, buffer);
1210 __ add(tmp, tmp, rscratch2);
1211 f.load_argument(0, rscratch2);
1212 __ str(rscratch2, Address(tmp, 0));
1213 __ b(done);
1214
1215 __ bind(runtime);
1216 __ push_call_clobbered_registers();
1217 f.load_argument(0, pre_val);
1218 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
1219 __ pop_call_clobbered_registers();
1220 __ bind(done);
1221 }
1222 break;
1223 case g1_post_barrier_slow_id:
1224 {
1225 StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1226
1227 // arg0: store_address
1228 Address store_addr(rfp, 2*BytesPerWord);
1229
1230 BarrierSet* bs = Universe::heap()->barrier_set();
1231 CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1232 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1233
1234 Label done;
1235 Label runtime;
1236
1237 // At this point we know new_value is non-NULL and the new_value crosses regions.
1238 // Must check to see if card is already dirty
1239
1240 const Register thread = rthread;
1241
1242 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1243 PtrQueue::byte_offset_of_index()));
1244 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1245 PtrQueue::byte_offset_of_buf()));
1246
1247 const Register card_offset = rscratch2;
1248 // LR is free here, so we can use it to hold the byte_map_base.
1249 const Register byte_map_base = lr;
1250
1251 assert_different_registers(card_offset, byte_map_base, rscratch1);
1252
1253 f.load_argument(0, card_offset);
1254 __ lsr(card_offset, card_offset, CardTableModRefBS::card_shift);
1255 __ load_byte_map_base(byte_map_base);
1256 __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1257 __ cmpw(rscratch1, (int)G1SATBCardTableModRefBS::g1_young_card_val());
1258 __ br(Assembler::EQ, done);
1259
1260 assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
1261
1262 __ membar(Assembler::StoreLoad);
1263 __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1264 __ cbzw(rscratch1, done);
1265
1266 // storing region crossing non-NULL, card is clean.
1267 // dirty card and log.
1268 __ strb(zr, Address(byte_map_base, card_offset));
1269
1270 // Convert card offset into an address in card_addr
1271 Register card_addr = card_offset;
1272 __ add(card_addr, byte_map_base, card_addr);
1273
1274 __ ldr(rscratch1, queue_index);
1275 __ cbz(rscratch1, runtime);
1276 __ sub(rscratch1, rscratch1, wordSize);
1277 __ str(rscratch1, queue_index);
1278
1279 // Reuse LR to hold buffer_addr
1280 const Register buffer_addr = lr;
1281
1282 __ ldr(buffer_addr, buffer);
1283 __ str(card_addr, Address(buffer_addr, rscratch1));
1284 __ b(done);
1285
1286 __ bind(runtime);
1287 __ push_call_clobbered_registers();
1288 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1289 __ pop_call_clobbered_registers();
1290 __ bind(done);
1291
1292 }
1293 break;
1294 #endif
1295
1296 case predicate_failed_trap_id:
1297 {
1298 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1299
1300 OopMap* map = save_live_registers(sasm);
1301
1302 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1303 oop_maps = new OopMapSet();
1304 oop_maps->add_gc_map(call_offset, map);
1305 restore_live_registers(sasm);
1306 __ leave();
1307 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1308 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1309
1310 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1311 }
1312 break;
1313
1314 case dtrace_object_alloc_id:
1315 { // c_rarg0: object
1316 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1317 save_live_registers(sasm);
1318
1319 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0);
1320
1321 restore_live_registers(sasm);
1322 }
1323 break;
1324
1325 default:
1326 { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1327 __ mov(r0, (int)id);
1328 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1329 __ should_not_reach_here();
1330 }
1331 break;
1332 }
1333 }
1334 return oop_maps;
1335 }
1336
1337 #undef __
1338
1339 // Simple helper to see if the caller of a runtime stub which
1340 // entered the VM has been deoptimized
1341
1342 static bool caller_is_deopted() {
1343 JavaThread* thread = JavaThread::current();
1344 RegisterMap reg_map(thread, false);
1345 frame runtime_frame = thread->last_frame();
1346 frame caller_frame = runtime_frame.sender(®_map);
1347 assert(caller_frame.is_compiled_frame(), "must be compiled");
1348 return caller_frame.is_deoptimized_frame();
1349 }
1350
1351 JRT_ENTRY(void, Runtime1::patch_code_aarch64(JavaThread* thread, Runtime1::StubID stub_id ))
1352 {
1353 RegisterMap reg_map(thread, false);
1354
1355 NOT_PRODUCT(_patch_code_slowcase_cnt++;)
1356 // According to the ARMv8 ARM, "Concurrent modification and
1357 // execution of instructions can lead to the resulting instruction
1358 // performing any behavior that can be achieved by executing any
1359 // sequence of instructions that can be executed from the same
1360 // Exception level, except where the instruction before
1361 // modification and the instruction after modification is a B, BL,
1362 // NOP, BKPT, SVC, HVC, or SMC instruction."
1363 //
1364 // This effectively makes the games we play when patching
1365 // impossible, so when we come across an access that needs
1366 // patching we must deoptimize.
1367
1368 if (TracePatching) {
1369 tty->print_cr("Deoptimizing because patch is needed");
1370 }
1371
1372 frame runtime_frame = thread->last_frame();
1373 frame caller_frame = runtime_frame.sender(®_map);
1374
1375 // It's possible the nmethod was invalidated in the last
1376 // safepoint, but if it's still alive then make it not_entrant.
1377 nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
1378 if (nm != NULL) {
1379 nm->make_not_entrant();
1380 }
1381
1382 Deoptimization::deoptimize_frame(thread, caller_frame.id());
1383
1384 // Return to the now deoptimized frame.
1385 }
1386 JRT_END
1387
1388 int Runtime1::access_field_patching(JavaThread* thread) {
1389 //
1390 // NOTE: we are still in Java
1391 //
1392 Thread* THREAD = thread;
1393 debug_only(NoHandleMark nhm;)
1394 {
1395 // Enter VM mode
1396
1397 ResetNoHandleMark rnhm;
1398 patch_code_aarch64(thread, access_field_patching_id);
1399 }
1400 // Back in JAVA, use no oops DON'T safepoint
1401
1402 // Return true if calling code is deoptimized
1403
1404 return caller_is_deopted();
1405 JRT_END
1406
1407
1408 int Runtime1::move_mirror_patching(JavaThread* thread) {
1409 //
1410 // NOTE: we are still in Java
1411 //
1412 Thread* THREAD = thread;
1413 debug_only(NoHandleMark nhm;)
1414 {
1415 // Enter VM mode
1416
1417 ResetNoHandleMark rnhm;
1418 patch_code_aarch64(thread, load_mirror_patching_id);
1419 }
1420 // Back in JAVA, use no oops DON'T safepoint
1421
1422 // Return true if calling code is deoptimized
1423
1424 return caller_is_deopted();
1425 }
1426
1427 int Runtime1::move_appendix_patching(JavaThread* thread) {
1428 //
1429 // NOTE: we are still in Java
1430 //
1431 Thread* THREAD = thread;
1432 debug_only(NoHandleMark nhm;)
1433 {
1434 // Enter VM mode
1435
1436 ResetNoHandleMark rnhm;
1437 patch_code_aarch64(thread, load_appendix_patching_id);
1438 }
1439 // Back in JAVA, use no oops DON'T safepoint
1440
1441 // Return true if calling code is deoptimized
1442
1443 return caller_is_deopted();
1444 }
1445
1446 int Runtime1::move_klass_patching(JavaThread* thread) {
1447 //
1448 // NOTE: we are still in Java
1449 //
1450 Thread* THREAD = thread;
1451 debug_only(NoHandleMark nhm;)
1452 {
1453 // Enter VM mode
1454
1455 ResetNoHandleMark rnhm;
1456 patch_code_aarch64(thread, load_klass_patching_id);
1457 }
1458 // Back in JAVA, use no oops DON'T safepoint
1459
1460 // Return true if calling code is deoptimized
1461
1462 return caller_is_deopted();
1463 }
1464
1465 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }