1 /*
2 * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "interpreter/bytecodeHistogram.hpp"
29 #include "interpreter/interp_masm.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "interpreter/interpreterRuntime.hpp"
32 #include "interpreter/templateInterpreterGenerator.hpp"
33 #include "interpreter/templateTable.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/methodData.hpp"
36 #include "oops/method.inline.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "oops/resolvedIndyEntry.hpp"
39 #include "oops/resolvedMethodEntry.hpp"
40 #include "prims/jvmtiExport.hpp"
41 #include "prims/jvmtiThreadState.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/arguments.hpp"
44 #include "runtime/deoptimization.hpp"
45 #include "runtime/frame.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/synchronizer.hpp"
50 #include "runtime/timer.hpp"
51 #include "runtime/vframeArray.hpp"
52 #include "utilities/align.hpp"
53 #include "utilities/debug.hpp"
54 #include "utilities/macros.hpp"
55
56 // Size of interpreter code. Increase if too small. Interpreter will
57 // fail with a guarantee ("not enough space for interpreter generation");
58 // if too small.
59 // Run with +PrintInterpreter to get the VM to print out the size.
60 // Max size with JVMTI
61 int TemplateInterpreter::InterpreterCodeSize = 180 * 1024;
62
63 #define __ _masm->
64
65 //------------------------------------------------------------------------------------------------------------------------
66
67 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
68 address entry = __ pc();
69
70 // callee-save register for saving LR, shared with generate_native_entry
71 const Register Rsaved_ret_addr = Rtmp_save0;
72
73 __ mov(Rsaved_ret_addr, LR);
74
75 __ mov(R1, Rmethod);
76 __ mov(R2, Rlocals);
77 __ mov(R3, SP);
78
79
80 // Safer to save R9 (when scratched) since callers may have been
81 // written assuming R9 survives. This is suboptimal but
82 // probably not important for this slow case call site.
83 // Note for R9 saving: slow_signature_handler may copy register
84 // arguments above the current SP (passed as R3). It is safe for
85 // call_VM to use push and pop to protect additional values on the
86 // stack if needed.
87 __ call_VM(CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), true /* save R9 if needed*/);
88 __ add(SP, SP, wordSize); // Skip R0
89 __ pop(RegisterSet(R1, R3)); // Load arguments passed in registers
90 #ifdef __ABI_HARD__
91 // Few alternatives to an always-load-FP-registers approach:
92 // - parse method signature to detect FP arguments
93 // - keep a counter/flag on a stack indicationg number of FP arguments in the method.
94 // The later has been originally implemented and tested but a conditional path could
95 // eliminate any gain imposed by avoiding 8 double word loads.
96 __ fldmiad(SP, FloatRegisterSet(D0, 8), writeback);
97 #endif // __ABI_HARD__
98
99 __ ret(Rsaved_ret_addr);
100
101 return entry;
102 }
103
104
105 //
106 // Various method entries (that c++ and asm interpreter agree upon)
107 //------------------------------------------------------------------------------------------------------------------------
108 //
109 //
110
111 // Abstract method entry
112 // Attempt to execute abstract method. Throw exception
113 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
114 address entry_point = __ pc();
115
116
117 __ empty_expression_stack();
118
119 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
120
121 DEBUG_ONLY(STOP("generate_abstract_entry");) // Should not reach here
122 return entry_point;
123 }
124
125 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
126 address entry_point = nullptr;
127 Register continuation = LR;
128 bool use_runtime_call = false;
129 switch (kind) {
130 case Interpreter::java_lang_math_abs:
131 entry_point = __ pc();
132 #ifdef __SOFTFP__
133 use_runtime_call = true;
134 __ ldrd(R0, Address(SP));
135 #else // !__SOFTFP__
136 __ ldr_double(D0, Address(SP));
137 __ abs_double(D0, D0);
138 #endif // __SOFTFP__
139 break;
140 case Interpreter::java_lang_math_sqrt:
141 entry_point = __ pc();
142 #ifdef __SOFTFP__
143 use_runtime_call = true;
144 __ ldrd(R0, Address(SP));
145 #else // !__SOFTFP__
146 __ ldr_double(D0, Address(SP));
147 __ sqrt_double(D0, D0);
148 #endif // __SOFTFP__
149 break;
150 case Interpreter::java_lang_math_sin:
151 case Interpreter::java_lang_math_cos:
152 case Interpreter::java_lang_math_tan:
153 case Interpreter::java_lang_math_log:
154 case Interpreter::java_lang_math_log10:
155 case Interpreter::java_lang_math_exp:
156 entry_point = __ pc();
157 use_runtime_call = true;
158 #ifdef __SOFTFP__
159 __ ldrd(R0, Address(SP));
160 #else // !__SOFTFP__
161 __ ldr_double(D0, Address(SP));
162 #endif // __SOFTFP__
163 break;
164 case Interpreter::java_lang_math_pow:
165 entry_point = __ pc();
166 use_runtime_call = true;
167 #ifdef __SOFTFP__
168 __ ldrd(R0, Address(SP, 2 * Interpreter::stackElementSize));
169 __ ldrd(R2, Address(SP));
170 #else // !__SOFTFP__
171 __ ldr_double(D0, Address(SP, 2 * Interpreter::stackElementSize));
172 __ ldr_double(D1, Address(SP));
173 #endif // __SOFTFP__
174 break;
175 case Interpreter::java_lang_math_fmaD:
176 case Interpreter::java_lang_math_fmaF:
177 case Interpreter::java_lang_math_sinh:
178 case Interpreter::java_lang_math_tanh:
179 case Interpreter::java_lang_math_cbrt:
180 // TODO: Implement intrinsic
181 break;
182 default:
183 ShouldNotReachHere();
184 }
185
186 if (entry_point != nullptr) {
187 __ mov(SP, Rsender_sp);
188 if (use_runtime_call) {
189 __ mov(Rtmp_save0, LR);
190 continuation = Rtmp_save0;
191 generate_math_runtime_call(kind);
192 }
193 __ ret(continuation);
194 }
195 return entry_point;
196 }
197
198 void TemplateInterpreterGenerator::generate_math_runtime_call(AbstractInterpreter::MethodKind kind) {
199 address fn;
200 switch (kind) {
201 #ifdef __SOFTFP__
202 case Interpreter::java_lang_math_abs:
203 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dabs);
204 break;
205 case Interpreter::java_lang_math_sqrt:
206 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsqrt);
207 break;
208 #endif // __SOFTFP__
209 case Interpreter::java_lang_math_sin:
210 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
211 break;
212 case Interpreter::java_lang_math_cos:
213 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
214 break;
215 case Interpreter::java_lang_math_tan:
216 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
217 break;
218 case Interpreter::java_lang_math_log:
219 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
220 break;
221 case Interpreter::java_lang_math_log10:
222 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
223 break;
224 case Interpreter::java_lang_math_exp:
225 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
226 break;
227 case Interpreter::java_lang_math_pow:
228 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow);
229 break;
230 default:
231 ShouldNotReachHere();
232 fn = nullptr; // silence "maybe uninitialized" compiler warnings
233 }
234 __ call_VM_leaf(fn);
235 }
236
237 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
238 address entry = __ pc();
239
240 // Note: There should be a minimal interpreter frame set up when stack
241 // overflow occurs since we check explicitly for it now.
242 //
243 #ifdef ASSERT
244 { Label L;
245 __ sub(Rtemp, FP, - frame::interpreter_frame_monitor_block_top_offset * wordSize);
246 __ cmp(SP, Rtemp); // Rtemp = maximal SP for current FP,
247 // (stack grows negative)
248 __ b(L, ls); // check if frame is complete
249 __ stop ("interpreter frame not set up");
250 __ bind(L);
251 }
252 #endif // ASSERT
253
254 // Restore bcp under the assumption that the current frame is still
255 // interpreted
256 __ restore_bcp();
257
258 // expression stack must be empty before entering the VM if an exception
259 // happened
260 __ empty_expression_stack();
261
262 // throw exception
263 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
264
265 __ should_not_reach_here();
266
267 return entry;
268 }
269
270 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
271 address entry = __ pc();
272
273 // index is in R4_ArrayIndexOutOfBounds_index
274
275 // expression stack must be empty before entering the VM if an exception happened
276 __ empty_expression_stack();
277
278 // setup parameters
279 // Array expected in R1.
280 __ mov(R2, R4_ArrayIndexOutOfBounds_index);
281
282 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), R1, R2);
283
284 __ nop(); // to avoid filling CPU pipeline with invalid instructions
285 __ nop();
286 __ should_not_reach_here();
287
288 return entry;
289 }
290
291 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
292 address entry = __ pc();
293
294 // object is in R2_ClassCastException_obj
295
296 // expression stack must be empty before entering the VM if an exception
297 // happened
298 __ empty_expression_stack();
299
300 __ mov(R1, R2_ClassCastException_obj);
301 __ call_VM(noreg,
302 CAST_FROM_FN_PTR(address,
303 InterpreterRuntime::throw_ClassCastException),
304 R1);
305
306 __ should_not_reach_here();
307
308 return entry;
309 }
310
311 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
312 assert(!pass_oop || message == nullptr, "either oop or message but not both");
313 address entry = __ pc();
314
315 InlinedString Lname(name);
316 InlinedString Lmessage(message);
317
318 if (pass_oop) {
319 // object is at TOS
320 __ pop_ptr(R2);
321 }
322
323 // expression stack must be empty before entering the VM if an exception happened
324 __ empty_expression_stack();
325
326 // setup parameters
327 __ ldr_literal(R1, Lname);
328
329 if (pass_oop) {
330 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), R1, R2);
331 } else {
332 if (message != nullptr) {
333 __ ldr_literal(R2, Lmessage);
334 } else {
335 __ mov(R2, 0);
336 }
337 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), R1, R2);
338 }
339
340 // throw exception
341 __ b(Interpreter::throw_exception_entry());
342
343 __ nop(); // to avoid filling CPU pipeline with invalid instructions
344 __ nop();
345 __ bind_literal(Lname);
346 if (!pass_oop && (message != nullptr)) {
347 __ bind_literal(Lmessage);
348 }
349
350 return entry;
351 }
352
353 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
354 address entry = __ pc();
355
356 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
357
358 // Restore stack bottom in case i2c adjusted stack
359 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
360 // and null it as marker that SP is now tos until next java call
361 __ mov(Rtemp, (int)NULL_WORD);
362 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
363
364 __ restore_method();
365 __ restore_bcp();
366 __ restore_dispatch();
367 __ restore_locals();
368
369 const Register Rcache = R2_tmp;
370 const Register Rindex = R3_tmp;
371
372 if (index_size == sizeof(u4)) {
373 __ load_resolved_indy_entry(Rcache, Rindex);
374 __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
375 } else {
376 // Pop N words from the stack
377 assert(index_size == sizeof(u2), "Can only be u2");
378 __ load_method_entry(Rcache, Rindex);
379 __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
380 }
381
382 __ check_stack_top();
383 __ add(Rstack_top, Rstack_top, AsmOperand(Rcache, lsl, Interpreter::logStackElementSize));
384
385 __ convert_retval_to_tos(state);
386
387 __ check_and_handle_popframe();
388 __ check_and_handle_earlyret();
389
390 __ dispatch_next(state, step);
391
392 return entry;
393 }
394
395 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {
396 address entry = __ pc();
397
398 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
399
400 // The stack is not extended by deopt but we must null last_sp as this
401 // entry is like a "return".
402 __ mov(Rtemp, 0);
403 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
404
405 __ restore_method();
406 __ restore_bcp();
407 __ restore_dispatch();
408 __ restore_locals();
409
410 // handle exceptions
411 { Label L;
412 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
413 __ cbz(Rtemp, L);
414 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
415 __ should_not_reach_here();
416 __ bind(L);
417 }
418
419 if (continuation == nullptr) {
420 __ dispatch_next(state, step);
421 } else {
422 __ jump_to_entry(continuation);
423 }
424
425 return entry;
426 }
427
428 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
429 address entry = __ pc();
430
431 switch (type) {
432 case T_CHAR : /* Nothing to do */ break;
433 case T_BYTE : /* Nothing to do */ break;
434 case T_SHORT : /* Nothing to do */ break;
435 case T_INT : /* Nothing to do */ break;
436 case T_LONG : /* Nothing to do */ break;
437 case T_VOID : /* Nothing to do */ break;
438 case T_DOUBLE : /* Nothing to do */ break;
439 case T_FLOAT : /* Nothing to do */ break;
440 case T_BOOLEAN : __ c2bool(R0); break;
441 case T_OBJECT :
442 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
443 __ verify_oop(R0);
444 break;
445 default : __ should_not_reach_here(); break;
446 }
447
448 __ ret();
449 return entry;
450 }
451
452 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
453 address entry = __ pc();
454 __ push(state);
455 __ call_VM(noreg, runtime_entry);
456
457 // load current bytecode
458 __ ldrb(R3_bytecode, Address(Rbcp));
459 __ dispatch_only_normal(vtos);
460 return entry;
461 }
462
463 address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() {
464 return nullptr;
465 }
466
467
468 // Helpers for commoning out cases in the various type of method entries.
469 //
470
471 // increment invocation count & check for overflow
472 //
473 // Note: checking for negative value instead of overflow
474 // so we have a 'sticky' overflow test
475 //
476 // In: Rmethod.
477 //
478 // Uses R0, R1, Rtemp.
479 //
480 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) {
481 Label done;
482 const Register Rcounters = Rtemp;
483 const Address invocation_counter(Rcounters,
484 MethodCounters::invocation_counter_offset() +
485 InvocationCounter::counter_offset());
486
487 // Note: In tiered we increment either counters in MethodCounters* or
488 // in MDO depending if we're profiling or not.
489 int increment = InvocationCounter::count_increment;
490 Label no_mdo;
491 if (ProfileInterpreter) {
492 // Are we profiling?
493 __ ldr(R1_tmp, Address(Rmethod, Method::method_data_offset()));
494 __ cbz(R1_tmp, no_mdo);
495 // Increment counter in the MDO
496 const Address mdo_invocation_counter(R1_tmp,
497 in_bytes(MethodData::invocation_counter_offset()) +
498 in_bytes(InvocationCounter::counter_offset()));
499 const Address mask(R1_tmp, in_bytes(MethodData::invoke_mask_offset()));
500 __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, R0_tmp, Rtemp, eq, overflow);
501 __ b(done);
502 }
503 __ bind(no_mdo);
504 __ get_method_counters(Rmethod, Rcounters, done);
505 const Address mask(Rcounters, in_bytes(MethodCounters::invoke_mask_offset()));
506 __ increment_mask_and_jump(invocation_counter, increment, mask, R0_tmp, R1_tmp, eq, overflow);
507 __ bind(done);
508 }
509
510 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
511 // InterpreterRuntime::frequency_counter_overflow takes one argument
512 // indicating if the counter overflow occurs at a backwards branch (non-null bcp).
513 // The call returns the address of the verified entry point for the method or null
514 // if the compilation did not complete (either went background or bailed out).
515 __ mov(R1, (int)false);
516 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);
517
518 // jump to the interpreted entry.
519 __ b(do_continue);
520 }
521
522 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
523 // Check if we've got enough room on the stack for
524 // - overhead;
525 // - locals;
526 // - expression stack.
527 //
528 // Registers on entry:
529 //
530 // R3 = number of additional locals
531 // Rthread
532 // Rmethod
533 // Registers used: R0, R1, R2, Rtemp.
534
535 const Register Radditional_locals = R3;
536 const Register RmaxStack = R2;
537
538 // monitor entry size
539 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
540
541 // total overhead size: entry_size + (saved registers, thru expr stack bottom).
542 // be sure to change this if you add/subtract anything to/from the overhead area
543 const int overhead_size = (frame::sender_sp_offset - frame::interpreter_frame_initial_sp_offset)*wordSize + entry_size;
544
545 // Pages reserved for VM runtime calls and subsequent Java calls.
546 const int reserved_pages = StackOverflow::stack_shadow_zone_size();
547
548 // Thread::stack_size() includes guard pages, and they should not be touched.
549 const int guard_pages = StackOverflow::stack_guard_zone_size();
550
551 __ ldr(R0, Address(Rthread, Thread::stack_base_offset()));
552 __ ldr(R1, Address(Rthread, Thread::stack_size_offset()));
553 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
554 __ ldrh(RmaxStack, Address(Rtemp, ConstMethod::max_stack_offset()));
555 __ sub_slow(Rtemp, SP, overhead_size + reserved_pages + guard_pages + Method::extra_stack_words());
556
557 // reserve space for additional locals
558 __ sub(Rtemp, Rtemp, AsmOperand(Radditional_locals, lsl, Interpreter::logStackElementSize));
559
560 // stack size
561 __ sub(R0, R0, R1);
562
563 // reserve space for expression stack
564 __ sub(Rtemp, Rtemp, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
565
566 __ cmp(Rtemp, R0);
567
568 __ mov(SP, Rsender_sp, ls); // restore SP
569 __ b(SharedRuntime::throw_StackOverflowError_entry(), ls);
570 }
571
572
573 // Allocate monitor and lock method (asm interpreter)
574 //
575 void TemplateInterpreterGenerator::lock_method() {
576 // synchronize method
577
578 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
579 assert ((entry_size % StackAlignmentInBytes) == 0, "should keep stack alignment");
580
581 #ifdef ASSERT
582 { Label L;
583 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
584 __ tbnz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
585 __ stop("method doesn't need synchronization");
586 __ bind(L);
587 }
588 #endif // ASSERT
589
590 // get synchronization object
591 { Label done;
592 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
593 __ tst(Rtemp, JVM_ACC_STATIC);
594 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0)), eq); // get receiver (assume this is frequent case)
595 __ b(done, eq);
596 __ load_mirror(R0, Rmethod, Rtemp);
597 __ bind(done);
598 }
599
600 // add space for monitor & lock
601
602
603 __ sub(Rstack_top, Rstack_top, entry_size);
604 __ check_stack_top_on_expansion();
605 // add space for a monitor entry
606 __ str(Rstack_top, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
607 // set new monitor block top
608 __ str(R0, Address(Rstack_top, BasicObjectLock::obj_offset()));
609 // store object
610 __ mov(R1, Rstack_top); // monitor entry address
611 __ lock_object(R1);
612 }
613
614
615 //
616 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
617 // and for native methods hence the shared code.
618
619 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
620 // Generates the following stack layout:
621 //
622 // [ expr. stack bottom ]
623 // [ saved Rbcp ]
624 // [ current Rlocals ]
625 // [ cache ]
626 // [ mdx ]
627 // [ Method* ]
628 // [ last_sp ]
629 // [ sender_sp ]
630 // [ saved FP ] <--- FP
631 // [ saved LR ]
632
633 // initialize fixed part of activation frame
634 __ push(LR); // save return address
635 __ push(FP); // save FP
636 __ mov(FP, SP); // establish new FP
637
638 __ push(Rsender_sp);
639
640 __ mov(R0, 0);
641 __ push(R0); // leave last_sp as null
642
643 // setup Rbcp
644 if (native_call) {
645 __ mov(Rbcp, 0); // bcp = 0 for native calls
646 } else {
647 __ ldr(Rtemp, Address(Rmethod, Method::const_offset())); // get ConstMethod*
648 __ add(Rbcp, Rtemp, ConstMethod::codes_offset()); // get codebase
649 }
650
651 __ push(Rmethod); // save Method*
652 // Get mirror and store it in the frame as GC root for this Method*
653 __ load_mirror(Rtemp, Rmethod, Rtemp);
654 __ push(Rtemp);
655
656 if (ProfileInterpreter) {
657 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
658 __ tst(Rtemp, Rtemp);
659 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()), ne);
660 __ push(Rtemp); // set the mdp (method data pointer)
661 } else {
662 __ push(R0);
663 }
664
665 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
666 __ ldr(Rtemp, Address(Rtemp, ConstMethod::constants_offset()));
667 __ ldr(Rtemp, Address(Rtemp, ConstantPool::cache_offset()));
668 __ push(Rtemp); // set constant pool cache
669 __ sub(Rtemp, Rlocals, FP);
670 __ logical_shift_right(Rtemp, Rtemp, Interpreter::logStackElementSize); // Rtemp = Rlocals - fp();
671 __ push(Rtemp); // set relativized Rlocals, see frame::interpreter_frame_locals()
672 __ push(Rbcp); // set bcp
673 __ push(R0); // reserve word for pointer to expression stack bottom
674 __ str(SP, Address(SP, 0)); // set expression stack bottom
675 }
676
677
678 // End of helpers
679
680 //------------------------------------------------------------------------------------------------------------------------
681 // Entry points
682 //
683 // Here we generate the various kind of entries into the interpreter.
684 // The two main entry type are generic bytecode methods and native call method.
685 // These both come in synchronized and non-synchronized versions but the
686 // frame layout they create is very similar. The other method entry
687 // types are really just special purpose entries that are really entry
688 // and interpretation all in one. These are for trivial methods like
689 // accessor, empty, or special math methods.
690 //
691 // When control flow reaches any of the entry types for the interpreter
692 // the following holds ->
693 //
694 // Arguments:
695 //
696 // Rmethod: Method*
697 // Rthread: thread
698 // Rsender_sp: sender sp
699 // Rparams (SP on 32-bit ARM): pointer to method parameters
700 //
701 // LR: return address
702 //
703 // Stack layout immediately at entry
704 //
705 // [ parameter n ] <--- Rparams (SP on 32-bit ARM)
706 // ...
707 // [ parameter 1 ]
708 // [ expression stack ] (caller's java expression stack)
709
710 // Assuming that we don't go to one of the trivial specialized
711 // entries the stack will look like below when we are ready to execute
712 // the first bytecode (or call the native routine). The register usage
713 // will be as the template based interpreter expects.
714 //
715 // local variables follow incoming parameters immediately; i.e.
716 // the return address is saved at the end of the locals.
717 //
718 // [ expr. stack ] <--- Rstack_top (SP on 32-bit ARM)
719 // [ monitor entry ]
720 // ...
721 // [ monitor entry ]
722 // [ expr. stack bottom ]
723 // [ saved Rbcp ]
724 // [ current Rlocals ]
725 // [ cache ]
726 // [ mdx ]
727 // [ mirror ]
728 // [ Method* ]
729 //
730 // 32-bit ARM:
731 // [ last_sp ]
732 //
733 // [ sender_sp ]
734 // [ saved FP ] <--- FP
735 // [ saved LR ]
736 // [ optional padding(*)]
737 // [ local variable m ]
738 // ...
739 // [ local variable 1 ]
740 // [ parameter n ]
741 // ...
742 // [ parameter 1 ] <--- Rlocals
743 //
744
745 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
746 // Code: _aload_0, _getfield, _areturn
747 // parameter size = 1
748 //
749 // The code that gets generated by this routine is split into 2 parts:
750 // 1. The "intrinsified" code performing an ON_WEAK_OOP_REF load,
751 // 2. The slow path - which is an expansion of the regular method entry.
752 //
753 // Notes:-
754 // * An intrinsic is always executed, where an ON_WEAK_OOP_REF load is performed.
755 // * We may jump to the slow path iff the receiver is null. If the
756 // Reference object is null then we no longer perform an ON_WEAK_OOP_REF load
757 // Thus we can use the regular method entry code to generate the NPE.
758 //
759 // Rmethod: Method*
760 // Rthread: thread
761 // Rsender_sp: sender sp, must be preserved for slow path, set SP to it on fast path
762 // Rparams: parameters
763
764 address entry = __ pc();
765 Label slow_path;
766 const Register Rthis = R0;
767 const Register Rret_addr = Rtmp_save1;
768 assert_different_registers(Rthis, Rret_addr, Rsender_sp);
769
770 const int referent_offset = java_lang_ref_Reference::referent_offset();
771
772 // Check if local 0 != nullptr
773 // If the receiver is null then it is OK to jump to the slow path.
774 __ ldr(Rthis, Address(Rparams));
775 __ cbz(Rthis, slow_path);
776
777 // Preserve LR
778 __ mov(Rret_addr, LR);
779
780 // Load the value of the referent field.
781 const Address field_address(Rthis, referent_offset);
782 __ load_heap_oop(R0, field_address, Rtemp, R1_tmp, R2_tmp, ON_WEAK_OOP_REF);
783
784 // _areturn
785 __ mov(SP, Rsender_sp);
786 __ ret(Rret_addr);
787
788 // generate a vanilla interpreter entry as the slow path
789 __ bind(slow_path);
790 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
791 return entry;
792 }
793
794 // Not supported
795 address TemplateInterpreterGenerator::generate_currentThread() { return nullptr; }
796 address TemplateInterpreterGenerator::generate_CRC32_update_entry() { return nullptr; }
797 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return nullptr; }
798 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return nullptr; }
799 address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() { return nullptr; }
800 address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() { return nullptr; }
801
802 //
803 // Interpreter stub for calling a native method. (asm interpreter)
804 // This sets up a somewhat different looking stack for calling the native method
805 // than the typical interpreter frame setup.
806 //
807
808 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
809 // determine code generation flags
810 bool inc_counter = UseCompiler || CountCompiledCalls;
811
812 // Incoming registers:
813 //
814 // Rmethod: Method*
815 // Rthread: thread
816 // Rsender_sp: sender sp
817 // Rparams: parameters
818
819 address entry_point = __ pc();
820
821 // Register allocation
822 const Register Rsize_of_params = R6;
823 const Register Rsig_handler = Rtmp_save0; // R4
824 const Register Rnative_code = Rtmp_save1; // R5
825 const Register Rresult_handler = R6;
826
827 const Register Rsaved_result_lo = Rtmp_save0; // R4
828 const Register Rsaved_result_hi = Rtmp_save1; // R5
829 FloatRegister saved_result_fp;
830
831
832 __ ldr(Rsize_of_params, Address(Rmethod, Method::const_offset()));
833 __ ldrh(Rsize_of_params, Address(Rsize_of_params, ConstMethod::size_of_parameters_offset()));
834
835 // native calls don't need the stack size check since they have no expression stack
836 // and the arguments are already on the stack and we only add a handful of words
837 // to the stack
838
839 // compute beginning of parameters (Rlocals)
840 __ sub(Rlocals, Rparams, wordSize);
841 __ add(Rlocals, Rlocals, AsmOperand(Rsize_of_params, lsl, Interpreter::logStackElementSize));
842
843 // reserve stack space for oop_temp
844 __ mov(R0, 0);
845 __ push(R0);
846
847 generate_fixed_frame(true); // Note: R9 is now saved in the frame
848
849 // make sure method is native & not abstract
850 #ifdef ASSERT
851 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
852 {
853 Label L;
854 __ tbnz(Rtemp, JVM_ACC_NATIVE_BIT, L);
855 __ stop("tried to execute non-native method as native");
856 __ bind(L);
857 }
858 { Label L;
859 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
860 __ stop("tried to execute abstract method in interpreter");
861 __ bind(L);
862 }
863 #endif
864
865 // increment invocation count & check for overflow
866 Label invocation_counter_overflow;
867 if (inc_counter) {
868 if (synchronized) {
869 // Avoid unlocking method's monitor in case of exception, as it has not
870 // been locked yet.
871 __ set_do_not_unlock_if_synchronized(true, Rtemp);
872 }
873 generate_counter_incr(&invocation_counter_overflow);
874 }
875
876 Label continue_after_compile;
877 __ bind(continue_after_compile);
878
879 if (inc_counter && synchronized) {
880 __ set_do_not_unlock_if_synchronized(false, Rtemp);
881 }
882
883 // check for synchronized methods
884 // Must happen AFTER invocation_counter check and stack overflow check,
885 // so method is not locked if overflows.
886 //
887 if (synchronized) {
888 lock_method();
889 } else {
890 // no synchronization necessary
891 #ifdef ASSERT
892 { Label L;
893 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
894 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
895 __ stop("method needs synchronization");
896 __ bind(L);
897 }
898 #endif
899 }
900
901 // start execution
902 #ifdef ASSERT
903 { Label L;
904 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
905 __ cmp(Rtemp, Rstack_top);
906 __ b(L, eq);
907 __ stop("broken stack frame setup in interpreter 3");
908 __ bind(L);
909 }
910 #endif
911 __ check_extended_sp(Rtemp);
912
913 // jvmti/dtrace support
914 __ notify_method_entry();
915 #if R9_IS_SCRATCHED
916 __ restore_method();
917 #endif
918
919 {
920 Label L;
921 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
922 __ cbnz(Rsig_handler, L);
923 __ mov(R1, Rmethod);
924 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1, true);
925 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
926 __ bind(L);
927 }
928
929 {
930 Label L;
931 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
932 __ cbnz(Rnative_code, L);
933 __ mov(R1, Rmethod);
934 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1);
935 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
936 __ bind(L);
937 }
938
939 // Allocate stack space for arguments
940
941
942 // C functions need aligned stack
943 __ bic(SP, SP, StackAlignmentInBytes - 1);
944 // Multiply by BytesPerLong instead of BytesPerWord, because calling convention
945 // may require empty slots due to long alignment, e.g. func(int, jlong, int, jlong)
946 __ sub(SP, SP, AsmOperand(Rsize_of_params, lsl, LogBytesPerLong));
947
948 #ifdef __ABI_HARD__
949 // Allocate more stack space to accommodate all GP as well as FP registers:
950 // 4 * wordSize
951 // 8 * BytesPerLong
952 int reg_arguments = align_up((4*wordSize) + (8*BytesPerLong), StackAlignmentInBytes);
953 #else
954 // Reserve at least 4 words on the stack for loading
955 // of parameters passed on registers (R0-R3).
956 // See generate_slow_signature_handler().
957 // It is also used for JNIEnv & class additional parameters.
958 int reg_arguments = 4 * wordSize;
959 #endif // __ABI_HARD__
960
961 __ sub(SP, SP, reg_arguments);
962
963
964 // Note: signature handler blows R4 besides all scratch registers.
965 // See AbstractInterpreterGenerator::generate_slow_signature_handler().
966 __ call(Rsig_handler);
967 #if R9_IS_SCRATCHED
968 __ restore_method();
969 #endif
970 __ mov(Rresult_handler, R0);
971
972 // Pass JNIEnv and mirror for static methods
973 {
974 Label L;
975 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
976 __ add(R0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
977 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, L);
978 __ load_mirror(Rtemp, Rmethod, Rtemp);
979 __ add(R1, FP, frame::interpreter_frame_oop_temp_offset * wordSize);
980 __ str(Rtemp, Address(R1, 0));
981 __ bind(L);
982 }
983
984 __ set_last_Java_frame(SP, FP, true, Rtemp);
985
986 // Changing state to _thread_in_native must be the last thing to do
987 // before the jump to native code. At this moment stack must be
988 // safepoint-safe and completely prepared for stack walking.
989 #ifdef ASSERT
990 {
991 Label L;
992 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
993 __ cmp_32(Rtemp, _thread_in_Java);
994 __ b(L, eq);
995 __ stop("invalid thread state");
996 __ bind(L);
997 }
998 #endif
999
1000 // Force all preceding writes to be observed prior to thread state change
1001 __ membar(MacroAssembler::StoreStore, Rtemp);
1002
1003 __ mov(Rtemp, _thread_in_native);
1004 __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1005
1006 __ call(Rnative_code);
1007 #if R9_IS_SCRATCHED
1008 __ restore_method();
1009 #endif
1010
1011 // Set FPSCR/FPCR to a known state
1012 if (AlwaysRestoreFPU) {
1013 __ restore_default_fp_mode();
1014 }
1015
1016 // Do safepoint check
1017 __ mov(Rtemp, _thread_in_native_trans);
1018 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1019
1020 // Force this write out before the read below
1021 if (!UseSystemMemoryBarrier) {
1022 __ membar(MacroAssembler::StoreLoad, Rtemp);
1023 }
1024
1025 // Protect the return value in the interleaved code: save it to callee-save registers.
1026 __ mov(Rsaved_result_lo, R0);
1027 __ mov(Rsaved_result_hi, R1);
1028 #ifdef __ABI_HARD__
1029 // preserve native FP result in a callee-saved register
1030 saved_result_fp = D8;
1031 __ fcpyd(saved_result_fp, D0);
1032 #else
1033 saved_result_fp = fnoreg;
1034 #endif // __ABI_HARD__
1035
1036 {
1037 Label call, skip_call;
1038 __ safepoint_poll(Rtemp, call);
1039 __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1040 __ cmp(R3, 0);
1041 __ b(skip_call, eq);
1042 __ bind(call);
1043 __ mov(R0, Rthread);
1044 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
1045 __ bind(skip_call);
1046
1047 #if R9_IS_SCRATCHED
1048 __ restore_method();
1049 #endif
1050 }
1051
1052 // Perform Native->Java thread transition
1053 __ mov(Rtemp, _thread_in_Java);
1054 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1055
1056 // Zero handles and last_java_sp
1057 __ reset_last_Java_frame(Rtemp);
1058 __ ldr(R3, Address(Rthread, JavaThread::active_handles_offset()));
1059 __ str_32(__ zero_register(Rtemp), Address(R3, JNIHandleBlock::top_offset()));
1060 if (CheckJNICalls) {
1061 __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1062 }
1063
1064 // Unbox oop result, e.g. JNIHandles::resolve result if it's an oop.
1065 {
1066 Label Lnot_oop;
1067 __ mov_slow(Rtemp, AbstractInterpreter::result_handler(T_OBJECT));
1068 __ cmp(Rtemp, Rresult_handler);
1069 __ b(Lnot_oop, ne);
1070 Register value = Rsaved_result_lo;
1071 __ resolve_jobject(value, // value
1072 Rtemp, // tmp1
1073 R1_tmp); // tmp2
1074 // Store resolved result in frame for GC visibility.
1075 __ str(value, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
1076 __ bind(Lnot_oop);
1077 }
1078
1079
1080 // reguard stack if StackOverflow exception happened while in native.
1081 {
1082 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::stack_guard_state_offset()));
1083 __ cmp_32(Rtemp, StackOverflow::stack_guard_yellow_reserved_disabled);
1084 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none, eq);
1085 #if R9_IS_SCRATCHED
1086 __ restore_method();
1087 #endif
1088 }
1089
1090 // check pending exceptions
1091 {
1092 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1093 __ cmp(Rtemp, 0);
1094 __ mov(Rexception_pc, PC, ne);
1095 __ b(StubRoutines::forward_exception_entry(), ne);
1096 }
1097
1098 if (synchronized) {
1099 // address of first monitor
1100 __ sub(R0, FP, - (frame::interpreter_frame_monitor_block_bottom_offset - frame::interpreter_frame_monitor_size()) * wordSize);
1101 __ unlock_object(R0);
1102 }
1103
1104 // jvmti/dtrace support
1105 // Note: This must happen _after_ handling/throwing any exceptions since
1106 // the exception handler code notifies the runtime of method exits
1107 // too. If this happens before, method entry/exit notifications are
1108 // not properly paired (was bug - gri 11/22/99).
1109 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result_lo, Rsaved_result_hi, saved_result_fp);
1110
1111 // Restore the result. Oop result is restored from the stack by the
1112 // result handler.
1113 __ mov(R0, Rsaved_result_lo);
1114 __ mov(R1, Rsaved_result_hi);
1115
1116 #ifdef __ABI_HARD__
1117 // reload native FP result
1118 __ fcpyd(D0, D8);
1119 #endif // __ABI_HARD__
1120
1121 __ blx(Rresult_handler);
1122
1123 // Restore FP/LR, sender_sp and return
1124 __ mov(Rtemp, FP);
1125 __ ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1126 __ ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1127
1128 __ ret();
1129
1130 if (inc_counter) {
1131 // Handle overflow of counter and compile method
1132 __ bind(invocation_counter_overflow);
1133 generate_counter_overflow(continue_after_compile);
1134 }
1135
1136 return entry_point;
1137 }
1138
1139 //
1140 // Generic interpreted method entry to (asm) interpreter
1141 //
1142 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1143 // determine code generation flags
1144 bool inc_counter = UseCompiler || CountCompiledCalls;
1145
1146 // Rmethod: Method*
1147 // Rthread: thread
1148 // Rsender_sp: sender sp (could differ from SP if we were called via c2i)
1149 // Rparams: pointer to the last parameter in the stack
1150
1151 address entry_point = __ pc();
1152
1153 const Register RconstMethod = R3;
1154
1155
1156 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
1157
1158 __ ldrh(R2, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
1159 __ ldrh(R3, Address(RconstMethod, ConstMethod::size_of_locals_offset()));
1160
1161 // setup Rlocals
1162 __ sub(Rlocals, Rparams, wordSize);
1163 __ add(Rlocals, Rlocals, AsmOperand(R2, lsl, Interpreter::logStackElementSize));
1164
1165 __ sub(R3, R3, R2); // number of additional locals
1166
1167
1168 // see if we've got enough room on the stack for locals plus overhead.
1169 generate_stack_overflow_check();
1170
1171 // allocate space for locals
1172 // explicitly initialize locals
1173
1174 // Loop is unrolled 4 times
1175 Label loop;
1176 __ mov(R0, 0);
1177 __ bind(loop);
1178
1179 // #1
1180 __ subs(R3, R3, 1);
1181 __ push(R0, ge);
1182
1183 // #2
1184 __ subs(R3, R3, 1, ge);
1185 __ push(R0, ge);
1186
1187 // #3
1188 __ subs(R3, R3, 1, ge);
1189 __ push(R0, ge);
1190
1191 // #4
1192 __ subs(R3, R3, 1, ge);
1193 __ push(R0, ge);
1194
1195 __ b(loop, gt);
1196
1197 // initialize fixed part of activation frame
1198 generate_fixed_frame(false);
1199
1200 __ restore_dispatch();
1201
1202 // make sure method is not native & not abstract
1203 #ifdef ASSERT
1204 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1205 {
1206 Label L;
1207 __ tbz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1208 __ stop("tried to execute native method as non-native");
1209 __ bind(L);
1210 }
1211 { Label L;
1212 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1213 __ stop("tried to execute abstract method in interpreter");
1214 __ bind(L);
1215 }
1216 #endif
1217
1218 // increment invocation count & check for overflow
1219 Label invocation_counter_overflow;
1220 if (inc_counter) {
1221 if (synchronized) {
1222 // Avoid unlocking method's monitor in case of exception, as it has not
1223 // been locked yet.
1224 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1225 }
1226 generate_counter_incr(&invocation_counter_overflow);
1227 }
1228 Label continue_after_compile;
1229 __ bind(continue_after_compile);
1230
1231 if (inc_counter && synchronized) {
1232 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1233 }
1234 #if R9_IS_SCRATCHED
1235 __ restore_method();
1236 #endif
1237
1238 // check for synchronized methods
1239 // Must happen AFTER invocation_counter check and stack overflow check,
1240 // so method is not locked if overflows.
1241 //
1242 if (synchronized) {
1243 // Allocate monitor and lock method
1244 lock_method();
1245 } else {
1246 // no synchronization necessary
1247 #ifdef ASSERT
1248 { Label L;
1249 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1250 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1251 __ stop("method needs synchronization");
1252 __ bind(L);
1253 }
1254 #endif
1255 }
1256
1257 // start execution
1258 #ifdef ASSERT
1259 { Label L;
1260 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1261 __ cmp(Rtemp, Rstack_top);
1262 __ b(L, eq);
1263 __ stop("broken stack frame setup in interpreter 4");
1264 __ bind(L);
1265 }
1266 #endif
1267 __ check_extended_sp(Rtemp);
1268
1269 // jvmti support
1270 __ notify_method_entry();
1271 #if R9_IS_SCRATCHED
1272 __ restore_method();
1273 #endif
1274
1275 __ dispatch_next(vtos);
1276
1277 // invocation counter overflow
1278 if (inc_counter) {
1279 // Handle overflow of counter and compile method
1280 __ bind(invocation_counter_overflow);
1281 generate_counter_overflow(continue_after_compile);
1282 }
1283
1284 return entry_point;
1285 }
1286
1287 //------------------------------------------------------------------------------------------------------------------------
1288 // Exceptions
1289
1290 void TemplateInterpreterGenerator::generate_throw_exception() {
1291 // Entry point in previous activation (i.e., if the caller was interpreted)
1292 Interpreter::_rethrow_exception_entry = __ pc();
1293 // Rexception_obj: exception
1294
1295 // Clear interpreter_frame_last_sp.
1296 __ mov(Rtemp, 0);
1297 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1298
1299 #if R9_IS_SCRATCHED
1300 __ restore_method();
1301 #endif
1302 __ restore_bcp();
1303 __ restore_dispatch();
1304 __ restore_locals();
1305
1306
1307 // Entry point for exceptions thrown within interpreter code
1308 Interpreter::_throw_exception_entry = __ pc();
1309
1310 // expression stack is undefined here
1311 // Rexception_obj: exception
1312 // Rbcp: exception bcp
1313 __ verify_oop(Rexception_obj);
1314
1315 // expression stack must be empty before entering the VM in case of an exception
1316 __ empty_expression_stack();
1317 // find exception handler address and preserve exception oop
1318 __ mov(R1, Rexception_obj);
1319 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), R1);
1320 // R0: exception handler entry point
1321 // Rexception_obj: preserved exception oop
1322 // Rbcp: bcp for exception handler
1323 __ push_ptr(Rexception_obj); // push exception which is now the only value on the stack
1324 __ jump(R0); // jump to exception handler (may be _remove_activation_entry!)
1325
1326 // If the exception is not handled in the current frame the frame is removed and
1327 // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1328 //
1329 // Note: At this point the bci is still the bxi for the instruction which caused
1330 // the exception and the expression stack is empty. Thus, for any VM calls
1331 // at this point, GC will find a legal oop map (with empty expression stack).
1332
1333 // In current activation
1334 // tos: exception
1335 // Rbcp: exception bcp
1336
1337 //
1338 // JVMTI PopFrame support
1339 //
1340 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1341
1342
1343 __ empty_expression_stack();
1344
1345 // Set the popframe_processing bit in _popframe_condition indicating that we are
1346 // currently handling popframe, so that call_VMs that may happen later do not trigger new
1347 // popframe handling cycles.
1348
1349 __ ldr_s32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1350 __ orr(Rtemp, Rtemp, (unsigned)JavaThread::popframe_processing_bit);
1351 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1352
1353 {
1354 // Check to see whether we are returning to a deoptimized frame.
1355 // (The PopFrame call ensures that the caller of the popped frame is
1356 // either interpreted or compiled and deoptimizes it if compiled.)
1357 // In this case, we can't call dispatch_next() after the frame is
1358 // popped, but instead must save the incoming arguments and restore
1359 // them after deoptimization has occurred.
1360 //
1361 // Note that we don't compare the return PC against the
1362 // deoptimization blob's unpack entry because of the presence of
1363 // adapter frames in C2.
1364 Label caller_not_deoptimized;
1365 __ ldr(R0, Address(FP, frame::return_addr_offset * wordSize));
1366 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), R0);
1367 __ cbnz_32(R0, caller_not_deoptimized);
1368
1369 // Compute size of arguments for saving when returning to deoptimized caller
1370 __ restore_method();
1371 __ ldr(R0, Address(Rmethod, Method::const_offset()));
1372 __ ldrh(R0, Address(R0, ConstMethod::size_of_parameters_offset()));
1373
1374 __ logical_shift_left(R1, R0, Interpreter::logStackElementSize);
1375 // Save these arguments
1376 __ restore_locals();
1377 __ sub(R2, Rlocals, R1);
1378 __ add(R2, R2, wordSize);
1379 __ mov(R0, Rthread);
1380 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), R0, R1, R2);
1381
1382 __ remove_activation(vtos, LR,
1383 /* throw_monitor_exception */ false,
1384 /* install_monitor_exception */ false,
1385 /* notify_jvmdi */ false);
1386
1387 // Inform deoptimization that it is responsible for restoring these arguments
1388 __ mov(Rtemp, JavaThread::popframe_force_deopt_reexecution_bit);
1389 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1390
1391 // Continue in deoptimization handler
1392 __ ret();
1393
1394 __ bind(caller_not_deoptimized);
1395 }
1396
1397 __ remove_activation(vtos, R4,
1398 /* throw_monitor_exception */ false,
1399 /* install_monitor_exception */ false,
1400 /* notify_jvmdi */ false);
1401
1402 // Finish with popframe handling
1403 // A previous I2C followed by a deoptimization might have moved the
1404 // outgoing arguments further up the stack. PopFrame expects the
1405 // mutations to those outgoing arguments to be preserved and other
1406 // constraints basically require this frame to look exactly as
1407 // though it had previously invoked an interpreted activation with
1408 // no space between the top of the expression stack (current
1409 // last_sp) and the top of stack. Rather than force deopt to
1410 // maintain this kind of invariant all the time we call a small
1411 // fixup routine to move the mutated arguments onto the top of our
1412 // expression stack if necessary.
1413 __ mov(R1, SP);
1414 __ ldr(R2, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1415 // PC must point into interpreter here
1416 __ set_last_Java_frame(SP, FP, true, Rtemp);
1417 __ mov(R0, Rthread);
1418 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), R0, R1, R2);
1419 __ reset_last_Java_frame(Rtemp);
1420
1421 // Restore the last_sp and null it out
1422 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1423 __ mov(Rtemp, (int)NULL_WORD);
1424 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1425
1426 __ restore_bcp();
1427 __ restore_dispatch();
1428 __ restore_locals();
1429 __ restore_method();
1430
1431 // The method data pointer was incremented already during
1432 // call profiling. We have to restore the mdp for the current bcp.
1433 if (ProfileInterpreter) {
1434 __ set_method_data_pointer_for_bcp();
1435 }
1436
1437 // Clear the popframe condition flag
1438 assert(JavaThread::popframe_inactive == 0, "adjust this code");
1439 __ str_32(__ zero_register(Rtemp), Address(Rthread, JavaThread::popframe_condition_offset()));
1440
1441 #if INCLUDE_JVMTI
1442 {
1443 Label L_done;
1444
1445 __ ldrb(Rtemp, Address(Rbcp, 0));
1446 __ cmp(Rtemp, Bytecodes::_invokestatic);
1447 __ b(L_done, ne);
1448
1449 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1450 // Detect such a case in the InterpreterRuntime function and return the member name argument, or null.
1451
1452 // get local0
1453 __ ldr(R1, Address(Rlocals, 0));
1454 __ mov(R2, Rmethod);
1455 __ mov(R3, Rbcp);
1456 __ call_VM(R0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), R1, R2, R3);
1457
1458 __ cbz(R0, L_done);
1459
1460 __ str(R0, Address(Rstack_top));
1461 __ bind(L_done);
1462 }
1463 #endif // INCLUDE_JVMTI
1464
1465 __ dispatch_next(vtos);
1466 // end of PopFrame support
1467
1468 Interpreter::_remove_activation_entry = __ pc();
1469
1470 // preserve exception over this code sequence
1471 __ pop_ptr(R0_tos);
1472 __ str(R0_tos, Address(Rthread, JavaThread::vm_result_oop_offset()));
1473 // remove the activation (without doing throws on illegalMonitorExceptions)
1474 __ remove_activation(vtos, Rexception_pc, false, true, false);
1475 // restore exception
1476 __ get_vm_result_oop(Rexception_obj, Rtemp);
1477
1478 // In between activations - previous activation type unknown yet
1479 // compute continuation point - the continuation point expects
1480 // the following registers set up:
1481 //
1482 // Rexception_obj: exception
1483 // Rexception_pc: return address/pc that threw exception
1484 // SP: expression stack of caller
1485 // FP: frame pointer of caller
1486 __ mov(c_rarg0, Rthread);
1487 __ mov(c_rarg1, Rexception_pc);
1488 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);
1489 // Note that an "issuing PC" is actually the next PC after the call
1490
1491 __ jump(R0); // jump to exception handler of caller
1492 }
1493
1494
1495 //
1496 // JVMTI ForceEarlyReturn support
1497 //
1498 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1499 address entry = __ pc();
1500
1501
1502 __ restore_bcp();
1503 __ restore_dispatch();
1504 __ restore_locals();
1505
1506 __ empty_expression_stack();
1507
1508 __ load_earlyret_value(state);
1509
1510 // Clear the earlyret state
1511 __ ldr(Rtemp, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
1512
1513 assert(JvmtiThreadState::earlyret_inactive == 0, "adjust this code");
1514 __ str_32(__ zero_register(R2), Address(Rtemp, JvmtiThreadState::earlyret_state_offset()));
1515
1516 __ remove_activation(state, LR,
1517 false, /* throw_monitor_exception */
1518 false, /* install_monitor_exception */
1519 true); /* notify_jvmdi */
1520
1521 // According to interpreter calling conventions, result is returned in R0/R1,
1522 // so ftos (S0) and dtos (D0) are moved to R0/R1.
1523 // This conversion should be done after remove_activation, as it uses
1524 // push(state) & pop(state) to preserve return value.
1525 __ convert_tos_to_retval(state);
1526 __ ret();
1527
1528 return entry;
1529 } // end of ForceEarlyReturn support
1530
1531
1532 //------------------------------------------------------------------------------------------------------------------------
1533 // Helper for vtos entry point generation
1534
1535 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1536 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1537 Label L;
1538
1539 #ifdef __SOFTFP__
1540 dep = __ pc(); // fall through
1541 #else
1542 fep = __ pc(); __ push(ftos); __ b(L);
1543 dep = __ pc(); __ push(dtos); __ b(L);
1544 #endif // __SOFTFP__
1545
1546 lep = __ pc(); __ push(ltos); __ b(L);
1547
1548 if (VerifyOops) { // can't share atos entry if VerifyOops
1549 aep = __ pc(); __ push(atos); __ b(L);
1550 } else {
1551 aep = __ pc(); // fall through
1552 }
1553
1554 #ifdef __SOFTFP__
1555 fep = __ pc(); // fall through
1556 #endif // __SOFTFP__
1557
1558 bep = cep = sep = // fall through
1559 iep = __ pc(); __ push(itos); // fall through
1560 vep = __ pc(); __ bind(L); // fall through
1561 generate_and_dispatch(t);
1562 }
1563
1564 //------------------------------------------------------------------------------------------------------------------------
1565
1566 // Non-product code
1567 #ifndef PRODUCT
1568 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1569 address entry = __ pc();
1570
1571 // prepare expression stack
1572 __ push(state); // save tosca
1573
1574 // pass tosca registers as arguments
1575 __ mov(R2, R0_tos);
1576 __ mov(R3, R1_tos_hi);
1577 __ mov(R1, LR); // save return address
1578
1579 // call tracer
1580 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), R1, R2, R3);
1581
1582 __ mov(LR, R0); // restore return address
1583 __ pop(state); // restore tosca
1584
1585 // return
1586 __ ret();
1587
1588 return entry;
1589 }
1590
1591
1592 void TemplateInterpreterGenerator::count_bytecode() {
1593 __ inc_global_counter((address) &BytecodeCounter::_counter_value, 0, Rtemp, R2_tmp, true);
1594 }
1595
1596
1597 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1598 __ inc_global_counter((address)&BytecodeHistogram::_counters[0], sizeof(BytecodeHistogram::_counters[0]) * t->bytecode(), Rtemp, R2_tmp, true);
1599 }
1600
1601
1602 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1603 const Register Rindex_addr = R2_tmp;
1604 Label Lcontinue;
1605 InlinedAddress Lcounters((address)BytecodePairHistogram::_counters);
1606 InlinedAddress Lindex((address)&BytecodePairHistogram::_index);
1607 const Register Rcounters_addr = R2_tmp;
1608 const Register Rindex = R4_tmp;
1609
1610 // calculate new index for counter:
1611 // index = (_index >> log2_number_of_codes) | (bytecode << log2_number_of_codes).
1612 // (_index >> log2_number_of_codes) is previous bytecode
1613
1614 __ ldr_literal(Rindex_addr, Lindex);
1615 __ ldr_s32(Rindex, Address(Rindex_addr));
1616 __ mov_slow(Rtemp, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1617 __ orr(Rindex, Rtemp, AsmOperand(Rindex, lsr, BytecodePairHistogram::log2_number_of_codes));
1618 __ str_32(Rindex, Address(Rindex_addr));
1619
1620 // Rindex (R4) contains index of counter
1621
1622 __ ldr_literal(Rcounters_addr, Lcounters);
1623 __ ldr_s32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1624 __ adds_32(Rtemp, Rtemp, 1);
1625 __ b(Lcontinue, mi); // avoid overflow
1626 __ str_32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1627
1628 __ b(Lcontinue);
1629
1630 __ bind_literal(Lindex);
1631 __ bind_literal(Lcounters);
1632
1633 __ bind(Lcontinue);
1634 }
1635
1636
1637 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1638 // Call a little run-time stub to avoid blow-up for each bytecode.
1639 // The run-time runtime saves the right registers, depending on
1640 // the tosca in-state for the given template.
1641 assert(Interpreter::trace_code(t->tos_in()) != nullptr,
1642 "entry must have been generated");
1643 address trace_entry = Interpreter::trace_code(t->tos_in());
1644 __ call(trace_entry, relocInfo::none);
1645 }
1646
1647
1648 void TemplateInterpreterGenerator::stop_interpreter_at() {
1649 Label Lcontinue;
1650 const Register stop_at = R2_tmp;
1651
1652 __ ldr_global_s32(Rtemp, (address) &BytecodeCounter::_counter_value);
1653 __ mov_slow(stop_at, StopInterpreterAt);
1654
1655 // test bytecode counter
1656 __ cmp(Rtemp, stop_at);
1657 __ b(Lcontinue, ne);
1658
1659 __ trace_state("stop_interpreter_at");
1660 __ breakpoint();
1661
1662 __ bind(Lcontinue);
1663 }
1664 #endif // !PRODUCT