1 /*
2 * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "interpreter/bytecodeHistogram.hpp"
29 #include "interpreter/interp_masm.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "interpreter/interpreterRuntime.hpp"
32 #include "interpreter/templateInterpreterGenerator.hpp"
33 #include "interpreter/templateTable.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/methodData.hpp"
36 #include "oops/method.inline.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "oops/resolvedIndyEntry.hpp"
39 #include "oops/resolvedMethodEntry.hpp"
40 #include "prims/jvmtiExport.hpp"
41 #include "prims/jvmtiThreadState.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/arguments.hpp"
44 #include "runtime/deoptimization.hpp"
45 #include "runtime/frame.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/synchronizer.hpp"
50 #include "runtime/timer.hpp"
51 #include "runtime/vframeArray.hpp"
52 #include "utilities/align.hpp"
53 #include "utilities/debug.hpp"
54 #include "utilities/macros.hpp"
55
56 // Size of interpreter code. Increase if too small. Interpreter will
57 // fail with a guarantee ("not enough space for interpreter generation");
58 // if too small.
59 // Run with +PrintInterpreter to get the VM to print out the size.
60 // Max size with JVMTI
61 int TemplateInterpreter::InterpreterCodeSize = 180 * 1024;
62
63 #define __ _masm->
64
65 //------------------------------------------------------------------------------------------------------------------------
66
67 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
68 address entry = __ pc();
69
70 // callee-save register for saving LR, shared with generate_native_entry
71 const Register Rsaved_ret_addr = Rtmp_save0;
72
73 __ mov(Rsaved_ret_addr, LR);
74
75 __ mov(R1, Rmethod);
76 __ mov(R2, Rlocals);
77 __ mov(R3, SP);
78
79
80 // Safer to save R9 (when scratched) since callers may have been
81 // written assuming R9 survives. This is suboptimal but
82 // probably not important for this slow case call site.
83 // Note for R9 saving: slow_signature_handler may copy register
84 // arguments above the current SP (passed as R3). It is safe for
85 // call_VM to use push and pop to protect additional values on the
86 // stack if needed.
87 __ call_VM(CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), true /* save R9 if needed*/);
88 __ add(SP, SP, wordSize); // Skip R0
89 __ pop(RegisterSet(R1, R3)); // Load arguments passed in registers
90 #ifdef __ABI_HARD__
91 // Few alternatives to an always-load-FP-registers approach:
92 // - parse method signature to detect FP arguments
93 // - keep a counter/flag on a stack indicationg number of FP arguments in the method.
94 // The later has been originally implemented and tested but a conditional path could
95 // eliminate any gain imposed by avoiding 8 double word loads.
96 __ fldmiad(SP, FloatRegisterSet(D0, 8), writeback);
97 #endif // __ABI_HARD__
98
99 __ ret(Rsaved_ret_addr);
100
101 return entry;
102 }
103
104
105 //
106 // Various method entries (that c++ and asm interpreter agree upon)
107 //------------------------------------------------------------------------------------------------------------------------
108 //
109 //
110
111 // Abstract method entry
112 // Attempt to execute abstract method. Throw exception
113 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
114 address entry_point = __ pc();
115
116
117 __ empty_expression_stack();
118
119 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
120
121 DEBUG_ONLY(STOP("generate_abstract_entry");) // Should not reach here
122 return entry_point;
123 }
124
125 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
126 address entry_point = nullptr;
127 Register continuation = LR;
128 bool use_runtime_call = false;
129 switch (kind) {
130 case Interpreter::java_lang_math_abs:
131 entry_point = __ pc();
132 #ifdef __SOFTFP__
133 use_runtime_call = true;
134 __ ldrd(R0, Address(SP));
135 #else // !__SOFTFP__
136 __ ldr_double(D0, Address(SP));
137 __ abs_double(D0, D0);
138 #endif // __SOFTFP__
139 break;
140 case Interpreter::java_lang_math_sqrt:
141 entry_point = __ pc();
142 #ifdef __SOFTFP__
143 use_runtime_call = true;
144 __ ldrd(R0, Address(SP));
145 #else // !__SOFTFP__
146 __ ldr_double(D0, Address(SP));
147 __ sqrt_double(D0, D0);
148 #endif // __SOFTFP__
149 break;
150 case Interpreter::java_lang_math_sin:
151 case Interpreter::java_lang_math_cos:
152 case Interpreter::java_lang_math_tan:
153 case Interpreter::java_lang_math_log:
154 case Interpreter::java_lang_math_log10:
155 case Interpreter::java_lang_math_exp:
156 entry_point = __ pc();
157 use_runtime_call = true;
158 #ifdef __SOFTFP__
159 __ ldrd(R0, Address(SP));
160 #else // !__SOFTFP__
161 __ ldr_double(D0, Address(SP));
162 #endif // __SOFTFP__
163 break;
164 case Interpreter::java_lang_math_pow:
165 entry_point = __ pc();
166 use_runtime_call = true;
167 #ifdef __SOFTFP__
168 __ ldrd(R0, Address(SP, 2 * Interpreter::stackElementSize));
169 __ ldrd(R2, Address(SP));
170 #else // !__SOFTFP__
171 __ ldr_double(D0, Address(SP, 2 * Interpreter::stackElementSize));
172 __ ldr_double(D1, Address(SP));
173 #endif // __SOFTFP__
174 break;
175 case Interpreter::java_lang_math_fmaD:
176 case Interpreter::java_lang_math_fmaF:
177 case Interpreter::java_lang_math_tanh:
178 // TODO: Implement intrinsic
179 break;
180 default:
181 ShouldNotReachHere();
182 }
183
184 if (entry_point != nullptr) {
185 __ mov(SP, Rsender_sp);
186 if (use_runtime_call) {
187 __ mov(Rtmp_save0, LR);
188 continuation = Rtmp_save0;
189 generate_math_runtime_call(kind);
190 }
191 __ ret(continuation);
192 }
193 return entry_point;
194 }
195
196 void TemplateInterpreterGenerator::generate_math_runtime_call(AbstractInterpreter::MethodKind kind) {
197 address fn;
198 switch (kind) {
199 #ifdef __SOFTFP__
200 case Interpreter::java_lang_math_abs:
201 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dabs);
202 break;
203 case Interpreter::java_lang_math_sqrt:
204 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsqrt);
205 break;
206 #endif // __SOFTFP__
207 case Interpreter::java_lang_math_sin:
208 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
209 break;
210 case Interpreter::java_lang_math_cos:
211 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
212 break;
213 case Interpreter::java_lang_math_tan:
214 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
215 break;
216 case Interpreter::java_lang_math_log:
217 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
218 break;
219 case Interpreter::java_lang_math_log10:
220 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
221 break;
222 case Interpreter::java_lang_math_exp:
223 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
224 break;
225 case Interpreter::java_lang_math_pow:
226 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow);
227 break;
228 default:
229 ShouldNotReachHere();
230 fn = nullptr; // silence "maybe uninitialized" compiler warnings
231 }
232 __ call_VM_leaf(fn);
233 }
234
235 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
236 address entry = __ pc();
237
238 // Note: There should be a minimal interpreter frame set up when stack
239 // overflow occurs since we check explicitly for it now.
240 //
241 #ifdef ASSERT
242 { Label L;
243 __ sub(Rtemp, FP, - frame::interpreter_frame_monitor_block_top_offset * wordSize);
244 __ cmp(SP, Rtemp); // Rtemp = maximal SP for current FP,
245 // (stack grows negative)
246 __ b(L, ls); // check if frame is complete
247 __ stop ("interpreter frame not set up");
248 __ bind(L);
249 }
250 #endif // ASSERT
251
252 // Restore bcp under the assumption that the current frame is still
253 // interpreted
254 __ restore_bcp();
255
256 // expression stack must be empty before entering the VM if an exception
257 // happened
258 __ empty_expression_stack();
259
260 // throw exception
261 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
262
263 __ should_not_reach_here();
264
265 return entry;
266 }
267
268 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
269 address entry = __ pc();
270
271 // index is in R4_ArrayIndexOutOfBounds_index
272
273 // expression stack must be empty before entering the VM if an exception happened
274 __ empty_expression_stack();
275
276 // setup parameters
277 // Array expected in R1.
278 __ mov(R2, R4_ArrayIndexOutOfBounds_index);
279
280 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), R1, R2);
281
282 __ nop(); // to avoid filling CPU pipeline with invalid instructions
283 __ nop();
284 __ should_not_reach_here();
285
286 return entry;
287 }
288
289 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
290 address entry = __ pc();
291
292 // object is in R2_ClassCastException_obj
293
294 // expression stack must be empty before entering the VM if an exception
295 // happened
296 __ empty_expression_stack();
297
298 __ mov(R1, R2_ClassCastException_obj);
299 __ call_VM(noreg,
300 CAST_FROM_FN_PTR(address,
301 InterpreterRuntime::throw_ClassCastException),
302 R1);
303
304 __ should_not_reach_here();
305
306 return entry;
307 }
308
309 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
310 assert(!pass_oop || message == nullptr, "either oop or message but not both");
311 address entry = __ pc();
312
313 InlinedString Lname(name);
314 InlinedString Lmessage(message);
315
316 if (pass_oop) {
317 // object is at TOS
318 __ pop_ptr(R2);
319 }
320
321 // expression stack must be empty before entering the VM if an exception happened
322 __ empty_expression_stack();
323
324 // setup parameters
325 __ ldr_literal(R1, Lname);
326
327 if (pass_oop) {
328 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), R1, R2);
329 } else {
330 if (message != nullptr) {
331 __ ldr_literal(R2, Lmessage);
332 } else {
333 __ mov(R2, 0);
334 }
335 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), R1, R2);
336 }
337
338 // throw exception
339 __ b(Interpreter::throw_exception_entry());
340
341 __ nop(); // to avoid filling CPU pipeline with invalid instructions
342 __ nop();
343 __ bind_literal(Lname);
344 if (!pass_oop && (message != nullptr)) {
345 __ bind_literal(Lmessage);
346 }
347
348 return entry;
349 }
350
351 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
352 address entry = __ pc();
353
354 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
355
356 // Restore stack bottom in case i2c adjusted stack
357 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
358 // and null it as marker that SP is now tos until next java call
359 __ mov(Rtemp, (int)NULL_WORD);
360 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
361
362 __ restore_method();
363 __ restore_bcp();
364 __ restore_dispatch();
365 __ restore_locals();
366
367 const Register Rcache = R2_tmp;
368 const Register Rindex = R3_tmp;
369
370 if (index_size == sizeof(u4)) {
371 __ load_resolved_indy_entry(Rcache, Rindex);
372 __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
373 } else {
374 // Pop N words from the stack
375 assert(index_size == sizeof(u2), "Can only be u2");
376 __ load_method_entry(Rcache, Rindex);
377 __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
378 }
379
380 __ check_stack_top();
381 __ add(Rstack_top, Rstack_top, AsmOperand(Rcache, lsl, Interpreter::logStackElementSize));
382
383 __ convert_retval_to_tos(state);
384
385 __ check_and_handle_popframe();
386 __ check_and_handle_earlyret();
387
388 __ dispatch_next(state, step);
389
390 return entry;
391 }
392
393 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {
394 address entry = __ pc();
395
396 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
397
398 // The stack is not extended by deopt but we must null last_sp as this
399 // entry is like a "return".
400 __ mov(Rtemp, 0);
401 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
402
403 __ restore_method();
404 __ restore_bcp();
405 __ restore_dispatch();
406 __ restore_locals();
407
408 // handle exceptions
409 { Label L;
410 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
411 __ cbz(Rtemp, L);
412 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
413 __ should_not_reach_here();
414 __ bind(L);
415 }
416
417 if (continuation == nullptr) {
418 __ dispatch_next(state, step);
419 } else {
420 __ jump_to_entry(continuation);
421 }
422
423 return entry;
424 }
425
426 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
427 address entry = __ pc();
428
429 switch (type) {
430 case T_CHAR : /* Nothing to do */ break;
431 case T_BYTE : /* Nothing to do */ break;
432 case T_SHORT : /* Nothing to do */ break;
433 case T_INT : /* Nothing to do */ break;
434 case T_LONG : /* Nothing to do */ break;
435 case T_VOID : /* Nothing to do */ break;
436 case T_DOUBLE : /* Nothing to do */ break;
437 case T_FLOAT : /* Nothing to do */ break;
438 case T_BOOLEAN : __ c2bool(R0); break;
439 case T_OBJECT :
440 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
441 __ verify_oop(R0);
442 break;
443 default : __ should_not_reach_here(); break;
444 }
445
446 __ ret();
447 return entry;
448 }
449
450 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
451 address entry = __ pc();
452 __ push(state);
453 __ call_VM(noreg, runtime_entry);
454
455 // load current bytecode
456 __ ldrb(R3_bytecode, Address(Rbcp));
457 __ dispatch_only_normal(vtos);
458 return entry;
459 }
460
461 address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() {
462 return nullptr;
463 }
464
465
466 // Helpers for commoning out cases in the various type of method entries.
467 //
468
469 // increment invocation count & check for overflow
470 //
471 // Note: checking for negative value instead of overflow
472 // so we have a 'sticky' overflow test
473 //
474 // In: Rmethod.
475 //
476 // Uses R0, R1, Rtemp.
477 //
478 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) {
479 Label done;
480 const Register Rcounters = Rtemp;
481 const Address invocation_counter(Rcounters,
482 MethodCounters::invocation_counter_offset() +
483 InvocationCounter::counter_offset());
484
485 // Note: In tiered we increment either counters in MethodCounters* or
486 // in MDO depending if we're profiling or not.
487 int increment = InvocationCounter::count_increment;
488 Label no_mdo;
489 if (ProfileInterpreter) {
490 // Are we profiling?
491 __ ldr(R1_tmp, Address(Rmethod, Method::method_data_offset()));
492 __ cbz(R1_tmp, no_mdo);
493 // Increment counter in the MDO
494 const Address mdo_invocation_counter(R1_tmp,
495 in_bytes(MethodData::invocation_counter_offset()) +
496 in_bytes(InvocationCounter::counter_offset()));
497 const Address mask(R1_tmp, in_bytes(MethodData::invoke_mask_offset()));
498 __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, R0_tmp, Rtemp, eq, overflow);
499 __ b(done);
500 }
501 __ bind(no_mdo);
502 __ get_method_counters(Rmethod, Rcounters, done);
503 const Address mask(Rcounters, in_bytes(MethodCounters::invoke_mask_offset()));
504 __ increment_mask_and_jump(invocation_counter, increment, mask, R0_tmp, R1_tmp, eq, overflow);
505 __ bind(done);
506 }
507
508 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
509 // InterpreterRuntime::frequency_counter_overflow takes one argument
510 // indicating if the counter overflow occurs at a backwards branch (non-null bcp).
511 // The call returns the address of the verified entry point for the method or null
512 // if the compilation did not complete (either went background or bailed out).
513 __ mov(R1, (int)false);
514 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);
515
516 // jump to the interpreted entry.
517 __ b(do_continue);
518 }
519
520 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
521 // Check if we've got enough room on the stack for
522 // - overhead;
523 // - locals;
524 // - expression stack.
525 //
526 // Registers on entry:
527 //
528 // R3 = number of additional locals
529 // Rthread
530 // Rmethod
531 // Registers used: R0, R1, R2, Rtemp.
532
533 const Register Radditional_locals = R3;
534 const Register RmaxStack = R2;
535
536 // monitor entry size
537 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
538
539 // total overhead size: entry_size + (saved registers, thru expr stack bottom).
540 // be sure to change this if you add/subtract anything to/from the overhead area
541 const int overhead_size = (frame::sender_sp_offset - frame::interpreter_frame_initial_sp_offset)*wordSize + entry_size;
542
543 // Pages reserved for VM runtime calls and subsequent Java calls.
544 const int reserved_pages = StackOverflow::stack_shadow_zone_size();
545
546 // Thread::stack_size() includes guard pages, and they should not be touched.
547 const int guard_pages = StackOverflow::stack_guard_zone_size();
548
549 __ ldr(R0, Address(Rthread, Thread::stack_base_offset()));
550 __ ldr(R1, Address(Rthread, Thread::stack_size_offset()));
551 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
552 __ ldrh(RmaxStack, Address(Rtemp, ConstMethod::max_stack_offset()));
553 __ sub_slow(Rtemp, SP, overhead_size + reserved_pages + guard_pages + Method::extra_stack_words());
554
555 // reserve space for additional locals
556 __ sub(Rtemp, Rtemp, AsmOperand(Radditional_locals, lsl, Interpreter::logStackElementSize));
557
558 // stack size
559 __ sub(R0, R0, R1);
560
561 // reserve space for expression stack
562 __ sub(Rtemp, Rtemp, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
563
564 __ cmp(Rtemp, R0);
565
566 __ mov(SP, Rsender_sp, ls); // restore SP
567 __ b(SharedRuntime::throw_StackOverflowError_entry(), ls);
568 }
569
570
571 // Allocate monitor and lock method (asm interpreter)
572 //
573 void TemplateInterpreterGenerator::lock_method() {
574 // synchronize method
575
576 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
577 assert ((entry_size % StackAlignmentInBytes) == 0, "should keep stack alignment");
578
579 #ifdef ASSERT
580 { Label L;
581 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
582 __ tbnz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
583 __ stop("method doesn't need synchronization");
584 __ bind(L);
585 }
586 #endif // ASSERT
587
588 // get synchronization object
589 { Label done;
590 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
591 __ tst(Rtemp, JVM_ACC_STATIC);
592 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0)), eq); // get receiver (assume this is frequent case)
593 __ b(done, eq);
594 __ load_mirror(R0, Rmethod, Rtemp);
595 __ bind(done);
596 }
597
598 // add space for monitor & lock
599
600
601 __ sub(Rstack_top, Rstack_top, entry_size);
602 __ check_stack_top_on_expansion();
603 // add space for a monitor entry
604 __ str(Rstack_top, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
605 // set new monitor block top
606 __ str(R0, Address(Rstack_top, BasicObjectLock::obj_offset()));
607 // store object
608 __ mov(R1, Rstack_top); // monitor entry address
609 __ lock_object(R1);
610 }
611
612
613 //
614 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
615 // and for native methods hence the shared code.
616
617 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
618 // Generates the following stack layout:
619 //
620 // [ expr. stack bottom ]
621 // [ saved Rbcp ]
622 // [ current Rlocals ]
623 // [ cache ]
624 // [ mdx ]
625 // [ Method* ]
626 // [ last_sp ]
627 // [ sender_sp ]
628 // [ saved FP ] <--- FP
629 // [ saved LR ]
630
631 // initialize fixed part of activation frame
632 __ push(LR); // save return address
633 __ push(FP); // save FP
634 __ mov(FP, SP); // establish new FP
635
636 __ push(Rsender_sp);
637
638 __ mov(R0, 0);
639 __ push(R0); // leave last_sp as null
640
641 // setup Rbcp
642 if (native_call) {
643 __ mov(Rbcp, 0); // bcp = 0 for native calls
644 } else {
645 __ ldr(Rtemp, Address(Rmethod, Method::const_offset())); // get ConstMethod*
646 __ add(Rbcp, Rtemp, ConstMethod::codes_offset()); // get codebase
647 }
648
649 __ push(Rmethod); // save Method*
650 // Get mirror and store it in the frame as GC root for this Method*
651 __ load_mirror(Rtemp, Rmethod, Rtemp);
652 __ push(Rtemp);
653
654 if (ProfileInterpreter) {
655 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
656 __ tst(Rtemp, Rtemp);
657 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()), ne);
658 __ push(Rtemp); // set the mdp (method data pointer)
659 } else {
660 __ push(R0);
661 }
662
663 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
664 __ ldr(Rtemp, Address(Rtemp, ConstMethod::constants_offset()));
665 __ ldr(Rtemp, Address(Rtemp, ConstantPool::cache_offset()));
666 __ push(Rtemp); // set constant pool cache
667 __ sub(Rtemp, Rlocals, FP);
668 __ logical_shift_right(Rtemp, Rtemp, Interpreter::logStackElementSize); // Rtemp = Rlocals - fp();
669 __ push(Rtemp); // set relativized Rlocals, see frame::interpreter_frame_locals()
670 __ push(Rbcp); // set bcp
671 __ push(R0); // reserve word for pointer to expression stack bottom
672 __ str(SP, Address(SP, 0)); // set expression stack bottom
673 }
674
675
676 // End of helpers
677
678 //------------------------------------------------------------------------------------------------------------------------
679 // Entry points
680 //
681 // Here we generate the various kind of entries into the interpreter.
682 // The two main entry type are generic bytecode methods and native call method.
683 // These both come in synchronized and non-synchronized versions but the
684 // frame layout they create is very similar. The other method entry
685 // types are really just special purpose entries that are really entry
686 // and interpretation all in one. These are for trivial methods like
687 // accessor, empty, or special math methods.
688 //
689 // When control flow reaches any of the entry types for the interpreter
690 // the following holds ->
691 //
692 // Arguments:
693 //
694 // Rmethod: Method*
695 // Rthread: thread
696 // Rsender_sp: sender sp
697 // Rparams (SP on 32-bit ARM): pointer to method parameters
698 //
699 // LR: return address
700 //
701 // Stack layout immediately at entry
702 //
703 // [ parameter n ] <--- Rparams (SP on 32-bit ARM)
704 // ...
705 // [ parameter 1 ]
706 // [ expression stack ] (caller's java expression stack)
707
708 // Assuming that we don't go to one of the trivial specialized
709 // entries the stack will look like below when we are ready to execute
710 // the first bytecode (or call the native routine). The register usage
711 // will be as the template based interpreter expects.
712 //
713 // local variables follow incoming parameters immediately; i.e.
714 // the return address is saved at the end of the locals.
715 //
716 // [ expr. stack ] <--- Rstack_top (SP on 32-bit ARM)
717 // [ monitor entry ]
718 // ...
719 // [ monitor entry ]
720 // [ expr. stack bottom ]
721 // [ saved Rbcp ]
722 // [ current Rlocals ]
723 // [ cache ]
724 // [ mdx ]
725 // [ mirror ]
726 // [ Method* ]
727 //
728 // 32-bit ARM:
729 // [ last_sp ]
730 //
731 // [ sender_sp ]
732 // [ saved FP ] <--- FP
733 // [ saved LR ]
734 // [ optional padding(*)]
735 // [ local variable m ]
736 // ...
737 // [ local variable 1 ]
738 // [ parameter n ]
739 // ...
740 // [ parameter 1 ] <--- Rlocals
741 //
742
743 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
744 // Code: _aload_0, _getfield, _areturn
745 // parameter size = 1
746 //
747 // The code that gets generated by this routine is split into 2 parts:
748 // 1. The "intrinsified" code performing an ON_WEAK_OOP_REF load,
749 // 2. The slow path - which is an expansion of the regular method entry.
750 //
751 // Notes:-
752 // * An intrinsic is always executed, where an ON_WEAK_OOP_REF load is performed.
753 // * We may jump to the slow path iff the receiver is null. If the
754 // Reference object is null then we no longer perform an ON_WEAK_OOP_REF load
755 // Thus we can use the regular method entry code to generate the NPE.
756 //
757 // Rmethod: Method*
758 // Rthread: thread
759 // Rsender_sp: sender sp, must be preserved for slow path, set SP to it on fast path
760 // Rparams: parameters
761
762 address entry = __ pc();
763 Label slow_path;
764 const Register Rthis = R0;
765 const Register Rret_addr = Rtmp_save1;
766 assert_different_registers(Rthis, Rret_addr, Rsender_sp);
767
768 const int referent_offset = java_lang_ref_Reference::referent_offset();
769
770 // Check if local 0 != nullptr
771 // If the receiver is null then it is OK to jump to the slow path.
772 __ ldr(Rthis, Address(Rparams));
773 __ cbz(Rthis, slow_path);
774
775 // Preserve LR
776 __ mov(Rret_addr, LR);
777
778 // Load the value of the referent field.
779 const Address field_address(Rthis, referent_offset);
780 __ load_heap_oop(R0, field_address, Rtemp, R1_tmp, R2_tmp, ON_WEAK_OOP_REF);
781
782 // _areturn
783 __ mov(SP, Rsender_sp);
784 __ ret(Rret_addr);
785
786 // generate a vanilla interpreter entry as the slow path
787 __ bind(slow_path);
788 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
789 return entry;
790 }
791
792 // Not supported
793 address TemplateInterpreterGenerator::generate_currentThread() { return nullptr; }
794 address TemplateInterpreterGenerator::generate_CRC32_update_entry() { return nullptr; }
795 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return nullptr; }
796 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return nullptr; }
797 address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() { return nullptr; }
798 address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() { return nullptr; }
799
800 //
801 // Interpreter stub for calling a native method. (asm interpreter)
802 // This sets up a somewhat different looking stack for calling the native method
803 // than the typical interpreter frame setup.
804 //
805
806 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized, bool runtime_upcalls) {
807 // determine code generation flags
808 bool inc_counter = (UseCompiler || CountCompiledCalls) && !PreloadOnly;
809
810 // Incoming registers:
811 //
812 // Rmethod: Method*
813 // Rthread: thread
814 // Rsender_sp: sender sp
815 // Rparams: parameters
816
817 address entry_point = __ pc();
818
819 // Register allocation
820 const Register Rsize_of_params = R6;
821 const Register Rsig_handler = Rtmp_save0; // R4
822 const Register Rnative_code = Rtmp_save1; // R5
823 const Register Rresult_handler = R6;
824
825 const Register Rsaved_result_lo = Rtmp_save0; // R4
826 const Register Rsaved_result_hi = Rtmp_save1; // R5
827 FloatRegister saved_result_fp;
828
829
830 __ ldr(Rsize_of_params, Address(Rmethod, Method::const_offset()));
831 __ ldrh(Rsize_of_params, Address(Rsize_of_params, ConstMethod::size_of_parameters_offset()));
832
833 // native calls don't need the stack size check since they have no expression stack
834 // and the arguments are already on the stack and we only add a handful of words
835 // to the stack
836
837 // compute beginning of parameters (Rlocals)
838 __ sub(Rlocals, Rparams, wordSize);
839 __ add(Rlocals, Rlocals, AsmOperand(Rsize_of_params, lsl, Interpreter::logStackElementSize));
840
841 // reserve stack space for oop_temp
842 __ mov(R0, 0);
843 __ push(R0);
844
845 generate_fixed_frame(true); // Note: R9 is now saved in the frame
846
847 // make sure method is native & not abstract
848 #ifdef ASSERT
849 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
850 {
851 Label L;
852 __ tbnz(Rtemp, JVM_ACC_NATIVE_BIT, L);
853 __ stop("tried to execute non-native method as native");
854 __ bind(L);
855 }
856 { Label L;
857 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
858 __ stop("tried to execute abstract method in interpreter");
859 __ bind(L);
860 }
861 #endif
862
863 // increment invocation count & check for overflow
864 Label invocation_counter_overflow;
865 if (inc_counter) {
866 if (synchronized) {
867 // Avoid unlocking method's monitor in case of exception, as it has not
868 // been locked yet.
869 __ set_do_not_unlock_if_synchronized(true, Rtemp);
870 }
871 generate_counter_incr(&invocation_counter_overflow);
872 }
873
874 Label continue_after_compile;
875 __ bind(continue_after_compile);
876
877 if (inc_counter && synchronized) {
878 __ set_do_not_unlock_if_synchronized(false, Rtemp);
879 }
880
881 // check for synchronized methods
882 // Must happen AFTER invocation_counter check and stack overflow check,
883 // so method is not locked if overflows.
884 //
885 if (synchronized) {
886 lock_method();
887 } else {
888 // no synchronization necessary
889 #ifdef ASSERT
890 { Label L;
891 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
892 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
893 __ stop("method needs synchronization");
894 __ bind(L);
895 }
896 #endif
897 }
898
899 // start execution
900 #ifdef ASSERT
901 { Label L;
902 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
903 __ cmp(Rtemp, Rstack_top);
904 __ b(L, eq);
905 __ stop("broken stack frame setup in interpreter 3");
906 __ bind(L);
907 }
908 #endif
909 __ check_extended_sp(Rtemp);
910
911 // jvmti/dtrace support
912 __ notify_method_entry();
913 #if R9_IS_SCRATCHED
914 __ restore_method();
915 #endif
916
917 {
918 Label L;
919 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
920 __ cbnz(Rsig_handler, L);
921 __ mov(R1, Rmethod);
922 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1, true);
923 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
924 __ bind(L);
925 }
926
927 {
928 Label L;
929 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
930 __ cbnz(Rnative_code, L);
931 __ mov(R1, Rmethod);
932 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1);
933 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
934 __ bind(L);
935 }
936
937 // Allocate stack space for arguments
938
939
940 // C functions need aligned stack
941 __ bic(SP, SP, StackAlignmentInBytes - 1);
942 // Multiply by BytesPerLong instead of BytesPerWord, because calling convention
943 // may require empty slots due to long alignment, e.g. func(int, jlong, int, jlong)
944 __ sub(SP, SP, AsmOperand(Rsize_of_params, lsl, LogBytesPerLong));
945
946 #ifdef __ABI_HARD__
947 // Allocate more stack space to accommodate all GP as well as FP registers:
948 // 4 * wordSize
949 // 8 * BytesPerLong
950 int reg_arguments = align_up((4*wordSize) + (8*BytesPerLong), StackAlignmentInBytes);
951 #else
952 // Reserve at least 4 words on the stack for loading
953 // of parameters passed on registers (R0-R3).
954 // See generate_slow_signature_handler().
955 // It is also used for JNIEnv & class additional parameters.
956 int reg_arguments = 4 * wordSize;
957 #endif // __ABI_HARD__
958
959 __ sub(SP, SP, reg_arguments);
960
961
962 // Note: signature handler blows R4 besides all scratch registers.
963 // See AbstractInterpreterGenerator::generate_slow_signature_handler().
964 __ call(Rsig_handler);
965 #if R9_IS_SCRATCHED
966 __ restore_method();
967 #endif
968 __ mov(Rresult_handler, R0);
969
970 // Pass JNIEnv and mirror for static methods
971 {
972 Label L;
973 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
974 __ add(R0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
975 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, L);
976 __ load_mirror(Rtemp, Rmethod, Rtemp);
977 __ add(R1, FP, frame::interpreter_frame_oop_temp_offset * wordSize);
978 __ str(Rtemp, Address(R1, 0));
979 __ bind(L);
980 }
981
982 __ set_last_Java_frame(SP, FP, true, Rtemp);
983
984 // Changing state to _thread_in_native must be the last thing to do
985 // before the jump to native code. At this moment stack must be
986 // safepoint-safe and completely prepared for stack walking.
987 #ifdef ASSERT
988 {
989 Label L;
990 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
991 __ cmp_32(Rtemp, _thread_in_Java);
992 __ b(L, eq);
993 __ stop("invalid thread state");
994 __ bind(L);
995 }
996 #endif
997
998 // Force all preceding writes to be observed prior to thread state change
999 __ membar(MacroAssembler::StoreStore, Rtemp);
1000
1001 __ mov(Rtemp, _thread_in_native);
1002 __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1003
1004 __ call(Rnative_code);
1005 #if R9_IS_SCRATCHED
1006 __ restore_method();
1007 #endif
1008
1009 // Set FPSCR/FPCR to a known state
1010 if (AlwaysRestoreFPU) {
1011 __ restore_default_fp_mode();
1012 }
1013
1014 // Do safepoint check
1015 __ mov(Rtemp, _thread_in_native_trans);
1016 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1017
1018 // Force this write out before the read below
1019 if (!UseSystemMemoryBarrier) {
1020 __ membar(MacroAssembler::StoreLoad, Rtemp);
1021 }
1022
1023 // Protect the return value in the interleaved code: save it to callee-save registers.
1024 __ mov(Rsaved_result_lo, R0);
1025 __ mov(Rsaved_result_hi, R1);
1026 #ifdef __ABI_HARD__
1027 // preserve native FP result in a callee-saved register
1028 saved_result_fp = D8;
1029 __ fcpyd(saved_result_fp, D0);
1030 #else
1031 saved_result_fp = fnoreg;
1032 #endif // __ABI_HARD__
1033
1034 {
1035 Label call, skip_call;
1036 __ safepoint_poll(Rtemp, call);
1037 __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1038 __ cmp(R3, 0);
1039 __ b(skip_call, eq);
1040 __ bind(call);
1041 __ mov(R0, Rthread);
1042 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
1043 __ bind(skip_call);
1044
1045 #if R9_IS_SCRATCHED
1046 __ restore_method();
1047 #endif
1048 }
1049
1050 // Perform Native->Java thread transition
1051 __ mov(Rtemp, _thread_in_Java);
1052 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1053
1054 // Zero handles and last_java_sp
1055 __ reset_last_Java_frame(Rtemp);
1056 __ ldr(R3, Address(Rthread, JavaThread::active_handles_offset()));
1057 __ str_32(__ zero_register(Rtemp), Address(R3, JNIHandleBlock::top_offset()));
1058 if (CheckJNICalls) {
1059 __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1060 }
1061
1062 // Unbox oop result, e.g. JNIHandles::resolve result if it's an oop.
1063 {
1064 Label Lnot_oop;
1065 __ mov_slow(Rtemp, AbstractInterpreter::result_handler(T_OBJECT));
1066 __ cmp(Rtemp, Rresult_handler);
1067 __ b(Lnot_oop, ne);
1068 Register value = Rsaved_result_lo;
1069 __ resolve_jobject(value, // value
1070 Rtemp, // tmp1
1071 R1_tmp); // tmp2
1072 // Store resolved result in frame for GC visibility.
1073 __ str(value, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
1074 __ bind(Lnot_oop);
1075 }
1076
1077
1078 // reguard stack if StackOverflow exception happened while in native.
1079 {
1080 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::stack_guard_state_offset()));
1081 __ cmp_32(Rtemp, StackOverflow::stack_guard_yellow_reserved_disabled);
1082 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none, eq);
1083 #if R9_IS_SCRATCHED
1084 __ restore_method();
1085 #endif
1086 }
1087
1088 // check pending exceptions
1089 {
1090 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1091 __ cmp(Rtemp, 0);
1092 __ mov(Rexception_pc, PC, ne);
1093 __ b(StubRoutines::forward_exception_entry(), ne);
1094 }
1095
1096 if (synchronized) {
1097 // address of first monitor
1098 __ sub(R0, FP, - (frame::interpreter_frame_monitor_block_bottom_offset - frame::interpreter_frame_monitor_size()) * wordSize);
1099 __ unlock_object(R0);
1100 }
1101
1102 // jvmti/dtrace support
1103 // Note: This must happen _after_ handling/throwing any exceptions since
1104 // the exception handler code notifies the runtime of method exits
1105 // too. If this happens before, method entry/exit notifications are
1106 // not properly paired (was bug - gri 11/22/99).
1107 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result_lo, Rsaved_result_hi, saved_result_fp);
1108
1109 // Restore the result. Oop result is restored from the stack by the
1110 // result handler.
1111 __ mov(R0, Rsaved_result_lo);
1112 __ mov(R1, Rsaved_result_hi);
1113
1114 #ifdef __ABI_HARD__
1115 // reload native FP result
1116 __ fcpyd(D0, D8);
1117 #endif // __ABI_HARD__
1118
1119 __ blx(Rresult_handler);
1120
1121 // Restore FP/LR, sender_sp and return
1122 __ mov(Rtemp, FP);
1123 __ ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1124 __ ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1125
1126 __ ret();
1127
1128 if (inc_counter) {
1129 // Handle overflow of counter and compile method
1130 __ bind(invocation_counter_overflow);
1131 generate_counter_overflow(continue_after_compile);
1132 }
1133
1134 return entry_point;
1135 }
1136
1137 //
1138 // Generic interpreted method entry to (asm) interpreter
1139 //
1140 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized, bool runtime_upcalls) {
1141 // determine code generation flags
1142 bool inc_counter = (UseCompiler || CountCompiledCalls) && !PreloadOnly;
1143
1144 // Rmethod: Method*
1145 // Rthread: thread
1146 // Rsender_sp: sender sp (could differ from SP if we were called via c2i)
1147 // Rparams: pointer to the last parameter in the stack
1148
1149 address entry_point = __ pc();
1150
1151 const Register RconstMethod = R3;
1152
1153
1154 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
1155
1156 __ ldrh(R2, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
1157 __ ldrh(R3, Address(RconstMethod, ConstMethod::size_of_locals_offset()));
1158
1159 // setup Rlocals
1160 __ sub(Rlocals, Rparams, wordSize);
1161 __ add(Rlocals, Rlocals, AsmOperand(R2, lsl, Interpreter::logStackElementSize));
1162
1163 __ sub(R3, R3, R2); // number of additional locals
1164
1165
1166 // see if we've got enough room on the stack for locals plus overhead.
1167 generate_stack_overflow_check();
1168
1169 // allocate space for locals
1170 // explicitly initialize locals
1171
1172 // Loop is unrolled 4 times
1173 Label loop;
1174 __ mov(R0, 0);
1175 __ bind(loop);
1176
1177 // #1
1178 __ subs(R3, R3, 1);
1179 __ push(R0, ge);
1180
1181 // #2
1182 __ subs(R3, R3, 1, ge);
1183 __ push(R0, ge);
1184
1185 // #3
1186 __ subs(R3, R3, 1, ge);
1187 __ push(R0, ge);
1188
1189 // #4
1190 __ subs(R3, R3, 1, ge);
1191 __ push(R0, ge);
1192
1193 __ b(loop, gt);
1194
1195 // initialize fixed part of activation frame
1196 generate_fixed_frame(false);
1197
1198 __ restore_dispatch();
1199
1200 // make sure method is not native & not abstract
1201 #ifdef ASSERT
1202 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1203 {
1204 Label L;
1205 __ tbz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1206 __ stop("tried to execute native method as non-native");
1207 __ bind(L);
1208 }
1209 { Label L;
1210 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1211 __ stop("tried to execute abstract method in interpreter");
1212 __ bind(L);
1213 }
1214 #endif
1215
1216 // increment invocation count & check for overflow
1217 Label invocation_counter_overflow;
1218 if (inc_counter) {
1219 if (synchronized) {
1220 // Avoid unlocking method's monitor in case of exception, as it has not
1221 // been locked yet.
1222 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1223 }
1224 generate_counter_incr(&invocation_counter_overflow);
1225 }
1226 Label continue_after_compile;
1227 __ bind(continue_after_compile);
1228
1229 if (inc_counter && synchronized) {
1230 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1231 }
1232 #if R9_IS_SCRATCHED
1233 __ restore_method();
1234 #endif
1235
1236 // check for synchronized methods
1237 // Must happen AFTER invocation_counter check and stack overflow check,
1238 // so method is not locked if overflows.
1239 //
1240 if (synchronized) {
1241 // Allocate monitor and lock method
1242 lock_method();
1243 } else {
1244 // no synchronization necessary
1245 #ifdef ASSERT
1246 { Label L;
1247 __ ldrh(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1248 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1249 __ stop("method needs synchronization");
1250 __ bind(L);
1251 }
1252 #endif
1253 }
1254
1255 // start execution
1256 #ifdef ASSERT
1257 { Label L;
1258 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1259 __ cmp(Rtemp, Rstack_top);
1260 __ b(L, eq);
1261 __ stop("broken stack frame setup in interpreter 4");
1262 __ bind(L);
1263 }
1264 #endif
1265 __ check_extended_sp(Rtemp);
1266
1267 // jvmti support
1268 __ notify_method_entry();
1269 #if R9_IS_SCRATCHED
1270 __ restore_method();
1271 #endif
1272
1273 __ dispatch_next(vtos);
1274
1275 // invocation counter overflow
1276 if (inc_counter) {
1277 // Handle overflow of counter and compile method
1278 __ bind(invocation_counter_overflow);
1279 generate_counter_overflow(continue_after_compile);
1280 }
1281
1282 return entry_point;
1283 }
1284
1285 //------------------------------------------------------------------------------------------------------------------------
1286 // Exceptions
1287
1288 void TemplateInterpreterGenerator::generate_throw_exception() {
1289 // Entry point in previous activation (i.e., if the caller was interpreted)
1290 Interpreter::_rethrow_exception_entry = __ pc();
1291 // Rexception_obj: exception
1292
1293 // Clear interpreter_frame_last_sp.
1294 __ mov(Rtemp, 0);
1295 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1296
1297 #if R9_IS_SCRATCHED
1298 __ restore_method();
1299 #endif
1300 __ restore_bcp();
1301 __ restore_dispatch();
1302 __ restore_locals();
1303
1304
1305 // Entry point for exceptions thrown within interpreter code
1306 Interpreter::_throw_exception_entry = __ pc();
1307
1308 // expression stack is undefined here
1309 // Rexception_obj: exception
1310 // Rbcp: exception bcp
1311 __ verify_oop(Rexception_obj);
1312
1313 // expression stack must be empty before entering the VM in case of an exception
1314 __ empty_expression_stack();
1315 // find exception handler address and preserve exception oop
1316 __ mov(R1, Rexception_obj);
1317 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), R1);
1318 // R0: exception handler entry point
1319 // Rexception_obj: preserved exception oop
1320 // Rbcp: bcp for exception handler
1321 __ push_ptr(Rexception_obj); // push exception which is now the only value on the stack
1322 __ jump(R0); // jump to exception handler (may be _remove_activation_entry!)
1323
1324 // If the exception is not handled in the current frame the frame is removed and
1325 // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1326 //
1327 // Note: At this point the bci is still the bxi for the instruction which caused
1328 // the exception and the expression stack is empty. Thus, for any VM calls
1329 // at this point, GC will find a legal oop map (with empty expression stack).
1330
1331 // In current activation
1332 // tos: exception
1333 // Rbcp: exception bcp
1334
1335 //
1336 // JVMTI PopFrame support
1337 //
1338 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1339
1340
1341 __ empty_expression_stack();
1342
1343 // Set the popframe_processing bit in _popframe_condition indicating that we are
1344 // currently handling popframe, so that call_VMs that may happen later do not trigger new
1345 // popframe handling cycles.
1346
1347 __ ldr_s32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1348 __ orr(Rtemp, Rtemp, (unsigned)JavaThread::popframe_processing_bit);
1349 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1350
1351 {
1352 // Check to see whether we are returning to a deoptimized frame.
1353 // (The PopFrame call ensures that the caller of the popped frame is
1354 // either interpreted or compiled and deoptimizes it if compiled.)
1355 // In this case, we can't call dispatch_next() after the frame is
1356 // popped, but instead must save the incoming arguments and restore
1357 // them after deoptimization has occurred.
1358 //
1359 // Note that we don't compare the return PC against the
1360 // deoptimization blob's unpack entry because of the presence of
1361 // adapter frames in C2.
1362 Label caller_not_deoptimized;
1363 __ ldr(R0, Address(FP, frame::return_addr_offset * wordSize));
1364 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), R0);
1365 __ cbnz_32(R0, caller_not_deoptimized);
1366
1367 // Compute size of arguments for saving when returning to deoptimized caller
1368 __ restore_method();
1369 __ ldr(R0, Address(Rmethod, Method::const_offset()));
1370 __ ldrh(R0, Address(R0, ConstMethod::size_of_parameters_offset()));
1371
1372 __ logical_shift_left(R1, R0, Interpreter::logStackElementSize);
1373 // Save these arguments
1374 __ restore_locals();
1375 __ sub(R2, Rlocals, R1);
1376 __ add(R2, R2, wordSize);
1377 __ mov(R0, Rthread);
1378 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), R0, R1, R2);
1379
1380 __ remove_activation(vtos, LR,
1381 /* throw_monitor_exception */ false,
1382 /* install_monitor_exception */ false,
1383 /* notify_jvmdi */ false);
1384
1385 // Inform deoptimization that it is responsible for restoring these arguments
1386 __ mov(Rtemp, JavaThread::popframe_force_deopt_reexecution_bit);
1387 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1388
1389 // Continue in deoptimization handler
1390 __ ret();
1391
1392 __ bind(caller_not_deoptimized);
1393 }
1394
1395 __ remove_activation(vtos, R4,
1396 /* throw_monitor_exception */ false,
1397 /* install_monitor_exception */ false,
1398 /* notify_jvmdi */ false);
1399
1400 // Finish with popframe handling
1401 // A previous I2C followed by a deoptimization might have moved the
1402 // outgoing arguments further up the stack. PopFrame expects the
1403 // mutations to those outgoing arguments to be preserved and other
1404 // constraints basically require this frame to look exactly as
1405 // though it had previously invoked an interpreted activation with
1406 // no space between the top of the expression stack (current
1407 // last_sp) and the top of stack. Rather than force deopt to
1408 // maintain this kind of invariant all the time we call a small
1409 // fixup routine to move the mutated arguments onto the top of our
1410 // expression stack if necessary.
1411 __ mov(R1, SP);
1412 __ ldr(R2, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1413 // PC must point into interpreter here
1414 __ set_last_Java_frame(SP, FP, true, Rtemp);
1415 __ mov(R0, Rthread);
1416 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), R0, R1, R2);
1417 __ reset_last_Java_frame(Rtemp);
1418
1419 // Restore the last_sp and null it out
1420 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1421 __ mov(Rtemp, (int)NULL_WORD);
1422 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1423
1424 __ restore_bcp();
1425 __ restore_dispatch();
1426 __ restore_locals();
1427 __ restore_method();
1428
1429 // The method data pointer was incremented already during
1430 // call profiling. We have to restore the mdp for the current bcp.
1431 if (ProfileInterpreter) {
1432 __ set_method_data_pointer_for_bcp();
1433 }
1434
1435 // Clear the popframe condition flag
1436 assert(JavaThread::popframe_inactive == 0, "adjust this code");
1437 __ str_32(__ zero_register(Rtemp), Address(Rthread, JavaThread::popframe_condition_offset()));
1438
1439 #if INCLUDE_JVMTI
1440 {
1441 Label L_done;
1442
1443 __ ldrb(Rtemp, Address(Rbcp, 0));
1444 __ cmp(Rtemp, Bytecodes::_invokestatic);
1445 __ b(L_done, ne);
1446
1447 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1448 // Detect such a case in the InterpreterRuntime function and return the member name argument, or null.
1449
1450 // get local0
1451 __ ldr(R1, Address(Rlocals, 0));
1452 __ mov(R2, Rmethod);
1453 __ mov(R3, Rbcp);
1454 __ call_VM(R0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), R1, R2, R3);
1455
1456 __ cbz(R0, L_done);
1457
1458 __ str(R0, Address(Rstack_top));
1459 __ bind(L_done);
1460 }
1461 #endif // INCLUDE_JVMTI
1462
1463 __ dispatch_next(vtos);
1464 // end of PopFrame support
1465
1466 Interpreter::_remove_activation_entry = __ pc();
1467
1468 // preserve exception over this code sequence
1469 __ pop_ptr(R0_tos);
1470 __ str(R0_tos, Address(Rthread, JavaThread::vm_result_offset()));
1471 // remove the activation (without doing throws on illegalMonitorExceptions)
1472 __ remove_activation(vtos, Rexception_pc, false, true, false);
1473 // restore exception
1474 __ get_vm_result(Rexception_obj, Rtemp);
1475
1476 // In between activations - previous activation type unknown yet
1477 // compute continuation point - the continuation point expects
1478 // the following registers set up:
1479 //
1480 // Rexception_obj: exception
1481 // Rexception_pc: return address/pc that threw exception
1482 // SP: expression stack of caller
1483 // FP: frame pointer of caller
1484 __ mov(c_rarg0, Rthread);
1485 __ mov(c_rarg1, Rexception_pc);
1486 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);
1487 // Note that an "issuing PC" is actually the next PC after the call
1488
1489 __ jump(R0); // jump to exception handler of caller
1490 }
1491
1492
1493 //
1494 // JVMTI ForceEarlyReturn support
1495 //
1496 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1497 address entry = __ pc();
1498
1499
1500 __ restore_bcp();
1501 __ restore_dispatch();
1502 __ restore_locals();
1503
1504 __ empty_expression_stack();
1505
1506 __ load_earlyret_value(state);
1507
1508 // Clear the earlyret state
1509 __ ldr(Rtemp, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
1510
1511 assert(JvmtiThreadState::earlyret_inactive == 0, "adjust this code");
1512 __ str_32(__ zero_register(R2), Address(Rtemp, JvmtiThreadState::earlyret_state_offset()));
1513
1514 __ remove_activation(state, LR,
1515 false, /* throw_monitor_exception */
1516 false, /* install_monitor_exception */
1517 true); /* notify_jvmdi */
1518
1519 // According to interpreter calling conventions, result is returned in R0/R1,
1520 // so ftos (S0) and dtos (D0) are moved to R0/R1.
1521 // This conversion should be done after remove_activation, as it uses
1522 // push(state) & pop(state) to preserve return value.
1523 __ convert_tos_to_retval(state);
1524 __ ret();
1525
1526 return entry;
1527 } // end of ForceEarlyReturn support
1528
1529
1530 //------------------------------------------------------------------------------------------------------------------------
1531 // Helper for vtos entry point generation
1532
1533 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1534 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1535 Label L;
1536
1537 #ifdef __SOFTFP__
1538 dep = __ pc(); // fall through
1539 #else
1540 fep = __ pc(); __ push(ftos); __ b(L);
1541 dep = __ pc(); __ push(dtos); __ b(L);
1542 #endif // __SOFTFP__
1543
1544 lep = __ pc(); __ push(ltos); __ b(L);
1545
1546 if (VerifyOops) { // can't share atos entry if VerifyOops
1547 aep = __ pc(); __ push(atos); __ b(L);
1548 } else {
1549 aep = __ pc(); // fall through
1550 }
1551
1552 #ifdef __SOFTFP__
1553 fep = __ pc(); // fall through
1554 #endif // __SOFTFP__
1555
1556 bep = cep = sep = // fall through
1557 iep = __ pc(); __ push(itos); // fall through
1558 vep = __ pc(); __ bind(L); // fall through
1559 generate_and_dispatch(t);
1560 }
1561
1562 //------------------------------------------------------------------------------------------------------------------------
1563
1564 void TemplateInterpreterGenerator::count_bytecode() {
1565 __ inc_global_counter((address) &BytecodeCounter::_counter_value, 0, Rtemp, R2_tmp, true);
1566 }
1567
1568
1569 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1570 __ inc_global_counter((address)&BytecodeHistogram::_counters[0], sizeof(BytecodeHistogram::_counters[0]) * t->bytecode(), Rtemp, R2_tmp, true);
1571 }
1572
1573 // Non-product code
1574 #ifndef PRODUCT
1575 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1576 address entry = __ pc();
1577
1578 // prepare expression stack
1579 __ push(state); // save tosca
1580
1581 // pass tosca registers as arguments
1582 __ mov(R2, R0_tos);
1583 __ mov(R3, R1_tos_hi);
1584 __ mov(R1, LR); // save return address
1585
1586 // call tracer
1587 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), R1, R2, R3);
1588
1589 __ mov(LR, R0); // restore return address
1590 __ pop(state); // restore tosca
1591
1592 // return
1593 __ ret();
1594
1595 return entry;
1596 }
1597
1598 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1599 const Register Rindex_addr = R2_tmp;
1600 Label Lcontinue;
1601 InlinedAddress Lcounters((address)BytecodePairHistogram::_counters);
1602 InlinedAddress Lindex((address)&BytecodePairHistogram::_index);
1603 const Register Rcounters_addr = R2_tmp;
1604 const Register Rindex = R4_tmp;
1605
1606 // calculate new index for counter:
1607 // index = (_index >> log2_number_of_codes) | (bytecode << log2_number_of_codes).
1608 // (_index >> log2_number_of_codes) is previous bytecode
1609
1610 __ ldr_literal(Rindex_addr, Lindex);
1611 __ ldr_s32(Rindex, Address(Rindex_addr));
1612 __ mov_slow(Rtemp, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1613 __ orr(Rindex, Rtemp, AsmOperand(Rindex, lsr, BytecodePairHistogram::log2_number_of_codes));
1614 __ str_32(Rindex, Address(Rindex_addr));
1615
1616 // Rindex (R4) contains index of counter
1617
1618 __ ldr_literal(Rcounters_addr, Lcounters);
1619 __ ldr_s32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1620 __ adds_32(Rtemp, Rtemp, 1);
1621 __ b(Lcontinue, mi); // avoid overflow
1622 __ str_32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1623
1624 __ b(Lcontinue);
1625
1626 __ bind_literal(Lindex);
1627 __ bind_literal(Lcounters);
1628
1629 __ bind(Lcontinue);
1630 }
1631
1632
1633 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1634 // Call a little run-time stub to avoid blow-up for each bytecode.
1635 // The run-time runtime saves the right registers, depending on
1636 // the tosca in-state for the given template.
1637 assert(Interpreter::trace_code(t->tos_in()) != nullptr,
1638 "entry must have been generated");
1639 address trace_entry = Interpreter::trace_code(t->tos_in());
1640 __ call(trace_entry, relocInfo::none);
1641 }
1642
1643
1644 void TemplateInterpreterGenerator::stop_interpreter_at() {
1645 Label Lcontinue;
1646 const Register stop_at = R2_tmp;
1647
1648 __ ldr_global_s32(Rtemp, (address) &BytecodeCounter::_counter_value);
1649 __ mov_slow(stop_at, StopInterpreterAt);
1650
1651 // test bytecode counter
1652 __ cmp(Rtemp, stop_at);
1653 __ b(Lcontinue, ne);
1654
1655 __ trace_state("stop_interpreter_at");
1656 __ breakpoint();
1657
1658 __ bind(Lcontinue);
1659 }
1660 #endif // !PRODUCT