1 /*
2 * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "compiler/compiler_globals.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "interpreter/bytecodeHistogram.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "interpreter/interpreterRuntime.hpp"
34 #include "interpreter/interp_masm.hpp"
35 #include "interpreter/templateInterpreterGenerator.hpp"
36 #include "interpreter/templateTable.hpp"
37 #include "interpreter/bytecodeTracer.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/arrayOop.hpp"
40 #include "oops/methodData.hpp"
41 #include "oops/method.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/inlineKlass.hpp"
44 #include "oops/resolvedIndyEntry.hpp"
45 #include "prims/jvmtiExport.hpp"
46 #include "prims/jvmtiThreadState.hpp"
47 #include "runtime/arguments.hpp"
48 #include "runtime/deoptimization.hpp"
49 #include "runtime/frame.inline.hpp"
50 #include "runtime/globals.hpp"
51 #include "runtime/jniHandles.hpp"
52 #include "runtime/sharedRuntime.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "runtime/synchronizer.hpp"
55 #include "runtime/timer.hpp"
56 #include "runtime/vframeArray.hpp"
57 #include "utilities/debug.hpp"
58 #include "utilities/powerOfTwo.hpp"
59 #include <sys/types.h>
60
61 // Size of interpreter code. Increase if too small. Interpreter will
62 // fail with a guarantee ("not enough space for interpreter generation");
63 // if too small.
64 // Run with +PrintInterpreter to get the VM to print out the size.
65 // Max size with JVMTI
66 int TemplateInterpreter::InterpreterCodeSize = 200 * 1024;
67
68 #define __ _masm->
69
70 //-----------------------------------------------------------------------------
71
72 extern "C" void entry(CodeBuffer*);
73
74 //-----------------------------------------------------------------------------
75
76 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
77 address entry = __ pc();
78
79 __ andr(esp, esp, -16);
80 __ mov(c_rarg3, esp);
81 // rmethod
82 // rlocals
83 // c_rarg3: first stack arg - wordSize
84
85 // adjust sp
86 __ sub(sp, c_rarg3, 18 * wordSize);
87 __ str(lr, Address(__ pre(sp, -2 * wordSize)));
88 __ call_VM(noreg,
89 CAST_FROM_FN_PTR(address,
90 InterpreterRuntime::slow_signature_handler),
91 rmethod, rlocals, c_rarg3);
92
93 // r0: result handler
94
95 // Stack layout:
96 // rsp: return address <- sp
97 // 1 garbage
98 // 8 integer args (if static first is unused)
99 // 1 float/double identifiers
100 // 8 double args
101 // stack args <- esp
102 // garbage
103 // expression stack bottom
104 // bcp (null)
105 // ...
106
107 // Restore LR
108 __ ldr(lr, Address(__ post(sp, 2 * wordSize)));
109
110 // Do FP first so we can use c_rarg3 as temp
111 __ ldrw(c_rarg3, Address(sp, 9 * wordSize)); // float/double identifiers
112
113 for (int i = 0; i < Argument::n_float_register_parameters_c; i++) {
114 const FloatRegister r = as_FloatRegister(i);
115
116 Label d, done;
117
118 __ tbnz(c_rarg3, i, d);
119 __ ldrs(r, Address(sp, (10 + i) * wordSize));
120 __ b(done);
121 __ bind(d);
122 __ ldrd(r, Address(sp, (10 + i) * wordSize));
123 __ bind(done);
124 }
125
126 // c_rarg0 contains the result from the call of
127 // InterpreterRuntime::slow_signature_handler so we don't touch it
128 // here. It will be loaded with the JNIEnv* later.
129 __ ldr(c_rarg1, Address(sp, 1 * wordSize));
130 for (int i = c_rarg2->encoding(); i <= c_rarg7->encoding(); i += 2) {
131 Register rm = as_Register(i), rn = as_Register(i+1);
132 __ ldp(rm, rn, Address(sp, i * wordSize));
133 }
134
135 __ add(sp, sp, 18 * wordSize);
136 __ ret(lr);
137
138 return entry;
139 }
140
141
142 //
143 // Various method entries
144 //
145
146 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
147 // rmethod: Method*
148 // r19_sender_sp: sender sp
149 // esp: args
150
151 // These don't need a safepoint check because they aren't virtually
152 // callable. We won't enter these intrinsics from compiled code.
153 // If in the future we added an intrinsic which was virtually callable
154 // we'd have to worry about how to safepoint so that this code is used.
155
156 // mathematical functions inlined by compiler
157 // (interpreter must provide identical implementation
158 // in order to avoid monotonicity bugs when switching
159 // from interpreter to compiler in the middle of some
160 // computation)
161 //
162 // stack:
163 // [ arg ] <-- esp
164 // [ arg ]
165 // retaddr in lr
166
167 address entry_point = nullptr;
168 Register continuation = lr;
169 switch (kind) {
170 case Interpreter::java_lang_math_abs:
171 entry_point = __ pc();
172 __ ldrd(v0, Address(esp));
173 __ fabsd(v0, v0);
174 __ mov(sp, r19_sender_sp); // Restore caller's SP
175 break;
176 case Interpreter::java_lang_math_sqrt:
177 entry_point = __ pc();
178 __ ldrd(v0, Address(esp));
179 __ fsqrtd(v0, v0);
180 __ mov(sp, r19_sender_sp);
181 break;
182 case Interpreter::java_lang_math_sin :
183 case Interpreter::java_lang_math_cos :
184 case Interpreter::java_lang_math_tan :
185 case Interpreter::java_lang_math_log :
186 case Interpreter::java_lang_math_log10 :
187 case Interpreter::java_lang_math_exp :
188 entry_point = __ pc();
189 __ ldrd(v0, Address(esp));
190 __ mov(sp, r19_sender_sp);
191 __ mov(r23, lr);
192 continuation = r23; // The first free callee-saved register
193 generate_transcendental_entry(kind, 1);
194 break;
195 case Interpreter::java_lang_math_pow :
196 entry_point = __ pc();
197 __ mov(r23, lr);
198 continuation = r23;
199 __ ldrd(v0, Address(esp, 2 * Interpreter::stackElementSize));
200 __ ldrd(v1, Address(esp));
201 __ mov(sp, r19_sender_sp);
202 generate_transcendental_entry(kind, 2);
203 break;
204 case Interpreter::java_lang_math_fmaD :
205 if (UseFMA) {
206 entry_point = __ pc();
207 __ ldrd(v0, Address(esp, 4 * Interpreter::stackElementSize));
208 __ ldrd(v1, Address(esp, 2 * Interpreter::stackElementSize));
209 __ ldrd(v2, Address(esp));
210 __ fmaddd(v0, v0, v1, v2);
211 __ mov(sp, r19_sender_sp); // Restore caller's SP
212 }
213 break;
214 case Interpreter::java_lang_math_fmaF :
215 if (UseFMA) {
216 entry_point = __ pc();
217 __ ldrs(v0, Address(esp, 2 * Interpreter::stackElementSize));
218 __ ldrs(v1, Address(esp, Interpreter::stackElementSize));
219 __ ldrs(v2, Address(esp));
220 __ fmadds(v0, v0, v1, v2);
221 __ mov(sp, r19_sender_sp); // Restore caller's SP
222 }
223 break;
224 default:
225 ;
226 }
227 if (entry_point) {
228 __ br(continuation);
229 }
230
231 return entry_point;
232 }
233
234 // double trigonometrics and transcendentals
235 // static jdouble dsin(jdouble x);
236 // static jdouble dcos(jdouble x);
237 // static jdouble dtan(jdouble x);
238 // static jdouble dlog(jdouble x);
239 // static jdouble dlog10(jdouble x);
240 // static jdouble dexp(jdouble x);
241 // static jdouble dpow(jdouble x, jdouble y);
242
243 void TemplateInterpreterGenerator::generate_transcendental_entry(AbstractInterpreter::MethodKind kind, int fpargs) {
244 address fn;
245 switch (kind) {
246 case Interpreter::java_lang_math_sin :
247 if (StubRoutines::dsin() == nullptr) {
248 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
249 } else {
250 fn = CAST_FROM_FN_PTR(address, StubRoutines::dsin());
251 }
252 break;
253 case Interpreter::java_lang_math_cos :
254 if (StubRoutines::dcos() == nullptr) {
255 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
256 } else {
257 fn = CAST_FROM_FN_PTR(address, StubRoutines::dcos());
258 }
259 break;
260 case Interpreter::java_lang_math_tan :
261 if (StubRoutines::dtan() == nullptr) {
262 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
263 } else {
264 fn = CAST_FROM_FN_PTR(address, StubRoutines::dtan());
265 }
266 break;
267 case Interpreter::java_lang_math_log :
268 if (StubRoutines::dlog() == nullptr) {
269 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
270 } else {
271 fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog());
272 }
273 break;
274 case Interpreter::java_lang_math_log10 :
275 if (StubRoutines::dlog10() == nullptr) {
276 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
277 } else {
278 fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog10());
279 }
280 break;
281 case Interpreter::java_lang_math_exp :
282 if (StubRoutines::dexp() == nullptr) {
283 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
284 } else {
285 fn = CAST_FROM_FN_PTR(address, StubRoutines::dexp());
286 }
287 break;
288 case Interpreter::java_lang_math_pow :
289 if (StubRoutines::dpow() == nullptr) {
290 fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow);
291 } else {
292 fn = CAST_FROM_FN_PTR(address, StubRoutines::dpow());
293 }
294 break;
295 default:
296 ShouldNotReachHere();
297 fn = nullptr; // unreachable
298 }
299 __ mov(rscratch1, fn);
300 __ blr(rscratch1);
301 }
302
303 address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() {
304 assert(VM_Version::supports_float16(), "this intrinsic is not supported");
305 // r19_sender_sp: sender sp
306 // stack:
307 // [ arg ] <-- esp
308 // [ arg ]
309 // retaddr in lr
310 // result in v0
311
312 address entry_point = __ pc();
313 __ ldrw(c_rarg0, Address(esp));
314 __ flt16_to_flt(v0, c_rarg0, v1);
315 __ mov(sp, r19_sender_sp); // Restore caller's SP
316 __ br(lr);
317 return entry_point;
318 }
319
320 address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() {
321 assert(VM_Version::supports_float16(), "this intrinsic is not supported");
322 // r19_sender_sp: sender sp
323 // stack:
324 // [ arg ] <-- esp
325 // [ arg ]
326 // retaddr in lr
327 // result in c_rarg0
328
329 address entry_point = __ pc();
330 __ ldrs(v0, Address(esp));
331 __ flt_to_flt16(c_rarg0, v0, v1);
332 __ mov(sp, r19_sender_sp); // Restore caller's SP
333 __ br(lr);
334 return entry_point;
335 }
336
337 // Abstract method entry
338 // Attempt to execute abstract method. Throw exception
339 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
340 // rmethod: Method*
341 // r19_sender_sp: sender SP
342
343 address entry_point = __ pc();
344
345 // abstract method entry
346
347 // pop return address, reset last_sp to null
348 __ empty_expression_stack();
349 __ restore_bcp(); // bcp must be correct for exception handler (was destroyed)
350 __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
351
352 // throw exception
353 __ call_VM(noreg, CAST_FROM_FN_PTR(address,
354 InterpreterRuntime::throw_AbstractMethodErrorWithMethod),
355 rmethod);
356 // the call_VM checks for exception, so we should never return here.
357 __ should_not_reach_here();
358
359 return entry_point;
360 }
361
362 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
363 address entry = __ pc();
364
365 #ifdef ASSERT
366 {
367 Label L;
368 __ ldr(rscratch1, Address(rfp,
369 frame::interpreter_frame_monitor_block_top_offset *
370 wordSize));
371 __ mov(rscratch2, sp);
372 __ cmp(rscratch1, rscratch2); // maximal rsp for current rfp (stack
373 // grows negative)
374 __ br(Assembler::HS, L); // check if frame is complete
375 __ stop ("interpreter frame not set up");
376 __ bind(L);
377 }
378 #endif // ASSERT
379 // Restore bcp under the assumption that the current frame is still
380 // interpreted
381 __ restore_bcp();
382
383 // expression stack must be empty before entering the VM if an
384 // exception happened
385 __ empty_expression_stack();
386 // throw exception
387 __ call_VM(noreg,
388 CAST_FROM_FN_PTR(address,
389 InterpreterRuntime::throw_StackOverflowError));
390 return entry;
391 }
392
393 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
394 address entry = __ pc();
395 // expression stack must be empty before entering the VM if an
396 // exception happened
397 __ empty_expression_stack();
398 // setup parameters
399
400 // ??? convention: expect aberrant index in register r1
401 __ movw(c_rarg2, r1);
402 // ??? convention: expect array in register r3
403 __ mov(c_rarg1, r3);
404 __ call_VM(noreg,
405 CAST_FROM_FN_PTR(address,
406 InterpreterRuntime::
407 throw_ArrayIndexOutOfBoundsException),
408 c_rarg1, c_rarg2);
409 return entry;
410 }
411
412 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
413 address entry = __ pc();
414
415 // object is at TOS
416 __ pop(c_rarg1);
417
418 // expression stack must be empty before entering the VM if an
419 // exception happened
420 __ empty_expression_stack();
421
422 __ call_VM(noreg,
423 CAST_FROM_FN_PTR(address,
424 InterpreterRuntime::
425 throw_ClassCastException),
426 c_rarg1);
427 return entry;
428 }
429
430 address TemplateInterpreterGenerator::generate_exception_handler_common(
431 const char* name, const char* message, bool pass_oop) {
432 assert(!pass_oop || message == nullptr, "either oop or message but not both");
433 address entry = __ pc();
434 if (pass_oop) {
435 // object is at TOS
436 __ pop(c_rarg2);
437 }
438 // expression stack must be empty before entering the VM if an
439 // exception happened
440 __ empty_expression_stack();
441 // setup parameters
442 __ lea(c_rarg1, Address((address)name));
443 if (pass_oop) {
444 __ call_VM(r0, CAST_FROM_FN_PTR(address,
445 InterpreterRuntime::
446 create_klass_exception),
447 c_rarg1, c_rarg2);
448 } else {
449 // kind of lame ExternalAddress can't take null because
450 // external_word_Relocation will assert.
451 if (message != nullptr) {
452 __ lea(c_rarg2, Address((address)message));
453 } else {
454 __ mov(c_rarg2, NULL_WORD);
455 }
456 __ call_VM(r0,
457 CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
458 c_rarg1, c_rarg2);
459 }
460 // throw exception
461 __ b(address(Interpreter::throw_exception_entry()));
462 return entry;
463 }
464
465 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
466 address entry = __ pc();
467
468 // Restore stack bottom in case i2c adjusted stack
469 __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
470 // and null it as marker that esp is now tos until next java call
471 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
472
473 if (state == atos && InlineTypeReturnedAsFields) {
474 __ store_inline_type_fields_to_buf(nullptr, true);
475 }
476
477 __ restore_bcp();
478 __ restore_locals();
479 __ restore_constant_pool_cache();
480 __ get_method(rmethod);
481
482 if (state == atos) {
483 Register obj = r0;
484 Register mdp = r1;
485 Register tmp = r2;
486 __ profile_return_type(mdp, obj, tmp);
487 }
488
489 const Register cache = r1;
490 const Register index = r2;
491
492 if (index_size == sizeof(u4)) {
493 __ load_resolved_indy_entry(cache, index);
494 __ load_unsigned_short(cache, Address(cache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
495 __ add(esp, esp, cache, Assembler::LSL, 3);
496 } else {
497 // Pop N words from the stack
498 __ get_cache_and_index_at_bcp(cache, index, 1, index_size);
499 __ ldr(cache, Address(cache, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
500 __ andr(cache, cache, ConstantPoolCacheEntry::parameter_size_mask);
501
502 __ add(esp, esp, cache, Assembler::LSL, 3);
503 }
504
505 // Restore machine SP
506 __ restore_sp_after_call();
507
508 __ check_and_handle_popframe(rthread);
509 __ check_and_handle_earlyret(rthread);
510
511 __ get_dispatch();
512 __ dispatch_next(state, step);
513
514 return entry;
515 }
516
517 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
518 int step,
519 address continuation) {
520 address entry = __ pc();
521 __ restore_bcp();
522 __ restore_locals();
523 __ restore_constant_pool_cache();
524 __ get_method(rmethod);
525 __ get_dispatch();
526
527 __ restore_sp_after_call(); // Restore SP to extended SP
528
529 // Restore expression stack pointer
530 __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
531 // null last_sp until next java call
532 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
533
534 #if INCLUDE_JVMCI
535 // Check if we need to take lock at entry of synchronized method. This can
536 // only occur on method entry so emit it only for vtos with step 0.
537 if (EnableJVMCI && state == vtos && step == 0) {
538 Label L;
539 __ ldrb(rscratch1, Address(rthread, JavaThread::pending_monitorenter_offset()));
540 __ cbz(rscratch1, L);
541 // Clear flag.
542 __ strb(zr, Address(rthread, JavaThread::pending_monitorenter_offset()));
543 // Take lock.
544 lock_method();
545 __ bind(L);
546 } else {
547 #ifdef ASSERT
548 if (EnableJVMCI) {
549 Label L;
550 __ ldrb(rscratch1, Address(rthread, JavaThread::pending_monitorenter_offset()));
551 __ cbz(rscratch1, L);
552 __ stop("unexpected pending monitor in deopt entry");
553 __ bind(L);
554 }
555 #endif
556 }
557 #endif
558 // handle exceptions
559 {
560 Label L;
561 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
562 __ cbz(rscratch1, L);
563 __ call_VM(noreg,
564 CAST_FROM_FN_PTR(address,
565 InterpreterRuntime::throw_pending_exception));
566 __ should_not_reach_here();
567 __ bind(L);
568 }
569
570 if (continuation == nullptr) {
571 __ dispatch_next(state, step);
572 } else {
573 __ jump_to_entry(continuation);
574 }
575 return entry;
576 }
577
578 address TemplateInterpreterGenerator::generate_result_handler_for(
579 BasicType type) {
580 address entry = __ pc();
581 switch (type) {
582 case T_BOOLEAN: __ c2bool(r0); break;
583 case T_CHAR : __ uxth(r0, r0); break;
584 case T_BYTE : __ sxtb(r0, r0); break;
585 case T_SHORT : __ sxth(r0, r0); break;
586 case T_INT : __ uxtw(r0, r0); break; // FIXME: We almost certainly don't need this
587 case T_LONG : /* nothing to do */ break;
588 case T_VOID : /* nothing to do */ break;
589 case T_FLOAT : /* nothing to do */ break;
590 case T_DOUBLE : /* nothing to do */ break;
591 case T_PRIMITIVE_OBJECT: // fall through (value types are handled with oops)
592 case T_OBJECT :
593 // retrieve result from frame
594 __ ldr(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize));
595 // and verify it
596 __ verify_oop(r0);
597 break;
598 default : ShouldNotReachHere();
599 }
600 __ ret(lr); // return from result handler
601 return entry;
602 }
603
604 address TemplateInterpreterGenerator::generate_safept_entry_for(
605 TosState state,
606 address runtime_entry) {
607 address entry = __ pc();
608 __ push(state);
609 __ push_cont_fastpath(rthread);
610 __ call_VM(noreg, runtime_entry);
611 __ pop_cont_fastpath(rthread);
612 __ membar(Assembler::AnyAny);
613 __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
614 return entry;
615 }
616
617 // Helpers for commoning out cases in the various type of method entries.
618 //
619
620
621 // increment invocation count & check for overflow
622 //
623 // Note: checking for negative value instead of overflow
624 // so we have a 'sticky' overflow test
625 //
626 // rmethod: method
627 //
628 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) {
629 Label done;
630 // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not.
631 int increment = InvocationCounter::count_increment;
632 Label no_mdo;
633 if (ProfileInterpreter) {
634 // Are we profiling?
635 __ ldr(r0, Address(rmethod, Method::method_data_offset()));
636 __ cbz(r0, no_mdo);
637 // Increment counter in the MDO
638 const Address mdo_invocation_counter(r0, in_bytes(MethodData::invocation_counter_offset()) +
639 in_bytes(InvocationCounter::counter_offset()));
640 const Address mask(r0, in_bytes(MethodData::invoke_mask_offset()));
641 __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rscratch1, rscratch2, false, Assembler::EQ, overflow);
642 __ b(done);
643 }
644 __ bind(no_mdo);
645 // Increment counter in MethodCounters
646 const Address invocation_counter(rscratch2,
647 MethodCounters::invocation_counter_offset() +
648 InvocationCounter::counter_offset());
649 __ get_method_counters(rmethod, rscratch2, done);
650 const Address mask(rscratch2, in_bytes(MethodCounters::invoke_mask_offset()));
651 __ increment_mask_and_jump(invocation_counter, increment, mask, rscratch1, r1, false, Assembler::EQ, overflow);
652 __ bind(done);
653 }
654
655 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
656
657 // Asm interpreter on entry
658 // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
659 // Everything as it was on entry
660
661 // InterpreterRuntime::frequency_counter_overflow takes two
662 // arguments, the first (thread) is passed by call_VM, the second
663 // indicates if the counter overflow occurs at a backwards branch
664 // (null bcp). We pass zero for it. The call returns the address
665 // of the verified entry point for the method or null if the
666 // compilation did not complete (either went background or bailed
667 // out).
668 __ mov(c_rarg1, 0);
669 __ call_VM(noreg,
670 CAST_FROM_FN_PTR(address,
671 InterpreterRuntime::frequency_counter_overflow),
672 c_rarg1);
673
674 __ b(do_continue);
675 }
676
677 // See if we've got enough room on the stack for locals plus overhead
678 // below JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError
679 // without going through the signal handler, i.e., reserved and yellow zones
680 // will not be made usable. The shadow zone must suffice to handle the
681 // overflow.
682 // The expression stack grows down incrementally, so the normal guard
683 // page mechanism will work for that.
684 //
685 // NOTE: Since the additional locals are also always pushed (wasn't
686 // obvious in generate_method_entry) so the guard should work for them
687 // too.
688 //
689 // Args:
690 // r3: number of additional locals this frame needs (what we must check)
691 // rmethod: Method*
692 //
693 // Kills:
694 // r0
695 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
696
697 // monitor entry size: see picture of stack set
698 // (generate_method_entry) and frame_amd64.hpp
699 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
700
701 // total overhead size: entry_size + (saved rbp through expr stack
702 // bottom). be sure to change this if you add/subtract anything
703 // to/from the overhead area
704 const int overhead_size =
705 -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
706
707 const size_t page_size = os::vm_page_size();
708
709 Label after_frame_check;
710
711 // see if the frame is greater than one page in size. If so,
712 // then we need to verify there is enough stack space remaining
713 // for the additional locals.
714 //
715 // Note that we use SUBS rather than CMP here because the immediate
716 // field of this instruction may overflow. SUBS can cope with this
717 // because it is a macro that will expand to some number of MOV
718 // instructions and a register operation.
719 __ subs(rscratch1, r3, (page_size - overhead_size) / Interpreter::stackElementSize);
720 __ br(Assembler::LS, after_frame_check);
721
722 // compute rsp as if this were going to be the last frame on
723 // the stack before the red zone
724
725 // locals + overhead, in bytes
726 __ mov(r0, overhead_size);
727 __ add(r0, r0, r3, Assembler::LSL, Interpreter::logStackElementSize); // 2 slots per parameter.
728
729 const Address stack_limit(rthread, JavaThread::stack_overflow_limit_offset());
730 __ ldr(rscratch1, stack_limit);
731
732 #ifdef ASSERT
733 Label limit_okay;
734 // Verify that thread stack limit is non-zero.
735 __ cbnz(rscratch1, limit_okay);
736 __ stop("stack overflow limit is zero");
737 __ bind(limit_okay);
738 #endif
739
740 // Add stack limit to locals.
741 __ add(r0, r0, rscratch1);
742
743 // Check against the current stack bottom.
744 __ cmp(sp, r0);
745 __ br(Assembler::HI, after_frame_check);
746
747 // Remove the incoming args, peeling the machine SP back to where it
748 // was in the caller. This is not strictly necessary, but unless we
749 // do so the stack frame may have a garbage FP; this ensures a
750 // correct call stack that we can always unwind. The ANDR should be
751 // unnecessary because the sender SP in r19 is always aligned, but
752 // it doesn't hurt.
753 __ andr(sp, r19_sender_sp, -16);
754
755 // Note: the restored frame is not necessarily interpreted.
756 // Use the shared runtime version of the StackOverflowError.
757 assert(StubRoutines::throw_StackOverflowError_entry() != nullptr, "stub not yet generated");
758 __ far_jump(RuntimeAddress(StubRoutines::throw_StackOverflowError_entry()));
759
760 // all done with frame size check
761 __ bind(after_frame_check);
762 }
763
764 // Allocate monitor and lock method (asm interpreter)
765 //
766 // Args:
767 // rmethod: Method*
768 // rlocals: locals
769 //
770 // Kills:
771 // r0
772 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
773 // rscratch1, rscratch2 (scratch regs)
774 void TemplateInterpreterGenerator::lock_method() {
775 // synchronize method
776 const Address access_flags(rmethod, Method::access_flags_offset());
777 const Address monitor_block_top(
778 rfp,
779 frame::interpreter_frame_monitor_block_top_offset * wordSize);
780 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
781
782 #ifdef ASSERT
783 {
784 Label L;
785 __ ldrw(r0, access_flags);
786 __ tst(r0, JVM_ACC_SYNCHRONIZED);
787 __ br(Assembler::NE, L);
788 __ stop("method doesn't need synchronization");
789 __ bind(L);
790 }
791 #endif // ASSERT
792
793 // get synchronization object
794 {
795 Label done;
796 __ ldrw(r0, access_flags);
797 __ tst(r0, JVM_ACC_STATIC);
798 // get receiver (assume this is frequent case)
799 __ ldr(r0, Address(rlocals, Interpreter::local_offset_in_bytes(0)));
800 __ br(Assembler::EQ, done);
801 __ load_mirror(r0, rmethod, r5, rscratch2);
802
803 #ifdef ASSERT
804 {
805 Label L;
806 __ cbnz(r0, L);
807 __ stop("synchronization object is null");
808 __ bind(L);
809 }
810 #endif // ASSERT
811
812 __ bind(done);
813 }
814
815 // add space for monitor & lock
816 __ check_extended_sp();
817 __ sub(sp, sp, entry_size); // add space for a monitor entry
818 __ sub(esp, esp, entry_size);
819 __ mov(rscratch1, sp);
820 __ str(rscratch1, Address(rfp, frame::interpreter_frame_extended_sp_offset * wordSize));
821 __ str(esp, monitor_block_top); // set new monitor block top
822 // store object
823 __ str(r0, Address(esp, BasicObjectLock::obj_offset()));
824 __ mov(c_rarg1, esp); // object address
825 __ lock_object(c_rarg1);
826 }
827
828 // Generate a fixed interpreter frame. This is identical setup for
829 // interpreted methods and for native methods hence the shared code.
830 //
831 // Args:
832 // lr: return address
833 // rmethod: Method*
834 // rlocals: pointer to locals
835 // rcpool: cp cache
836 // stack_pointer: previous sp
837 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
838 // initialize fixed part of activation frame
839 if (native_call) {
840 __ sub(esp, sp, 14 * wordSize);
841 __ mov(rbcp, zr);
842 __ stp(esp, zr, Address(__ pre(sp, -14 * wordSize)));
843 // add 2 zero-initialized slots for native calls
844 __ stp(zr, zr, Address(sp, 12 * wordSize));
845 } else {
846 __ sub(esp, sp, 12 * wordSize);
847 __ ldr(rscratch1, Address(rmethod, Method::const_offset())); // get ConstMethod
848 __ add(rbcp, rscratch1, in_bytes(ConstMethod::codes_offset())); // get codebase
849 __ stp(esp, rbcp, Address(__ pre(sp, -12 * wordSize)));
850 }
851
852 if (ProfileInterpreter) {
853 Label method_data_continue;
854 __ ldr(rscratch1, Address(rmethod, Method::method_data_offset()));
855 __ cbz(rscratch1, method_data_continue);
856 __ lea(rscratch1, Address(rscratch1, in_bytes(MethodData::data_offset())));
857 __ bind(method_data_continue);
858 __ stp(rscratch1, rmethod, Address(sp, 6 * wordSize)); // save Method* and mdp (method data pointer)
859 } else {
860 __ stp(zr, rmethod, Address(sp, 6 * wordSize)); // save Method* (no mdp)
861 }
862
863 __ protect_return_address();
864 __ stp(rfp, lr, Address(sp, 10 * wordSize));
865 __ lea(rfp, Address(sp, 10 * wordSize));
866
867 __ ldr(rcpool, Address(rmethod, Method::const_offset()));
868 __ ldr(rcpool, Address(rcpool, ConstMethod::constants_offset()));
869 __ ldr(rcpool, Address(rcpool, ConstantPool::cache_offset()));
870 __ sub(rscratch1, rlocals, rfp);
871 __ lsr(rscratch1, rscratch1, Interpreter::logStackElementSize); // rscratch1 = rlocals - fp();
872 // Store relativized rlocals, see frame::interpreter_frame_locals().
873 __ stp(rscratch1, rcpool, Address(sp, 2 * wordSize));
874
875 // set sender sp
876 // leave last_sp as null
877 __ stp(zr, r19_sender_sp, Address(sp, 8 * wordSize));
878
879 // Get mirror
880 __ load_mirror(r10, rmethod, r5, rscratch2);
881 if (! native_call) {
882 __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
883 __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
884 __ add(rscratch1, rscratch1, MAX2(3, Method::extra_stack_entries()));
885 __ sub(rscratch1, sp, rscratch1, ext::uxtw, 3);
886 __ andr(rscratch1, rscratch1, -16);
887 // Store extended SP and mirror
888 __ stp(r10, rscratch1, Address(sp, 4 * wordSize));
889 // Move SP out of the way
890 __ mov(sp, rscratch1);
891 } else {
892 // Make sure there is room for the exception oop pushed in case method throws
893 // an exception (see TemplateInterpreterGenerator::generate_throw_exception())
894 __ sub(rscratch1, sp, 2 * wordSize);
895 __ stp(r10, rscratch1, Address(sp, 4 * wordSize));
896 __ mov(sp, rscratch1);
897 }
898 }
899
900 // End of helpers
901
902 // Various method entries
903 //------------------------------------------------------------------------------------------------------------------------
904 //
905 //
906
907 // Method entry for java.lang.ref.Reference.get.
908 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
909 // Code: _aload_0, _getfield, _areturn
910 // parameter size = 1
911 //
912 // The code that gets generated by this routine is split into 2 parts:
913 // 1. The "intrinsified" code for G1 (or any SATB based GC),
914 // 2. The slow path - which is an expansion of the regular method entry.
915 //
916 // Notes:-
917 // * In the G1 code we do not check whether we need to block for
918 // a safepoint. If G1 is enabled then we must execute the specialized
919 // code for Reference.get (except when the Reference object is null)
920 // so that we can log the value in the referent field with an SATB
921 // update buffer.
922 // If the code for the getfield template is modified so that the
923 // G1 pre-barrier code is executed when the current method is
924 // Reference.get() then going through the normal method entry
925 // will be fine.
926 // * The G1 code can, however, check the receiver object (the instance
927 // of java.lang.Reference) and jump to the slow path if null. If the
928 // Reference object is null then we obviously cannot fetch the referent
929 // and so we don't need to call the G1 pre-barrier. Thus we can use the
930 // regular method entry code to generate the NPE.
931 //
932 // This code is based on generate_accessor_entry.
933 //
934 // rmethod: Method*
935 // r19_sender_sp: senderSP must preserve for slow path, set SP to it on fast path
936
937 // LR is live. It must be saved around calls.
938
939 address entry = __ pc();
940
941 const int referent_offset = java_lang_ref_Reference::referent_offset();
942
943 Label slow_path;
944 const Register local_0 = c_rarg0;
945 // Check if local 0 != null
946 // If the receiver is null then it is OK to jump to the slow path.
947 __ ldr(local_0, Address(esp, 0));
948 __ cbz(local_0, slow_path);
949
950 // Load the value of the referent field.
951 const Address field_address(local_0, referent_offset);
952 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
953 bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT, local_0, field_address, /*tmp1*/ rscratch1, /*tmp2*/ rscratch2);
954
955 // areturn
956 __ andr(sp, r19_sender_sp, -16); // done with stack
957 __ ret(lr);
958
959 // generate a vanilla interpreter entry as the slow path
960 __ bind(slow_path);
961 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
962 return entry;
963
964 }
965
966 /**
967 * Method entry for static native methods:
968 * int java.util.zip.CRC32.update(int crc, int b)
969 */
970 address TemplateInterpreterGenerator::generate_CRC32_update_entry() {
971 assert(UseCRC32Intrinsics, "this intrinsic is not supported");
972 address entry = __ pc();
973
974 // rmethod: Method*
975 // r19_sender_sp: senderSP must preserved for slow path
976 // esp: args
977
978 Label slow_path;
979 // If we need a safepoint check, generate full interpreter entry.
980 __ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */);
981
982 // We don't generate local frame and don't align stack because
983 // we call stub code and there is no safepoint on this path.
984
985 // Load parameters
986 const Register crc = c_rarg0; // crc
987 const Register val = c_rarg1; // source java byte value
988 const Register tbl = c_rarg2; // scratch
989
990 // Arguments are reversed on java expression stack
991 __ ldrw(val, Address(esp, 0)); // byte value
992 __ ldrw(crc, Address(esp, wordSize)); // Initial CRC
993
994 uint64_t offset;
995 __ adrp(tbl, ExternalAddress(StubRoutines::crc_table_addr()), offset);
996 __ add(tbl, tbl, offset);
997
998 __ mvnw(crc, crc); // ~crc
999 __ update_byte_crc32(crc, val, tbl);
1000 __ mvnw(crc, crc); // ~crc
1001
1002 // result in c_rarg0
1003
1004 __ andr(sp, r19_sender_sp, -16);
1005 __ ret(lr);
1006
1007 // generate a vanilla native entry as the slow path
1008 __ bind(slow_path);
1009 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
1010 return entry;
1011 }
1012
1013 /**
1014 * Method entry for static native methods:
1015 * int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len)
1016 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
1017 */
1018 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
1019 assert(UseCRC32Intrinsics, "this intrinsic is not supported");
1020 address entry = __ pc();
1021
1022 // rmethod,: Method*
1023 // r19_sender_sp: senderSP must preserved for slow path
1024
1025 Label slow_path;
1026 // If we need a safepoint check, generate full interpreter entry.
1027 __ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */);
1028
1029 // We don't generate local frame and don't align stack because
1030 // we call stub code and there is no safepoint on this path.
1031
1032 // Load parameters
1033 const Register crc = c_rarg0; // crc
1034 const Register buf = c_rarg1; // source java byte array address
1035 const Register len = c_rarg2; // length
1036 const Register off = len; // offset (never overlaps with 'len')
1037
1038 // Arguments are reversed on java expression stack
1039 // Calculate address of start element
1040 if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
1041 __ ldr(buf, Address(esp, 2*wordSize)); // long buf
1042 __ ldrw(off, Address(esp, wordSize)); // offset
1043 __ add(buf, buf, off); // + offset
1044 __ ldrw(crc, Address(esp, 4*wordSize)); // Initial CRC
1045 } else {
1046 __ ldr(buf, Address(esp, 2*wordSize)); // byte[] array
1047 __ add(buf, buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
1048 __ ldrw(off, Address(esp, wordSize)); // offset
1049 __ add(buf, buf, off); // + offset
1050 __ ldrw(crc, Address(esp, 3*wordSize)); // Initial CRC
1051 }
1052 // Can now load 'len' since we're finished with 'off'
1053 __ ldrw(len, Address(esp, 0x0)); // Length
1054
1055 __ andr(sp, r19_sender_sp, -16); // Restore the caller's SP
1056
1057 // We are frameless so we can just jump to the stub.
1058 __ b(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32()));
1059
1060 // generate a vanilla native entry as the slow path
1061 __ bind(slow_path);
1062 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
1063 return entry;
1064 }
1065
1066 /**
1067 * Method entry for intrinsic-candidate (non-native) methods:
1068 * int java.util.zip.CRC32C.updateBytes(int crc, byte[] b, int off, int end)
1069 * int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end)
1070 * Unlike CRC32, CRC32C does not have any methods marked as native
1071 * CRC32C also uses an "end" variable instead of the length variable CRC32 uses
1072 */
1073 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
1074 assert(UseCRC32CIntrinsics, "this intrinsic is not supported");
1075 address entry = __ pc();
1076
1077 // Prepare jump to stub using parameters from the stack
1078 const Register crc = c_rarg0; // initial crc
1079 const Register buf = c_rarg1; // source java byte array address
1080 const Register len = c_rarg2; // len argument to the kernel
1081
1082 const Register end = len; // index of last element to process
1083 const Register off = crc; // offset
1084
1085 __ ldrw(end, Address(esp)); // int end
1086 __ ldrw(off, Address(esp, wordSize)); // int offset
1087 __ sub(len, end, off);
1088 __ ldr(buf, Address(esp, 2*wordSize)); // byte[] buf | long buf
1089 __ add(buf, buf, off); // + offset
1090 if (kind == Interpreter::java_util_zip_CRC32C_updateDirectByteBuffer) {
1091 __ ldrw(crc, Address(esp, 4*wordSize)); // long crc
1092 } else {
1093 __ add(buf, buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
1094 __ ldrw(crc, Address(esp, 3*wordSize)); // long crc
1095 }
1096
1097 __ andr(sp, r19_sender_sp, -16); // Restore the caller's SP
1098
1099 // Jump to the stub.
1100 __ b(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32C()));
1101
1102 return entry;
1103 }
1104
1105 void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
1106 // See more discussion in stackOverflow.hpp.
1107
1108 const int shadow_zone_size = checked_cast<int>(StackOverflow::stack_shadow_zone_size());
1109 const int page_size = (int)os::vm_page_size();
1110 const int n_shadow_pages = shadow_zone_size / page_size;
1111
1112 #ifdef ASSERT
1113 Label L_good_limit;
1114 __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit()));
1115 __ cbnz(rscratch1, L_good_limit);
1116 __ stop("shadow zone safe limit is not initialized");
1117 __ bind(L_good_limit);
1118
1119 Label L_good_watermark;
1120 __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark()));
1121 __ cbnz(rscratch1, L_good_watermark);
1122 __ stop("shadow zone growth watermark is not initialized");
1123 __ bind(L_good_watermark);
1124 #endif
1125
1126 Label L_done;
1127
1128 __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark()));
1129 __ cmp(sp, rscratch1);
1130 __ br(Assembler::HI, L_done);
1131
1132 for (int p = 1; p <= n_shadow_pages; p++) {
1133 __ sub(rscratch2, sp, p*page_size);
1134 __ str(zr, Address(rscratch2));
1135 }
1136
1137 // Record the new watermark, but only if the update is above the safe limit.
1138 // Otherwise, the next time around the check above would pass the safe limit.
1139 __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit()));
1140 __ cmp(sp, rscratch1);
1141 __ br(Assembler::LS, L_done);
1142 __ mov(rscratch1, sp);
1143 __ str(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark()));
1144
1145 __ bind(L_done);
1146 }
1147
1148 // Interpreter stub for calling a native method. (asm interpreter)
1149 // This sets up a somewhat different looking stack for calling the
1150 // native method than the typical interpreter frame setup.
1151 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
1152 // determine code generation flags
1153 bool inc_counter = UseCompiler || CountCompiledCalls;
1154
1155 // r1: Method*
1156 // rscratch1: sender sp
1157
1158 address entry_point = __ pc();
1159
1160 const Address constMethod (rmethod, Method::const_offset());
1161 const Address access_flags (rmethod, Method::access_flags_offset());
1162 const Address size_of_parameters(r2, ConstMethod::
1163 size_of_parameters_offset());
1164
1165 // get parameter size (always needed)
1166 __ ldr(r2, constMethod);
1167 __ load_unsigned_short(r2, size_of_parameters);
1168
1169 // Native calls don't need the stack size check since they have no
1170 // expression stack and the arguments are already on the stack and
1171 // we only add a handful of words to the stack.
1172
1173 // rmethod: Method*
1174 // r2: size of parameters
1175 // rscratch1: sender sp
1176
1177 // for natives the size of locals is zero
1178
1179 // compute beginning of parameters (rlocals)
1180 __ add(rlocals, esp, r2, ext::uxtx, 3);
1181 __ add(rlocals, rlocals, -wordSize);
1182
1183 // Pull SP back to minimum size: this avoids holes in the stack
1184 __ andr(sp, esp, -16);
1185
1186 // initialize fixed part of activation frame
1187 generate_fixed_frame(true);
1188
1189 // make sure method is native & not abstract
1190 #ifdef ASSERT
1191 __ ldrw(r0, access_flags);
1192 {
1193 Label L;
1194 __ tst(r0, JVM_ACC_NATIVE);
1195 __ br(Assembler::NE, L);
1196 __ stop("tried to execute non-native method as native");
1197 __ bind(L);
1198 }
1199 {
1200 Label L;
1201 __ tst(r0, JVM_ACC_ABSTRACT);
1202 __ br(Assembler::EQ, L);
1203 __ stop("tried to execute abstract method in interpreter");
1204 __ bind(L);
1205 }
1206 #endif
1207
1208 // Since at this point in the method invocation the exception
1209 // handler would try to exit the monitor of synchronized methods
1210 // which hasn't been entered yet, we set the thread local variable
1211 // _do_not_unlock_if_synchronized to true. The remove_activation
1212 // will check this flag.
1213
1214 const Address do_not_unlock_if_synchronized(rthread,
1215 in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1216 __ mov(rscratch2, true);
1217 __ strb(rscratch2, do_not_unlock_if_synchronized);
1218
1219 // increment invocation count & check for overflow
1220 Label invocation_counter_overflow;
1221 if (inc_counter) {
1222 generate_counter_incr(&invocation_counter_overflow);
1223 }
1224
1225 Label continue_after_compile;
1226 __ bind(continue_after_compile);
1227
1228 bang_stack_shadow_pages(true);
1229
1230 // reset the _do_not_unlock_if_synchronized flag
1231 __ strb(zr, do_not_unlock_if_synchronized);
1232
1233 // check for synchronized methods
1234 // Must happen AFTER invocation_counter check and stack overflow check,
1235 // so method is not locked if overflows.
1236 if (synchronized) {
1237 lock_method();
1238 } else {
1239 // no synchronization necessary
1240 #ifdef ASSERT
1241 {
1242 Label L;
1243 __ ldrw(r0, access_flags);
1244 __ tst(r0, JVM_ACC_SYNCHRONIZED);
1245 __ br(Assembler::EQ, L);
1246 __ stop("method needs synchronization");
1247 __ bind(L);
1248 }
1249 #endif
1250 }
1251
1252 // start execution
1253 #ifdef ASSERT
1254 {
1255 Label L;
1256 const Address monitor_block_top(rfp,
1257 frame::interpreter_frame_monitor_block_top_offset * wordSize);
1258 __ ldr(rscratch1, monitor_block_top);
1259 __ cmp(esp, rscratch1);
1260 __ br(Assembler::EQ, L);
1261 __ stop("broken stack frame setup in interpreter 1");
1262 __ bind(L);
1263 }
1264 #endif
1265
1266 // jvmti support
1267 __ notify_method_entry();
1268
1269 // work registers
1270 const Register t = r17;
1271 const Register result_handler = r19;
1272
1273 // allocate space for parameters
1274 __ ldr(t, Address(rmethod, Method::const_offset()));
1275 __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
1276
1277 __ sub(rscratch1, esp, t, ext::uxtx, Interpreter::logStackElementSize);
1278 __ andr(sp, rscratch1, -16);
1279 __ mov(esp, rscratch1);
1280
1281 // get signature handler
1282 {
1283 Label L;
1284 __ ldr(t, Address(rmethod, Method::signature_handler_offset()));
1285 __ cbnz(t, L);
1286 __ call_VM(noreg,
1287 CAST_FROM_FN_PTR(address,
1288 InterpreterRuntime::prepare_native_call),
1289 rmethod);
1290 __ ldr(t, Address(rmethod, Method::signature_handler_offset()));
1291 __ bind(L);
1292 }
1293
1294 // call signature handler
1295 assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rlocals,
1296 "adjust this code");
1297 assert(InterpreterRuntime::SignatureHandlerGenerator::to() == sp,
1298 "adjust this code");
1299 assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
1300 "adjust this code");
1301
1302 // The generated handlers do not touch rmethod (the method).
1303 // However, large signatures cannot be cached and are generated
1304 // each time here. The slow-path generator can do a GC on return,
1305 // so we must reload it after the call.
1306 __ blr(t);
1307 __ get_method(rmethod); // slow path can do a GC, reload rmethod
1308
1309
1310 // result handler is in r0
1311 // set result handler
1312 __ mov(result_handler, r0);
1313 // pass mirror handle if static call
1314 {
1315 Label L;
1316 __ ldrw(t, Address(rmethod, Method::access_flags_offset()));
1317 __ tbz(t, exact_log2(JVM_ACC_STATIC), L);
1318 // get mirror
1319 __ load_mirror(t, rmethod, r10, rscratch2);
1320 // copy mirror into activation frame
1321 __ str(t, Address(rfp, frame::interpreter_frame_oop_temp_offset * wordSize));
1322 // pass handle to mirror
1323 __ add(c_rarg1, rfp, frame::interpreter_frame_oop_temp_offset * wordSize);
1324 __ bind(L);
1325 }
1326
1327 // get native function entry point in r10
1328 {
1329 Label L;
1330 __ ldr(r10, Address(rmethod, Method::native_function_offset()));
1331 address unsatisfied = (SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
1332 __ mov(rscratch2, unsatisfied);
1333 __ ldr(rscratch2, rscratch2);
1334 __ cmp(r10, rscratch2);
1335 __ br(Assembler::NE, L);
1336 __ call_VM(noreg,
1337 CAST_FROM_FN_PTR(address,
1338 InterpreterRuntime::prepare_native_call),
1339 rmethod);
1340 __ get_method(rmethod);
1341 __ ldr(r10, Address(rmethod, Method::native_function_offset()));
1342 __ bind(L);
1343 }
1344
1345 // pass JNIEnv
1346 __ add(c_rarg0, rthread, in_bytes(JavaThread::jni_environment_offset()));
1347
1348 // Set the last Java PC in the frame anchor to be the return address from
1349 // the call to the native method: this will allow the debugger to
1350 // generate an accurate stack trace.
1351 Label native_return;
1352 __ set_last_Java_frame(esp, rfp, native_return, rscratch1);
1353
1354 // change thread state
1355 #ifdef ASSERT
1356 {
1357 Label L;
1358 __ ldrw(t, Address(rthread, JavaThread::thread_state_offset()));
1359 __ cmp(t, (u1)_thread_in_Java);
1360 __ br(Assembler::EQ, L);
1361 __ stop("Wrong thread state in native stub");
1362 __ bind(L);
1363 }
1364 #endif
1365
1366 // Change state to native
1367 __ mov(rscratch1, _thread_in_native);
1368 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1369 __ stlrw(rscratch1, rscratch2);
1370
1371 // Call the native method.
1372 __ blr(r10);
1373 __ bind(native_return);
1374 __ get_method(rmethod);
1375 // result potentially in r0 or v0
1376
1377 // make room for the pushes we're about to do
1378 __ sub(rscratch1, esp, 4 * wordSize);
1379 __ andr(sp, rscratch1, -16);
1380
1381 // NOTE: The order of these pushes is known to frame::interpreter_frame_result
1382 // in order to extract the result of a method call. If the order of these
1383 // pushes change or anything else is added to the stack then the code in
1384 // interpreter_frame_result must also change.
1385 __ push(dtos);
1386 __ push(ltos);
1387
1388 __ verify_sve_vector_length();
1389
1390 // change thread state
1391 __ mov(rscratch1, _thread_in_native_trans);
1392 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1393 __ stlrw(rscratch1, rscratch2);
1394
1395 // Force this write out before the read below
1396 if (!UseSystemMemoryBarrier) {
1397 __ dmb(Assembler::ISH);
1398 }
1399
1400 // check for safepoint operation in progress and/or pending suspend requests
1401 {
1402 Label L, Continue;
1403
1404 // We need an acquire here to ensure that any subsequent load of the
1405 // global SafepointSynchronize::_state flag is ordered after this load
1406 // of the thread-local polling word. We don't want this poll to
1407 // return false (i.e. not safepointing) and a later poll of the global
1408 // SafepointSynchronize::_state spuriously to return true.
1409 //
1410 // This is to avoid a race when we're in a native->Java transition
1411 // racing the code which wakes up from a safepoint.
1412 __ safepoint_poll(L, true /* at_return */, true /* acquire */, false /* in_nmethod */);
1413 __ ldrw(rscratch2, Address(rthread, JavaThread::suspend_flags_offset()));
1414 __ cbz(rscratch2, Continue);
1415 __ bind(L);
1416
1417 // Don't use call_VM as it will see a possible pending exception
1418 // and forward it and never return here preventing us from
1419 // clearing _last_native_pc down below. So we do a runtime call by
1420 // hand.
1421 //
1422 __ mov(c_rarg0, rthread);
1423 __ mov(rscratch2, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
1424 __ blr(rscratch2);
1425 __ get_method(rmethod);
1426 __ reinit_heapbase();
1427 __ bind(Continue);
1428 }
1429
1430 // change thread state
1431 __ mov(rscratch1, _thread_in_Java);
1432 __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1433 __ stlrw(rscratch1, rscratch2);
1434
1435 // reset_last_Java_frame
1436 __ reset_last_Java_frame(true);
1437
1438 if (CheckJNICalls) {
1439 // clear_pending_jni_exception_check
1440 __ str(zr, Address(rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1441 }
1442
1443 // reset handle block
1444 __ ldr(t, Address(rthread, JavaThread::active_handles_offset()));
1445 __ str(zr, Address(t, JNIHandleBlock::top_offset()));
1446
1447 // If result is an oop unbox and store it in frame where gc will see it
1448 // and result handler will pick it up
1449
1450 {
1451 Label no_oop;
1452 __ adr(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1453 __ cmp(t, result_handler);
1454 __ br(Assembler::NE, no_oop);
1455 // Unbox oop result, e.g. JNIHandles::resolve result.
1456 __ pop(ltos);
1457 __ resolve_jobject(r0, t, rscratch2);
1458 __ str(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize));
1459 // keep stack depth as expected by pushing oop which will eventually be discarded
1460 __ push(ltos);
1461 __ bind(no_oop);
1462 }
1463
1464 {
1465 Label no_reguard;
1466 __ lea(rscratch1, Address(rthread, in_bytes(JavaThread::stack_guard_state_offset())));
1467 __ ldrw(rscratch1, Address(rscratch1));
1468 __ cmp(rscratch1, (u1)StackOverflow::stack_guard_yellow_reserved_disabled);
1469 __ br(Assembler::NE, no_reguard);
1470
1471 __ push_call_clobbered_registers();
1472 __ mov(c_rarg0, rthread);
1473 __ mov(rscratch2, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1474 __ blr(rscratch2);
1475 __ pop_call_clobbered_registers();
1476
1477 __ bind(no_reguard);
1478 }
1479
1480 // The method register is junk from after the thread_in_native transition
1481 // until here. Also can't call_VM until the bcp has been
1482 // restored. Need bcp for throwing exception below so get it now.
1483 __ get_method(rmethod);
1484
1485 // restore bcp to have legal interpreter frame, i.e., bci == 0 <=>
1486 // rbcp == code_base()
1487 __ ldr(rbcp, Address(rmethod, Method::const_offset())); // get ConstMethod*
1488 __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset())); // get codebase
1489 // handle exceptions (exception handling will handle unlocking!)
1490 {
1491 Label L;
1492 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
1493 __ cbz(rscratch1, L);
1494 // Note: At some point we may want to unify this with the code
1495 // used in call_VM_base(); i.e., we should use the
1496 // StubRoutines::forward_exception code. For now this doesn't work
1497 // here because the rsp is not correctly set at this point.
1498 __ MacroAssembler::call_VM(noreg,
1499 CAST_FROM_FN_PTR(address,
1500 InterpreterRuntime::throw_pending_exception));
1501 __ should_not_reach_here();
1502 __ bind(L);
1503 }
1504
1505 // do unlocking if necessary
1506 {
1507 Label L;
1508 __ ldrw(t, Address(rmethod, Method::access_flags_offset()));
1509 __ tbz(t, exact_log2(JVM_ACC_SYNCHRONIZED), L);
1510 // the code below should be shared with interpreter macro
1511 // assembler implementation
1512 {
1513 Label unlock;
1514 // BasicObjectLock will be first in list, since this is a
1515 // synchronized method. However, need to check that the object
1516 // has not been unlocked by an explicit monitorexit bytecode.
1517
1518 // monitor expect in c_rarg1 for slow unlock path
1519 __ lea (c_rarg1, Address(rfp, // address of first monitor
1520 (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1521 wordSize - sizeof(BasicObjectLock))));
1522
1523 __ ldr(t, Address(c_rarg1, BasicObjectLock::obj_offset()));
1524 __ cbnz(t, unlock);
1525
1526 // Entry already unlocked, need to throw exception
1527 __ MacroAssembler::call_VM(noreg,
1528 CAST_FROM_FN_PTR(address,
1529 InterpreterRuntime::throw_illegal_monitor_state_exception));
1530 __ should_not_reach_here();
1531
1532 __ bind(unlock);
1533 __ unlock_object(c_rarg1);
1534 }
1535 __ bind(L);
1536 }
1537
1538 // jvmti support
1539 // Note: This must happen _after_ handling/throwing any exceptions since
1540 // the exception handler code notifies the runtime of method exits
1541 // too. If this happens before, method entry/exit notifications are
1542 // not properly paired (was bug - gri 11/22/99).
1543 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1544
1545 // restore potential result in r0:d0, call result handler to
1546 // restore potential result in ST0 & handle result
1547
1548 __ pop(ltos);
1549 __ pop(dtos);
1550
1551 __ blr(result_handler);
1552
1553 // remove activation
1554 __ ldr(esp, Address(rfp,
1555 frame::interpreter_frame_sender_sp_offset *
1556 wordSize)); // get sender sp
1557 // remove frame anchor
1558 __ leave();
1559
1560 // restore sender sp
1561 __ mov(sp, esp);
1562
1563 __ ret(lr);
1564
1565 if (inc_counter) {
1566 // Handle overflow of counter and compile method
1567 __ bind(invocation_counter_overflow);
1568 generate_counter_overflow(continue_after_compile);
1569 }
1570
1571 return entry_point;
1572 }
1573
1574 //
1575 // Generic interpreted method entry to (asm) interpreter
1576 //
1577 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1578 // determine code generation flags
1579 bool inc_counter = UseCompiler || CountCompiledCalls;
1580
1581 // rscratch1: sender sp
1582 address entry_point = __ pc();
1583
1584 const Address constMethod(rmethod, Method::const_offset());
1585 const Address access_flags(rmethod, Method::access_flags_offset());
1586 const Address size_of_parameters(r3,
1587 ConstMethod::size_of_parameters_offset());
1588 const Address size_of_locals(r3, ConstMethod::size_of_locals_offset());
1589
1590 // get parameter size (always needed)
1591 // need to load the const method first
1592 __ ldr(r3, constMethod);
1593 __ load_unsigned_short(r2, size_of_parameters);
1594
1595 // r2: size of parameters
1596
1597 __ load_unsigned_short(r3, size_of_locals); // get size of locals in words
1598 __ sub(r3, r3, r2); // r3 = no. of additional locals
1599
1600 // see if we've got enough room on the stack for locals plus overhead.
1601 generate_stack_overflow_check();
1602
1603 // compute beginning of parameters (rlocals)
1604 __ add(rlocals, esp, r2, ext::uxtx, 3);
1605 __ sub(rlocals, rlocals, wordSize);
1606
1607 __ mov(rscratch1, esp);
1608
1609 // r3 - # of additional locals
1610 // allocate space for locals
1611 // explicitly initialize locals
1612 // Initializing memory allocated for locals in the same direction as
1613 // the stack grows to ensure page initialization order according
1614 // to windows-aarch64 stack page growth requirement (see
1615 // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-160#stack)
1616 {
1617 Label exit, loop;
1618 __ ands(zr, r3, r3);
1619 __ br(Assembler::LE, exit); // do nothing if r3 <= 0
1620 __ bind(loop);
1621 __ str(zr, Address(__ pre(rscratch1, -wordSize)));
1622 __ sub(r3, r3, 1); // until everything initialized
1623 __ cbnz(r3, loop);
1624 __ bind(exit);
1625 }
1626
1627 // Padding between locals and fixed part of activation frame to ensure
1628 // SP is always 16-byte aligned.
1629 __ andr(sp, rscratch1, -16);
1630
1631 // And the base dispatch table
1632 __ get_dispatch();
1633
1634 // initialize fixed part of activation frame
1635 generate_fixed_frame(false);
1636
1637 // make sure method is not native & not abstract
1638 #ifdef ASSERT
1639 __ ldrw(r0, access_flags);
1640 {
1641 Label L;
1642 __ tst(r0, JVM_ACC_NATIVE);
1643 __ br(Assembler::EQ, L);
1644 __ stop("tried to execute native method as non-native");
1645 __ bind(L);
1646 }
1647 {
1648 Label L;
1649 __ tst(r0, JVM_ACC_ABSTRACT);
1650 __ br(Assembler::EQ, L);
1651 __ stop("tried to execute abstract method in interpreter");
1652 __ bind(L);
1653 }
1654 #endif
1655
1656 // Since at this point in the method invocation the exception
1657 // handler would try to exit the monitor of synchronized methods
1658 // which hasn't been entered yet, we set the thread local variable
1659 // _do_not_unlock_if_synchronized to true. The remove_activation
1660 // will check this flag.
1661
1662 const Address do_not_unlock_if_synchronized(rthread,
1663 in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1664 __ mov(rscratch2, true);
1665 __ strb(rscratch2, do_not_unlock_if_synchronized);
1666
1667 Register mdp = r3;
1668 __ profile_parameters_type(mdp, r1, r2);
1669
1670 // increment invocation count & check for overflow
1671 Label invocation_counter_overflow;
1672 if (inc_counter) {
1673 generate_counter_incr(&invocation_counter_overflow);
1674 }
1675
1676 Label continue_after_compile;
1677 __ bind(continue_after_compile);
1678
1679 bang_stack_shadow_pages(false);
1680
1681 // reset the _do_not_unlock_if_synchronized flag
1682 __ strb(zr, do_not_unlock_if_synchronized);
1683
1684 // check for synchronized methods
1685 // Must happen AFTER invocation_counter check and stack overflow check,
1686 // so method is not locked if overflows.
1687 if (synchronized) {
1688 // Allocate monitor and lock method
1689 lock_method();
1690 } else {
1691 // no synchronization necessary
1692 #ifdef ASSERT
1693 {
1694 Label L;
1695 __ ldrw(r0, access_flags);
1696 __ tst(r0, JVM_ACC_SYNCHRONIZED);
1697 __ br(Assembler::EQ, L);
1698 __ stop("method needs synchronization");
1699 __ bind(L);
1700 }
1701 #endif
1702 }
1703
1704 // start execution
1705 #ifdef ASSERT
1706 {
1707 Label L;
1708 const Address monitor_block_top (rfp,
1709 frame::interpreter_frame_monitor_block_top_offset * wordSize);
1710 __ ldr(rscratch1, monitor_block_top);
1711 __ cmp(esp, rscratch1);
1712 __ br(Assembler::EQ, L);
1713 __ stop("broken stack frame setup in interpreter 2");
1714 __ bind(L);
1715 }
1716 #endif
1717
1718 // jvmti support
1719 __ notify_method_entry();
1720
1721 __ dispatch_next(vtos);
1722
1723 // invocation counter overflow
1724 if (inc_counter) {
1725 // Handle overflow of counter and compile method
1726 __ bind(invocation_counter_overflow);
1727 generate_counter_overflow(continue_after_compile);
1728 }
1729
1730 return entry_point;
1731 }
1732
1733 // Method entry for java.lang.Thread.currentThread
1734 address TemplateInterpreterGenerator::generate_currentThread() {
1735 address entry_point = __ pc();
1736
1737 __ ldr(r0, Address(rthread, JavaThread::vthread_offset()));
1738 __ resolve_oop_handle(r0, rscratch1, rscratch2);
1739 __ ret(lr);
1740
1741 return entry_point;
1742 }
1743
1744 // Not supported
1745 address TemplateInterpreterGenerator::generate_Float_intBitsToFloat_entry() { return nullptr; }
1746 address TemplateInterpreterGenerator::generate_Float_floatToRawIntBits_entry() { return nullptr; }
1747 address TemplateInterpreterGenerator::generate_Double_longBitsToDouble_entry() { return nullptr; }
1748 address TemplateInterpreterGenerator::generate_Double_doubleToRawLongBits_entry() { return nullptr; }
1749
1750 //-----------------------------------------------------------------------------
1751 // Exceptions
1752
1753 void TemplateInterpreterGenerator::generate_throw_exception() {
1754 // Entry point in previous activation (i.e., if the caller was
1755 // interpreted)
1756 Interpreter::_rethrow_exception_entry = __ pc();
1757 // Restore sp to interpreter_frame_last_sp even though we are going
1758 // to empty the expression stack for the exception processing.
1759 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1760 // r0: exception
1761 // r3: return address/pc that threw exception
1762 __ restore_bcp(); // rbcp points to call/send
1763 __ restore_locals();
1764 __ restore_constant_pool_cache();
1765 __ reinit_heapbase(); // restore rheapbase as heapbase.
1766 __ get_dispatch();
1767
1768 // Entry point for exceptions thrown within interpreter code
1769 Interpreter::_throw_exception_entry = __ pc();
1770 // If we came here via a NullPointerException on the receiver of a
1771 // method, rmethod may be corrupt.
1772 __ get_method(rmethod);
1773 // expression stack is undefined here
1774 // r0: exception
1775 // rbcp: exception bcp
1776 __ verify_oop(r0);
1777 __ mov(c_rarg1, r0);
1778
1779 // expression stack must be empty before entering the VM in case of
1780 // an exception
1781 __ empty_expression_stack();
1782 // find exception handler address and preserve exception oop
1783 __ call_VM(r3,
1784 CAST_FROM_FN_PTR(address,
1785 InterpreterRuntime::exception_handler_for_exception),
1786 c_rarg1);
1787
1788 // Restore machine SP
1789 __ restore_sp_after_call();
1790
1791 // r0: exception handler entry point
1792 // r3: preserved exception oop
1793 // rbcp: bcp for exception handler
1794 __ push_ptr(r3); // push exception which is now the only value on the stack
1795 __ br(r0); // jump to exception handler (may be _remove_activation_entry!)
1796
1797 // If the exception is not handled in the current frame the frame is
1798 // removed and the exception is rethrown (i.e. exception
1799 // continuation is _rethrow_exception).
1800 //
1801 // Note: At this point the bci is still the bxi for the instruction
1802 // which caused the exception and the expression stack is
1803 // empty. Thus, for any VM calls at this point, GC will find a legal
1804 // oop map (with empty expression stack).
1805
1806 //
1807 // JVMTI PopFrame support
1808 //
1809
1810 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1811 __ empty_expression_stack();
1812 // Set the popframe_processing bit in pending_popframe_condition
1813 // indicating that we are currently handling popframe, so that
1814 // call_VMs that may happen later do not trigger new popframe
1815 // handling cycles.
1816 __ ldrw(r3, Address(rthread, JavaThread::popframe_condition_offset()));
1817 __ orr(r3, r3, JavaThread::popframe_processing_bit);
1818 __ strw(r3, Address(rthread, JavaThread::popframe_condition_offset()));
1819
1820 {
1821 // Check to see whether we are returning to a deoptimized frame.
1822 // (The PopFrame call ensures that the caller of the popped frame is
1823 // either interpreted or compiled and deoptimizes it if compiled.)
1824 // In this case, we can't call dispatch_next() after the frame is
1825 // popped, but instead must save the incoming arguments and restore
1826 // them after deoptimization has occurred.
1827 //
1828 // Note that we don't compare the return PC against the
1829 // deoptimization blob's unpack entry because of the presence of
1830 // adapter frames in C2.
1831 Label caller_not_deoptimized;
1832 __ ldr(c_rarg1, Address(rfp, frame::return_addr_offset * wordSize));
1833 // This is a return address, so requires authenticating for PAC.
1834 __ authenticate_return_address(c_rarg1, rscratch1);
1835 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1836 InterpreterRuntime::interpreter_contains), c_rarg1);
1837 __ cbnz(r0, caller_not_deoptimized);
1838
1839 // Compute size of arguments for saving when returning to
1840 // deoptimized caller
1841 __ get_method(r0);
1842 __ ldr(r0, Address(r0, Method::const_offset()));
1843 __ load_unsigned_short(r0, Address(r0, in_bytes(ConstMethod::
1844 size_of_parameters_offset())));
1845 __ lsl(r0, r0, Interpreter::logStackElementSize);
1846 __ restore_locals(); // XXX do we need this?
1847 __ sub(rlocals, rlocals, r0);
1848 __ add(rlocals, rlocals, wordSize);
1849 // Save these arguments
1850 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1851 Deoptimization::
1852 popframe_preserve_args),
1853 rthread, r0, rlocals);
1854
1855 __ remove_activation(vtos,
1856 /* throw_monitor_exception */ false,
1857 /* install_monitor_exception */ false,
1858 /* notify_jvmdi */ false);
1859
1860 // Inform deoptimization that it is responsible for restoring
1861 // these arguments
1862 __ mov(rscratch1, JavaThread::popframe_force_deopt_reexecution_bit);
1863 __ strw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset()));
1864
1865 // Continue in deoptimization handler
1866 __ ret(lr);
1867
1868 __ bind(caller_not_deoptimized);
1869 }
1870
1871 __ remove_activation(vtos,
1872 /* throw_monitor_exception */ false,
1873 /* install_monitor_exception */ false,
1874 /* notify_jvmdi */ false);
1875
1876 // Restore the last_sp and null it out
1877 __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1878 __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1879
1880 __ restore_bcp();
1881 __ restore_locals();
1882 __ restore_constant_pool_cache();
1883 __ get_method(rmethod);
1884 __ get_dispatch();
1885
1886 // The method data pointer was incremented already during
1887 // call profiling. We have to restore the mdp for the current bcp.
1888 if (ProfileInterpreter) {
1889 __ set_method_data_pointer_for_bcp();
1890 }
1891
1892 // Clear the popframe condition flag
1893 __ strw(zr, Address(rthread, JavaThread::popframe_condition_offset()));
1894 assert(JavaThread::popframe_inactive == 0, "fix popframe_inactive");
1895
1896 #if INCLUDE_JVMTI
1897 {
1898 Label L_done;
1899
1900 __ ldrb(rscratch1, Address(rbcp, 0));
1901 __ cmpw(rscratch1, Bytecodes::_invokestatic);
1902 __ br(Assembler::NE, L_done);
1903
1904 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1905 // Detect such a case in the InterpreterRuntime function and return the member name argument, or null.
1906
1907 __ ldr(c_rarg0, Address(rlocals, 0));
1908 __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), c_rarg0, rmethod, rbcp);
1909
1910 __ cbz(r0, L_done);
1911
1912 __ str(r0, Address(esp, 0));
1913 __ bind(L_done);
1914 }
1915 #endif // INCLUDE_JVMTI
1916
1917 // Restore machine SP
1918 __ restore_sp_after_call();
1919
1920 __ dispatch_next(vtos);
1921 // end of PopFrame support
1922
1923 Interpreter::_remove_activation_entry = __ pc();
1924
1925 // preserve exception over this code sequence
1926 __ pop_ptr(r0);
1927 __ str(r0, Address(rthread, JavaThread::vm_result_offset()));
1928 // remove the activation (without doing throws on illegalMonitorExceptions)
1929 __ remove_activation(vtos, false, true, false);
1930 // restore exception
1931 __ get_vm_result(r0, rthread);
1932
1933 // In between activations - previous activation type unknown yet
1934 // compute continuation point - the continuation point expects the
1935 // following registers set up:
1936 //
1937 // r0: exception
1938 // lr: return address/pc that threw exception
1939 // esp: expression stack of caller
1940 // rfp: fp of caller
1941 __ stp(r0, lr, Address(__ pre(sp, -2 * wordSize))); // save exception & return address
1942 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1943 SharedRuntime::exception_handler_for_return_address),
1944 rthread, lr);
1945 __ mov(r1, r0); // save exception handler
1946 __ ldp(r0, lr, Address(__ post(sp, 2 * wordSize))); // restore exception & return address
1947 // We might be returning to a deopt handler that expects r3 to
1948 // contain the exception pc
1949 __ mov(r3, lr);
1950 // Note that an "issuing PC" is actually the next PC after the call
1951 __ br(r1); // jump to exception
1952 // handler of caller
1953 }
1954
1955
1956 //
1957 // JVMTI ForceEarlyReturn support
1958 //
1959 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1960 address entry = __ pc();
1961
1962 __ restore_bcp();
1963 __ restore_locals();
1964 __ empty_expression_stack();
1965 __ load_earlyret_value(state);
1966
1967 __ ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
1968 Address cond_addr(rscratch1, JvmtiThreadState::earlyret_state_offset());
1969
1970 // Clear the earlyret state
1971 assert(JvmtiThreadState::earlyret_inactive == 0, "should be");
1972 __ str(zr, cond_addr);
1973
1974 __ remove_activation(state,
1975 false, /* throw_monitor_exception */
1976 false, /* install_monitor_exception */
1977 true); /* notify_jvmdi */
1978 __ ret(lr);
1979
1980 return entry;
1981 } // end of ForceEarlyReturn support
1982
1983
1984
1985 //-----------------------------------------------------------------------------
1986 // Helper for vtos entry point generation
1987
1988 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1989 address& bep,
1990 address& cep,
1991 address& sep,
1992 address& aep,
1993 address& iep,
1994 address& lep,
1995 address& fep,
1996 address& dep,
1997 address& vep) {
1998 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1999 Label L;
2000 aep = __ pc(); __ push_ptr(); __ b(L);
2001 fep = __ pc(); __ push_f(); __ b(L);
2002 dep = __ pc(); __ push_d(); __ b(L);
2003 lep = __ pc(); __ push_l(); __ b(L);
2004 bep = cep = sep =
2005 iep = __ pc(); __ push_i();
2006 vep = __ pc();
2007 __ bind(L);
2008 generate_and_dispatch(t);
2009 }
2010
2011 //-----------------------------------------------------------------------------
2012
2013 // Non-product code
2014 #ifndef PRODUCT
2015 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
2016 address entry = __ pc();
2017
2018 __ protect_return_address();
2019 __ push(lr);
2020 __ push(state);
2021 __ push(RegSet::range(r0, r15), sp);
2022 __ mov(c_rarg2, r0); // Pass itos
2023 __ call_VM(noreg,
2024 CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode),
2025 c_rarg1, c_rarg2, c_rarg3);
2026 __ pop(RegSet::range(r0, r15), sp);
2027 __ pop(state);
2028 __ pop(lr);
2029 __ authenticate_return_address();
2030 __ ret(lr); // return from result handler
2031
2032 return entry;
2033 }
2034
2035 void TemplateInterpreterGenerator::count_bytecode() {
2036 __ mov(r10, (address) &BytecodeCounter::_counter_value);
2037 __ atomic_addw(noreg, 1, r10);
2038 }
2039
2040 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
2041 __ mov(r10, (address) &BytecodeHistogram::_counters[t->bytecode()]);
2042 __ atomic_addw(noreg, 1, r10);
2043 }
2044
2045 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
2046 // Calculate new index for counter:
2047 // _index = (_index >> log2_number_of_codes) |
2048 // (bytecode << log2_number_of_codes);
2049 Register index_addr = rscratch1;
2050 Register index = rscratch2;
2051 __ mov(index_addr, (address) &BytecodePairHistogram::_index);
2052 __ ldrw(index, index_addr);
2053 __ mov(r10,
2054 ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
2055 __ orrw(index, r10, index, Assembler::LSR,
2056 BytecodePairHistogram::log2_number_of_codes);
2057 __ strw(index, index_addr);
2058
2059 // Bump bucket contents:
2060 // _counters[_index] ++;
2061 Register counter_addr = rscratch1;
2062 __ mov(r10, (address) &BytecodePairHistogram::_counters);
2063 __ lea(counter_addr, Address(r10, index, Address::lsl(LogBytesPerInt)));
2064 __ atomic_addw(noreg, 1, counter_addr);
2065 }
2066
2067 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
2068 // Call a little run-time stub to avoid blow-up for each bytecode.
2069 // The run-time runtime saves the right registers, depending on
2070 // the tosca in-state for the given template.
2071
2072 assert(Interpreter::trace_code(t->tos_in()) != nullptr,
2073 "entry must have been generated");
2074 __ bl(Interpreter::trace_code(t->tos_in()));
2075 __ reinit_heapbase();
2076 }
2077
2078
2079 void TemplateInterpreterGenerator::stop_interpreter_at() {
2080 Label L;
2081 __ push(rscratch1);
2082 __ mov(rscratch1, (address) &BytecodeCounter::_counter_value);
2083 __ ldr(rscratch1, Address(rscratch1));
2084 __ mov(rscratch2, StopInterpreterAt);
2085 __ cmpw(rscratch1, rscratch2);
2086 __ br(Assembler::NE, L);
2087 __ brk(0);
2088 __ bind(L);
2089 __ pop(rscratch1);
2090 }
2091
2092 #endif // !PRODUCT