1 /*
2 * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "compiler/compiler_globals.hpp"
29 #include "compiler/disassembler.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "interpreter/bytecodeHistogram.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "interpreter/interpreter.hpp"
34 #include "interpreter/interpreterRuntime.hpp"
35 #include "interpreter/templateInterpreterGenerator.hpp"
36 #include "interpreter/templateTable.hpp"
37 #include "oops/arrayOop.hpp"
38 #include "oops/methodCounters.hpp"
39 #include "oops/methodData.hpp"
40 #include "oops/method.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/resolvedIndyEntry.hpp"
43 #include "oops/resolvedMethodEntry.hpp"
44 #include "prims/jvmtiExport.hpp"
45 #include "prims/jvmtiThreadState.hpp"
46 #include "runtime/continuation.hpp"
47 #include "runtime/deoptimization.hpp"
48 #include "runtime/frame.inline.hpp"
49 #include "runtime/globals.hpp"
50 #include "runtime/jniHandles.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/synchronizer.hpp"
54 #include "runtime/timer.hpp"
55 #include "runtime/vframeArray.hpp"
56 #include "utilities/checkedCast.hpp"
57 #include "utilities/debug.hpp"
58 #include "utilities/macros.hpp"
59
60 #define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)->
61
62 // Size of interpreter code. Increase if too small. Interpreter will
63 // fail with a guarantee ("not enough space for interpreter generation");
64 // if too small.
65 // Run with +PrintInterpreter to get the VM to print out the size.
66 // Max size with JVMTI
67 #ifdef AMD64
68 int TemplateInterpreter::InterpreterCodeSize = JVMCI_ONLY(268) NOT_JVMCI(256) * 1024;
69 #else
70 int TemplateInterpreter::InterpreterCodeSize = 224 * 1024;
71 #endif // AMD64
72
73 // Global Register Names
74 static const Register rbcp = LP64_ONLY(r13) NOT_LP64(rsi);
75 static const Register rlocals = LP64_ONLY(r14) NOT_LP64(rdi);
76
77 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
78 const int bcp_offset = frame::interpreter_frame_bcp_offset * wordSize;
79 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
80
81
82 //-----------------------------------------------------------------------------
83
84 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
85 address entry = __ pc();
86
87 #ifdef ASSERT
88 {
89 Label L;
90 __ movptr(rax, Address(rbp,
91 frame::interpreter_frame_monitor_block_top_offset *
92 wordSize));
93 __ lea(rax, Address(rbp, rax, Address::times_ptr));
94 __ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack
95 // grows negative)
96 __ jcc(Assembler::aboveEqual, L); // check if frame is complete
97 __ stop ("interpreter frame not set up");
98 __ bind(L);
99 }
100 #endif // ASSERT
101 // Restore bcp under the assumption that the current frame is still
102 // interpreted
103 __ restore_bcp();
104
105 // expression stack must be empty before entering the VM if an
106 // exception happened
107 __ empty_expression_stack();
108 // throw exception
109 __ call_VM(noreg,
110 CAST_FROM_FN_PTR(address,
111 InterpreterRuntime::throw_StackOverflowError));
112 return entry;
113 }
114
115 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
116 address entry = __ pc();
117 // The expression stack must be empty before entering the VM if an
118 // exception happened.
119 __ empty_expression_stack();
120
121 // Setup parameters.
122 // ??? convention: expect aberrant index in register ebx/rbx.
123 // Pass array to create more detailed exceptions.
124 Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);
125 __ call_VM(noreg,
126 CAST_FROM_FN_PTR(address,
127 InterpreterRuntime::
128 throw_ArrayIndexOutOfBoundsException),
129 rarg, rbx);
130 return entry;
131 }
132
133 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
134 address entry = __ pc();
135
136 // object is at TOS
137 Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);
138 __ pop(rarg);
139
140 // expression stack must be empty before entering the VM if an
141 // exception happened
142 __ empty_expression_stack();
143
144 __ call_VM(noreg,
145 CAST_FROM_FN_PTR(address,
146 InterpreterRuntime::
147 throw_ClassCastException),
148 rarg);
149 return entry;
150 }
151
152 address TemplateInterpreterGenerator::generate_exception_handler_common(
153 const char* name, const char* message, bool pass_oop) {
154 assert(!pass_oop || message == nullptr, "either oop or message but not both");
155 address entry = __ pc();
156
157 Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);
158 Register rarg2 = NOT_LP64(rbx) LP64_ONLY(c_rarg2);
159
160 if (pass_oop) {
161 // object is at TOS
162 __ pop(rarg2);
163 }
164 // expression stack must be empty before entering the VM if an
165 // exception happened
166 __ empty_expression_stack();
167 // setup parameters
168 __ lea(rarg, ExternalAddress((address)name));
169 if (pass_oop) {
170 __ call_VM(rax, CAST_FROM_FN_PTR(address,
171 InterpreterRuntime::
172 create_klass_exception),
173 rarg, rarg2);
174 } else {
175 __ lea(rarg2, ExternalAddress((address)message));
176 __ call_VM(rax,
177 CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
178 rarg, rarg2);
179 }
180 // throw exception
181 __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
182 return entry;
183 }
184
185 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
186 address entry = __ pc();
187
188 #ifndef _LP64
189 #ifdef COMPILER2
190 // The FPU stack is clean if UseSSE >= 2 but must be cleaned in other cases
191 if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
192 for (int i = 1; i < 8; i++) {
193 __ ffree(i);
194 }
195 } else if (UseSSE < 2) {
196 __ empty_FPU_stack();
197 }
198 #endif // COMPILER2
199 if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
200 __ MacroAssembler::verify_FPU(1, "generate_return_entry_for compiled");
201 } else {
202 __ MacroAssembler::verify_FPU(0, "generate_return_entry_for compiled");
203 }
204
205 if (state == ftos) {
206 __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_return_entry_for in interpreter");
207 } else if (state == dtos) {
208 __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_return_entry_for in interpreter");
209 }
210 #endif // _LP64
211
212 // Restore stack bottom in case i2c adjusted stack
213 __ movptr(rcx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
214 __ lea(rsp, Address(rbp, rcx, Address::times_ptr));
215 // and null it as marker that esp is now tos until next java call
216 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
217
218 __ restore_bcp();
219 __ restore_locals();
220
221 if (state == atos) {
222 Register mdp = rbx;
223 Register tmp = rcx;
224 __ profile_return_type(mdp, rax, tmp);
225 }
226
227 const Register cache = rbx;
228 const Register index = rcx;
229 if (index_size == sizeof(u4)) {
230 __ load_resolved_indy_entry(cache, index);
231 __ load_unsigned_short(cache, Address(cache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
232 __ lea(rsp, Address(rsp, cache, Interpreter::stackElementScale()));
233 } else {
234 assert(index_size == sizeof(u2), "Can only be u2");
235 __ load_method_entry(cache, index);
236 __ load_unsigned_short(cache, Address(cache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
237 __ lea(rsp, Address(rsp, cache, Interpreter::stackElementScale()));
238 }
239
240 const Register java_thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);
241 if (JvmtiExport::can_pop_frame()) {
242 NOT_LP64(__ get_thread(java_thread));
243 __ check_and_handle_popframe(java_thread);
244 }
245 if (JvmtiExport::can_force_early_return()) {
246 NOT_LP64(__ get_thread(java_thread));
247 __ check_and_handle_earlyret(java_thread);
248 }
249
250 __ dispatch_next(state, step);
251
252 return entry;
253 }
254
255
256 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {
257 address entry = __ pc();
258
259 #ifndef _LP64
260 if (state == ftos) {
261 __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_deopt_entry_for in interpreter");
262 } else if (state == dtos) {
263 __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_deopt_entry_for in interpreter");
264 }
265 #endif // _LP64
266
267 // null last_sp until next java call
268 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
269 __ restore_bcp();
270 __ restore_locals();
271 const Register thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);
272 NOT_LP64(__ get_thread(thread));
273 #if INCLUDE_JVMCI
274 // Check if we need to take lock at entry of synchronized method. This can
275 // only occur on method entry so emit it only for vtos with step 0.
276 if (EnableJVMCI && state == vtos && step == 0) {
277 Label L;
278 __ cmpb(Address(thread, JavaThread::pending_monitorenter_offset()), 0);
279 __ jcc(Assembler::zero, L);
280 // Clear flag.
281 __ movb(Address(thread, JavaThread::pending_monitorenter_offset()), 0);
282 // Satisfy calling convention for lock_method().
283 __ get_method(rbx);
284 // Take lock.
285 lock_method();
286 __ bind(L);
287 } else {
288 #ifdef ASSERT
289 if (EnableJVMCI) {
290 Label L;
291 __ cmpb(Address(r15_thread, JavaThread::pending_monitorenter_offset()), 0);
292 __ jcc(Assembler::zero, L);
293 __ stop("unexpected pending monitor in deopt entry");
294 __ bind(L);
295 }
296 #endif
297 }
298 #endif
299 // handle exceptions
300 {
301 Label L;
302 __ cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
303 __ jcc(Assembler::zero, L);
304 __ call_VM(noreg,
305 CAST_FROM_FN_PTR(address,
306 InterpreterRuntime::throw_pending_exception));
307 __ should_not_reach_here();
308 __ bind(L);
309 }
310 if (continuation == nullptr) {
311 __ dispatch_next(state, step);
312 } else {
313 __ jump_to_entry(continuation);
314 }
315 return entry;
316 }
317
318 address TemplateInterpreterGenerator::generate_result_handler_for(
319 BasicType type) {
320 address entry = __ pc();
321 switch (type) {
322 case T_BOOLEAN: __ c2bool(rax); break;
323 #ifndef _LP64
324 case T_CHAR : __ andptr(rax, 0xFFFF); break;
325 #else
326 case T_CHAR : __ movzwl(rax, rax); break;
327 #endif // _LP64
328 case T_BYTE : __ sign_extend_byte(rax); break;
329 case T_SHORT : __ sign_extend_short(rax); break;
330 case T_INT : /* nothing to do */ break;
331 case T_LONG : /* nothing to do */ break;
332 case T_VOID : /* nothing to do */ break;
333 #ifndef _LP64
334 case T_DOUBLE :
335 case T_FLOAT :
336 { const Register t = InterpreterRuntime::SignatureHandlerGenerator::temp();
337 __ pop(t); // remove return address first
338 // Must return a result for interpreter or compiler. In SSE
339 // mode, results are returned in xmm0 and the FPU stack must
340 // be empty.
341 if (type == T_FLOAT && UseSSE >= 1) {
342 // Load ST0
343 __ fld_d(Address(rsp, 0));
344 // Store as float and empty fpu stack
345 __ fstp_s(Address(rsp, 0));
346 // and reload
347 __ movflt(xmm0, Address(rsp, 0));
348 } else if (type == T_DOUBLE && UseSSE >= 2 ) {
349 __ movdbl(xmm0, Address(rsp, 0));
350 } else {
351 // restore ST0
352 __ fld_d(Address(rsp, 0));
353 }
354 // and pop the temp
355 __ addptr(rsp, 2 * wordSize);
356 __ push(t); // restore return address
357 }
358 break;
359 #else
360 case T_FLOAT : /* nothing to do */ break;
361 case T_DOUBLE : /* nothing to do */ break;
362 #endif // _LP64
363
364 case T_OBJECT :
365 // retrieve result from frame
366 __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
367 // and verify it
368 __ verify_oop(rax);
369 break;
370 default : ShouldNotReachHere();
371 }
372 __ ret(0); // return from result handler
373 return entry;
374 }
375
376 address TemplateInterpreterGenerator::generate_safept_entry_for(
377 TosState state,
378 address runtime_entry) {
379 address entry = __ pc();
380
381 __ push(state);
382 __ push_cont_fastpath();
383 __ call_VM(noreg, runtime_entry);
384 __ pop_cont_fastpath();
385
386 __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
387 return entry;
388 }
389
390
391
392 // Helpers for commoning out cases in the various type of method entries.
393 //
394
395
396 // increment invocation count & check for overflow
397 //
398 // Note: checking for negative value instead of overflow
399 // so we have a 'sticky' overflow test
400 //
401 // rbx: method
402 // rcx: invocation counter
403 //
404 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) {
405 Label done;
406 // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not.
407 Label no_mdo;
408 if (ProfileInterpreter) {
409 // Are we profiling?
410 __ movptr(rax, Address(rbx, Method::method_data_offset()));
411 __ testptr(rax, rax);
412 __ jccb(Assembler::zero, no_mdo);
413 // Increment counter in the MDO
414 const Address mdo_invocation_counter(rax, in_bytes(MethodData::invocation_counter_offset()) +
415 in_bytes(InvocationCounter::counter_offset()));
416 const Address mask(rax, in_bytes(MethodData::invoke_mask_offset()));
417 __ increment_mask_and_jump(mdo_invocation_counter, mask, rcx, overflow);
418 __ jmp(done);
419 }
420 __ bind(no_mdo);
421 // Increment counter in MethodCounters
422 const Address invocation_counter(rax,
423 MethodCounters::invocation_counter_offset() +
424 InvocationCounter::counter_offset());
425 __ get_method_counters(rbx, rax, done);
426 const Address mask(rax, in_bytes(MethodCounters::invoke_mask_offset()));
427 __ increment_mask_and_jump(invocation_counter, mask, rcx, overflow);
428 __ bind(done);
429 }
430
431 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
432
433 // Asm interpreter on entry
434 // r14/rdi - locals
435 // r13/rsi - bcp
436 // rbx - method
437 // rdx - cpool --- DOES NOT APPEAR TO BE TRUE
438 // rbp - interpreter frame
439
440 // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
441 // Everything as it was on entry
442 // rdx is not restored. Doesn't appear to really be set.
443
444 // InterpreterRuntime::frequency_counter_overflow takes two
445 // arguments, the first (thread) is passed by call_VM, the second
446 // indicates if the counter overflow occurs at a backwards branch
447 // (null bcp). We pass zero for it. The call returns the address
448 // of the verified entry point for the method or null if the
449 // compilation did not complete (either went background or bailed
450 // out).
451 Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);
452 __ movl(rarg, 0);
453 __ call_VM(noreg,
454 CAST_FROM_FN_PTR(address,
455 InterpreterRuntime::frequency_counter_overflow),
456 rarg);
457
458 __ movptr(rbx, Address(rbp, method_offset)); // restore Method*
459 // Preserve invariant that r13/r14 contain bcp/locals of sender frame
460 // and jump to the interpreted entry.
461 __ jmp(do_continue, relocInfo::none);
462 }
463
464 // See if we've got enough room on the stack for locals plus overhead below
465 // JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError
466 // without going through the signal handler, i.e., reserved and yellow zones
467 // will not be made usable. The shadow zone must suffice to handle the
468 // overflow.
469 // The expression stack grows down incrementally, so the normal guard
470 // page mechanism will work for that.
471 //
472 // NOTE: Since the additional locals are also always pushed (wasn't
473 // obvious in generate_fixed_frame) so the guard should work for them
474 // too.
475 //
476 // Args:
477 // rdx: number of additional locals this frame needs (what we must check)
478 // rbx: Method*
479 //
480 // Kills:
481 // rax
482 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
483
484 // monitor entry size: see picture of stack in frame_x86.hpp
485 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
486
487 // total overhead size: entry_size + (saved rbp through expr stack
488 // bottom). be sure to change this if you add/subtract anything
489 // to/from the overhead area
490 const int overhead_size =
491 -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
492
493 const int page_size = (int)os::vm_page_size();
494
495 Label after_frame_check;
496
497 // see if the frame is greater than one page in size. If so,
498 // then we need to verify there is enough stack space remaining
499 // for the additional locals.
500 __ cmpl(rdx, (page_size - overhead_size) / Interpreter::stackElementSize);
501 __ jcc(Assembler::belowEqual, after_frame_check);
502
503 // compute rsp as if this were going to be the last frame on
504 // the stack before the red zone
505
506 Label after_frame_check_pop;
507 const Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
508 #ifndef _LP64
509 __ push(thread);
510 __ get_thread(thread);
511 #endif
512
513 const Address stack_limit(thread, JavaThread::stack_overflow_limit_offset());
514
515 // locals + overhead, in bytes
516 __ mov(rax, rdx);
517 __ shlptr(rax, Interpreter::logStackElementSize); // Convert parameter count to bytes.
518 __ addptr(rax, overhead_size);
519
520 #ifdef ASSERT
521 Label limit_okay;
522 // Verify that thread stack overflow limit is non-zero.
523 __ cmpptr(stack_limit, NULL_WORD);
524 __ jcc(Assembler::notEqual, limit_okay);
525 __ stop("stack overflow limit is zero");
526 __ bind(limit_okay);
527 #endif
528
529 // Add locals/frame size to stack limit.
530 __ addptr(rax, stack_limit);
531
532 // Check against the current stack bottom.
533 __ cmpptr(rsp, rax);
534
535 __ jcc(Assembler::above, after_frame_check_pop);
536 NOT_LP64(__ pop(rsi)); // get saved bcp
537
538 // Restore sender's sp as SP. This is necessary if the sender's
539 // frame is an extended compiled frame (see gen_c2i_adapter())
540 // and safer anyway in case of JSR292 adaptations.
541
542 __ pop(rax); // return address must be moved if SP is changed
543 __ mov(rsp, rbcp);
544 __ push(rax);
545
546 // Note: the restored frame is not necessarily interpreted.
547 // Use the shared runtime version of the StackOverflowError.
548 assert(StubRoutines::throw_StackOverflowError_entry() != nullptr, "stub not yet generated");
549 __ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
550 // all done with frame size check
551 __ bind(after_frame_check_pop);
552 NOT_LP64(__ pop(rsi));
553
554 // all done with frame size check
555 __ bind(after_frame_check);
556 }
557
558 // Allocate monitor and lock method (asm interpreter)
559 //
560 // Args:
561 // rbx: Method*
562 // r14/rdi: locals
563 //
564 // Kills:
565 // rax
566 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
567 // rscratch1, rscratch2 (scratch regs)
568 void TemplateInterpreterGenerator::lock_method() {
569 // synchronize method
570 const Address access_flags(rbx, Method::access_flags_offset());
571 const Address monitor_block_top(
572 rbp,
573 frame::interpreter_frame_monitor_block_top_offset * wordSize);
574 const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
575
576 #ifdef ASSERT
577 {
578 Label L;
579 __ movl(rax, access_flags);
580 __ testl(rax, JVM_ACC_SYNCHRONIZED);
581 __ jcc(Assembler::notZero, L);
582 __ stop("method doesn't need synchronization");
583 __ bind(L);
584 }
585 #endif // ASSERT
586
587 // get synchronization object
588 {
589 Label done;
590 __ movl(rax, access_flags);
591 __ testl(rax, JVM_ACC_STATIC);
592 // get receiver (assume this is frequent case)
593 __ movptr(rax, Address(rlocals, Interpreter::local_offset_in_bytes(0)));
594 __ jcc(Assembler::zero, done);
595 __ load_mirror(rax, rbx, rscratch2);
596
597 #ifdef ASSERT
598 {
599 Label L;
600 __ testptr(rax, rax);
601 __ jcc(Assembler::notZero, L);
602 __ stop("synchronization object is null");
603 __ bind(L);
604 }
605 #endif // ASSERT
606
607 __ bind(done);
608 }
609
610 // add space for monitor & lock
611 __ subptr(rsp, entry_size); // add space for a monitor entry
612 __ subptr(monitor_block_top, entry_size / wordSize); // set new monitor block top
613 // store object
614 __ movptr(Address(rsp, BasicObjectLock::obj_offset()), rax);
615 const Register lockreg = NOT_LP64(rdx) LP64_ONLY(c_rarg1);
616 __ movptr(lockreg, rsp); // object address
617 __ lock_object(lockreg);
618 }
619
620 // Generate a fixed interpreter frame. This is identical setup for
621 // interpreted methods and for native methods hence the shared code.
622 //
623 // Args:
624 // rax: return address
625 // rbx: Method*
626 // r14/rdi: pointer to locals
627 // r13/rsi: sender sp
628 // rdx: cp cache
629 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
630 // initialize fixed part of activation frame
631 __ push(rax); // save return address
632 __ enter(); // save old & set new rbp
633 __ push(rbcp); // set sender sp
634 __ push(NULL_WORD); // leave last_sp as null
635 __ movptr(rbcp, Address(rbx, Method::const_offset())); // get ConstMethod*
636 __ lea(rbcp, Address(rbcp, ConstMethod::codes_offset())); // get codebase
637 __ push(rbx); // save Method*
638 // Get mirror and store it in the frame as GC root for this Method*
639 __ load_mirror(rdx, rbx, rscratch2);
640 __ push(rdx);
641 if (ProfileInterpreter) {
642 Label method_data_continue;
643 __ movptr(rdx, Address(rbx, in_bytes(Method::method_data_offset())));
644 __ testptr(rdx, rdx);
645 __ jcc(Assembler::zero, method_data_continue);
646 __ addptr(rdx, in_bytes(MethodData::data_offset()));
647 __ bind(method_data_continue);
648 __ push(rdx); // set the mdp (method data pointer)
649 } else {
650 __ push(0);
651 }
652
653 __ movptr(rdx, Address(rbx, Method::const_offset()));
654 __ movptr(rdx, Address(rdx, ConstMethod::constants_offset()));
655 __ movptr(rdx, Address(rdx, ConstantPool::cache_offset()));
656 __ push(rdx); // set constant pool cache
657
658 __ movptr(rax, rlocals);
659 __ subptr(rax, rbp);
660 __ shrptr(rax, Interpreter::logStackElementSize); // rax = rlocals - fp();
661 __ push(rax); // set relativized rlocals, see frame::interpreter_frame_locals()
662
663 if (native_call) {
664 __ push(0); // no bcp
665 } else {
666 __ push(rbcp); // set bcp
667 }
668 // initialize relativized pointer to expression stack bottom
669 __ push(frame::interpreter_frame_initial_sp_offset);
670 }
671
672 // End of helpers
673
674 // Method entry for java.lang.ref.Reference.get.
675 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
676 // Code: _aload_0, _getfield, _areturn
677 // parameter size = 1
678 //
679 // The code that gets generated by this routine is split into 2 parts:
680 // 1. The "intrinsified" code performing an ON_WEAK_OOP_REF load,
681 // 2. The slow path - which is an expansion of the regular method entry.
682 //
683 // Notes:-
684 // * An intrinsic is always executed, where an ON_WEAK_OOP_REF load is performed.
685 // * We may jump to the slow path iff the receiver is null. If the
686 // Reference object is null then we no longer perform an ON_WEAK_OOP_REF load
687 // Thus we can use the regular method entry code to generate the NPE.
688 //
689 // rbx: Method*
690
691 // r13: senderSP must preserve for slow path, set SP to it on fast path
692
693 address entry = __ pc();
694
695 const int referent_offset = java_lang_ref_Reference::referent_offset();
696
697 Label slow_path;
698 // rbx: method
699
700 // Check if local 0 != null
701 // If the receiver is null then it is OK to jump to the slow path.
702 __ movptr(rax, Address(rsp, wordSize));
703
704 __ testptr(rax, rax);
705 __ jcc(Assembler::zero, slow_path);
706
707 // rax: local 0
708 // rbx: method (but can be used as scratch now)
709 // rdx: scratch
710 // rdi: scratch
711
712 // Preserve the sender sp in case the load barrier
713 // calls the runtime
714 NOT_LP64(__ push(rsi));
715
716 // Load the value of the referent field.
717 const Address field_address(rax, referent_offset);
718 __ load_heap_oop(rax, field_address, /*tmp1*/ rbx, /*tmp_thread*/ rdx, ON_WEAK_OOP_REF);
719
720 // _areturn
721 const Register sender_sp = NOT_LP64(rsi) LP64_ONLY(r13);
722 NOT_LP64(__ pop(rsi)); // get sender sp
723 __ pop(rdi); // get return address
724 __ mov(rsp, sender_sp); // set sp to sender sp
725 __ jmp(rdi);
726 __ ret(0);
727
728 // generate a vanilla interpreter entry as the slow path
729 __ bind(slow_path);
730 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
731 return entry;
732 }
733
734 void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
735 // See more discussion in stackOverflow.hpp.
736
737 // Note that we do the banging after the frame is setup, since the exception
738 // handling code expects to find a valid interpreter frame on the stack.
739 // Doing the banging earlier fails if the caller frame is not an interpreter
740 // frame.
741 // (Also, the exception throwing code expects to unlock any synchronized
742 // method receiver, so do the banging after locking the receiver.)
743
744 const int shadow_zone_size = checked_cast<int>(StackOverflow::stack_shadow_zone_size());
745 const int page_size = (int)os::vm_page_size();
746 const int n_shadow_pages = shadow_zone_size / page_size;
747
748 const Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
749 #ifndef _LP64
750 __ push(thread);
751 __ get_thread(thread);
752 #endif
753
754 #ifdef ASSERT
755 Label L_good_limit;
756 __ cmpptr(Address(thread, JavaThread::shadow_zone_safe_limit()), NULL_WORD);
757 __ jcc(Assembler::notEqual, L_good_limit);
758 __ stop("shadow zone safe limit is not initialized");
759 __ bind(L_good_limit);
760
761 Label L_good_watermark;
762 __ cmpptr(Address(thread, JavaThread::shadow_zone_growth_watermark()), NULL_WORD);
763 __ jcc(Assembler::notEqual, L_good_watermark);
764 __ stop("shadow zone growth watermark is not initialized");
765 __ bind(L_good_watermark);
766 #endif
767
768 Label L_done;
769
770 __ cmpptr(rsp, Address(thread, JavaThread::shadow_zone_growth_watermark()));
771 __ jcc(Assembler::above, L_done);
772
773 for (int p = 1; p <= n_shadow_pages; p++) {
774 __ bang_stack_with_offset(p*page_size);
775 }
776
777 // Record the new watermark, but only if update is above the safe limit.
778 // Otherwise, the next time around the check above would pass the safe limit.
779 __ cmpptr(rsp, Address(thread, JavaThread::shadow_zone_safe_limit()));
780 __ jccb(Assembler::belowEqual, L_done);
781 __ movptr(Address(thread, JavaThread::shadow_zone_growth_watermark()), rsp);
782
783 __ bind(L_done);
784
785 #ifndef _LP64
786 __ pop(thread);
787 #endif
788 }
789
790 // Interpreter stub for calling a native method. (asm interpreter)
791 // This sets up a somewhat different looking stack for calling the
792 // native method than the typical interpreter frame setup.
793 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
794 // determine code generation flags
795 bool inc_counter = UseCompiler || CountCompiledCalls;
796
797 // rbx: Method*
798 // rbcp: sender sp
799
800 address entry_point = __ pc();
801
802 const Address constMethod (rbx, Method::const_offset());
803 const Address access_flags (rbx, Method::access_flags_offset());
804 const Address size_of_parameters(rcx, ConstMethod::
805 size_of_parameters_offset());
806
807
808 // get parameter size (always needed)
809 __ movptr(rcx, constMethod);
810 __ load_unsigned_short(rcx, size_of_parameters);
811
812 // native calls don't need the stack size check since they have no
813 // expression stack and the arguments are already on the stack and
814 // we only add a handful of words to the stack
815
816 // rbx: Method*
817 // rcx: size of parameters
818 // rbcp: sender sp
819 __ pop(rax); // get return address
820
821 // for natives the size of locals is zero
822
823 // compute beginning of parameters
824 __ lea(rlocals, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
825
826 // add 2 zero-initialized slots for native calls
827 // initialize result_handler slot
828 __ push(NULL_WORD);
829 // slot for oop temp
830 // (static native method holder mirror/jni oop result)
831 __ push(NULL_WORD);
832
833 // initialize fixed part of activation frame
834 generate_fixed_frame(true);
835
836 // make sure method is native & not abstract
837 #ifdef ASSERT
838 __ movl(rax, access_flags);
839 {
840 Label L;
841 __ testl(rax, JVM_ACC_NATIVE);
842 __ jcc(Assembler::notZero, L);
843 __ stop("tried to execute non-native method as native");
844 __ bind(L);
845 }
846 {
847 Label L;
848 __ testl(rax, JVM_ACC_ABSTRACT);
849 __ jcc(Assembler::zero, L);
850 __ stop("tried to execute abstract method in interpreter");
851 __ bind(L);
852 }
853 #endif
854
855 // Since at this point in the method invocation the exception handler
856 // would try to exit the monitor of synchronized methods which hasn't
857 // been entered yet, we set the thread local variable
858 // _do_not_unlock_if_synchronized to true. The remove_activation will
859 // check this flag.
860
861 const Register thread1 = NOT_LP64(rax) LP64_ONLY(r15_thread);
862 NOT_LP64(__ get_thread(thread1));
863 const Address do_not_unlock_if_synchronized(thread1,
864 in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
865 __ movbool(do_not_unlock_if_synchronized, true);
866
867 // increment invocation count & check for overflow
868 Label invocation_counter_overflow;
869 if (inc_counter) {
870 generate_counter_incr(&invocation_counter_overflow);
871 }
872
873 Label continue_after_compile;
874 __ bind(continue_after_compile);
875
876 bang_stack_shadow_pages(true);
877
878 // reset the _do_not_unlock_if_synchronized flag
879 NOT_LP64(__ get_thread(thread1));
880 __ movbool(do_not_unlock_if_synchronized, false);
881
882 // check for synchronized methods
883 // Must happen AFTER invocation_counter check and stack overflow check,
884 // so method is not locked if overflows.
885 if (synchronized) {
886 lock_method();
887 } else {
888 // no synchronization necessary
889 #ifdef ASSERT
890 {
891 Label L;
892 __ movl(rax, access_flags);
893 __ testl(rax, JVM_ACC_SYNCHRONIZED);
894 __ jcc(Assembler::zero, L);
895 __ stop("method needs synchronization");
896 __ bind(L);
897 }
898 #endif
899 }
900
901 // start execution
902 #ifdef ASSERT
903 {
904 Label L;
905 const Address monitor_block_top(rbp,
906 frame::interpreter_frame_monitor_block_top_offset * wordSize);
907 __ movptr(rax, monitor_block_top);
908 __ lea(rax, Address(rbp, rax, Address::times_ptr));
909 __ cmpptr(rax, rsp);
910 __ jcc(Assembler::equal, L);
911 __ stop("broken stack frame setup in interpreter 5");
912 __ bind(L);
913 }
914 #endif
915
916 // jvmti support
917 __ notify_method_entry();
918
919 // work registers
920 const Register method = rbx;
921 const Register thread = NOT_LP64(rdi) LP64_ONLY(r15_thread);
922 const Register t = NOT_LP64(rcx) LP64_ONLY(r11);
923
924 // allocate space for parameters
925 __ get_method(method);
926 __ movptr(t, Address(method, Method::const_offset()));
927 __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
928
929 #ifndef _LP64
930 __ shlptr(t, Interpreter::logStackElementSize); // Convert parameter count to bytes.
931 __ addptr(t, 2*wordSize); // allocate two more slots for JNIEnv and possible mirror
932 __ subptr(rsp, t);
933 __ andptr(rsp, -(StackAlignmentInBytes)); // gcc needs 16 byte aligned stacks to do XMM intrinsics
934 #else
935 __ shll(t, Interpreter::logStackElementSize);
936
937 __ subptr(rsp, t);
938 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
939 __ andptr(rsp, -16); // must be 16 byte boundary (see amd64 ABI)
940 #endif // _LP64
941
942 // get signature handler
943 {
944 Label L;
945 __ movptr(t, Address(method, Method::signature_handler_offset()));
946 __ testptr(t, t);
947 __ jcc(Assembler::notZero, L);
948 __ call_VM(noreg,
949 CAST_FROM_FN_PTR(address,
950 InterpreterRuntime::prepare_native_call),
951 method);
952 __ get_method(method);
953 __ movptr(t, Address(method, Method::signature_handler_offset()));
954 __ bind(L);
955 }
956
957 // call signature handler
958 assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rlocals,
959 "adjust this code");
960 assert(InterpreterRuntime::SignatureHandlerGenerator::to() == rsp,
961 "adjust this code");
962 assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == NOT_LP64(t) LP64_ONLY(rscratch1),
963 "adjust this code");
964
965 // The generated handlers do not touch RBX (the method).
966 // However, large signatures cannot be cached and are generated
967 // each time here. The slow-path generator can do a GC on return,
968 // so we must reload it after the call.
969 __ call(t);
970 __ get_method(method); // slow path can do a GC, reload RBX
971
972
973 // result handler is in rax
974 // set result handler
975 __ movptr(Address(rbp,
976 (frame::interpreter_frame_result_handler_offset) * wordSize),
977 rax);
978
979 // pass mirror handle if static call
980 {
981 Label L;
982 __ movl(t, Address(method, Method::access_flags_offset()));
983 __ testl(t, JVM_ACC_STATIC);
984 __ jcc(Assembler::zero, L);
985 // get mirror
986 __ load_mirror(t, method, rax);
987 // copy mirror into activation frame
988 __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize),
989 t);
990 // pass handle to mirror
991 #ifndef _LP64
992 __ lea(t, Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
993 __ movptr(Address(rsp, wordSize), t);
994 #else
995 __ lea(c_rarg1,
996 Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
997 #endif // _LP64
998 __ bind(L);
999 }
1000
1001 // get native function entry point
1002 {
1003 Label L;
1004 __ movptr(rax, Address(method, Method::native_function_offset()));
1005 ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
1006 __ cmpptr(rax, unsatisfied.addr(), rscratch1);
1007 __ jcc(Assembler::notEqual, L);
1008 __ call_VM(noreg,
1009 CAST_FROM_FN_PTR(address,
1010 InterpreterRuntime::prepare_native_call),
1011 method);
1012 __ get_method(method);
1013 __ movptr(rax, Address(method, Method::native_function_offset()));
1014 __ bind(L);
1015 }
1016
1017 // pass JNIEnv
1018 #ifndef _LP64
1019 __ get_thread(thread);
1020 __ lea(t, Address(thread, JavaThread::jni_environment_offset()));
1021 __ movptr(Address(rsp, 0), t);
1022
1023 // set_last_Java_frame_before_call
1024 // It is enough that the pc()
1025 // points into the right code segment. It does not have to be the correct return pc.
1026 __ set_last_Java_frame(thread, noreg, rbp, __ pc(), noreg);
1027 #else
1028 __ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
1029
1030 // It is enough that the pc() points into the right code
1031 // segment. It does not have to be the correct return pc.
1032 __ set_last_Java_frame(rsp, rbp, (address) __ pc(), rscratch1);
1033 #endif // _LP64
1034
1035 // change thread state
1036 #ifdef ASSERT
1037 {
1038 Label L;
1039 __ movl(t, Address(thread, JavaThread::thread_state_offset()));
1040 __ cmpl(t, _thread_in_Java);
1041 __ jcc(Assembler::equal, L);
1042 __ stop("Wrong thread state in native stub");
1043 __ bind(L);
1044 }
1045 #endif
1046
1047 // Change state to native
1048
1049 __ movl(Address(thread, JavaThread::thread_state_offset()),
1050 _thread_in_native);
1051
1052 // Call the native method.
1053 __ call(rax);
1054 // 32: result potentially in rdx:rax or ST0
1055 // 64: result potentially in rax or xmm0
1056
1057 // Verify or restore cpu control state after JNI call
1058 __ restore_cpu_control_state_after_jni(rscratch1);
1059
1060 // NOTE: The order of these pushes is known to frame::interpreter_frame_result
1061 // in order to extract the result of a method call. If the order of these
1062 // pushes change or anything else is added to the stack then the code in
1063 // interpreter_frame_result must also change.
1064
1065 #ifndef _LP64
1066 // save potential result in ST(0) & rdx:rax
1067 // (if result handler is the T_FLOAT or T_DOUBLE handler, result must be in ST0 -
1068 // the check is necessary to avoid potential Intel FPU overflow problems by saving/restoring 'empty' FPU registers)
1069 // It is safe to do this push because state is _thread_in_native and return address will be found
1070 // via _last_native_pc and not via _last_jave_sp
1071
1072 // NOTE: the order of these push(es) is known to frame::interpreter_frame_result.
1073 // If the order changes or anything else is added to the stack the code in
1074 // interpreter_frame_result will have to be changed.
1075
1076 { Label L;
1077 Label push_double;
1078 ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT));
1079 ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE));
1080 __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1081 float_handler.addr(), noreg);
1082 __ jcc(Assembler::equal, push_double);
1083 __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1084 double_handler.addr(), noreg);
1085 __ jcc(Assembler::notEqual, L);
1086 __ bind(push_double);
1087 __ push_d(); // FP values are returned using the FPU, so push FPU contents (even if UseSSE > 0).
1088 __ bind(L);
1089 }
1090 #else
1091 __ push(dtos);
1092 #endif // _LP64
1093
1094 __ push(ltos);
1095
1096 // change thread state
1097 NOT_LP64(__ get_thread(thread));
1098 __ movl(Address(thread, JavaThread::thread_state_offset()),
1099 _thread_in_native_trans);
1100
1101 // Force this write out before the read below
1102 if (!UseSystemMemoryBarrier) {
1103 __ membar(Assembler::Membar_mask_bits(
1104 Assembler::LoadLoad | Assembler::LoadStore |
1105 Assembler::StoreLoad | Assembler::StoreStore));
1106 }
1107 #ifndef _LP64
1108 if (AlwaysRestoreFPU) {
1109 // Make sure the control word is correct.
1110 __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
1111 }
1112 #endif // _LP64
1113
1114 // check for safepoint operation in progress and/or pending suspend requests
1115 {
1116 Label Continue;
1117 Label slow_path;
1118
1119 __ safepoint_poll(slow_path, thread, true /* at_return */, false /* in_nmethod */);
1120
1121 __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
1122 __ jcc(Assembler::equal, Continue);
1123 __ bind(slow_path);
1124
1125 // Don't use call_VM as it will see a possible pending exception
1126 // and forward it and never return here preventing us from
1127 // clearing _last_native_pc down below. Also can't use
1128 // call_VM_leaf either as it will check to see if r13 & r14 are
1129 // preserved and correspond to the bcp/locals pointers. So we do a
1130 // runtime call by hand.
1131 //
1132 #ifndef _LP64
1133 __ push(thread);
1134 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
1135 JavaThread::check_special_condition_for_native_trans)));
1136 __ increment(rsp, wordSize);
1137 __ get_thread(thread);
1138 #else
1139 __ mov(c_rarg0, r15_thread);
1140 __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
1141 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1142 __ andptr(rsp, -16); // align stack as required by ABI
1143 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
1144 __ mov(rsp, r12); // restore sp
1145 __ reinit_heapbase();
1146 #endif // _LP64
1147 __ bind(Continue);
1148 }
1149
1150 // change thread state
1151 __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
1152
1153 // reset_last_Java_frame
1154 __ reset_last_Java_frame(thread, true);
1155
1156 if (CheckJNICalls) {
1157 // clear_pending_jni_exception_check
1158 __ movptr(Address(thread, JavaThread::pending_jni_exception_check_fn_offset()), NULL_WORD);
1159 }
1160
1161 // reset handle block
1162 __ movptr(t, Address(thread, JavaThread::active_handles_offset()));
1163 __ movl(Address(t, JNIHandleBlock::top_offset()), NULL_WORD);
1164
1165 // If result is an oop unbox and store it in frame where gc will see it
1166 // and result handler will pick it up
1167
1168 {
1169 Label no_oop;
1170 __ lea(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1171 __ cmpptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1172 __ jcc(Assembler::notEqual, no_oop);
1173 // retrieve result
1174 __ pop(ltos);
1175 // Unbox oop result, e.g. JNIHandles::resolve value.
1176 __ resolve_jobject(rax /* value */,
1177 thread /* thread */,
1178 t /* tmp */);
1179 __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize), rax);
1180 // keep stack depth as expected by pushing oop which will eventually be discarded
1181 __ push(ltos);
1182 __ bind(no_oop);
1183 }
1184
1185
1186 {
1187 Label no_reguard;
1188 __ cmpl(Address(thread, JavaThread::stack_guard_state_offset()),
1189 StackOverflow::stack_guard_yellow_reserved_disabled);
1190 __ jcc(Assembler::notEqual, no_reguard);
1191
1192 __ pusha(); // XXX only save smashed registers
1193 #ifndef _LP64
1194 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1195 __ popa();
1196 #else
1197 __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
1198 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1199 __ andptr(rsp, -16); // align stack as required by ABI
1200 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1201 __ mov(rsp, r12); // restore sp
1202 __ popa(); // XXX only restore smashed registers
1203 __ reinit_heapbase();
1204 #endif // _LP64
1205
1206 __ bind(no_reguard);
1207 }
1208
1209
1210 // The method register is junk from after the thread_in_native transition
1211 // until here. Also can't call_VM until the bcp has been
1212 // restored. Need bcp for throwing exception below so get it now.
1213 __ get_method(method);
1214
1215 // restore to have legal interpreter frame, i.e., bci == 0 <=> code_base()
1216 __ movptr(rbcp, Address(method, Method::const_offset())); // get ConstMethod*
1217 __ lea(rbcp, Address(rbcp, ConstMethod::codes_offset())); // get codebase
1218
1219 // handle exceptions (exception handling will handle unlocking!)
1220 {
1221 Label L;
1222 __ cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
1223 __ jcc(Assembler::zero, L);
1224 // Note: At some point we may want to unify this with the code
1225 // used in call_VM_base(); i.e., we should use the
1226 // StubRoutines::forward_exception code. For now this doesn't work
1227 // here because the rsp is not correctly set at this point.
1228 __ MacroAssembler::call_VM(noreg,
1229 CAST_FROM_FN_PTR(address,
1230 InterpreterRuntime::throw_pending_exception));
1231 __ should_not_reach_here();
1232 __ bind(L);
1233 }
1234
1235 // do unlocking if necessary
1236 {
1237 Label L;
1238 __ movl(t, Address(method, Method::access_flags_offset()));
1239 __ testl(t, JVM_ACC_SYNCHRONIZED);
1240 __ jcc(Assembler::zero, L);
1241 // the code below should be shared with interpreter macro
1242 // assembler implementation
1243 {
1244 Label unlock;
1245 // BasicObjectLock will be first in list, since this is a
1246 // synchronized method. However, need to check that the object
1247 // has not been unlocked by an explicit monitorexit bytecode.
1248 const Address monitor(rbp,
1249 (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1250 wordSize - (int)sizeof(BasicObjectLock)));
1251
1252 const Register regmon = NOT_LP64(rdx) LP64_ONLY(c_rarg1);
1253
1254 // monitor expect in c_rarg1 for slow unlock path
1255 __ lea(regmon, monitor); // address of first monitor
1256
1257 __ movptr(t, Address(regmon, BasicObjectLock::obj_offset()));
1258 __ testptr(t, t);
1259 __ jcc(Assembler::notZero, unlock);
1260
1261 // Entry already unlocked, need to throw exception
1262 __ MacroAssembler::call_VM(noreg,
1263 CAST_FROM_FN_PTR(address,
1264 InterpreterRuntime::throw_illegal_monitor_state_exception));
1265 __ should_not_reach_here();
1266
1267 __ bind(unlock);
1268 __ unlock_object(regmon);
1269 }
1270 __ bind(L);
1271 }
1272
1273 // jvmti support
1274 // Note: This must happen _after_ handling/throwing any exceptions since
1275 // the exception handler code notifies the runtime of method exits
1276 // too. If this happens before, method entry/exit notifications are
1277 // not properly paired (was bug - gri 11/22/99).
1278 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1279
1280 // restore potential result in edx:eax, call result handler to
1281 // restore potential result in ST0 & handle result
1282
1283 __ pop(ltos);
1284 LP64_ONLY( __ pop(dtos));
1285
1286 __ movptr(t, Address(rbp,
1287 (frame::interpreter_frame_result_handler_offset) * wordSize));
1288 __ call(t);
1289
1290 // remove activation
1291 __ movptr(t, Address(rbp,
1292 frame::interpreter_frame_sender_sp_offset *
1293 wordSize)); // get sender sp
1294 __ leave(); // remove frame anchor
1295 __ pop(rdi); // get return address
1296 __ mov(rsp, t); // set sp to sender sp
1297 __ jmp(rdi);
1298
1299 if (inc_counter) {
1300 // Handle overflow of counter and compile method
1301 __ bind(invocation_counter_overflow);
1302 generate_counter_overflow(continue_after_compile);
1303 }
1304
1305 return entry_point;
1306 }
1307
1308 // Abstract method entry
1309 // Attempt to execute abstract method. Throw exception
1310 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
1311
1312 address entry_point = __ pc();
1313
1314 // abstract method entry
1315
1316 // pop return address, reset last_sp to null
1317 __ empty_expression_stack();
1318 __ restore_bcp(); // rsi must be correct for exception handler (was destroyed)
1319 __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
1320
1321 // throw exception
1322 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorWithMethod), rbx);
1323 // the call_VM checks for exception, so we should never return here.
1324 __ should_not_reach_here();
1325
1326 return entry_point;
1327 }
1328
1329 //
1330 // Generic interpreted method entry to (asm) interpreter
1331 //
1332 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1333 // determine code generation flags
1334 bool inc_counter = UseCompiler || CountCompiledCalls;
1335
1336 // ebx: Method*
1337 // rbcp: sender sp (set in InterpreterMacroAssembler::prepare_to_jump_from_interpreted / generate_call_stub)
1338 address entry_point = __ pc();
1339
1340 const Address constMethod(rbx, Method::const_offset());
1341 const Address access_flags(rbx, Method::access_flags_offset());
1342 const Address size_of_parameters(rdx,
1343 ConstMethod::size_of_parameters_offset());
1344 const Address size_of_locals(rdx, ConstMethod::size_of_locals_offset());
1345
1346
1347 // get parameter size (always needed)
1348 __ movptr(rdx, constMethod);
1349 __ load_unsigned_short(rcx, size_of_parameters);
1350
1351 // rbx: Method*
1352 // rcx: size of parameters
1353 // rbcp: sender_sp (could differ from sp+wordSize if we were called via c2i )
1354
1355 __ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
1356 __ subl(rdx, rcx); // rdx = no. of additional locals
1357
1358 // YYY
1359 // __ incrementl(rdx);
1360 // __ andl(rdx, -2);
1361
1362 // see if we've got enough room on the stack for locals plus overhead.
1363 generate_stack_overflow_check();
1364
1365 // get return address
1366 __ pop(rax);
1367
1368 // compute beginning of parameters
1369 __ lea(rlocals, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
1370
1371 // rdx - # of additional locals
1372 // allocate space for locals
1373 // explicitly initialize locals
1374 {
1375 Label exit, loop;
1376 __ testl(rdx, rdx);
1377 __ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
1378 __ bind(loop);
1379 __ push(NULL_WORD); // initialize local variables
1380 __ decrementl(rdx); // until everything initialized
1381 __ jcc(Assembler::greater, loop);
1382 __ bind(exit);
1383 }
1384
1385 // initialize fixed part of activation frame
1386 generate_fixed_frame(false);
1387
1388 // make sure method is not native & not abstract
1389 #ifdef ASSERT
1390 __ movl(rax, access_flags);
1391 {
1392 Label L;
1393 __ testl(rax, JVM_ACC_NATIVE);
1394 __ jcc(Assembler::zero, L);
1395 __ stop("tried to execute native method as non-native");
1396 __ bind(L);
1397 }
1398 {
1399 Label L;
1400 __ testl(rax, JVM_ACC_ABSTRACT);
1401 __ jcc(Assembler::zero, L);
1402 __ stop("tried to execute abstract method in interpreter");
1403 __ bind(L);
1404 }
1405 #endif
1406
1407 // Since at this point in the method invocation the exception
1408 // handler would try to exit the monitor of synchronized methods
1409 // which hasn't been entered yet, we set the thread local variable
1410 // _do_not_unlock_if_synchronized to true. The remove_activation
1411 // will check this flag.
1412
1413 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1414 NOT_LP64(__ get_thread(thread));
1415 const Address do_not_unlock_if_synchronized(thread,
1416 in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1417 __ movbool(do_not_unlock_if_synchronized, true);
1418
1419 __ profile_parameters_type(rax, rcx, rdx);
1420 // increment invocation count & check for overflow
1421 Label invocation_counter_overflow;
1422 if (inc_counter) {
1423 generate_counter_incr(&invocation_counter_overflow);
1424 }
1425
1426 Label continue_after_compile;
1427 __ bind(continue_after_compile);
1428
1429 // check for synchronized interpreted methods
1430 bang_stack_shadow_pages(false);
1431
1432 // reset the _do_not_unlock_if_synchronized flag
1433 NOT_LP64(__ get_thread(thread));
1434 __ movbool(do_not_unlock_if_synchronized, false);
1435
1436 // check for synchronized methods
1437 // Must happen AFTER invocation_counter check and stack overflow check,
1438 // so method is not locked if overflows.
1439 if (synchronized) {
1440 // Allocate monitor and lock method
1441 lock_method();
1442 } else {
1443 // no synchronization necessary
1444 #ifdef ASSERT
1445 {
1446 Label L;
1447 __ movl(rax, access_flags);
1448 __ testl(rax, JVM_ACC_SYNCHRONIZED);
1449 __ jcc(Assembler::zero, L);
1450 __ stop("method needs synchronization");
1451 __ bind(L);
1452 }
1453 #endif
1454 }
1455
1456 // start execution
1457 #ifdef ASSERT
1458 {
1459 Label L;
1460 const Address monitor_block_top (rbp,
1461 frame::interpreter_frame_monitor_block_top_offset * wordSize);
1462 __ movptr(rax, monitor_block_top);
1463 __ lea(rax, Address(rbp, rax, Address::times_ptr));
1464 __ cmpptr(rax, rsp);
1465 __ jcc(Assembler::equal, L);
1466 __ stop("broken stack frame setup in interpreter 6");
1467 __ bind(L);
1468 }
1469 #endif
1470
1471 // jvmti support
1472 __ notify_method_entry();
1473
1474 __ dispatch_next(vtos);
1475
1476 // invocation counter overflow
1477 if (inc_counter) {
1478 // Handle overflow of counter and compile method
1479 __ bind(invocation_counter_overflow);
1480 generate_counter_overflow(continue_after_compile);
1481 }
1482
1483 return entry_point;
1484 }
1485
1486 //-----------------------------------------------------------------------------
1487 // Exceptions
1488
1489 void TemplateInterpreterGenerator::generate_throw_exception() {
1490 // Entry point in previous activation (i.e., if the caller was
1491 // interpreted)
1492 Interpreter::_rethrow_exception_entry = __ pc();
1493 // Restore sp to interpreter_frame_last_sp even though we are going
1494 // to empty the expression stack for the exception processing.
1495 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1496 // rax: exception
1497 // rdx: return address/pc that threw exception
1498 __ restore_bcp(); // r13/rsi points to call/send
1499 __ restore_locals();
1500 LP64_ONLY(__ reinit_heapbase()); // restore r12 as heapbase.
1501 // Entry point for exceptions thrown within interpreter code
1502 Interpreter::_throw_exception_entry = __ pc();
1503 // expression stack is undefined here
1504 // rax: exception
1505 // r13/rsi: exception bcp
1506 __ verify_oop(rax);
1507 Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);
1508 LP64_ONLY(__ mov(c_rarg1, rax));
1509
1510 // expression stack must be empty before entering the VM in case of
1511 // an exception
1512 __ empty_expression_stack();
1513 // find exception handler address and preserve exception oop
1514 __ call_VM(rdx,
1515 CAST_FROM_FN_PTR(address,
1516 InterpreterRuntime::exception_handler_for_exception),
1517 rarg);
1518 // rax: exception handler entry point
1519 // rdx: preserved exception oop
1520 // r13/rsi: bcp for exception handler
1521 __ push_ptr(rdx); // push exception which is now the only value on the stack
1522 __ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
1523
1524 // If the exception is not handled in the current frame the frame is
1525 // removed and the exception is rethrown (i.e. exception
1526 // continuation is _rethrow_exception).
1527 //
1528 // Note: At this point the bci is still the bxi for the instruction
1529 // which caused the exception and the expression stack is
1530 // empty. Thus, for any VM calls at this point, GC will find a legal
1531 // oop map (with empty expression stack).
1532
1533 // In current activation
1534 // tos: exception
1535 // esi: exception bcp
1536
1537 //
1538 // JVMTI PopFrame support
1539 //
1540
1541 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1542 __ empty_expression_stack();
1543 // Set the popframe_processing bit in pending_popframe_condition
1544 // indicating that we are currently handling popframe, so that
1545 // call_VMs that may happen later do not trigger new popframe
1546 // handling cycles.
1547 const Register thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);
1548 NOT_LP64(__ get_thread(thread));
1549 __ movl(rdx, Address(thread, JavaThread::popframe_condition_offset()));
1550 __ orl(rdx, JavaThread::popframe_processing_bit);
1551 __ movl(Address(thread, JavaThread::popframe_condition_offset()), rdx);
1552
1553 {
1554 // Check to see whether we are returning to a deoptimized frame.
1555 // (The PopFrame call ensures that the caller of the popped frame is
1556 // either interpreted or compiled and deoptimizes it if compiled.)
1557 // In this case, we can't call dispatch_next() after the frame is
1558 // popped, but instead must save the incoming arguments and restore
1559 // them after deoptimization has occurred.
1560 //
1561 // Note that we don't compare the return PC against the
1562 // deoptimization blob's unpack entry because of the presence of
1563 // adapter frames in C2.
1564 Label caller_not_deoptimized;
1565 Register rarg = NOT_LP64(rdx) LP64_ONLY(c_rarg1);
1566 __ movptr(rarg, Address(rbp, frame::return_addr_offset * wordSize));
1567 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1568 InterpreterRuntime::interpreter_contains), rarg);
1569 __ testl(rax, rax);
1570 __ jcc(Assembler::notZero, caller_not_deoptimized);
1571
1572 // Compute size of arguments for saving when returning to
1573 // deoptimized caller
1574 __ get_method(rax);
1575 __ movptr(rax, Address(rax, Method::const_offset()));
1576 __ load_unsigned_short(rax, Address(rax, in_bytes(ConstMethod::
1577 size_of_parameters_offset())));
1578 __ shll(rax, Interpreter::logStackElementSize);
1579 __ restore_locals();
1580 __ subptr(rlocals, rax);
1581 __ addptr(rlocals, wordSize);
1582 // Save these arguments
1583 NOT_LP64(__ get_thread(thread));
1584 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1585 Deoptimization::
1586 popframe_preserve_args),
1587 thread, rax, rlocals);
1588
1589 __ remove_activation(vtos, rdx,
1590 /* throw_monitor_exception */ false,
1591 /* install_monitor_exception */ false,
1592 /* notify_jvmdi */ false);
1593
1594 // Inform deoptimization that it is responsible for restoring
1595 // these arguments
1596 NOT_LP64(__ get_thread(thread));
1597 __ movl(Address(thread, JavaThread::popframe_condition_offset()),
1598 JavaThread::popframe_force_deopt_reexecution_bit);
1599
1600 // Continue in deoptimization handler
1601 __ jmp(rdx);
1602
1603 __ bind(caller_not_deoptimized);
1604 }
1605
1606 __ remove_activation(vtos, rdx, /* rdx result (retaddr) is not used */
1607 /* throw_monitor_exception */ false,
1608 /* install_monitor_exception */ false,
1609 /* notify_jvmdi */ false);
1610
1611 // Finish with popframe handling
1612 // A previous I2C followed by a deoptimization might have moved the
1613 // outgoing arguments further up the stack. PopFrame expects the
1614 // mutations to those outgoing arguments to be preserved and other
1615 // constraints basically require this frame to look exactly as
1616 // though it had previously invoked an interpreted activation with
1617 // no space between the top of the expression stack (current
1618 // last_sp) and the top of stack. Rather than force deopt to
1619 // maintain this kind of invariant all the time we call a small
1620 // fixup routine to move the mutated arguments onto the top of our
1621 // expression stack if necessary.
1622 #ifndef _LP64
1623 __ mov(rax, rsp);
1624 __ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1625 __ lea(rbx, Address(rbp, rbx, Address::times_ptr));
1626 __ get_thread(thread);
1627 // PC must point into interpreter here
1628 __ set_last_Java_frame(thread, noreg, rbp, __ pc(), noreg);
1629 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
1630 __ get_thread(thread);
1631 #else
1632 __ mov(c_rarg1, rsp);
1633 __ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1634 __ lea(c_rarg2, Address(rbp, c_rarg2, Address::times_ptr));
1635 // PC must point into interpreter here
1636 __ set_last_Java_frame(noreg, rbp, __ pc(), rscratch1);
1637 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
1638 #endif
1639 __ reset_last_Java_frame(thread, true);
1640
1641 // Restore the last_sp and null it out
1642 __ movptr(rcx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1643 __ lea(rsp, Address(rbp, rcx, Address::times_ptr));
1644 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1645
1646 __ restore_bcp();
1647 __ restore_locals();
1648 // The method data pointer was incremented already during
1649 // call profiling. We have to restore the mdp for the current bcp.
1650 if (ProfileInterpreter) {
1651 __ set_method_data_pointer_for_bcp();
1652 }
1653
1654 // Clear the popframe condition flag
1655 NOT_LP64(__ get_thread(thread));
1656 __ movl(Address(thread, JavaThread::popframe_condition_offset()),
1657 JavaThread::popframe_inactive);
1658
1659 #if INCLUDE_JVMTI
1660 {
1661 Label L_done;
1662 const Register local0 = rlocals;
1663
1664 __ cmpb(Address(rbcp, 0), Bytecodes::_invokestatic);
1665 __ jcc(Assembler::notEqual, L_done);
1666
1667 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1668 // Detect such a case in the InterpreterRuntime function and return the member name argument, or null.
1669
1670 __ get_method(rdx);
1671 __ movptr(rax, Address(local0, 0));
1672 __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), rax, rdx, rbcp);
1673
1674 __ testptr(rax, rax);
1675 __ jcc(Assembler::zero, L_done);
1676
1677 __ movptr(Address(rbx, 0), rax);
1678 __ bind(L_done);
1679 }
1680 #endif // INCLUDE_JVMTI
1681
1682 __ dispatch_next(vtos);
1683 // end of PopFrame support
1684
1685 Interpreter::_remove_activation_entry = __ pc();
1686
1687 // preserve exception over this code sequence
1688 __ pop_ptr(rax);
1689 NOT_LP64(__ get_thread(thread));
1690 __ movptr(Address(thread, JavaThread::vm_result_offset()), rax);
1691 // remove the activation (without doing throws on illegalMonitorExceptions)
1692 __ remove_activation(vtos, rdx, false, true, false);
1693 // restore exception
1694 NOT_LP64(__ get_thread(thread));
1695 __ get_vm_result(rax, thread);
1696
1697 // In between activations - previous activation type unknown yet
1698 // compute continuation point - the continuation point expects the
1699 // following registers set up:
1700 //
1701 // rax: exception
1702 // rdx: return address/pc that threw exception
1703 // rsp: expression stack of caller
1704 // rbp: ebp of caller
1705 __ push(rax); // save exception
1706 __ push(rdx); // save return address
1707 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1708 SharedRuntime::exception_handler_for_return_address),
1709 thread, rdx);
1710 __ mov(rbx, rax); // save exception handler
1711 __ pop(rdx); // restore return address
1712 __ pop(rax); // restore exception
1713 // Note that an "issuing PC" is actually the next PC after the call
1714 __ jmp(rbx); // jump to exception
1715 // handler of caller
1716 }
1717
1718
1719 //
1720 // JVMTI ForceEarlyReturn support
1721 //
1722 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1723 address entry = __ pc();
1724
1725 __ restore_bcp();
1726 __ restore_locals();
1727 __ empty_expression_stack();
1728 __ load_earlyret_value(state); // 32 bits returns value in rdx, so don't reuse
1729
1730 const Register thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);
1731 NOT_LP64(__ get_thread(thread));
1732 __ movptr(rcx, Address(thread, JavaThread::jvmti_thread_state_offset()));
1733 Address cond_addr(rcx, JvmtiThreadState::earlyret_state_offset());
1734
1735 // Clear the earlyret state
1736 __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1737
1738 __ remove_activation(state, rsi,
1739 false, /* throw_monitor_exception */
1740 false, /* install_monitor_exception */
1741 true); /* notify_jvmdi */
1742 __ jmp(rsi);
1743
1744 return entry;
1745 } // end of ForceEarlyReturn support
1746
1747
1748 //-----------------------------------------------------------------------------
1749 // Helper for vtos entry point generation
1750
1751 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1752 address& bep,
1753 address& cep,
1754 address& sep,
1755 address& aep,
1756 address& iep,
1757 address& lep,
1758 address& fep,
1759 address& dep,
1760 address& vep) {
1761 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1762 Label L;
1763 #ifndef _LP64
1764 fep = __ pc(); // ftos entry point
1765 __ push(ftos);
1766 __ jmpb(L);
1767 dep = __ pc(); // dtos entry point
1768 __ push(dtos);
1769 __ jmpb(L);
1770 #else
1771 fep = __ pc(); // ftos entry point
1772 __ push_f(xmm0);
1773 __ jmpb(L);
1774 dep = __ pc(); // dtos entry point
1775 __ push_d(xmm0);
1776 __ jmpb(L);
1777 #endif // _LP64
1778 lep = __ pc(); // ltos entry point
1779 __ push_l();
1780 __ jmpb(L);
1781 aep = bep = cep = sep = iep = __ pc(); // [abcsi]tos entry point
1782 __ push_i_or_ptr();
1783 vep = __ pc(); // vtos entry point
1784 __ bind(L);
1785 generate_and_dispatch(t);
1786 }
1787
1788 //-----------------------------------------------------------------------------
1789
1790 // Non-product code
1791 #ifndef PRODUCT
1792
1793 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1794 address entry = __ pc();
1795
1796 #ifndef _LP64
1797 // prepare expression stack
1798 __ pop(rcx); // pop return address so expression stack is 'pure'
1799 __ push(state); // save tosca
1800
1801 // pass tosca registers as arguments & call tracer
1802 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), rcx, rax, rdx);
1803 __ mov(rcx, rax); // make sure return address is not destroyed by pop(state)
1804 __ pop(state); // restore tosca
1805
1806 // return
1807 __ jmp(rcx);
1808 #else
1809 __ push(state);
1810 __ push(c_rarg0);
1811 __ push(c_rarg1);
1812 __ push(c_rarg2);
1813 __ push(c_rarg3);
1814 __ mov(c_rarg2, rax); // Pass itos
1815 #ifdef _WIN64
1816 __ movflt(xmm3, xmm0); // Pass ftos
1817 #endif
1818 __ call_VM(noreg,
1819 CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode),
1820 c_rarg1, c_rarg2, c_rarg3);
1821 __ pop(c_rarg3);
1822 __ pop(c_rarg2);
1823 __ pop(c_rarg1);
1824 __ pop(c_rarg0);
1825 __ pop(state);
1826 __ ret(0); // return from result handler
1827 #endif // _LP64
1828
1829 return entry;
1830 }
1831
1832 void TemplateInterpreterGenerator::count_bytecode() {
1833 __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value), rscratch1);
1834 }
1835
1836 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1837 __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]), rscratch1);
1838 }
1839
1840 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1841 __ mov32(rbx, ExternalAddress((address) &BytecodePairHistogram::_index));
1842 __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1843 __ orl(rbx,
1844 ((int) t->bytecode()) <<
1845 BytecodePairHistogram::log2_number_of_codes);
1846 __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx, rscratch1);
1847 __ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
1848 __ incrementl(Address(rscratch1, rbx, Address::times_4));
1849 }
1850
1851
1852 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1853 // Call a little run-time stub to avoid blow-up for each bytecode.
1854 // The run-time runtime saves the right registers, depending on
1855 // the tosca in-state for the given template.
1856
1857 assert(Interpreter::trace_code(t->tos_in()) != nullptr,
1858 "entry must have been generated");
1859 #ifndef _LP64
1860 __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1861 #else
1862 __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
1863 __ andptr(rsp, -16); // align stack as required by ABI
1864 __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1865 __ mov(rsp, r12); // restore sp
1866 __ reinit_heapbase();
1867 #endif // _LP64
1868 }
1869
1870
1871 void TemplateInterpreterGenerator::stop_interpreter_at() {
1872 Label L;
1873 __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1874 StopInterpreterAt,
1875 rscratch1);
1876 __ jcc(Assembler::notEqual, L);
1877 __ int3();
1878 __ bind(L);
1879 }
1880 #endif // !PRODUCT