1 /*
2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/classLoader.hpp"
26 #include "classfile/javaClasses.inline.hpp"
27 #include "classfile/stringTable.hpp"
28 #include "classfile/vmClasses.hpp"
29 #include "classfile/vmSymbols.hpp"
30 #include "code/codeCache.hpp"
31 #include "code/compiledIC.hpp"
32 #include "code/nmethod.inline.hpp"
33 #include "code/scopeDesc.hpp"
34 #include "code/vtableStubs.hpp"
35 #include "compiler/abstractCompiler.hpp"
36 #include "compiler/compileBroker.hpp"
37 #include "compiler/disassembler.hpp"
38 #include "gc/shared/barrierSet.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "interpreter/interpreterRuntime.hpp"
42 #include "jvm.h"
43 #include "jfr/jfrEvents.hpp"
44 #include "logging/log.hpp"
45 #include "memory/resourceArea.hpp"
46 #include "memory/universe.hpp"
47 #include "metaprogramming/primitiveConversions.hpp"
48 #include "oops/klass.hpp"
49 #include "oops/method.inline.hpp"
50 #include "oops/objArrayKlass.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "prims/forte.hpp"
53 #include "prims/jvmtiExport.hpp"
54 #include "prims/jvmtiThreadState.hpp"
55 #include "prims/methodHandles.hpp"
56 #include "prims/nativeLookup.hpp"
57 #include "runtime/arguments.hpp"
58 #include "runtime/atomic.hpp"
59 #include "runtime/basicLock.inline.hpp"
60 #include "runtime/frame.inline.hpp"
61 #include "runtime/handles.inline.hpp"
62 #include "runtime/init.hpp"
63 #include "runtime/interfaceSupport.inline.hpp"
64 #include "runtime/java.hpp"
65 #include "runtime/javaCalls.hpp"
66 #include "runtime/jniHandles.inline.hpp"
67 #include "runtime/perfData.hpp"
68 #include "runtime/sharedRuntime.hpp"
69 #include "runtime/stackWatermarkSet.hpp"
70 #include "runtime/stubRoutines.hpp"
71 #include "runtime/synchronizer.inline.hpp"
72 #include "runtime/timerTrace.hpp"
73 #include "runtime/vframe.inline.hpp"
74 #include "runtime/vframeArray.hpp"
75 #include "runtime/vm_version.hpp"
76 #include "utilities/copy.hpp"
77 #include "utilities/dtrace.hpp"
78 #include "utilities/events.hpp"
79 #include "utilities/globalDefinitions.hpp"
80 #include "utilities/resourceHash.hpp"
81 #include "utilities/macros.hpp"
82 #include "utilities/xmlstream.hpp"
83 #ifdef COMPILER1
84 #include "c1/c1_Runtime1.hpp"
85 #endif
86 #if INCLUDE_JFR
87 #include "jfr/jfr.hpp"
88 #endif
89
90 // Shared runtime stub routines reside in their own unique blob with a
91 // single entry point
92
93
94 #define SHARED_STUB_FIELD_DEFINE(name, type) \
95 type SharedRuntime::BLOB_FIELD_NAME(name);
96 SHARED_STUBS_DO(SHARED_STUB_FIELD_DEFINE)
97 #undef SHARED_STUB_FIELD_DEFINE
98
99 nmethod* SharedRuntime::_cont_doYield_stub;
100
101 #define SHARED_STUB_NAME_DECLARE(name, type) "Shared Runtime " # name "_blob",
102 const char *SharedRuntime::_stub_names[] = {
103 SHARED_STUBS_DO(SHARED_STUB_NAME_DECLARE)
104 };
105
106 //----------------------------generate_stubs-----------------------------------
107 void SharedRuntime::generate_initial_stubs() {
108 // Build this early so it's available for the interpreter.
109 _throw_StackOverflowError_blob =
110 generate_throw_exception(SharedStubId::throw_StackOverflowError_id,
111 CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
112 }
113
114 void SharedRuntime::generate_stubs() {
115 _wrong_method_blob =
116 generate_resolve_blob(SharedStubId::wrong_method_id,
117 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method));
118 _wrong_method_abstract_blob =
119 generate_resolve_blob(SharedStubId::wrong_method_abstract_id,
120 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract));
121 _ic_miss_blob =
122 generate_resolve_blob(SharedStubId::ic_miss_id,
123 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss));
124 _resolve_opt_virtual_call_blob =
125 generate_resolve_blob(SharedStubId::resolve_opt_virtual_call_id,
126 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C));
127 _resolve_virtual_call_blob =
128 generate_resolve_blob(SharedStubId::resolve_virtual_call_id,
129 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C));
130 _resolve_static_call_blob =
131 generate_resolve_blob(SharedStubId::resolve_static_call_id,
132 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C));
133
134 _throw_delayed_StackOverflowError_blob =
135 generate_throw_exception(SharedStubId::throw_delayed_StackOverflowError_id,
136 CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
137
138 _throw_AbstractMethodError_blob =
139 generate_throw_exception(SharedStubId::throw_AbstractMethodError_id,
140 CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
141
142 _throw_IncompatibleClassChangeError_blob =
143 generate_throw_exception(SharedStubId::throw_IncompatibleClassChangeError_id,
144 CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
145
146 _throw_NullPointerException_at_call_blob =
147 generate_throw_exception(SharedStubId::throw_NullPointerException_at_call_id,
148 CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
149
150 AdapterHandlerLibrary::initialize();
151
152 #if COMPILER2_OR_JVMCI
153 // Vectors are generated only by C2 and JVMCI.
154 bool support_wide = is_wide_vector(MaxVectorSize);
155 if (support_wide) {
156 _polling_page_vectors_safepoint_handler_blob =
157 generate_handler_blob(SharedStubId::polling_page_vectors_safepoint_handler_id,
158 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
159 }
160 #endif // COMPILER2_OR_JVMCI
161 _polling_page_safepoint_handler_blob =
162 generate_handler_blob(SharedStubId::polling_page_safepoint_handler_id,
163 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
164 _polling_page_return_handler_blob =
165 generate_handler_blob(SharedStubId::polling_page_return_handler_id,
166 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
167
168 generate_deopt_blob();
169 }
170
171 #if INCLUDE_JFR
172 //------------------------------generate jfr runtime stubs ------
173 void SharedRuntime::generate_jfr_stubs() {
174 ResourceMark rm;
175 const char* timer_msg = "SharedRuntime generate_jfr_stubs";
176 TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime));
177
178 _jfr_write_checkpoint_blob = generate_jfr_write_checkpoint();
179 _jfr_return_lease_blob = generate_jfr_return_lease();
180 }
181
182 #endif // INCLUDE_JFR
183
184 #include <math.h>
185
186 // Implementation of SharedRuntime
187
188 #ifndef PRODUCT
189 // For statistics
190 uint SharedRuntime::_ic_miss_ctr = 0;
191 uint SharedRuntime::_wrong_method_ctr = 0;
192 uint SharedRuntime::_resolve_static_ctr = 0;
193 uint SharedRuntime::_resolve_virtual_ctr = 0;
194 uint SharedRuntime::_resolve_opt_virtual_ctr = 0;
195 uint SharedRuntime::_implicit_null_throws = 0;
196 uint SharedRuntime::_implicit_div0_throws = 0;
197
198 int64_t SharedRuntime::_nof_normal_calls = 0;
199 int64_t SharedRuntime::_nof_inlined_calls = 0;
200 int64_t SharedRuntime::_nof_megamorphic_calls = 0;
201 int64_t SharedRuntime::_nof_static_calls = 0;
202 int64_t SharedRuntime::_nof_inlined_static_calls = 0;
203 int64_t SharedRuntime::_nof_interface_calls = 0;
204 int64_t SharedRuntime::_nof_inlined_interface_calls = 0;
205
206 uint SharedRuntime::_new_instance_ctr=0;
207 uint SharedRuntime::_new_array_ctr=0;
208 uint SharedRuntime::_multi2_ctr=0;
209 uint SharedRuntime::_multi3_ctr=0;
210 uint SharedRuntime::_multi4_ctr=0;
211 uint SharedRuntime::_multi5_ctr=0;
212 uint SharedRuntime::_mon_enter_stub_ctr=0;
213 uint SharedRuntime::_mon_exit_stub_ctr=0;
214 uint SharedRuntime::_mon_enter_ctr=0;
215 uint SharedRuntime::_mon_exit_ctr=0;
216 uint SharedRuntime::_partial_subtype_ctr=0;
217 uint SharedRuntime::_jbyte_array_copy_ctr=0;
218 uint SharedRuntime::_jshort_array_copy_ctr=0;
219 uint SharedRuntime::_jint_array_copy_ctr=0;
220 uint SharedRuntime::_jlong_array_copy_ctr=0;
221 uint SharedRuntime::_oop_array_copy_ctr=0;
222 uint SharedRuntime::_checkcast_array_copy_ctr=0;
223 uint SharedRuntime::_unsafe_array_copy_ctr=0;
224 uint SharedRuntime::_generic_array_copy_ctr=0;
225 uint SharedRuntime::_slow_array_copy_ctr=0;
226 uint SharedRuntime::_find_handler_ctr=0;
227 uint SharedRuntime::_rethrow_ctr=0;
228 uint SharedRuntime::_unsafe_set_memory_ctr=0;
229
230 int SharedRuntime::_ICmiss_index = 0;
231 int SharedRuntime::_ICmiss_count[SharedRuntime::maxICmiss_count];
232 address SharedRuntime::_ICmiss_at[SharedRuntime::maxICmiss_count];
233
234
235 void SharedRuntime::trace_ic_miss(address at) {
236 for (int i = 0; i < _ICmiss_index; i++) {
237 if (_ICmiss_at[i] == at) {
238 _ICmiss_count[i]++;
239 return;
240 }
241 }
242 int index = _ICmiss_index++;
243 if (_ICmiss_index >= maxICmiss_count) _ICmiss_index = maxICmiss_count - 1;
244 _ICmiss_at[index] = at;
245 _ICmiss_count[index] = 1;
246 }
247
248 void SharedRuntime::print_ic_miss_histogram() {
249 if (ICMissHistogram) {
250 tty->print_cr("IC Miss Histogram:");
251 int tot_misses = 0;
252 for (int i = 0; i < _ICmiss_index; i++) {
253 tty->print_cr(" at: " INTPTR_FORMAT " nof: %d", p2i(_ICmiss_at[i]), _ICmiss_count[i]);
254 tot_misses += _ICmiss_count[i];
255 }
256 tty->print_cr("Total IC misses: %7d", tot_misses);
257 }
258 }
259 #endif // PRODUCT
260
261
262 JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x))
263 return x * y;
264 JRT_END
265
266
267 JRT_LEAF(jlong, SharedRuntime::ldiv(jlong y, jlong x))
268 if (x == min_jlong && y == CONST64(-1)) {
269 return x;
270 } else {
271 return x / y;
272 }
273 JRT_END
274
275
276 JRT_LEAF(jlong, SharedRuntime::lrem(jlong y, jlong x))
277 if (x == min_jlong && y == CONST64(-1)) {
278 return 0;
279 } else {
280 return x % y;
281 }
282 JRT_END
283
284
285 #ifdef _WIN64
286 const juint float_sign_mask = 0x7FFFFFFF;
287 const juint float_infinity = 0x7F800000;
288 const julong double_sign_mask = CONST64(0x7FFFFFFFFFFFFFFF);
289 const julong double_infinity = CONST64(0x7FF0000000000000);
290 #endif
291
292 #if !defined(X86)
293 JRT_LEAF(jfloat, SharedRuntime::frem(jfloat x, jfloat y))
294 #ifdef _WIN64
295 // 64-bit Windows on amd64 returns the wrong values for
296 // infinity operands.
297 juint xbits = PrimitiveConversions::cast<juint>(x);
298 juint ybits = PrimitiveConversions::cast<juint>(y);
299 // x Mod Infinity == x unless x is infinity
300 if (((xbits & float_sign_mask) != float_infinity) &&
301 ((ybits & float_sign_mask) == float_infinity) ) {
302 return x;
303 }
304 return ((jfloat)fmod_winx64((double)x, (double)y));
305 #else
306 return ((jfloat)fmod((double)x,(double)y));
307 #endif
308 JRT_END
309
310 JRT_LEAF(jdouble, SharedRuntime::drem(jdouble x, jdouble y))
311 #ifdef _WIN64
312 julong xbits = PrimitiveConversions::cast<julong>(x);
313 julong ybits = PrimitiveConversions::cast<julong>(y);
314 // x Mod Infinity == x unless x is infinity
315 if (((xbits & double_sign_mask) != double_infinity) &&
316 ((ybits & double_sign_mask) == double_infinity) ) {
317 return x;
318 }
319 return ((jdouble)fmod_winx64((double)x, (double)y));
320 #else
321 return ((jdouble)fmod((double)x,(double)y));
322 #endif
323 JRT_END
324 #endif // !X86
325
326 JRT_LEAF(jfloat, SharedRuntime::i2f(jint x))
327 return (jfloat)x;
328 JRT_END
329
330 #ifdef __SOFTFP__
331 JRT_LEAF(jfloat, SharedRuntime::fadd(jfloat x, jfloat y))
332 return x + y;
333 JRT_END
334
335 JRT_LEAF(jfloat, SharedRuntime::fsub(jfloat x, jfloat y))
336 return x - y;
337 JRT_END
338
339 JRT_LEAF(jfloat, SharedRuntime::fmul(jfloat x, jfloat y))
340 return x * y;
341 JRT_END
342
343 JRT_LEAF(jfloat, SharedRuntime::fdiv(jfloat x, jfloat y))
344 return x / y;
345 JRT_END
346
347 JRT_LEAF(jdouble, SharedRuntime::dadd(jdouble x, jdouble y))
348 return x + y;
349 JRT_END
350
351 JRT_LEAF(jdouble, SharedRuntime::dsub(jdouble x, jdouble y))
352 return x - y;
353 JRT_END
354
355 JRT_LEAF(jdouble, SharedRuntime::dmul(jdouble x, jdouble y))
356 return x * y;
357 JRT_END
358
359 JRT_LEAF(jdouble, SharedRuntime::ddiv(jdouble x, jdouble y))
360 return x / y;
361 JRT_END
362
363 JRT_LEAF(jdouble, SharedRuntime::i2d(jint x))
364 return (jdouble)x;
365 JRT_END
366
367 JRT_LEAF(jdouble, SharedRuntime::f2d(jfloat x))
368 return (jdouble)x;
369 JRT_END
370
371 JRT_LEAF(int, SharedRuntime::fcmpl(float x, float y))
372 return x>y ? 1 : (x==y ? 0 : -1); /* x<y or is_nan*/
373 JRT_END
374
375 JRT_LEAF(int, SharedRuntime::fcmpg(float x, float y))
376 return x<y ? -1 : (x==y ? 0 : 1); /* x>y or is_nan */
377 JRT_END
378
379 JRT_LEAF(int, SharedRuntime::dcmpl(double x, double y))
380 return x>y ? 1 : (x==y ? 0 : -1); /* x<y or is_nan */
381 JRT_END
382
383 JRT_LEAF(int, SharedRuntime::dcmpg(double x, double y))
384 return x<y ? -1 : (x==y ? 0 : 1); /* x>y or is_nan */
385 JRT_END
386
387 // Functions to return the opposite of the aeabi functions for nan.
388 JRT_LEAF(int, SharedRuntime::unordered_fcmplt(float x, float y))
389 return (x < y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
390 JRT_END
391
392 JRT_LEAF(int, SharedRuntime::unordered_dcmplt(double x, double y))
393 return (x < y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
394 JRT_END
395
396 JRT_LEAF(int, SharedRuntime::unordered_fcmple(float x, float y))
397 return (x <= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
398 JRT_END
399
400 JRT_LEAF(int, SharedRuntime::unordered_dcmple(double x, double y))
401 return (x <= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
402 JRT_END
403
404 JRT_LEAF(int, SharedRuntime::unordered_fcmpge(float x, float y))
405 return (x >= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
406 JRT_END
407
408 JRT_LEAF(int, SharedRuntime::unordered_dcmpge(double x, double y))
409 return (x >= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
410 JRT_END
411
412 JRT_LEAF(int, SharedRuntime::unordered_fcmpgt(float x, float y))
413 return (x > y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
414 JRT_END
415
416 JRT_LEAF(int, SharedRuntime::unordered_dcmpgt(double x, double y))
417 return (x > y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
418 JRT_END
419
420 // Intrinsics make gcc generate code for these.
421 float SharedRuntime::fneg(float f) {
422 return -f;
423 }
424
425 double SharedRuntime::dneg(double f) {
426 return -f;
427 }
428
429 #endif // __SOFTFP__
430
431 #if defined(__SOFTFP__) || defined(E500V2)
432 // Intrinsics make gcc generate code for these.
433 double SharedRuntime::dabs(double f) {
434 return (f <= (double)0.0) ? (double)0.0 - f : f;
435 }
436
437 #endif
438
439 #if defined(__SOFTFP__) || defined(PPC)
440 double SharedRuntime::dsqrt(double f) {
441 return sqrt(f);
442 }
443 #endif
444
445 JRT_LEAF(jint, SharedRuntime::f2i(jfloat x))
446 if (g_isnan(x))
447 return 0;
448 if (x >= (jfloat) max_jint)
449 return max_jint;
450 if (x <= (jfloat) min_jint)
451 return min_jint;
452 return (jint) x;
453 JRT_END
454
455
456 JRT_LEAF(jlong, SharedRuntime::f2l(jfloat x))
457 if (g_isnan(x))
458 return 0;
459 if (x >= (jfloat) max_jlong)
460 return max_jlong;
461 if (x <= (jfloat) min_jlong)
462 return min_jlong;
463 return (jlong) x;
464 JRT_END
465
466
467 JRT_LEAF(jint, SharedRuntime::d2i(jdouble x))
468 if (g_isnan(x))
469 return 0;
470 if (x >= (jdouble) max_jint)
471 return max_jint;
472 if (x <= (jdouble) min_jint)
473 return min_jint;
474 return (jint) x;
475 JRT_END
476
477
478 JRT_LEAF(jlong, SharedRuntime::d2l(jdouble x))
479 if (g_isnan(x))
480 return 0;
481 if (x >= (jdouble) max_jlong)
482 return max_jlong;
483 if (x <= (jdouble) min_jlong)
484 return min_jlong;
485 return (jlong) x;
486 JRT_END
487
488
489 JRT_LEAF(jfloat, SharedRuntime::d2f(jdouble x))
490 return (jfloat)x;
491 JRT_END
492
493
494 JRT_LEAF(jfloat, SharedRuntime::l2f(jlong x))
495 return (jfloat)x;
496 JRT_END
497
498
499 JRT_LEAF(jdouble, SharedRuntime::l2d(jlong x))
500 return (jdouble)x;
501 JRT_END
502
503
504 // Exception handling across interpreter/compiler boundaries
505 //
506 // exception_handler_for_return_address(...) returns the continuation address.
507 // The continuation address is the entry point of the exception handler of the
508 // previous frame depending on the return address.
509
510 address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* current, address return_address) {
511 // Note: This is called when we have unwound the frame of the callee that did
512 // throw an exception. So far, no check has been performed by the StackWatermarkSet.
513 // Notably, the stack is not walkable at this point, and hence the check must
514 // be deferred until later. Specifically, any of the handlers returned here in
515 // this function, will get dispatched to, and call deferred checks to
516 // StackWatermarkSet::after_unwind at a point where the stack is walkable.
517 assert(frame::verify_return_pc(return_address), "must be a return address: " INTPTR_FORMAT, p2i(return_address));
518 assert(current->frames_to_pop_failed_realloc() == 0 || Interpreter::contains(return_address), "missed frames to pop?");
519
520 // Reset method handle flag.
521 current->set_is_method_handle_return(false);
522
523 #if INCLUDE_JVMCI
524 // JVMCI's ExceptionHandlerStub expects the thread local exception PC to be clear
525 // and other exception handler continuations do not read it
526 current->set_exception_pc(nullptr);
527 #endif // INCLUDE_JVMCI
528
529 if (Continuation::is_return_barrier_entry(return_address)) {
530 return StubRoutines::cont_returnBarrierExc();
531 }
532
533 // The fastest case first
534 CodeBlob* blob = CodeCache::find_blob(return_address);
535 nmethod* nm = (blob != nullptr) ? blob->as_nmethod_or_null() : nullptr;
536 if (nm != nullptr) {
537 // Set flag if return address is a method handle call site.
538 current->set_is_method_handle_return(nm->is_method_handle_return(return_address));
539 // native nmethods don't have exception handlers
540 assert(!nm->is_native_method() || nm->method()->is_continuation_enter_intrinsic(), "no exception handler");
541 assert(nm->header_begin() != nm->exception_begin(), "no exception handler");
542 if (nm->is_deopt_pc(return_address)) {
543 // If we come here because of a stack overflow, the stack may be
544 // unguarded. Reguard the stack otherwise if we return to the
545 // deopt blob and the stack bang causes a stack overflow we
546 // crash.
547 StackOverflow* overflow_state = current->stack_overflow_state();
548 bool guard_pages_enabled = overflow_state->reguard_stack_if_needed();
549 if (overflow_state->reserved_stack_activation() != current->stack_base()) {
550 overflow_state->set_reserved_stack_activation(current->stack_base());
551 }
552 assert(guard_pages_enabled, "stack banging in deopt blob may cause crash");
553 // The deferred StackWatermarkSet::after_unwind check will be performed in
554 // Deoptimization::fetch_unroll_info (with exec_mode == Unpack_exception)
555 return SharedRuntime::deopt_blob()->unpack_with_exception();
556 } else {
557 // The deferred StackWatermarkSet::after_unwind check will be performed in
558 // * OptoRuntime::handle_exception_C_helper for C2 code
559 // * exception_handler_for_pc_helper via Runtime1::handle_exception_from_callee_id for C1 code
560 return nm->exception_begin();
561 }
562 }
563
564 // Entry code
565 if (StubRoutines::returns_to_call_stub(return_address)) {
566 // The deferred StackWatermarkSet::after_unwind check will be performed in
567 // JavaCallWrapper::~JavaCallWrapper
568 return StubRoutines::catch_exception_entry();
569 }
570 if (blob != nullptr && blob->is_upcall_stub()) {
571 return StubRoutines::upcall_stub_exception_handler();
572 }
573 // Interpreted code
574 if (Interpreter::contains(return_address)) {
575 // The deferred StackWatermarkSet::after_unwind check will be performed in
576 // InterpreterRuntime::exception_handler_for_exception
577 return Interpreter::rethrow_exception_entry();
578 }
579
580 guarantee(blob == nullptr || !blob->is_runtime_stub(), "caller should have skipped stub");
581 guarantee(!VtableStubs::contains(return_address), "null exceptions in vtables should have been handled already!");
582
583 #ifndef PRODUCT
584 { ResourceMark rm;
585 tty->print_cr("No exception handler found for exception at " INTPTR_FORMAT " - potential problems:", p2i(return_address));
586 os::print_location(tty, (intptr_t)return_address);
587 tty->print_cr("a) exception happened in (new?) code stubs/buffers that is not handled here");
588 tty->print_cr("b) other problem");
589 }
590 #endif // PRODUCT
591 ShouldNotReachHere();
592 return nullptr;
593 }
594
595
596 JRT_LEAF(address, SharedRuntime::exception_handler_for_return_address(JavaThread* current, address return_address))
597 return raw_exception_handler_for_return_address(current, return_address);
598 JRT_END
599
600
601 address SharedRuntime::get_poll_stub(address pc) {
602 address stub;
603 // Look up the code blob
604 CodeBlob *cb = CodeCache::find_blob(pc);
605
606 // Should be an nmethod
607 guarantee(cb != nullptr && cb->is_nmethod(), "safepoint polling: pc must refer to an nmethod");
608
609 // Look up the relocation information
610 assert(cb->as_nmethod()->is_at_poll_or_poll_return(pc),
611 "safepoint polling: type must be poll at pc " INTPTR_FORMAT, p2i(pc));
612
613 #ifdef ASSERT
614 if (!((NativeInstruction*)pc)->is_safepoint_poll()) {
615 tty->print_cr("bad pc: " PTR_FORMAT, p2i(pc));
616 Disassembler::decode(cb);
617 fatal("Only polling locations are used for safepoint");
618 }
619 #endif
620
621 bool at_poll_return = cb->as_nmethod()->is_at_poll_return(pc);
622 bool has_wide_vectors = cb->as_nmethod()->has_wide_vectors();
623 if (at_poll_return) {
624 assert(SharedRuntime::polling_page_return_handler_blob() != nullptr,
625 "polling page return stub not created yet");
626 stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
627 } else if (has_wide_vectors) {
628 assert(SharedRuntime::polling_page_vectors_safepoint_handler_blob() != nullptr,
629 "polling page vectors safepoint stub not created yet");
630 stub = SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point();
631 } else {
632 assert(SharedRuntime::polling_page_safepoint_handler_blob() != nullptr,
633 "polling page safepoint stub not created yet");
634 stub = SharedRuntime::polling_page_safepoint_handler_blob()->entry_point();
635 }
636 log_debug(safepoint)("... found polling page %s exception at pc = "
637 INTPTR_FORMAT ", stub =" INTPTR_FORMAT,
638 at_poll_return ? "return" : "loop",
639 (intptr_t)pc, (intptr_t)stub);
640 return stub;
641 }
642
643 void SharedRuntime::throw_and_post_jvmti_exception(JavaThread* current, Handle h_exception) {
644 if (JvmtiExport::can_post_on_exceptions()) {
645 vframeStream vfst(current, true);
646 methodHandle method = methodHandle(current, vfst.method());
647 address bcp = method()->bcp_from(vfst.bci());
648 JvmtiExport::post_exception_throw(current, method(), bcp, h_exception());
649 }
650
651 #if INCLUDE_JVMCI
652 if (EnableJVMCI) {
653 vframeStream vfst(current, true);
654 methodHandle method = methodHandle(current, vfst.method());
655 int bci = vfst.bci();
656 MethodData* trap_mdo = method->method_data();
657 if (trap_mdo != nullptr) {
658 // Set exception_seen if the exceptional bytecode is an invoke
659 Bytecode_invoke call = Bytecode_invoke_check(method, bci);
660 if (call.is_valid()) {
661 ResourceMark rm(current);
662
663 // Lock to read ProfileData, and ensure lock is not broken by a safepoint
664 MutexLocker ml(trap_mdo->extra_data_lock(), Mutex::_no_safepoint_check_flag);
665
666 ProfileData* pdata = trap_mdo->allocate_bci_to_data(bci, nullptr);
667 if (pdata != nullptr && pdata->is_BitData()) {
668 BitData* bit_data = (BitData*) pdata;
669 bit_data->set_exception_seen();
670 }
671 }
672 }
673 }
674 #endif
675
676 Exceptions::_throw(current, __FILE__, __LINE__, h_exception);
677 }
678
679 void SharedRuntime::throw_and_post_jvmti_exception(JavaThread* current, Symbol* name, const char *message) {
680 Handle h_exception = Exceptions::new_exception(current, name, message);
681 throw_and_post_jvmti_exception(current, h_exception);
682 }
683
684 #if INCLUDE_JVMTI
685 JRT_ENTRY(void, SharedRuntime::notify_jvmti_vthread_start(oopDesc* vt, jboolean hide, JavaThread* current))
686 assert(hide == JNI_FALSE, "must be VTMS transition finish");
687 jobject vthread = JNIHandles::make_local(const_cast<oopDesc*>(vt));
688 JvmtiVTMSTransitionDisabler::VTMS_vthread_start(vthread);
689 JNIHandles::destroy_local(vthread);
690 JRT_END
691
692 JRT_ENTRY(void, SharedRuntime::notify_jvmti_vthread_end(oopDesc* vt, jboolean hide, JavaThread* current))
693 assert(hide == JNI_TRUE, "must be VTMS transition start");
694 jobject vthread = JNIHandles::make_local(const_cast<oopDesc*>(vt));
695 JvmtiVTMSTransitionDisabler::VTMS_vthread_end(vthread);
696 JNIHandles::destroy_local(vthread);
697 JRT_END
698
699 JRT_ENTRY(void, SharedRuntime::notify_jvmti_vthread_mount(oopDesc* vt, jboolean hide, JavaThread* current))
700 jobject vthread = JNIHandles::make_local(const_cast<oopDesc*>(vt));
701 JvmtiVTMSTransitionDisabler::VTMS_vthread_mount(vthread, hide);
702 JNIHandles::destroy_local(vthread);
703 JRT_END
704
705 JRT_ENTRY(void, SharedRuntime::notify_jvmti_vthread_unmount(oopDesc* vt, jboolean hide, JavaThread* current))
706 jobject vthread = JNIHandles::make_local(const_cast<oopDesc*>(vt));
707 JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount(vthread, hide);
708 JNIHandles::destroy_local(vthread);
709 JRT_END
710 #endif // INCLUDE_JVMTI
711
712 // The interpreter code to call this tracing function is only
713 // called/generated when UL is on for redefine, class and has the right level
714 // and tags. Since obsolete methods are never compiled, we don't have
715 // to modify the compilers to generate calls to this function.
716 //
717 JRT_LEAF(int, SharedRuntime::rc_trace_method_entry(
718 JavaThread* thread, Method* method))
719 if (method->is_obsolete()) {
720 // We are calling an obsolete method, but this is not necessarily
721 // an error. Our method could have been redefined just after we
722 // fetched the Method* from the constant pool.
723 ResourceMark rm;
724 log_trace(redefine, class, obsolete)("calling obsolete method '%s'", method->name_and_sig_as_C_string());
725 }
726 return 0;
727 JRT_END
728
729 // ret_pc points into caller; we are returning caller's exception handler
730 // for given exception
731 // Note that the implementation of this method assumes it's only called when an exception has actually occured
732 address SharedRuntime::compute_compiled_exc_handler(nmethod* nm, address ret_pc, Handle& exception,
733 bool force_unwind, bool top_frame_only, bool& recursive_exception_occurred) {
734 assert(nm != nullptr, "must exist");
735 ResourceMark rm;
736
737 #if INCLUDE_JVMCI
738 if (nm->is_compiled_by_jvmci()) {
739 // lookup exception handler for this pc
740 int catch_pco = pointer_delta_as_int(ret_pc, nm->code_begin());
741 ExceptionHandlerTable table(nm);
742 HandlerTableEntry *t = table.entry_for(catch_pco, -1, 0);
743 if (t != nullptr) {
744 return nm->code_begin() + t->pco();
745 } else {
746 return Deoptimization::deoptimize_for_missing_exception_handler(nm);
747 }
748 }
749 #endif // INCLUDE_JVMCI
750
751 ScopeDesc* sd = nm->scope_desc_at(ret_pc);
752 // determine handler bci, if any
753 EXCEPTION_MARK;
754
755 int handler_bci = -1;
756 int scope_depth = 0;
757 if (!force_unwind) {
758 int bci = sd->bci();
759 bool recursive_exception = false;
760 do {
761 bool skip_scope_increment = false;
762 // exception handler lookup
763 Klass* ek = exception->klass();
764 methodHandle mh(THREAD, sd->method());
765 handler_bci = Method::fast_exception_handler_bci_for(mh, ek, bci, THREAD);
766 if (HAS_PENDING_EXCEPTION) {
767 recursive_exception = true;
768 // We threw an exception while trying to find the exception handler.
769 // Transfer the new exception to the exception handle which will
770 // be set into thread local storage, and do another lookup for an
771 // exception handler for this exception, this time starting at the
772 // BCI of the exception handler which caused the exception to be
773 // thrown (bugs 4307310 and 4546590). Set "exception" reference
774 // argument to ensure that the correct exception is thrown (4870175).
775 recursive_exception_occurred = true;
776 exception = Handle(THREAD, PENDING_EXCEPTION);
777 CLEAR_PENDING_EXCEPTION;
778 if (handler_bci >= 0) {
779 bci = handler_bci;
780 handler_bci = -1;
781 skip_scope_increment = true;
782 }
783 }
784 else {
785 recursive_exception = false;
786 }
787 if (!top_frame_only && handler_bci < 0 && !skip_scope_increment) {
788 sd = sd->sender();
789 if (sd != nullptr) {
790 bci = sd->bci();
791 }
792 ++scope_depth;
793 }
794 } while (recursive_exception || (!top_frame_only && handler_bci < 0 && sd != nullptr));
795 }
796
797 // found handling method => lookup exception handler
798 int catch_pco = pointer_delta_as_int(ret_pc, nm->code_begin());
799
800 ExceptionHandlerTable table(nm);
801 HandlerTableEntry *t = table.entry_for(catch_pco, handler_bci, scope_depth);
802 if (t == nullptr && (nm->is_compiled_by_c1() || handler_bci != -1)) {
803 // Allow abbreviated catch tables. The idea is to allow a method
804 // to materialize its exceptions without committing to the exact
805 // routing of exceptions. In particular this is needed for adding
806 // a synthetic handler to unlock monitors when inlining
807 // synchronized methods since the unlock path isn't represented in
808 // the bytecodes.
809 t = table.entry_for(catch_pco, -1, 0);
810 }
811
812 #ifdef COMPILER1
813 if (t == nullptr && nm->is_compiled_by_c1()) {
814 assert(nm->unwind_handler_begin() != nullptr, "");
815 return nm->unwind_handler_begin();
816 }
817 #endif
818
819 if (t == nullptr) {
820 ttyLocker ttyl;
821 tty->print_cr("MISSING EXCEPTION HANDLER for pc " INTPTR_FORMAT " and handler bci %d, catch_pco: %d", p2i(ret_pc), handler_bci, catch_pco);
822 tty->print_cr(" Exception:");
823 exception->print();
824 tty->cr();
825 tty->print_cr(" Compiled exception table :");
826 table.print();
827 nm->print();
828 nm->print_code();
829 guarantee(false, "missing exception handler");
830 return nullptr;
831 }
832
833 if (handler_bci != -1) { // did we find a handler in this method?
834 sd->method()->set_exception_handler_entered(handler_bci); // profile
835 }
836 return nm->code_begin() + t->pco();
837 }
838
839 JRT_ENTRY(void, SharedRuntime::throw_AbstractMethodError(JavaThread* current))
840 // These errors occur only at call sites
841 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_AbstractMethodError());
842 JRT_END
843
844 JRT_ENTRY(void, SharedRuntime::throw_IncompatibleClassChangeError(JavaThread* current))
845 // These errors occur only at call sites
846 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IncompatibleClassChangeError(), "vtable stub");
847 JRT_END
848
849 JRT_ENTRY(void, SharedRuntime::throw_ArithmeticException(JavaThread* current))
850 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ArithmeticException(), "/ by zero");
851 JRT_END
852
853 JRT_ENTRY(void, SharedRuntime::throw_NullPointerException(JavaThread* current))
854 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException(), nullptr);
855 JRT_END
856
857 JRT_ENTRY(void, SharedRuntime::throw_NullPointerException_at_call(JavaThread* current))
858 // This entry point is effectively only used for NullPointerExceptions which occur at inline
859 // cache sites (when the callee activation is not yet set up) so we are at a call site
860 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException(), nullptr);
861 JRT_END
862
863 JRT_ENTRY(void, SharedRuntime::throw_StackOverflowError(JavaThread* current))
864 throw_StackOverflowError_common(current, false);
865 JRT_END
866
867 JRT_ENTRY(void, SharedRuntime::throw_delayed_StackOverflowError(JavaThread* current))
868 throw_StackOverflowError_common(current, true);
869 JRT_END
870
871 void SharedRuntime::throw_StackOverflowError_common(JavaThread* current, bool delayed) {
872 // We avoid using the normal exception construction in this case because
873 // it performs an upcall to Java, and we're already out of stack space.
874 JavaThread* THREAD = current; // For exception macros.
875 Klass* k = vmClasses::StackOverflowError_klass();
876 oop exception_oop = InstanceKlass::cast(k)->allocate_instance(CHECK);
877 if (delayed) {
878 java_lang_Throwable::set_message(exception_oop,
879 Universe::delayed_stack_overflow_error_message());
880 }
881 Handle exception (current, exception_oop);
882 if (StackTraceInThrowable) {
883 java_lang_Throwable::fill_in_stack_trace(exception);
884 }
885 // Remove the ScopedValue bindings in case we got a
886 // StackOverflowError while we were trying to remove ScopedValue
887 // bindings.
888 current->clear_scopedValueBindings();
889 // Increment counter for hs_err file reporting
890 Atomic::inc(&Exceptions::_stack_overflow_errors);
891 throw_and_post_jvmti_exception(current, exception);
892 }
893
894 address SharedRuntime::continuation_for_implicit_exception(JavaThread* current,
895 address pc,
896 ImplicitExceptionKind exception_kind)
897 {
898 address target_pc = nullptr;
899
900 if (Interpreter::contains(pc)) {
901 switch (exception_kind) {
902 case IMPLICIT_NULL: return Interpreter::throw_NullPointerException_entry();
903 case IMPLICIT_DIVIDE_BY_ZERO: return Interpreter::throw_ArithmeticException_entry();
904 case STACK_OVERFLOW: return Interpreter::throw_StackOverflowError_entry();
905 default: ShouldNotReachHere();
906 }
907 } else {
908 switch (exception_kind) {
909 case STACK_OVERFLOW: {
910 // Stack overflow only occurs upon frame setup; the callee is
911 // going to be unwound. Dispatch to a shared runtime stub
912 // which will cause the StackOverflowError to be fabricated
913 // and processed.
914 // Stack overflow should never occur during deoptimization:
915 // the compiled method bangs the stack by as much as the
916 // interpreter would need in case of a deoptimization. The
917 // deoptimization blob and uncommon trap blob bang the stack
918 // in a debug VM to verify the correctness of the compiled
919 // method stack banging.
920 assert(current->deopt_mark() == nullptr, "no stack overflow from deopt blob/uncommon trap");
921 Events::log_exception(current, "StackOverflowError at " INTPTR_FORMAT, p2i(pc));
922 return SharedRuntime::throw_StackOverflowError_entry();
923 }
924
925 case IMPLICIT_NULL: {
926 if (VtableStubs::contains(pc)) {
927 // We haven't yet entered the callee frame. Fabricate an
928 // exception and begin dispatching it in the caller. Since
929 // the caller was at a call site, it's safe to destroy all
930 // caller-saved registers, as these entry points do.
931 VtableStub* vt_stub = VtableStubs::stub_containing(pc);
932
933 // If vt_stub is null, then return null to signal handler to report the SEGV error.
934 if (vt_stub == nullptr) return nullptr;
935
936 if (vt_stub->is_abstract_method_error(pc)) {
937 assert(!vt_stub->is_vtable_stub(), "should never see AbstractMethodErrors from vtable-type VtableStubs");
938 Events::log_exception(current, "AbstractMethodError at " INTPTR_FORMAT, p2i(pc));
939 // Instead of throwing the abstract method error here directly, we re-resolve
940 // and will throw the AbstractMethodError during resolve. As a result, we'll
941 // get a more detailed error message.
942 return SharedRuntime::get_handle_wrong_method_stub();
943 } else {
944 Events::log_exception(current, "NullPointerException at vtable entry " INTPTR_FORMAT, p2i(pc));
945 // Assert that the signal comes from the expected location in stub code.
946 assert(vt_stub->is_null_pointer_exception(pc),
947 "obtained signal from unexpected location in stub code");
948 return SharedRuntime::throw_NullPointerException_at_call_entry();
949 }
950 } else {
951 CodeBlob* cb = CodeCache::find_blob(pc);
952
953 // If code blob is null, then return null to signal handler to report the SEGV error.
954 if (cb == nullptr) return nullptr;
955
956 // Exception happened in CodeCache. Must be either:
957 // 1. Inline-cache check in C2I handler blob,
958 // 2. Inline-cache check in nmethod, or
959 // 3. Implicit null exception in nmethod
960
961 if (!cb->is_nmethod()) {
962 bool is_in_blob = cb->is_adapter_blob() || cb->is_method_handles_adapter_blob();
963 if (!is_in_blob) {
964 // Allow normal crash reporting to handle this
965 return nullptr;
966 }
967 Events::log_exception(current, "NullPointerException in code blob at " INTPTR_FORMAT, p2i(pc));
968 // There is no handler here, so we will simply unwind.
969 return SharedRuntime::throw_NullPointerException_at_call_entry();
970 }
971
972 // Otherwise, it's a compiled method. Consult its exception handlers.
973 nmethod* nm = cb->as_nmethod();
974 if (nm->inlinecache_check_contains(pc)) {
975 // exception happened inside inline-cache check code
976 // => the nmethod is not yet active (i.e., the frame
977 // is not set up yet) => use return address pushed by
978 // caller => don't push another return address
979 Events::log_exception(current, "NullPointerException in IC check " INTPTR_FORMAT, p2i(pc));
980 return SharedRuntime::throw_NullPointerException_at_call_entry();
981 }
982
983 if (nm->method()->is_method_handle_intrinsic()) {
984 // exception happened inside MH dispatch code, similar to a vtable stub
985 Events::log_exception(current, "NullPointerException in MH adapter " INTPTR_FORMAT, p2i(pc));
986 return SharedRuntime::throw_NullPointerException_at_call_entry();
987 }
988
989 #ifndef PRODUCT
990 _implicit_null_throws++;
991 #endif
992 target_pc = nm->continuation_for_implicit_null_exception(pc);
993 // If there's an unexpected fault, target_pc might be null,
994 // in which case we want to fall through into the normal
995 // error handling code.
996 }
997
998 break; // fall through
999 }
1000
1001
1002 case IMPLICIT_DIVIDE_BY_ZERO: {
1003 nmethod* nm = CodeCache::find_nmethod(pc);
1004 guarantee(nm != nullptr, "must have containing compiled method for implicit division-by-zero exceptions");
1005 #ifndef PRODUCT
1006 _implicit_div0_throws++;
1007 #endif
1008 target_pc = nm->continuation_for_implicit_div0_exception(pc);
1009 // If there's an unexpected fault, target_pc might be null,
1010 // in which case we want to fall through into the normal
1011 // error handling code.
1012 break; // fall through
1013 }
1014
1015 default: ShouldNotReachHere();
1016 }
1017
1018 assert(exception_kind == IMPLICIT_NULL || exception_kind == IMPLICIT_DIVIDE_BY_ZERO, "wrong implicit exception kind");
1019
1020 if (exception_kind == IMPLICIT_NULL) {
1021 #ifndef PRODUCT
1022 // for AbortVMOnException flag
1023 Exceptions::debug_check_abort("java.lang.NullPointerException");
1024 #endif //PRODUCT
1025 Events::log_exception(current, "Implicit null exception at " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(pc), p2i(target_pc));
1026 } else {
1027 #ifndef PRODUCT
1028 // for AbortVMOnException flag
1029 Exceptions::debug_check_abort("java.lang.ArithmeticException");
1030 #endif //PRODUCT
1031 Events::log_exception(current, "Implicit division by zero exception at " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(pc), p2i(target_pc));
1032 }
1033 return target_pc;
1034 }
1035
1036 ShouldNotReachHere();
1037 return nullptr;
1038 }
1039
1040
1041 /**
1042 * Throws an java/lang/UnsatisfiedLinkError. The address of this method is
1043 * installed in the native function entry of all native Java methods before
1044 * they get linked to their actual native methods.
1045 *
1046 * \note
1047 * This method actually never gets called! The reason is because
1048 * the interpreter's native entries call NativeLookup::lookup() which
1049 * throws the exception when the lookup fails. The exception is then
1050 * caught and forwarded on the return from NativeLookup::lookup() call
1051 * before the call to the native function. This might change in the future.
1052 */
1053 JNI_ENTRY(void*, throw_unsatisfied_link_error(JNIEnv* env, ...))
1054 {
1055 // We return a bad value here to make sure that the exception is
1056 // forwarded before we look at the return value.
1057 THROW_(vmSymbols::java_lang_UnsatisfiedLinkError(), (void*)badAddress);
1058 }
1059 JNI_END
1060
1061 address SharedRuntime::native_method_throw_unsatisfied_link_error_entry() {
1062 return CAST_FROM_FN_PTR(address, &throw_unsatisfied_link_error);
1063 }
1064
1065 JRT_ENTRY_NO_ASYNC(void, SharedRuntime::register_finalizer(JavaThread* current, oopDesc* obj))
1066 #if INCLUDE_JVMCI
1067 if (!obj->klass()->has_finalizer()) {
1068 return;
1069 }
1070 #endif // INCLUDE_JVMCI
1071 assert(oopDesc::is_oop(obj), "must be a valid oop");
1072 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
1073 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
1074 JRT_END
1075
1076 jlong SharedRuntime::get_java_tid(JavaThread* thread) {
1077 assert(thread != nullptr, "No thread");
1078 if (thread == nullptr) {
1079 return 0;
1080 }
1081 guarantee(Thread::current() != thread || thread->is_oop_safe(),
1082 "current cannot touch oops after its GC barrier is detached.");
1083 oop obj = thread->threadObj();
1084 return (obj == nullptr) ? 0 : java_lang_Thread::thread_id(obj);
1085 }
1086
1087 /**
1088 * This function ought to be a void function, but cannot be because
1089 * it gets turned into a tail-call on sparc, which runs into dtrace bug
1090 * 6254741. Once that is fixed we can remove the dummy return value.
1091 */
1092 int SharedRuntime::dtrace_object_alloc(oopDesc* o) {
1093 return dtrace_object_alloc(JavaThread::current(), o, o->size());
1094 }
1095
1096 int SharedRuntime::dtrace_object_alloc(JavaThread* thread, oopDesc* o) {
1097 return dtrace_object_alloc(thread, o, o->size());
1098 }
1099
1100 int SharedRuntime::dtrace_object_alloc(JavaThread* thread, oopDesc* o, size_t size) {
1101 assert(DTraceAllocProbes, "wrong call");
1102 Klass* klass = o->klass();
1103 Symbol* name = klass->name();
1104 HOTSPOT_OBJECT_ALLOC(
1105 get_java_tid(thread),
1106 (char *) name->bytes(), name->utf8_length(), size * HeapWordSize);
1107 return 0;
1108 }
1109
1110 JRT_LEAF(int, SharedRuntime::dtrace_method_entry(
1111 JavaThread* current, Method* method))
1112 assert(current == JavaThread::current(), "pre-condition");
1113
1114 assert(DTraceMethodProbes, "wrong call");
1115 Symbol* kname = method->klass_name();
1116 Symbol* name = method->name();
1117 Symbol* sig = method->signature();
1118 HOTSPOT_METHOD_ENTRY(
1119 get_java_tid(current),
1120 (char *) kname->bytes(), kname->utf8_length(),
1121 (char *) name->bytes(), name->utf8_length(),
1122 (char *) sig->bytes(), sig->utf8_length());
1123 return 0;
1124 JRT_END
1125
1126 JRT_LEAF(int, SharedRuntime::dtrace_method_exit(
1127 JavaThread* current, Method* method))
1128 assert(current == JavaThread::current(), "pre-condition");
1129 assert(DTraceMethodProbes, "wrong call");
1130 Symbol* kname = method->klass_name();
1131 Symbol* name = method->name();
1132 Symbol* sig = method->signature();
1133 HOTSPOT_METHOD_RETURN(
1134 get_java_tid(current),
1135 (char *) kname->bytes(), kname->utf8_length(),
1136 (char *) name->bytes(), name->utf8_length(),
1137 (char *) sig->bytes(), sig->utf8_length());
1138 return 0;
1139 JRT_END
1140
1141
1142 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode)
1143 // for a call current in progress, i.e., arguments has been pushed on stack
1144 // put callee has not been invoked yet. Used by: resolve virtual/static,
1145 // vtable updates, etc. Caller frame must be compiled.
1146 Handle SharedRuntime::find_callee_info(Bytecodes::Code& bc, CallInfo& callinfo, TRAPS) {
1147 JavaThread* current = THREAD;
1148 ResourceMark rm(current);
1149
1150 // last java frame on stack (which includes native call frames)
1151 vframeStream vfst(current, true); // Do not skip and javaCalls
1152
1153 return find_callee_info_helper(vfst, bc, callinfo, THREAD);
1154 }
1155
1156 Method* SharedRuntime::extract_attached_method(vframeStream& vfst) {
1157 nmethod* caller = vfst.nm();
1158
1159 address pc = vfst.frame_pc();
1160 { // Get call instruction under lock because another thread may be busy patching it.
1161 CompiledICLocker ic_locker(caller);
1162 return caller->attached_method_before_pc(pc);
1163 }
1164 return nullptr;
1165 }
1166
1167 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode
1168 // for a call current in progress, i.e., arguments has been pushed on stack
1169 // but callee has not been invoked yet. Caller frame must be compiled.
1170 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1171 CallInfo& callinfo, TRAPS) {
1172 Handle receiver;
1173 Handle nullHandle; // create a handy null handle for exception returns
1174 JavaThread* current = THREAD;
1175
1176 assert(!vfst.at_end(), "Java frame must exist");
1177
1178 // Find caller and bci from vframe
1179 methodHandle caller(current, vfst.method());
1180 int bci = vfst.bci();
1181
1182 if (caller->is_continuation_enter_intrinsic()) {
1183 bc = Bytecodes::_invokestatic;
1184 LinkResolver::resolve_continuation_enter(callinfo, CHECK_NH);
1185 return receiver;
1186 }
1187
1188 Bytecode_invoke bytecode(caller, bci);
1189 int bytecode_index = bytecode.index();
1190 bc = bytecode.invoke_code();
1191
1192 methodHandle attached_method(current, extract_attached_method(vfst));
1193 if (attached_method.not_null()) {
1194 Method* callee = bytecode.static_target(CHECK_NH);
1195 vmIntrinsics::ID id = callee->intrinsic_id();
1196 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1197 // it attaches statically resolved method to the call site.
1198 if (MethodHandles::is_signature_polymorphic(id) &&
1199 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1200 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1201
1202 // Adjust invocation mode according to the attached method.
1203 switch (bc) {
1204 case Bytecodes::_invokevirtual:
1205 if (attached_method->method_holder()->is_interface()) {
1206 bc = Bytecodes::_invokeinterface;
1207 }
1208 break;
1209 case Bytecodes::_invokeinterface:
1210 if (!attached_method->method_holder()->is_interface()) {
1211 bc = Bytecodes::_invokevirtual;
1212 }
1213 break;
1214 case Bytecodes::_invokehandle:
1215 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1216 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1217 : Bytecodes::_invokevirtual;
1218 }
1219 break;
1220 default:
1221 break;
1222 }
1223 }
1224 }
1225
1226 assert(bc != Bytecodes::_illegal, "not initialized");
1227
1228 bool has_receiver = bc != Bytecodes::_invokestatic &&
1229 bc != Bytecodes::_invokedynamic &&
1230 bc != Bytecodes::_invokehandle;
1231
1232 // Find receiver for non-static call
1233 if (has_receiver) {
1234 // This register map must be update since we need to find the receiver for
1235 // compiled frames. The receiver might be in a register.
1236 RegisterMap reg_map2(current,
1237 RegisterMap::UpdateMap::include,
1238 RegisterMap::ProcessFrames::include,
1239 RegisterMap::WalkContinuation::skip);
1240 frame stubFrame = current->last_frame();
1241 // Caller-frame is a compiled frame
1242 frame callerFrame = stubFrame.sender(®_map2);
1243
1244 if (attached_method.is_null()) {
1245 Method* callee = bytecode.static_target(CHECK_NH);
1246 if (callee == nullptr) {
1247 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1248 }
1249 }
1250
1251 // Retrieve from a compiled argument list
1252 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1253 assert(oopDesc::is_oop_or_null(receiver()), "");
1254
1255 if (receiver.is_null()) {
1256 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1257 }
1258 }
1259
1260 // Resolve method
1261 if (attached_method.not_null()) {
1262 // Parameterized by attached method.
1263 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, CHECK_NH);
1264 } else {
1265 // Parameterized by bytecode.
1266 constantPoolHandle constants(current, caller->constants());
1267 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1268 }
1269
1270 #ifdef ASSERT
1271 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1272 if (has_receiver) {
1273 assert(receiver.not_null(), "should have thrown exception");
1274 Klass* receiver_klass = receiver->klass();
1275 Klass* rk = nullptr;
1276 if (attached_method.not_null()) {
1277 // In case there's resolved method attached, use its holder during the check.
1278 rk = attached_method->method_holder();
1279 } else {
1280 // Klass is already loaded.
1281 constantPoolHandle constants(current, caller->constants());
1282 rk = constants->klass_ref_at(bytecode_index, bc, CHECK_NH);
1283 }
1284 Klass* static_receiver_klass = rk;
1285 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1286 "actual receiver must be subclass of static receiver klass");
1287 if (receiver_klass->is_instance_klass()) {
1288 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1289 tty->print_cr("ERROR: Klass not yet initialized!!");
1290 receiver_klass->print();
1291 }
1292 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1293 }
1294 }
1295 #endif
1296
1297 return receiver;
1298 }
1299
1300 methodHandle SharedRuntime::find_callee_method(TRAPS) {
1301 JavaThread* current = THREAD;
1302 ResourceMark rm(current);
1303 // We need first to check if any Java activations (compiled, interpreted)
1304 // exist on the stack since last JavaCall. If not, we need
1305 // to get the target method from the JavaCall wrapper.
1306 vframeStream vfst(current, true); // Do not skip any javaCalls
1307 methodHandle callee_method;
1308 if (vfst.at_end()) {
1309 // No Java frames were found on stack since we did the JavaCall.
1310 // Hence the stack can only contain an entry_frame. We need to
1311 // find the target method from the stub frame.
1312 RegisterMap reg_map(current,
1313 RegisterMap::UpdateMap::skip,
1314 RegisterMap::ProcessFrames::include,
1315 RegisterMap::WalkContinuation::skip);
1316 frame fr = current->last_frame();
1317 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1318 fr = fr.sender(®_map);
1319 assert(fr.is_entry_frame(), "must be");
1320 // fr is now pointing to the entry frame.
1321 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1322 } else {
1323 Bytecodes::Code bc;
1324 CallInfo callinfo;
1325 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1326 callee_method = methodHandle(current, callinfo.selected_method());
1327 }
1328 assert(callee_method()->is_method(), "must be");
1329 return callee_method;
1330 }
1331
1332 // Resolves a call.
1333 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, TRAPS) {
1334 JavaThread* current = THREAD;
1335 ResourceMark rm(current);
1336 RegisterMap cbl_map(current,
1337 RegisterMap::UpdateMap::skip,
1338 RegisterMap::ProcessFrames::include,
1339 RegisterMap::WalkContinuation::skip);
1340 frame caller_frame = current->last_frame().sender(&cbl_map);
1341
1342 CodeBlob* caller_cb = caller_frame.cb();
1343 guarantee(caller_cb != nullptr && caller_cb->is_nmethod(), "must be called from compiled method");
1344 nmethod* caller_nm = caller_cb->as_nmethod();
1345
1346 // determine call info & receiver
1347 // note: a) receiver is null for static calls
1348 // b) an exception is thrown if receiver is null for non-static calls
1349 CallInfo call_info;
1350 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1351 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1352
1353 NoSafepointVerifier nsv;
1354
1355 methodHandle callee_method(current, call_info.selected_method());
1356
1357 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1358 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1359 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1360 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1361 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1362
1363 assert(!caller_nm->is_unloading(), "It should not be unloading");
1364
1365 #ifndef PRODUCT
1366 // tracing/debugging/statistics
1367 uint *addr = (is_optimized) ? (&_resolve_opt_virtual_ctr) :
1368 (is_virtual) ? (&_resolve_virtual_ctr) :
1369 (&_resolve_static_ctr);
1370 Atomic::inc(addr);
1371
1372 if (TraceCallFixup) {
1373 ResourceMark rm(current);
1374 tty->print("resolving %s%s (%s) call to",
1375 (is_optimized) ? "optimized " : "", (is_virtual) ? "virtual" : "static",
1376 Bytecodes::name(invoke_code));
1377 callee_method->print_short_name(tty);
1378 tty->print_cr(" at pc: " INTPTR_FORMAT " to code: " INTPTR_FORMAT,
1379 p2i(caller_frame.pc()), p2i(callee_method->code()));
1380 }
1381 #endif
1382
1383 if (invoke_code == Bytecodes::_invokestatic) {
1384 assert(callee_method->method_holder()->is_initialized() ||
1385 callee_method->method_holder()->is_reentrant_initialization(current),
1386 "invalid class initialization state for invoke_static");
1387 if (!VM_Version::supports_fast_class_init_checks() && callee_method->needs_clinit_barrier()) {
1388 // In order to keep class initialization check, do not patch call
1389 // site for static call when the class is not fully initialized.
1390 // Proper check is enforced by call site re-resolution on every invocation.
1391 //
1392 // When fast class initialization checks are supported (VM_Version::supports_fast_class_init_checks() == true),
1393 // explicit class initialization check is put in nmethod entry (VEP).
1394 assert(callee_method->method_holder()->is_linked(), "must be");
1395 return callee_method;
1396 }
1397 }
1398
1399
1400 // JSR 292 key invariant:
1401 // If the resolved method is a MethodHandle invoke target, the call
1402 // site must be a MethodHandle call site, because the lambda form might tail-call
1403 // leaving the stack in a state unknown to either caller or callee
1404
1405 // Compute entry points. The computation of the entry points is independent of
1406 // patching the call.
1407
1408 // Make sure the callee nmethod does not get deoptimized and removed before
1409 // we are done patching the code.
1410
1411
1412 CompiledICLocker ml(caller_nm);
1413 if (is_virtual && !is_optimized) {
1414 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1415 inline_cache->update(&call_info, receiver->klass());
1416 } else {
1417 // Callsite is a direct call - set it to the destination method
1418 CompiledDirectCall* callsite = CompiledDirectCall::before(caller_frame.pc());
1419 callsite->set(callee_method);
1420 }
1421
1422 return callee_method;
1423 }
1424
1425 // Inline caches exist only in compiled code
1426 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1427 #ifdef ASSERT
1428 RegisterMap reg_map(current,
1429 RegisterMap::UpdateMap::skip,
1430 RegisterMap::ProcessFrames::include,
1431 RegisterMap::WalkContinuation::skip);
1432 frame stub_frame = current->last_frame();
1433 assert(stub_frame.is_runtime_frame(), "sanity check");
1434 frame caller_frame = stub_frame.sender(®_map);
1435 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_upcall_stub_frame(), "unexpected frame");
1436 #endif /* ASSERT */
1437
1438 methodHandle callee_method;
1439 JRT_BLOCK
1440 callee_method = SharedRuntime::handle_ic_miss_helper(CHECK_NULL);
1441 // Return Method* through TLS
1442 current->set_vm_result_2(callee_method());
1443 JRT_BLOCK_END
1444 // return compiled code entry point after potential safepoints
1445 return get_resolved_entry(current, callee_method);
1446 JRT_END
1447
1448
1449 // Handle call site that has been made non-entrant
1450 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1451 // 6243940 We might end up in here if the callee is deoptimized
1452 // as we race to call it. We don't want to take a safepoint if
1453 // the caller was interpreted because the caller frame will look
1454 // interpreted to the stack walkers and arguments are now
1455 // "compiled" so it is much better to make this transition
1456 // invisible to the stack walking code. The i2c path will
1457 // place the callee method in the callee_target. It is stashed
1458 // there because if we try and find the callee by normal means a
1459 // safepoint is possible and have trouble gc'ing the compiled args.
1460 RegisterMap reg_map(current,
1461 RegisterMap::UpdateMap::skip,
1462 RegisterMap::ProcessFrames::include,
1463 RegisterMap::WalkContinuation::skip);
1464 frame stub_frame = current->last_frame();
1465 assert(stub_frame.is_runtime_frame(), "sanity check");
1466 frame caller_frame = stub_frame.sender(®_map);
1467
1468 if (caller_frame.is_interpreted_frame() ||
1469 caller_frame.is_entry_frame() ||
1470 caller_frame.is_upcall_stub_frame()) {
1471 Method* callee = current->callee_target();
1472 guarantee(callee != nullptr && callee->is_method(), "bad handshake");
1473 current->set_vm_result_2(callee);
1474 current->set_callee_target(nullptr);
1475 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1476 // Bypass class initialization checks in c2i when caller is in native.
1477 // JNI calls to static methods don't have class initialization checks.
1478 // Fast class initialization checks are present in c2i adapters and call into
1479 // SharedRuntime::handle_wrong_method() on the slow path.
1480 //
1481 // JVM upcalls may land here as well, but there's a proper check present in
1482 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1483 // so bypassing it in c2i adapter is benign.
1484 return callee->get_c2i_no_clinit_check_entry();
1485 } else {
1486 return callee->get_c2i_entry();
1487 }
1488 }
1489
1490 // Must be compiled to compiled path which is safe to stackwalk
1491 methodHandle callee_method;
1492 JRT_BLOCK
1493 // Force resolving of caller (if we called from compiled frame)
1494 callee_method = SharedRuntime::reresolve_call_site(CHECK_NULL);
1495 current->set_vm_result_2(callee_method());
1496 JRT_BLOCK_END
1497 // return compiled code entry point after potential safepoints
1498 return get_resolved_entry(current, callee_method);
1499 JRT_END
1500
1501 // Handle abstract method call
1502 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1503 // Verbose error message for AbstractMethodError.
1504 // Get the called method from the invoke bytecode.
1505 vframeStream vfst(current, true);
1506 assert(!vfst.at_end(), "Java frame must exist");
1507 methodHandle caller(current, vfst.method());
1508 Bytecode_invoke invoke(caller, vfst.bci());
1509 DEBUG_ONLY( invoke.verify(); )
1510
1511 // Find the compiled caller frame.
1512 RegisterMap reg_map(current,
1513 RegisterMap::UpdateMap::include,
1514 RegisterMap::ProcessFrames::include,
1515 RegisterMap::WalkContinuation::skip);
1516 frame stubFrame = current->last_frame();
1517 assert(stubFrame.is_runtime_frame(), "must be");
1518 frame callerFrame = stubFrame.sender(®_map);
1519 assert(callerFrame.is_compiled_frame(), "must be");
1520
1521 // Install exception and return forward entry.
1522 address res = SharedRuntime::throw_AbstractMethodError_entry();
1523 JRT_BLOCK
1524 methodHandle callee(current, invoke.static_target(current));
1525 if (!callee.is_null()) {
1526 oop recv = callerFrame.retrieve_receiver(®_map);
1527 Klass *recv_klass = (recv != nullptr) ? recv->klass() : nullptr;
1528 res = StubRoutines::forward_exception_entry();
1529 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1530 }
1531 JRT_BLOCK_END
1532 return res;
1533 JRT_END
1534
1535 // return verified_code_entry if interp_only_mode is not set for the current thread;
1536 // otherwise return c2i entry.
1537 address SharedRuntime::get_resolved_entry(JavaThread* current, methodHandle callee_method) {
1538 if (current->is_interp_only_mode() && !callee_method->is_special_native_intrinsic()) {
1539 // In interp_only_mode we need to go to the interpreted entry
1540 // The c2i won't patch in this mode -- see fixup_callers_callsite
1541 return callee_method->get_c2i_entry();
1542 }
1543 assert(callee_method->verified_code_entry() != nullptr, " Jump to zero!");
1544 return callee_method->verified_code_entry();
1545 }
1546
1547 // resolve a static call and patch code
1548 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1549 methodHandle callee_method;
1550 bool enter_special = false;
1551 JRT_BLOCK
1552 callee_method = SharedRuntime::resolve_helper(false, false, CHECK_NULL);
1553 current->set_vm_result_2(callee_method());
1554 JRT_BLOCK_END
1555 // return compiled code entry point after potential safepoints
1556 return get_resolved_entry(current, callee_method);
1557 JRT_END
1558
1559 // resolve virtual call and update inline cache to monomorphic
1560 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1561 methodHandle callee_method;
1562 JRT_BLOCK
1563 callee_method = SharedRuntime::resolve_helper(true, false, CHECK_NULL);
1564 current->set_vm_result_2(callee_method());
1565 JRT_BLOCK_END
1566 // return compiled code entry point after potential safepoints
1567 return get_resolved_entry(current, callee_method);
1568 JRT_END
1569
1570
1571 // Resolve a virtual call that can be statically bound (e.g., always
1572 // monomorphic, so it has no inline cache). Patch code to resolved target.
1573 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1574 methodHandle callee_method;
1575 JRT_BLOCK
1576 callee_method = SharedRuntime::resolve_helper(true, true, CHECK_NULL);
1577 current->set_vm_result_2(callee_method());
1578 JRT_BLOCK_END
1579 // return compiled code entry point after potential safepoints
1580 return get_resolved_entry(current, callee_method);
1581 JRT_END
1582
1583 methodHandle SharedRuntime::handle_ic_miss_helper(TRAPS) {
1584 JavaThread* current = THREAD;
1585 ResourceMark rm(current);
1586 CallInfo call_info;
1587 Bytecodes::Code bc;
1588
1589 // receiver is null for static calls. An exception is thrown for null
1590 // receivers for non-static calls
1591 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1592
1593 methodHandle callee_method(current, call_info.selected_method());
1594
1595 #ifndef PRODUCT
1596 Atomic::inc(&_ic_miss_ctr);
1597
1598 // Statistics & Tracing
1599 if (TraceCallFixup) {
1600 ResourceMark rm(current);
1601 tty->print("IC miss (%s) call to", Bytecodes::name(bc));
1602 callee_method->print_short_name(tty);
1603 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1604 }
1605
1606 if (ICMissHistogram) {
1607 MutexLocker m(VMStatistic_lock);
1608 RegisterMap reg_map(current,
1609 RegisterMap::UpdateMap::skip,
1610 RegisterMap::ProcessFrames::include,
1611 RegisterMap::WalkContinuation::skip);
1612 frame f = current->last_frame().real_sender(®_map);// skip runtime stub
1613 // produce statistics under the lock
1614 trace_ic_miss(f.pc());
1615 }
1616 #endif
1617
1618 // install an event collector so that when a vtable stub is created the
1619 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1620 // event can't be posted when the stub is created as locks are held
1621 // - instead the event will be deferred until the event collector goes
1622 // out of scope.
1623 JvmtiDynamicCodeEventCollector event_collector;
1624
1625 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1626 RegisterMap reg_map(current,
1627 RegisterMap::UpdateMap::skip,
1628 RegisterMap::ProcessFrames::include,
1629 RegisterMap::WalkContinuation::skip);
1630 frame caller_frame = current->last_frame().sender(®_map);
1631 CodeBlob* cb = caller_frame.cb();
1632 nmethod* caller_nm = cb->as_nmethod();
1633
1634 CompiledICLocker ml(caller_nm);
1635 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1636 inline_cache->update(&call_info, receiver()->klass());
1637
1638 return callee_method;
1639 }
1640
1641 //
1642 // Resets a call-site in compiled code so it will get resolved again.
1643 // This routines handles both virtual call sites, optimized virtual call
1644 // sites, and static call sites. Typically used to change a call sites
1645 // destination from compiled to interpreted.
1646 //
1647 methodHandle SharedRuntime::reresolve_call_site(TRAPS) {
1648 JavaThread* current = THREAD;
1649 ResourceMark rm(current);
1650 RegisterMap reg_map(current,
1651 RegisterMap::UpdateMap::skip,
1652 RegisterMap::ProcessFrames::include,
1653 RegisterMap::WalkContinuation::skip);
1654 frame stub_frame = current->last_frame();
1655 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1656 frame caller = stub_frame.sender(®_map);
1657
1658 // Do nothing if the frame isn't a live compiled frame.
1659 // nmethod could be deoptimized by the time we get here
1660 // so no update to the caller is needed.
1661
1662 if ((caller.is_compiled_frame() && !caller.is_deoptimized_frame()) ||
1663 (caller.is_native_frame() && caller.cb()->as_nmethod()->method()->is_continuation_enter_intrinsic())) {
1664
1665 address pc = caller.pc();
1666
1667 nmethod* caller_nm = CodeCache::find_nmethod(pc);
1668 assert(caller_nm != nullptr, "did not find caller nmethod");
1669
1670 // Default call_addr is the location of the "basic" call.
1671 // Determine the address of the call we a reresolving. With
1672 // Inline Caches we will always find a recognizable call.
1673 // With Inline Caches disabled we may or may not find a
1674 // recognizable call. We will always find a call for static
1675 // calls and for optimized virtual calls. For vanilla virtual
1676 // calls it depends on the state of the UseInlineCaches switch.
1677 //
1678 // With Inline Caches disabled we can get here for a virtual call
1679 // for two reasons:
1680 // 1 - calling an abstract method. The vtable for abstract methods
1681 // will run us thru handle_wrong_method and we will eventually
1682 // end up in the interpreter to throw the ame.
1683 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1684 // call and between the time we fetch the entry address and
1685 // we jump to it the target gets deoptimized. Similar to 1
1686 // we will wind up in the interprter (thru a c2i with c2).
1687 //
1688 CompiledICLocker ml(caller_nm);
1689 address call_addr = caller_nm->call_instruction_address(pc);
1690
1691 if (call_addr != nullptr) {
1692 // On x86 the logic for finding a call instruction is blindly checking for a call opcode 5
1693 // bytes back in the instruction stream so we must also check for reloc info.
1694 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1695 bool ret = iter.next(); // Get item
1696 if (ret) {
1697 switch (iter.type()) {
1698 case relocInfo::static_call_type:
1699 case relocInfo::opt_virtual_call_type: {
1700 CompiledDirectCall* cdc = CompiledDirectCall::at(call_addr);
1701 cdc->set_to_clean();
1702 break;
1703 }
1704
1705 case relocInfo::virtual_call_type: {
1706 // compiled, dispatched call (which used to call an interpreted method)
1707 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1708 inline_cache->set_to_clean();
1709 break;
1710 }
1711 default:
1712 break;
1713 }
1714 }
1715 }
1716 }
1717
1718 methodHandle callee_method = find_callee_method(CHECK_(methodHandle()));
1719
1720
1721 #ifndef PRODUCT
1722 Atomic::inc(&_wrong_method_ctr);
1723
1724 if (TraceCallFixup) {
1725 ResourceMark rm(current);
1726 tty->print("handle_wrong_method reresolving call to");
1727 callee_method->print_short_name(tty);
1728 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1729 }
1730 #endif
1731
1732 return callee_method;
1733 }
1734
1735 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1736 // The faulting unsafe accesses should be changed to throw the error
1737 // synchronously instead. Meanwhile the faulting instruction will be
1738 // skipped over (effectively turning it into a no-op) and an
1739 // asynchronous exception will be raised which the thread will
1740 // handle at a later point. If the instruction is a load it will
1741 // return garbage.
1742
1743 // Request an async exception.
1744 thread->set_pending_unsafe_access_error();
1745
1746 // Return address of next instruction to execute.
1747 return next_pc;
1748 }
1749
1750 #ifdef ASSERT
1751 void SharedRuntime::check_member_name_argument_is_last_argument(const methodHandle& method,
1752 const BasicType* sig_bt,
1753 const VMRegPair* regs) {
1754 ResourceMark rm;
1755 const int total_args_passed = method->size_of_parameters();
1756 const VMRegPair* regs_with_member_name = regs;
1757 VMRegPair* regs_without_member_name = NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed - 1);
1758
1759 const int member_arg_pos = total_args_passed - 1;
1760 assert(member_arg_pos >= 0 && member_arg_pos < total_args_passed, "oob");
1761 assert(sig_bt[member_arg_pos] == T_OBJECT, "dispatch argument must be an object");
1762
1763 java_calling_convention(sig_bt, regs_without_member_name, total_args_passed - 1);
1764
1765 for (int i = 0; i < member_arg_pos; i++) {
1766 VMReg a = regs_with_member_name[i].first();
1767 VMReg b = regs_without_member_name[i].first();
1768 assert(a->value() == b->value(), "register allocation mismatch: a= %d, b= %d", a->value(), b->value());
1769 }
1770 assert(regs_with_member_name[member_arg_pos].first()->is_valid(), "bad member arg");
1771 }
1772 #endif
1773
1774 // ---------------------------------------------------------------------------
1775 // We are calling the interpreter via a c2i. Normally this would mean that
1776 // we were called by a compiled method. However we could have lost a race
1777 // where we went int -> i2c -> c2i and so the caller could in fact be
1778 // interpreted. If the caller is compiled we attempt to patch the caller
1779 // so he no longer calls into the interpreter.
1780 JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address caller_pc))
1781 AARCH64_PORT_ONLY(assert(pauth_ptr_is_raw(caller_pc), "should be raw"));
1782
1783 // It's possible that deoptimization can occur at a call site which hasn't
1784 // been resolved yet, in which case this function will be called from
1785 // an nmethod that has been patched for deopt and we can ignore the
1786 // request for a fixup.
1787 // Also it is possible that we lost a race in that from_compiled_entry
1788 // is now back to the i2c in that case we don't need to patch and if
1789 // we did we'd leap into space because the callsite needs to use
1790 // "to interpreter" stub in order to load up the Method*. Don't
1791 // ask me how I know this...
1792
1793 // Result from nmethod::is_unloading is not stable across safepoints.
1794 NoSafepointVerifier nsv;
1795
1796 nmethod* callee = method->code();
1797 if (callee == nullptr) {
1798 return;
1799 }
1800
1801 // write lock needed because we might patch call site by set_to_clean()
1802 // and is_unloading() can modify nmethod's state
1803 MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, JavaThread::current()));
1804
1805 CodeBlob* cb = CodeCache::find_blob(caller_pc);
1806 if (cb == nullptr || !cb->is_nmethod() || !callee->is_in_use() || callee->is_unloading()) {
1807 return;
1808 }
1809
1810 // The check above makes sure this is an nmethod.
1811 nmethod* caller = cb->as_nmethod();
1812
1813 // Get the return PC for the passed caller PC.
1814 address return_pc = caller_pc + frame::pc_return_offset;
1815
1816 if (!caller->is_in_use() || !NativeCall::is_call_before(return_pc)) {
1817 return;
1818 }
1819
1820 // Expect to find a native call there (unless it was no-inline cache vtable dispatch)
1821 CompiledICLocker ic_locker(caller);
1822 ResourceMark rm;
1823
1824 // If we got here through a static call or opt_virtual call, then we know where the
1825 // call address would be; let's peek at it
1826 address callsite_addr = (address)nativeCall_before(return_pc);
1827 RelocIterator iter(caller, callsite_addr, callsite_addr + 1);
1828 if (!iter.next()) {
1829 // No reloc entry found; not a static or optimized virtual call
1830 return;
1831 }
1832
1833 relocInfo::relocType type = iter.reloc()->type();
1834 if (type != relocInfo::static_call_type &&
1835 type != relocInfo::opt_virtual_call_type) {
1836 return;
1837 }
1838
1839 CompiledDirectCall* callsite = CompiledDirectCall::before(return_pc);
1840 callsite->set_to_clean();
1841 JRT_END
1842
1843
1844 // same as JVM_Arraycopy, but called directly from compiled code
1845 JRT_ENTRY(void, SharedRuntime::slow_arraycopy_C(oopDesc* src, jint src_pos,
1846 oopDesc* dest, jint dest_pos,
1847 jint length,
1848 JavaThread* current)) {
1849 #ifndef PRODUCT
1850 _slow_array_copy_ctr++;
1851 #endif
1852 // Check if we have null pointers
1853 if (src == nullptr || dest == nullptr) {
1854 THROW(vmSymbols::java_lang_NullPointerException());
1855 }
1856 // Do the copy. The casts to arrayOop are necessary to the copy_array API,
1857 // even though the copy_array API also performs dynamic checks to ensure
1858 // that src and dest are truly arrays (and are conformable).
1859 // The copy_array mechanism is awkward and could be removed, but
1860 // the compilers don't call this function except as a last resort,
1861 // so it probably doesn't matter.
1862 src->klass()->copy_array((arrayOopDesc*)src, src_pos,
1863 (arrayOopDesc*)dest, dest_pos,
1864 length, current);
1865 }
1866 JRT_END
1867
1868 // The caller of generate_class_cast_message() (or one of its callers)
1869 // must use a ResourceMark in order to correctly free the result.
1870 char* SharedRuntime::generate_class_cast_message(
1871 JavaThread* thread, Klass* caster_klass) {
1872
1873 // Get target class name from the checkcast instruction
1874 vframeStream vfst(thread, true);
1875 assert(!vfst.at_end(), "Java frame must exist");
1876 Bytecode_checkcast cc(vfst.method(), vfst.method()->bcp_from(vfst.bci()));
1877 constantPoolHandle cpool(thread, vfst.method()->constants());
1878 Klass* target_klass = ConstantPool::klass_at_if_loaded(cpool, cc.index());
1879 Symbol* target_klass_name = nullptr;
1880 if (target_klass == nullptr) {
1881 // This klass should be resolved, but just in case, get the name in the klass slot.
1882 target_klass_name = cpool->klass_name_at(cc.index());
1883 }
1884 return generate_class_cast_message(caster_klass, target_klass, target_klass_name);
1885 }
1886
1887
1888 // The caller of generate_class_cast_message() (or one of its callers)
1889 // must use a ResourceMark in order to correctly free the result.
1890 char* SharedRuntime::generate_class_cast_message(
1891 Klass* caster_klass, Klass* target_klass, Symbol* target_klass_name) {
1892 const char* caster_name = caster_klass->external_name();
1893
1894 assert(target_klass != nullptr || target_klass_name != nullptr, "one must be provided");
1895 const char* target_name = target_klass == nullptr ? target_klass_name->as_klass_external_name() :
1896 target_klass->external_name();
1897
1898 size_t msglen = strlen(caster_name) + strlen("class ") + strlen(" cannot be cast to class ") + strlen(target_name) + 1;
1899
1900 const char* caster_klass_description = "";
1901 const char* target_klass_description = "";
1902 const char* klass_separator = "";
1903 if (target_klass != nullptr && caster_klass->module() == target_klass->module()) {
1904 caster_klass_description = caster_klass->joint_in_module_of_loader(target_klass);
1905 } else {
1906 caster_klass_description = caster_klass->class_in_module_of_loader();
1907 target_klass_description = (target_klass != nullptr) ? target_klass->class_in_module_of_loader() : "";
1908 klass_separator = (target_klass != nullptr) ? "; " : "";
1909 }
1910
1911 // add 3 for parenthesis and preceding space
1912 msglen += strlen(caster_klass_description) + strlen(target_klass_description) + strlen(klass_separator) + 3;
1913
1914 char* message = NEW_RESOURCE_ARRAY_RETURN_NULL(char, msglen);
1915 if (message == nullptr) {
1916 // Shouldn't happen, but don't cause even more problems if it does
1917 message = const_cast<char*>(caster_klass->external_name());
1918 } else {
1919 jio_snprintf(message,
1920 msglen,
1921 "class %s cannot be cast to class %s (%s%s%s)",
1922 caster_name,
1923 target_name,
1924 caster_klass_description,
1925 klass_separator,
1926 target_klass_description
1927 );
1928 }
1929 return message;
1930 }
1931
1932 JRT_LEAF(void, SharedRuntime::reguard_yellow_pages())
1933 (void) JavaThread::current()->stack_overflow_state()->reguard_stack();
1934 JRT_END
1935
1936 void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
1937 if (!SafepointSynchronize::is_synchronizing()) {
1938 // Only try quick_enter() if we're not trying to reach a safepoint
1939 // so that the calling thread reaches the safepoint more quickly.
1940 if (ObjectSynchronizer::quick_enter(obj, lock, current)) {
1941 return;
1942 }
1943 }
1944 // NO_ASYNC required because an async exception on the state transition destructor
1945 // would leave you with the lock held and it would never be released.
1946 // The normal monitorenter NullPointerException is thrown without acquiring a lock
1947 // and the model is that an exception implies the method failed.
1948 JRT_BLOCK_NO_ASYNC
1949 Handle h_obj(THREAD, obj);
1950 ObjectSynchronizer::enter(h_obj, lock, current);
1951 assert(!HAS_PENDING_EXCEPTION, "Should have no exception here");
1952 JRT_BLOCK_END
1953 }
1954
1955 // Handles the uncommon case in locking, i.e., contention or an inflated lock.
1956 JRT_BLOCK_ENTRY(void, SharedRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current))
1957 SharedRuntime::monitor_enter_helper(obj, lock, current);
1958 JRT_END
1959
1960 void SharedRuntime::monitor_exit_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
1961 assert(JavaThread::current() == current, "invariant");
1962 // Exit must be non-blocking, and therefore no exceptions can be thrown.
1963 ExceptionMark em(current);
1964
1965 // Check if C2_MacroAssembler::fast_unlock() or
1966 // C2_MacroAssembler::fast_unlock_lightweight() unlocked an inflated
1967 // monitor before going slow path. Since there is no safepoint
1968 // polling when calling into the VM, we can be sure that the monitor
1969 // hasn't been deallocated.
1970 ObjectMonitor* m = current->unlocked_inflated_monitor();
1971 if (m != nullptr) {
1972 assert(!m->has_owner(current), "must be");
1973 current->clear_unlocked_inflated_monitor();
1974
1975 // We need to reacquire the lock before we can call ObjectSynchronizer::exit().
1976 if (!m->try_enter(current, /*check_for_recursion*/ false)) {
1977 // Some other thread acquired the lock (or the monitor was
1978 // deflated). Either way we are done.
1979 current->dec_held_monitor_count();
1980 return;
1981 }
1982 }
1983
1984 // The object could become unlocked through a JNI call, which we have no other checks for.
1985 // Give a fatal message if CheckJNICalls. Otherwise we ignore it.
1986 if (obj->is_unlocked()) {
1987 if (CheckJNICalls) {
1988 fatal("Object has been unlocked by JNI");
1989 }
1990 return;
1991 }
1992 ObjectSynchronizer::exit(obj, lock, current);
1993 }
1994
1995 // Handles the uncommon cases of monitor unlocking in compiled code
1996 JRT_LEAF(void, SharedRuntime::complete_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* current))
1997 assert(current == JavaThread::current(), "pre-condition");
1998 SharedRuntime::monitor_exit_helper(obj, lock, current);
1999 JRT_END
2000
2001 // This is only called when CheckJNICalls is true, and only
2002 // for virtual thread termination.
2003 JRT_LEAF(void, SharedRuntime::log_jni_monitor_still_held())
2004 assert(CheckJNICalls, "Only call this when checking JNI usage");
2005 if (log_is_enabled(Debug, jni)) {
2006 JavaThread* current = JavaThread::current();
2007 int64_t vthread_id = java_lang_Thread::thread_id(current->vthread());
2008 int64_t carrier_id = java_lang_Thread::thread_id(current->threadObj());
2009 log_debug(jni)("VirtualThread (tid: " INT64_FORMAT ", carrier id: " INT64_FORMAT
2010 ") exiting with Objects still locked by JNI MonitorEnter.",
2011 vthread_id, carrier_id);
2012 }
2013 JRT_END
2014
2015 #ifndef PRODUCT
2016
2017 void SharedRuntime::print_statistics() {
2018 ttyLocker ttyl;
2019 if (xtty != nullptr) xtty->head("statistics type='SharedRuntime'");
2020
2021 SharedRuntime::print_ic_miss_histogram();
2022
2023 // Dump the JRT_ENTRY counters
2024 if (_new_instance_ctr) tty->print_cr("%5u new instance requires GC", _new_instance_ctr);
2025 if (_new_array_ctr) tty->print_cr("%5u new array requires GC", _new_array_ctr);
2026 if (_multi2_ctr) tty->print_cr("%5u multianewarray 2 dim", _multi2_ctr);
2027 if (_multi3_ctr) tty->print_cr("%5u multianewarray 3 dim", _multi3_ctr);
2028 if (_multi4_ctr) tty->print_cr("%5u multianewarray 4 dim", _multi4_ctr);
2029 if (_multi5_ctr) tty->print_cr("%5u multianewarray 5 dim", _multi5_ctr);
2030
2031 tty->print_cr("%5u inline cache miss in compiled", _ic_miss_ctr);
2032 tty->print_cr("%5u wrong method", _wrong_method_ctr);
2033 tty->print_cr("%5u unresolved static call site", _resolve_static_ctr);
2034 tty->print_cr("%5u unresolved virtual call site", _resolve_virtual_ctr);
2035 tty->print_cr("%5u unresolved opt virtual call site", _resolve_opt_virtual_ctr);
2036
2037 if (_mon_enter_stub_ctr) tty->print_cr("%5u monitor enter stub", _mon_enter_stub_ctr);
2038 if (_mon_exit_stub_ctr) tty->print_cr("%5u monitor exit stub", _mon_exit_stub_ctr);
2039 if (_mon_enter_ctr) tty->print_cr("%5u monitor enter slow", _mon_enter_ctr);
2040 if (_mon_exit_ctr) tty->print_cr("%5u monitor exit slow", _mon_exit_ctr);
2041 if (_partial_subtype_ctr) tty->print_cr("%5u slow partial subtype", _partial_subtype_ctr);
2042 if (_jbyte_array_copy_ctr) tty->print_cr("%5u byte array copies", _jbyte_array_copy_ctr);
2043 if (_jshort_array_copy_ctr) tty->print_cr("%5u short array copies", _jshort_array_copy_ctr);
2044 if (_jint_array_copy_ctr) tty->print_cr("%5u int array copies", _jint_array_copy_ctr);
2045 if (_jlong_array_copy_ctr) tty->print_cr("%5u long array copies", _jlong_array_copy_ctr);
2046 if (_oop_array_copy_ctr) tty->print_cr("%5u oop array copies", _oop_array_copy_ctr);
2047 if (_checkcast_array_copy_ctr) tty->print_cr("%5u checkcast array copies", _checkcast_array_copy_ctr);
2048 if (_unsafe_array_copy_ctr) tty->print_cr("%5u unsafe array copies", _unsafe_array_copy_ctr);
2049 if (_generic_array_copy_ctr) tty->print_cr("%5u generic array copies", _generic_array_copy_ctr);
2050 if (_slow_array_copy_ctr) tty->print_cr("%5u slow array copies", _slow_array_copy_ctr);
2051 if (_find_handler_ctr) tty->print_cr("%5u find exception handler", _find_handler_ctr);
2052 if (_rethrow_ctr) tty->print_cr("%5u rethrow handler", _rethrow_ctr);
2053 if (_unsafe_set_memory_ctr) tty->print_cr("%5u unsafe set memorys", _unsafe_set_memory_ctr);
2054
2055 AdapterHandlerLibrary::print_statistics();
2056
2057 if (xtty != nullptr) xtty->tail("statistics");
2058 }
2059
2060 inline double percent(int64_t x, int64_t y) {
2061 return 100.0 * (double)x / (double)MAX2(y, (int64_t)1);
2062 }
2063
2064 class MethodArityHistogram {
2065 public:
2066 enum { MAX_ARITY = 256 };
2067 private:
2068 static uint64_t _arity_histogram[MAX_ARITY]; // histogram of #args
2069 static uint64_t _size_histogram[MAX_ARITY]; // histogram of arg size in words
2070 static uint64_t _total_compiled_calls;
2071 static uint64_t _max_compiled_calls_per_method;
2072 static int _max_arity; // max. arity seen
2073 static int _max_size; // max. arg size seen
2074
2075 static void add_method_to_histogram(nmethod* nm) {
2076 Method* method = (nm == nullptr) ? nullptr : nm->method();
2077 if (method != nullptr) {
2078 ArgumentCount args(method->signature());
2079 int arity = args.size() + (method->is_static() ? 0 : 1);
2080 int argsize = method->size_of_parameters();
2081 arity = MIN2(arity, MAX_ARITY-1);
2082 argsize = MIN2(argsize, MAX_ARITY-1);
2083 uint64_t count = (uint64_t)method->compiled_invocation_count();
2084 _max_compiled_calls_per_method = count > _max_compiled_calls_per_method ? count : _max_compiled_calls_per_method;
2085 _total_compiled_calls += count;
2086 _arity_histogram[arity] += count;
2087 _size_histogram[argsize] += count;
2088 _max_arity = MAX2(_max_arity, arity);
2089 _max_size = MAX2(_max_size, argsize);
2090 }
2091 }
2092
2093 void print_histogram_helper(int n, uint64_t* histo, const char* name) {
2094 const int N = MIN2(9, n);
2095 double sum = 0;
2096 double weighted_sum = 0;
2097 for (int i = 0; i <= n; i++) { sum += (double)histo[i]; weighted_sum += (double)(i*histo[i]); }
2098 if (sum >= 1) { // prevent divide by zero or divide overflow
2099 double rest = sum;
2100 double percent = sum / 100;
2101 for (int i = 0; i <= N; i++) {
2102 rest -= (double)histo[i];
2103 tty->print_cr("%4d: " UINT64_FORMAT_W(12) " (%5.1f%%)", i, histo[i], (double)histo[i] / percent);
2104 }
2105 tty->print_cr("rest: " INT64_FORMAT_W(12) " (%5.1f%%)", (int64_t)rest, rest / percent);
2106 tty->print_cr("(avg. %s = %3.1f, max = %d)", name, weighted_sum / sum, n);
2107 tty->print_cr("(total # of compiled calls = " INT64_FORMAT_W(14) ")", _total_compiled_calls);
2108 tty->print_cr("(max # of compiled calls = " INT64_FORMAT_W(14) ")", _max_compiled_calls_per_method);
2109 } else {
2110 tty->print_cr("Histogram generation failed for %s. n = %d, sum = %7.5f", name, n, sum);
2111 }
2112 }
2113
2114 void print_histogram() {
2115 tty->print_cr("\nHistogram of call arity (incl. rcvr, calls to compiled methods only):");
2116 print_histogram_helper(_max_arity, _arity_histogram, "arity");
2117 tty->print_cr("\nHistogram of parameter block size (in words, incl. rcvr):");
2118 print_histogram_helper(_max_size, _size_histogram, "size");
2119 tty->cr();
2120 }
2121
2122 public:
2123 MethodArityHistogram() {
2124 // Take the Compile_lock to protect against changes in the CodeBlob structures
2125 MutexLocker mu1(Compile_lock, Mutex::_safepoint_check_flag);
2126 // Take the CodeCache_lock to protect against changes in the CodeHeap structure
2127 MutexLocker mu2(CodeCache_lock, Mutex::_no_safepoint_check_flag);
2128 _max_arity = _max_size = 0;
2129 _total_compiled_calls = 0;
2130 _max_compiled_calls_per_method = 0;
2131 for (int i = 0; i < MAX_ARITY; i++) _arity_histogram[i] = _size_histogram[i] = 0;
2132 CodeCache::nmethods_do(add_method_to_histogram);
2133 print_histogram();
2134 }
2135 };
2136
2137 uint64_t MethodArityHistogram::_arity_histogram[MethodArityHistogram::MAX_ARITY];
2138 uint64_t MethodArityHistogram::_size_histogram[MethodArityHistogram::MAX_ARITY];
2139 uint64_t MethodArityHistogram::_total_compiled_calls;
2140 uint64_t MethodArityHistogram::_max_compiled_calls_per_method;
2141 int MethodArityHistogram::_max_arity;
2142 int MethodArityHistogram::_max_size;
2143
2144 void SharedRuntime::print_call_statistics(uint64_t comp_total) {
2145 tty->print_cr("Calls from compiled code:");
2146 int64_t total = _nof_normal_calls + _nof_interface_calls + _nof_static_calls;
2147 int64_t mono_c = _nof_normal_calls - _nof_megamorphic_calls;
2148 int64_t mono_i = _nof_interface_calls;
2149 tty->print_cr("\t" INT64_FORMAT_W(12) " (100%%) total non-inlined ", total);
2150 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- virtual calls ", _nof_normal_calls, percent(_nof_normal_calls, total));
2151 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_calls, percent(_nof_inlined_calls, _nof_normal_calls));
2152 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- monomorphic ", mono_c, percent(mono_c, _nof_normal_calls));
2153 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- megamorphic ", _nof_megamorphic_calls, percent(_nof_megamorphic_calls, _nof_normal_calls));
2154 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- interface calls ", _nof_interface_calls, percent(_nof_interface_calls, total));
2155 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_interface_calls, percent(_nof_inlined_interface_calls, _nof_interface_calls));
2156 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- monomorphic ", mono_i, percent(mono_i, _nof_interface_calls));
2157 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- static/special calls", _nof_static_calls, percent(_nof_static_calls, total));
2158 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_static_calls, percent(_nof_inlined_static_calls, _nof_static_calls));
2159 tty->cr();
2160 tty->print_cr("Note 1: counter updates are not MT-safe.");
2161 tty->print_cr("Note 2: %% in major categories are relative to total non-inlined calls;");
2162 tty->print_cr(" %% in nested categories are relative to their category");
2163 tty->print_cr(" (and thus add up to more than 100%% with inlining)");
2164 tty->cr();
2165
2166 MethodArityHistogram h;
2167 }
2168 #endif
2169
2170 #ifndef PRODUCT
2171 static int _lookups; // number of calls to lookup
2172 static int _equals; // number of buckets checked with matching hash
2173 static int _hits; // number of successful lookups
2174 static int _compact; // number of equals calls with compact signature
2175 #endif
2176
2177 // A simple wrapper class around the calling convention information
2178 // that allows sharing of adapters for the same calling convention.
2179 class AdapterFingerPrint : public CHeapObj<mtCode> {
2180 private:
2181 enum {
2182 _basic_type_bits = 4,
2183 _basic_type_mask = right_n_bits(_basic_type_bits),
2184 _basic_types_per_int = BitsPerInt / _basic_type_bits,
2185 _compact_int_count = 3
2186 };
2187 // TO DO: Consider integrating this with a more global scheme for compressing signatures.
2188 // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive.
2189
2190 union {
2191 int _compact[_compact_int_count];
2192 int* _fingerprint;
2193 } _value;
2194 int _length; // A negative length indicates the fingerprint is in the compact form,
2195 // Otherwise _value._fingerprint is the array.
2196
2197 // Remap BasicTypes that are handled equivalently by the adapters.
2198 // These are correct for the current system but someday it might be
2199 // necessary to make this mapping platform dependent.
2200 static int adapter_encoding(BasicType in) {
2201 switch (in) {
2202 case T_BOOLEAN:
2203 case T_BYTE:
2204 case T_SHORT:
2205 case T_CHAR:
2206 // There are all promoted to T_INT in the calling convention
2207 return T_INT;
2208
2209 case T_OBJECT:
2210 case T_ARRAY:
2211 // In other words, we assume that any register good enough for
2212 // an int or long is good enough for a managed pointer.
2213 #ifdef _LP64
2214 return T_LONG;
2215 #else
2216 return T_INT;
2217 #endif
2218
2219 case T_INT:
2220 case T_LONG:
2221 case T_FLOAT:
2222 case T_DOUBLE:
2223 case T_VOID:
2224 return in;
2225
2226 default:
2227 ShouldNotReachHere();
2228 return T_CONFLICT;
2229 }
2230 }
2231
2232 public:
2233 AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) {
2234 // The fingerprint is based on the BasicType signature encoded
2235 // into an array of ints with eight entries per int.
2236 int* ptr;
2237 int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int;
2238 if (len <= _compact_int_count) {
2239 assert(_compact_int_count == 3, "else change next line");
2240 _value._compact[0] = _value._compact[1] = _value._compact[2] = 0;
2241 // Storing the signature encoded as signed chars hits about 98%
2242 // of the time.
2243 _length = -len;
2244 ptr = _value._compact;
2245 } else {
2246 _length = len;
2247 _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode);
2248 ptr = _value._fingerprint;
2249 }
2250
2251 // Now pack the BasicTypes with 8 per int
2252 int sig_index = 0;
2253 for (int index = 0; index < len; index++) {
2254 int value = 0;
2255 for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) {
2256 int bt = adapter_encoding(sig_bt[sig_index++]);
2257 assert((bt & _basic_type_mask) == bt, "must fit in 4 bits");
2258 value = (value << _basic_type_bits) | bt;
2259 }
2260 ptr[index] = value;
2261 }
2262 }
2263
2264 ~AdapterFingerPrint() {
2265 if (_length > 0) {
2266 FREE_C_HEAP_ARRAY(int, _value._fingerprint);
2267 }
2268 }
2269
2270 int value(int index) {
2271 if (_length < 0) {
2272 return _value._compact[index];
2273 }
2274 return _value._fingerprint[index];
2275 }
2276 int length() {
2277 if (_length < 0) return -_length;
2278 return _length;
2279 }
2280
2281 bool is_compact() {
2282 return _length <= 0;
2283 }
2284
2285 unsigned int compute_hash() {
2286 int hash = 0;
2287 for (int i = 0; i < length(); i++) {
2288 int v = value(i);
2289 hash = (hash << 8) ^ v ^ (hash >> 5);
2290 }
2291 return (unsigned int)hash;
2292 }
2293
2294 const char* as_string() {
2295 stringStream st;
2296 st.print("0x");
2297 for (int i = 0; i < length(); i++) {
2298 st.print("%x", value(i));
2299 }
2300 return st.as_string();
2301 }
2302
2303 #ifndef PRODUCT
2304 // Reconstitutes the basic type arguments from the fingerprint,
2305 // producing strings like LIJDF
2306 const char* as_basic_args_string() {
2307 stringStream st;
2308 bool long_prev = false;
2309 for (int i = 0; i < length(); i++) {
2310 unsigned val = (unsigned)value(i);
2311 // args are packed so that first/lower arguments are in the highest
2312 // bits of each int value, so iterate from highest to the lowest
2313 for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) {
2314 unsigned v = (val >> j) & _basic_type_mask;
2315 if (v == 0) {
2316 assert(i == length() - 1, "Only expect zeroes in the last word");
2317 continue;
2318 }
2319 if (long_prev) {
2320 long_prev = false;
2321 if (v == T_VOID) {
2322 st.print("J");
2323 } else {
2324 st.print("L");
2325 }
2326 }
2327 switch (v) {
2328 case T_INT: st.print("I"); break;
2329 case T_LONG: long_prev = true; break;
2330 case T_FLOAT: st.print("F"); break;
2331 case T_DOUBLE: st.print("D"); break;
2332 case T_VOID: break;
2333 default: ShouldNotReachHere();
2334 }
2335 }
2336 }
2337 if (long_prev) {
2338 st.print("L");
2339 }
2340 return st.as_string();
2341 }
2342 #endif // !product
2343
2344 bool equals(AdapterFingerPrint* other) {
2345 if (other->_length != _length) {
2346 return false;
2347 }
2348 if (_length < 0) {
2349 assert(_compact_int_count == 3, "else change next line");
2350 return _value._compact[0] == other->_value._compact[0] &&
2351 _value._compact[1] == other->_value._compact[1] &&
2352 _value._compact[2] == other->_value._compact[2];
2353 } else {
2354 for (int i = 0; i < _length; i++) {
2355 if (_value._fingerprint[i] != other->_value._fingerprint[i]) {
2356 return false;
2357 }
2358 }
2359 }
2360 return true;
2361 }
2362
2363 static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) {
2364 NOT_PRODUCT(_equals++);
2365 return fp1->equals(fp2);
2366 }
2367
2368 static unsigned int compute_hash(AdapterFingerPrint* const& fp) {
2369 return fp->compute_hash();
2370 }
2371 };
2372
2373 // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries
2374 using AdapterHandlerTable = ResourceHashtable<AdapterFingerPrint*, AdapterHandlerEntry*, 293,
2375 AnyObj::C_HEAP, mtCode,
2376 AdapterFingerPrint::compute_hash,
2377 AdapterFingerPrint::equals>;
2378 static AdapterHandlerTable* _adapter_handler_table;
2379
2380 // Find a entry with the same fingerprint if it exists
2381 static AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) {
2382 NOT_PRODUCT(_lookups++);
2383 assert_lock_strong(AdapterHandlerLibrary_lock);
2384 AdapterFingerPrint fp(total_args_passed, sig_bt);
2385 AdapterHandlerEntry** entry = _adapter_handler_table->get(&fp);
2386 if (entry != nullptr) {
2387 #ifndef PRODUCT
2388 if (fp.is_compact()) _compact++;
2389 _hits++;
2390 #endif
2391 return *entry;
2392 }
2393 return nullptr;
2394 }
2395
2396 #ifndef PRODUCT
2397 static void print_table_statistics() {
2398 auto size = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) {
2399 return sizeof(*key) + sizeof(*a);
2400 };
2401 TableStatistics ts = _adapter_handler_table->statistics_calculate(size);
2402 ts.print(tty, "AdapterHandlerTable");
2403 tty->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)",
2404 _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries());
2405 tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d compact %d",
2406 _lookups, _equals, _hits, _compact);
2407 }
2408 #endif
2409
2410 // ---------------------------------------------------------------------------
2411 // Implementation of AdapterHandlerLibrary
2412 AdapterHandlerEntry* AdapterHandlerLibrary::_abstract_method_handler = nullptr;
2413 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = nullptr;
2414 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr;
2415 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr;
2416 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr;
2417 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr;
2418 const int AdapterHandlerLibrary_size = 16*K;
2419 BufferBlob* AdapterHandlerLibrary::_buffer = nullptr;
2420
2421 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2422 return _buffer;
2423 }
2424
2425 static void post_adapter_creation(const AdapterBlob* new_adapter,
2426 const AdapterHandlerEntry* entry) {
2427 if (Forte::is_enabled() || JvmtiExport::should_post_dynamic_code_generated()) {
2428 char blob_id[256];
2429 jio_snprintf(blob_id,
2430 sizeof(blob_id),
2431 "%s(%s)",
2432 new_adapter->name(),
2433 entry->fingerprint()->as_string());
2434 if (Forte::is_enabled()) {
2435 Forte::register_stub(blob_id, new_adapter->content_begin(), new_adapter->content_end());
2436 }
2437
2438 if (JvmtiExport::should_post_dynamic_code_generated()) {
2439 JvmtiExport::post_dynamic_code_generated(blob_id, new_adapter->content_begin(), new_adapter->content_end());
2440 }
2441 }
2442 }
2443
2444 void AdapterHandlerLibrary::initialize() {
2445 ResourceMark rm;
2446 AdapterBlob* no_arg_blob = nullptr;
2447 AdapterBlob* int_arg_blob = nullptr;
2448 AdapterBlob* obj_arg_blob = nullptr;
2449 AdapterBlob* obj_int_arg_blob = nullptr;
2450 AdapterBlob* obj_obj_arg_blob = nullptr;
2451 {
2452 _adapter_handler_table = new (mtCode) AdapterHandlerTable();
2453 MutexLocker mu(AdapterHandlerLibrary_lock);
2454
2455 // Create a special handler for abstract methods. Abstract methods
2456 // are never compiled so an i2c entry is somewhat meaningless, but
2457 // throw AbstractMethodError just in case.
2458 // Pass wrong_method_abstract for the c2i transitions to return
2459 // AbstractMethodError for invalid invocations.
2460 address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
2461 _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, nullptr),
2462 SharedRuntime::throw_AbstractMethodError_entry(),
2463 wrong_method_abstract, wrong_method_abstract);
2464
2465 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2466 _no_arg_handler = create_adapter(no_arg_blob, 0, nullptr, true);
2467
2468 BasicType obj_args[] = { T_OBJECT };
2469 _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true);
2470
2471 BasicType int_args[] = { T_INT };
2472 _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true);
2473
2474 BasicType obj_int_args[] = { T_OBJECT, T_INT };
2475 _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true);
2476
2477 BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT };
2478 _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true);
2479
2480 assert(no_arg_blob != nullptr &&
2481 obj_arg_blob != nullptr &&
2482 int_arg_blob != nullptr &&
2483 obj_int_arg_blob != nullptr &&
2484 obj_obj_arg_blob != nullptr, "Initial adapters must be properly created");
2485 }
2486
2487 // Outside of the lock
2488 post_adapter_creation(no_arg_blob, _no_arg_handler);
2489 post_adapter_creation(obj_arg_blob, _obj_arg_handler);
2490 post_adapter_creation(int_arg_blob, _int_arg_handler);
2491 post_adapter_creation(obj_int_arg_blob, _obj_int_arg_handler);
2492 post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler);
2493 }
2494
2495 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint,
2496 address i2c_entry,
2497 address c2i_entry,
2498 address c2i_unverified_entry,
2499 address c2i_no_clinit_check_entry) {
2500 // Insert an entry into the table
2501 return new AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry,
2502 c2i_no_clinit_check_entry);
2503 }
2504
2505 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2506 if (method->is_abstract()) {
2507 return _abstract_method_handler;
2508 }
2509 int total_args_passed = method->size_of_parameters(); // All args on stack
2510 if (total_args_passed == 0) {
2511 return _no_arg_handler;
2512 } else if (total_args_passed == 1) {
2513 if (!method->is_static()) {
2514 return _obj_arg_handler;
2515 }
2516 switch (method->signature()->char_at(1)) {
2517 case JVM_SIGNATURE_CLASS:
2518 case JVM_SIGNATURE_ARRAY:
2519 return _obj_arg_handler;
2520 case JVM_SIGNATURE_INT:
2521 case JVM_SIGNATURE_BOOLEAN:
2522 case JVM_SIGNATURE_CHAR:
2523 case JVM_SIGNATURE_BYTE:
2524 case JVM_SIGNATURE_SHORT:
2525 return _int_arg_handler;
2526 }
2527 } else if (total_args_passed == 2 &&
2528 !method->is_static()) {
2529 switch (method->signature()->char_at(1)) {
2530 case JVM_SIGNATURE_CLASS:
2531 case JVM_SIGNATURE_ARRAY:
2532 return _obj_obj_arg_handler;
2533 case JVM_SIGNATURE_INT:
2534 case JVM_SIGNATURE_BOOLEAN:
2535 case JVM_SIGNATURE_CHAR:
2536 case JVM_SIGNATURE_BYTE:
2537 case JVM_SIGNATURE_SHORT:
2538 return _obj_int_arg_handler;
2539 }
2540 }
2541 return nullptr;
2542 }
2543
2544 class AdapterSignatureIterator : public SignatureIterator {
2545 private:
2546 BasicType stack_sig_bt[16];
2547 BasicType* sig_bt;
2548 int index;
2549
2550 public:
2551 AdapterSignatureIterator(Symbol* signature,
2552 fingerprint_t fingerprint,
2553 bool is_static,
2554 int total_args_passed) :
2555 SignatureIterator(signature, fingerprint),
2556 index(0)
2557 {
2558 sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
2559 if (!is_static) { // Pass in receiver first
2560 sig_bt[index++] = T_OBJECT;
2561 }
2562 do_parameters_on(this);
2563 }
2564
2565 BasicType* basic_types() {
2566 return sig_bt;
2567 }
2568
2569 #ifdef ASSERT
2570 int slots() {
2571 return index;
2572 }
2573 #endif
2574
2575 private:
2576
2577 friend class SignatureIterator; // so do_parameters_on can call do_type
2578 void do_type(BasicType type) {
2579 sig_bt[index++] = type;
2580 if (type == T_LONG || type == T_DOUBLE) {
2581 sig_bt[index++] = T_VOID; // Longs & doubles take 2 Java slots
2582 }
2583 }
2584 };
2585
2586 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
2587 // Use customized signature handler. Need to lock around updates to
2588 // the _adapter_handler_table (it is not safe for concurrent readers
2589 // and a single writer: this could be fixed if it becomes a
2590 // problem).
2591
2592 // Fast-path for trivial adapters
2593 AdapterHandlerEntry* entry = get_simple_adapter(method);
2594 if (entry != nullptr) {
2595 return entry;
2596 }
2597
2598 ResourceMark rm;
2599 AdapterBlob* new_adapter = nullptr;
2600
2601 // Fill in the signature array, for the calling-convention call.
2602 int total_args_passed = method->size_of_parameters(); // All args on stack
2603
2604 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
2605 method->is_static(), total_args_passed);
2606 assert(si.slots() == total_args_passed, "");
2607 BasicType* sig_bt = si.basic_types();
2608 {
2609 MutexLocker mu(AdapterHandlerLibrary_lock);
2610
2611 // Lookup method signature's fingerprint
2612 entry = lookup(total_args_passed, sig_bt);
2613
2614 if (entry != nullptr) {
2615 #ifdef ASSERT
2616 if (VerifyAdapterSharing) {
2617 AdapterBlob* comparison_blob = nullptr;
2618 AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false);
2619 assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison");
2620 assert(comparison_entry->compare_code(entry), "code must match");
2621 // Release the one just created and return the original
2622 delete comparison_entry;
2623 }
2624 #endif
2625 return entry;
2626 }
2627
2628 entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true);
2629 }
2630
2631 // Outside of the lock
2632 if (new_adapter != nullptr) {
2633 post_adapter_creation(new_adapter, entry);
2634 }
2635 return entry;
2636 }
2637
2638 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter,
2639 int total_args_passed,
2640 BasicType* sig_bt,
2641 bool allocate_code_blob) {
2642 if (log_is_enabled(Info, perf, class, link)) {
2643 ClassLoader::perf_method_adapters_count()->inc();
2644 }
2645
2646 // StubRoutines::_final_stubs_code is initialized after this function can be called. As a result,
2647 // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated prior
2648 // to all StubRoutines::_final_stubs_code being set. Checks refer to runtime range checks generated
2649 // in an I2C stub that ensure that an I2C stub is called from an interpreter frame or stubs.
2650 bool contains_all_checks = StubRoutines::final_stubs_code() != nullptr;
2651
2652 VMRegPair stack_regs[16];
2653 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
2654
2655 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
2656 int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
2657 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
2658 CodeBuffer buffer(buf);
2659 short buffer_locs[20];
2660 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
2661 sizeof(buffer_locs)/sizeof(relocInfo));
2662
2663 // Make a C heap allocated version of the fingerprint to store in the adapter
2664 AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt);
2665 MacroAssembler _masm(&buffer);
2666 AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm,
2667 total_args_passed,
2668 comp_args_on_stack,
2669 sig_bt,
2670 regs,
2671 fingerprint);
2672
2673 #ifdef ASSERT
2674 if (VerifyAdapterSharing) {
2675 entry->save_code(buf->code_begin(), buffer.insts_size());
2676 if (!allocate_code_blob) {
2677 return entry;
2678 }
2679 }
2680 #endif
2681
2682 new_adapter = AdapterBlob::create(&buffer);
2683 NOT_PRODUCT(int insts_size = buffer.insts_size());
2684 if (new_adapter == nullptr) {
2685 // CodeCache is full, disable compilation
2686 // Ought to log this but compile log is only per compile thread
2687 // and we're some non descript Java thread.
2688 return nullptr;
2689 }
2690 entry->relocate(new_adapter->content_begin());
2691 #ifndef PRODUCT
2692 // debugging support
2693 if (PrintAdapterHandlers || PrintStubCode) {
2694 ttyLocker ttyl;
2695 entry->print_adapter_on(tty);
2696 tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)",
2697 _adapter_handler_table->number_of_entries(), fingerprint->as_basic_args_string(),
2698 fingerprint->as_string(), insts_size);
2699 tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(entry->get_c2i_entry()));
2700 if (Verbose || PrintStubCode) {
2701 address first_pc = entry->base_address();
2702 if (first_pc != nullptr) {
2703 Disassembler::decode(first_pc, first_pc + insts_size, tty
2704 NOT_PRODUCT(COMMA &new_adapter->asm_remarks()));
2705 tty->cr();
2706 }
2707 }
2708 }
2709 #endif
2710
2711 // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp)
2712 // The checks are inserted only if -XX:+VerifyAdapterCalls is specified.
2713 if (contains_all_checks || !VerifyAdapterCalls) {
2714 assert_lock_strong(AdapterHandlerLibrary_lock);
2715 _adapter_handler_table->put(fingerprint, entry);
2716 }
2717 return entry;
2718 }
2719
2720 address AdapterHandlerEntry::base_address() {
2721 address base = _i2c_entry;
2722 if (base == nullptr) base = _c2i_entry;
2723 assert(base <= _c2i_entry || _c2i_entry == nullptr, "");
2724 assert(base <= _c2i_unverified_entry || _c2i_unverified_entry == nullptr, "");
2725 assert(base <= _c2i_no_clinit_check_entry || _c2i_no_clinit_check_entry == nullptr, "");
2726 return base;
2727 }
2728
2729 void AdapterHandlerEntry::relocate(address new_base) {
2730 address old_base = base_address();
2731 assert(old_base != nullptr, "");
2732 ptrdiff_t delta = new_base - old_base;
2733 if (_i2c_entry != nullptr)
2734 _i2c_entry += delta;
2735 if (_c2i_entry != nullptr)
2736 _c2i_entry += delta;
2737 if (_c2i_unverified_entry != nullptr)
2738 _c2i_unverified_entry += delta;
2739 if (_c2i_no_clinit_check_entry != nullptr)
2740 _c2i_no_clinit_check_entry += delta;
2741 assert(base_address() == new_base, "");
2742 }
2743
2744
2745 AdapterHandlerEntry::~AdapterHandlerEntry() {
2746 delete _fingerprint;
2747 #ifdef ASSERT
2748 FREE_C_HEAP_ARRAY(unsigned char, _saved_code);
2749 #endif
2750 }
2751
2752
2753 #ifdef ASSERT
2754 // Capture the code before relocation so that it can be compared
2755 // against other versions. If the code is captured after relocation
2756 // then relative instructions won't be equivalent.
2757 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
2758 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
2759 _saved_code_length = length;
2760 memcpy(_saved_code, buffer, length);
2761 }
2762
2763
2764 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
2765 assert(_saved_code != nullptr && other->_saved_code != nullptr, "code not saved");
2766
2767 if (other->_saved_code_length != _saved_code_length) {
2768 return false;
2769 }
2770
2771 return memcmp(other->_saved_code, _saved_code, _saved_code_length) == 0;
2772 }
2773 #endif
2774
2775
2776 /**
2777 * Create a native wrapper for this native method. The wrapper converts the
2778 * Java-compiled calling convention to the native convention, handles
2779 * arguments, and transitions to native. On return from the native we transition
2780 * back to java blocking if a safepoint is in progress.
2781 */
2782 void AdapterHandlerLibrary::create_native_wrapper(const methodHandle& method) {
2783 ResourceMark rm;
2784 nmethod* nm = nullptr;
2785
2786 // Check if memory should be freed before allocation
2787 CodeCache::gc_on_allocation();
2788
2789 assert(method->is_native(), "must be native");
2790 assert(method->is_special_native_intrinsic() ||
2791 method->has_native_function(), "must have something valid to call!");
2792
2793 {
2794 // Perform the work while holding the lock, but perform any printing outside the lock
2795 MutexLocker mu(AdapterHandlerLibrary_lock);
2796 // See if somebody beat us to it
2797 if (method->code() != nullptr) {
2798 return;
2799 }
2800
2801 const int compile_id = CompileBroker::assign_compile_id(method, CompileBroker::standard_entry_bci);
2802 assert(compile_id > 0, "Must generate native wrapper");
2803
2804
2805 ResourceMark rm;
2806 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
2807 if (buf != nullptr) {
2808 CodeBuffer buffer(buf);
2809
2810 if (method->is_continuation_enter_intrinsic()) {
2811 buffer.initialize_stubs_size(192);
2812 }
2813
2814 struct { double data[20]; } locs_buf;
2815 struct { double data[20]; } stubs_locs_buf;
2816 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
2817 #if defined(AARCH64) || defined(PPC64)
2818 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
2819 // in the constant pool to ensure ordering between the barrier and oops
2820 // accesses. For native_wrappers we need a constant.
2821 // On PPC64 the continuation enter intrinsic needs the constant pool for the compiled
2822 // static java call that is resolved in the runtime.
2823 if (PPC64_ONLY(method->is_continuation_enter_intrinsic() &&) true) {
2824 buffer.initialize_consts_size(8 PPC64_ONLY(+ 24));
2825 }
2826 #endif
2827 buffer.stubs()->initialize_shared_locs((relocInfo*)&stubs_locs_buf, sizeof(stubs_locs_buf) / sizeof(relocInfo));
2828 MacroAssembler _masm(&buffer);
2829
2830 // Fill in the signature array, for the calling-convention call.
2831 const int total_args_passed = method->size_of_parameters();
2832
2833 VMRegPair stack_regs[16];
2834 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
2835
2836 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
2837 method->is_static(), total_args_passed);
2838 BasicType* sig_bt = si.basic_types();
2839 assert(si.slots() == total_args_passed, "");
2840 BasicType ret_type = si.return_type();
2841
2842 // Now get the compiled-Java arguments layout.
2843 SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
2844
2845 // Generate the compiled-to-native wrapper code
2846 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type);
2847
2848 if (nm != nullptr) {
2849 {
2850 MutexLocker pl(NMethodState_lock, Mutex::_no_safepoint_check_flag);
2851 if (nm->make_in_use()) {
2852 method->set_code(method, nm);
2853 }
2854 }
2855
2856 DirectiveSet* directive = DirectivesStack::getMatchingDirective(method, CompileBroker::compiler(CompLevel_simple));
2857 if (directive->PrintAssemblyOption) {
2858 nm->print_code();
2859 }
2860 DirectivesStack::release(directive);
2861 }
2862 }
2863 } // Unlock AdapterHandlerLibrary_lock
2864
2865
2866 // Install the generated code.
2867 if (nm != nullptr) {
2868 const char *msg = method->is_static() ? "(static)" : "";
2869 CompileTask::print_ul(nm, msg);
2870 if (PrintCompilation) {
2871 ttyLocker ttyl;
2872 CompileTask::print(tty, nm, msg);
2873 }
2874 nm->post_compiled_method_load_event();
2875 }
2876 }
2877
2878 // -------------------------------------------------------------------------
2879 // Java-Java calling convention
2880 // (what you use when Java calls Java)
2881
2882 //------------------------------name_for_receiver----------------------------------
2883 // For a given signature, return the VMReg for parameter 0.
2884 VMReg SharedRuntime::name_for_receiver() {
2885 VMRegPair regs;
2886 BasicType sig_bt = T_OBJECT;
2887 (void) java_calling_convention(&sig_bt, ®s, 1);
2888 // Return argument 0 register. In the LP64 build pointers
2889 // take 2 registers, but the VM wants only the 'main' name.
2890 return regs.first();
2891 }
2892
2893 VMRegPair *SharedRuntime::find_callee_arguments(Symbol* sig, bool has_receiver, bool has_appendix, int* arg_size) {
2894 // This method is returning a data structure allocating as a
2895 // ResourceObject, so do not put any ResourceMarks in here.
2896
2897 BasicType *sig_bt = NEW_RESOURCE_ARRAY(BasicType, 256);
2898 VMRegPair *regs = NEW_RESOURCE_ARRAY(VMRegPair, 256);
2899 int cnt = 0;
2900 if (has_receiver) {
2901 sig_bt[cnt++] = T_OBJECT; // Receiver is argument 0; not in signature
2902 }
2903
2904 for (SignatureStream ss(sig); !ss.at_return_type(); ss.next()) {
2905 BasicType type = ss.type();
2906 sig_bt[cnt++] = type;
2907 if (is_double_word_type(type))
2908 sig_bt[cnt++] = T_VOID;
2909 }
2910
2911 if (has_appendix) {
2912 sig_bt[cnt++] = T_OBJECT;
2913 }
2914
2915 assert(cnt < 256, "grow table size");
2916
2917 int comp_args_on_stack;
2918 comp_args_on_stack = java_calling_convention(sig_bt, regs, cnt);
2919
2920 // the calling convention doesn't count out_preserve_stack_slots so
2921 // we must add that in to get "true" stack offsets.
2922
2923 if (comp_args_on_stack) {
2924 for (int i = 0; i < cnt; i++) {
2925 VMReg reg1 = regs[i].first();
2926 if (reg1->is_stack()) {
2927 // Yuck
2928 reg1 = reg1->bias(out_preserve_stack_slots());
2929 }
2930 VMReg reg2 = regs[i].second();
2931 if (reg2->is_stack()) {
2932 // Yuck
2933 reg2 = reg2->bias(out_preserve_stack_slots());
2934 }
2935 regs[i].set_pair(reg2, reg1);
2936 }
2937 }
2938
2939 // results
2940 *arg_size = cnt;
2941 return regs;
2942 }
2943
2944 // OSR Migration Code
2945 //
2946 // This code is used convert interpreter frames into compiled frames. It is
2947 // called from very start of a compiled OSR nmethod. A temp array is
2948 // allocated to hold the interesting bits of the interpreter frame. All
2949 // active locks are inflated to allow them to move. The displaced headers and
2950 // active interpreter locals are copied into the temp buffer. Then we return
2951 // back to the compiled code. The compiled code then pops the current
2952 // interpreter frame off the stack and pushes a new compiled frame. Then it
2953 // copies the interpreter locals and displaced headers where it wants.
2954 // Finally it calls back to free the temp buffer.
2955 //
2956 // All of this is done NOT at any Safepoint, nor is any safepoint or GC allowed.
2957
2958 JRT_LEAF(intptr_t*, SharedRuntime::OSR_migration_begin( JavaThread *current) )
2959 assert(current == JavaThread::current(), "pre-condition");
2960
2961 // During OSR migration, we unwind the interpreted frame and replace it with a compiled
2962 // frame. The stack watermark code below ensures that the interpreted frame is processed
2963 // before it gets unwound. This is helpful as the size of the compiled frame could be
2964 // larger than the interpreted frame, which could result in the new frame not being
2965 // processed correctly.
2966 StackWatermarkSet::before_unwind(current);
2967
2968 //
2969 // This code is dependent on the memory layout of the interpreter local
2970 // array and the monitors. On all of our platforms the layout is identical
2971 // so this code is shared. If some platform lays the their arrays out
2972 // differently then this code could move to platform specific code or
2973 // the code here could be modified to copy items one at a time using
2974 // frame accessor methods and be platform independent.
2975
2976 frame fr = current->last_frame();
2977 assert(fr.is_interpreted_frame(), "");
2978 assert(fr.interpreter_frame_expression_stack_size()==0, "only handle empty stacks");
2979
2980 // Figure out how many monitors are active.
2981 int active_monitor_count = 0;
2982 for (BasicObjectLock *kptr = fr.interpreter_frame_monitor_end();
2983 kptr < fr.interpreter_frame_monitor_begin();
2984 kptr = fr.next_monitor_in_interpreter_frame(kptr) ) {
2985 if (kptr->obj() != nullptr) active_monitor_count++;
2986 }
2987
2988 // QQQ we could place number of active monitors in the array so that compiled code
2989 // could double check it.
2990
2991 Method* moop = fr.interpreter_frame_method();
2992 int max_locals = moop->max_locals();
2993 // Allocate temp buffer, 1 word per local & 2 per active monitor
2994 int buf_size_words = max_locals + active_monitor_count * BasicObjectLock::size();
2995 intptr_t *buf = NEW_C_HEAP_ARRAY(intptr_t,buf_size_words, mtCode);
2996
2997 // Copy the locals. Order is preserved so that loading of longs works.
2998 // Since there's no GC I can copy the oops blindly.
2999 assert(sizeof(HeapWord)==sizeof(intptr_t), "fix this code");
3000 Copy::disjoint_words((HeapWord*)fr.interpreter_frame_local_at(max_locals-1),
3001 (HeapWord*)&buf[0],
3002 max_locals);
3003
3004 // Inflate locks. Copy the displaced headers. Be careful, there can be holes.
3005 int i = max_locals;
3006 for (BasicObjectLock *kptr2 = fr.interpreter_frame_monitor_end();
3007 kptr2 < fr.interpreter_frame_monitor_begin();
3008 kptr2 = fr.next_monitor_in_interpreter_frame(kptr2) ) {
3009 if (kptr2->obj() != nullptr) { // Avoid 'holes' in the monitor array
3010 BasicLock *lock = kptr2->lock();
3011 if (LockingMode == LM_LEGACY) {
3012 // Inflate so the object's header no longer refers to the BasicLock.
3013 if (lock->displaced_header().is_unlocked()) {
3014 // The object is locked and the resulting ObjectMonitor* will also be
3015 // locked so it can't be async deflated until ownership is dropped.
3016 // See the big comment in basicLock.cpp: BasicLock::move_to().
3017 ObjectSynchronizer::inflate_helper(kptr2->obj());
3018 }
3019 // Now the displaced header is free to move because the
3020 // object's header no longer refers to it.
3021 buf[i] = (intptr_t)lock->displaced_header().value();
3022 } else if (UseObjectMonitorTable) {
3023 buf[i] = (intptr_t)lock->object_monitor_cache();
3024 }
3025 #ifdef ASSERT
3026 else {
3027 buf[i] = badDispHeaderOSR;
3028 }
3029 #endif
3030 i++;
3031 buf[i++] = cast_from_oop<intptr_t>(kptr2->obj());
3032 }
3033 }
3034 assert(i - max_locals == active_monitor_count*2, "found the expected number of monitors");
3035
3036 RegisterMap map(current,
3037 RegisterMap::UpdateMap::skip,
3038 RegisterMap::ProcessFrames::include,
3039 RegisterMap::WalkContinuation::skip);
3040 frame sender = fr.sender(&map);
3041 if (sender.is_interpreted_frame()) {
3042 current->push_cont_fastpath(sender.sp());
3043 }
3044
3045 return buf;
3046 JRT_END
3047
3048 JRT_LEAF(void, SharedRuntime::OSR_migration_end( intptr_t* buf) )
3049 FREE_C_HEAP_ARRAY(intptr_t, buf);
3050 JRT_END
3051
3052 bool AdapterHandlerLibrary::contains(const CodeBlob* b) {
3053 bool found = false;
3054 auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) {
3055 return (found = (b == CodeCache::find_blob(a->get_i2c_entry())));
3056 };
3057 assert_locked_or_safepoint(AdapterHandlerLibrary_lock);
3058 _adapter_handler_table->iterate(findblob);
3059 return found;
3060 }
3061
3062 void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) {
3063 bool found = false;
3064 auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) {
3065 if (b == CodeCache::find_blob(a->get_i2c_entry())) {
3066 found = true;
3067 st->print("Adapter for signature: ");
3068 a->print_adapter_on(st);
3069 return true;
3070 } else {
3071 return false; // keep looking
3072 }
3073 };
3074 assert_locked_or_safepoint(AdapterHandlerLibrary_lock);
3075 _adapter_handler_table->iterate(findblob);
3076 assert(found, "Should have found handler");
3077 }
3078
3079 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3080 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3081 if (get_i2c_entry() != nullptr) {
3082 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3083 }
3084 if (get_c2i_entry() != nullptr) {
3085 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3086 }
3087 if (get_c2i_unverified_entry() != nullptr) {
3088 st->print(" c2iUV: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3089 }
3090 if (get_c2i_no_clinit_check_entry() != nullptr) {
3091 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3092 }
3093 st->cr();
3094 }
3095
3096 #ifndef PRODUCT
3097
3098 void AdapterHandlerLibrary::print_statistics() {
3099 print_table_statistics();
3100 }
3101
3102 #endif /* PRODUCT */
3103
3104 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3105 assert(current == JavaThread::current(), "pre-condition");
3106 StackOverflow* overflow_state = current->stack_overflow_state();
3107 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3108 overflow_state->set_reserved_stack_activation(current->stack_base());
3109 JRT_END
3110
3111 frame SharedRuntime::look_for_reserved_stack_annotated_method(JavaThread* current, frame fr) {
3112 ResourceMark rm(current);
3113 frame activation;
3114 nmethod* nm = nullptr;
3115 int count = 1;
3116
3117 assert(fr.is_java_frame(), "Must start on Java frame");
3118
3119 RegisterMap map(JavaThread::current(),
3120 RegisterMap::UpdateMap::skip,
3121 RegisterMap::ProcessFrames::skip,
3122 RegisterMap::WalkContinuation::skip); // don't walk continuations
3123 for (; !fr.is_first_frame(); fr = fr.sender(&map)) {
3124 if (!fr.is_java_frame()) {
3125 continue;
3126 }
3127
3128 Method* method = nullptr;
3129 bool found = false;
3130 if (fr.is_interpreted_frame()) {
3131 method = fr.interpreter_frame_method();
3132 if (method != nullptr && method->has_reserved_stack_access()) {
3133 found = true;
3134 }
3135 } else {
3136 CodeBlob* cb = fr.cb();
3137 if (cb != nullptr && cb->is_nmethod()) {
3138 nm = cb->as_nmethod();
3139 method = nm->method();
3140 // scope_desc_near() must be used, instead of scope_desc_at() because on
3141 // SPARC, the pcDesc can be on the delay slot after the call instruction.
3142 for (ScopeDesc *sd = nm->scope_desc_near(fr.pc()); sd != nullptr; sd = sd->sender()) {
3143 method = sd->method();
3144 if (method != nullptr && method->has_reserved_stack_access()) {
3145 found = true;
3146 }
3147 }
3148 }
3149 }
3150 if (found) {
3151 activation = fr;
3152 warning("Potentially dangerous stack overflow in "
3153 "ReservedStackAccess annotated method %s [%d]",
3154 method->name_and_sig_as_C_string(), count++);
3155 EventReservedStackActivation event;
3156 if (event.should_commit()) {
3157 event.set_method(method);
3158 event.commit();
3159 }
3160 }
3161 }
3162 return activation;
3163 }
3164
3165 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
3166 // After any safepoint, just before going back to compiled code,
3167 // we inform the GC that we will be doing initializing writes to
3168 // this object in the future without emitting card-marks, so
3169 // GC may take any compensating steps.
3170
3171 oop new_obj = current->vm_result();
3172 if (new_obj == nullptr) return;
3173
3174 BarrierSet *bs = BarrierSet::barrier_set();
3175 bs->on_slowpath_allocation_exit(current, new_obj);
3176 }