1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "cds/aotCompressedPointers.hpp"
26 #include "cds/archiveBuilder.hpp"
27 #include "cds/archiveUtils.inline.hpp"
28 #include "classfile/classLoader.hpp"
29 #include "classfile/compactHashtable.hpp"
30 #include "classfile/javaClasses.inline.hpp"
31 #include "classfile/stringTable.hpp"
32 #include "classfile/vmClasses.hpp"
33 #include "classfile/vmSymbols.hpp"
34 #include "code/aotCodeCache.hpp"
35 #include "code/codeCache.hpp"
36 #include "code/compiledIC.hpp"
37 #include "code/nmethod.inline.hpp"
38 #include "code/scopeDesc.hpp"
39 #include "code/vtableStubs.hpp"
40 #include "compiler/abstractCompiler.hpp"
41 #include "compiler/compileBroker.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "gc/shared/barrierSet.hpp"
44 #include "gc/shared/collectedHeap.hpp"
45 #include "interpreter/interpreter.hpp"
46 #include "interpreter/interpreterRuntime.hpp"
47 #include "jfr/jfrEvents.hpp"
48 #include "jvm.h"
49 #include "logging/log.hpp"
50 #include "memory/oopFactory.hpp"
51 #include "memory/resourceArea.hpp"
52 #include "memory/universe.hpp"
53 #include "metaprogramming/primitiveConversions.hpp"
54 #include "oops/access.hpp"
55 #include "oops/fieldStreams.inline.hpp"
56 #include "oops/inlineKlass.inline.hpp"
57 #include "oops/klass.hpp"
58 #include "oops/method.inline.hpp"
59 #include "oops/objArrayKlass.hpp"
60 #include "oops/objArrayOop.inline.hpp"
61 #include "oops/oop.inline.hpp"
62 #include "prims/forte.hpp"
63 #include "prims/jvmtiExport.hpp"
64 #include "prims/jvmtiThreadState.hpp"
65 #include "prims/methodHandles.hpp"
66 #include "prims/nativeLookup.hpp"
67 #include "runtime/arguments.hpp"
68 #include "runtime/atomicAccess.hpp"
69 #include "runtime/basicLock.inline.hpp"
70 #include "runtime/frame.inline.hpp"
71 #include "runtime/handles.inline.hpp"
72 #include "runtime/init.hpp"
73 #include "runtime/interfaceSupport.inline.hpp"
74 #include "runtime/java.hpp"
75 #include "runtime/javaCalls.hpp"
76 #include "runtime/jniHandles.inline.hpp"
77 #include "runtime/osThread.hpp"
78 #include "runtime/perfData.hpp"
79 #include "runtime/sharedRuntime.hpp"
80 #include "runtime/signature.hpp"
81 #include "runtime/stackWatermarkSet.hpp"
82 #include "runtime/stubRoutines.hpp"
83 #include "runtime/synchronizer.hpp"
84 #include "runtime/timerTrace.hpp"
85 #include "runtime/vframe.inline.hpp"
86 #include "runtime/vframeArray.hpp"
87 #include "runtime/vm_version.hpp"
88 #include "utilities/copy.hpp"
89 #include "utilities/dtrace.hpp"
90 #include "utilities/events.hpp"
91 #include "utilities/exceptions.hpp"
92 #include "utilities/globalDefinitions.hpp"
93 #include "utilities/hashTable.hpp"
94 #include "utilities/macros.hpp"
95 #include "utilities/xmlstream.hpp"
96 #ifdef COMPILER1
97 #include "c1/c1_Runtime1.hpp"
98 #endif
99 #ifdef COMPILER2
100 #include "opto/runtime.hpp"
101 #endif
102 #if INCLUDE_JFR
103 #include "jfr/jfr.inline.hpp"
104 #endif
105
106 // Shared runtime stub routines reside in their own unique blob with a
107 // single entry point
108
109
110 #define SHARED_STUB_FIELD_DEFINE(name, type) \
111 type* SharedRuntime::BLOB_FIELD_NAME(name);
112 SHARED_STUBS_DO(SHARED_STUB_FIELD_DEFINE)
113 #undef SHARED_STUB_FIELD_DEFINE
114
115 nmethod* SharedRuntime::_cont_doYield_stub;
116
117 //----------------------------generate_stubs-----------------------------------
118 void SharedRuntime::generate_initial_stubs() {
119 // Build this early so it's available for the interpreter.
120 _throw_StackOverflowError_blob =
121 generate_throw_exception(StubId::shared_throw_StackOverflowError_id,
122 CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
123
124 if (InlineTypeReturnedAsFields) {
125 _store_inline_type_fields_to_buf_blob =
126 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf));
127 }
128 }
129
130 void SharedRuntime::generate_stubs() {
131 _wrong_method_blob =
132 generate_resolve_blob(StubId::shared_wrong_method_id,
133 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method));
134 _wrong_method_abstract_blob =
135 generate_resolve_blob(StubId::shared_wrong_method_abstract_id,
136 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract));
137 _ic_miss_blob =
138 generate_resolve_blob(StubId::shared_ic_miss_id,
139 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss));
140 _resolve_opt_virtual_call_blob =
141 generate_resolve_blob(StubId::shared_resolve_opt_virtual_call_id,
142 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C));
143 _resolve_virtual_call_blob =
144 generate_resolve_blob(StubId::shared_resolve_virtual_call_id,
145 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C));
146 _resolve_static_call_blob =
147 generate_resolve_blob(StubId::shared_resolve_static_call_id,
148 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C));
149
150 _throw_delayed_StackOverflowError_blob =
151 generate_throw_exception(StubId::shared_throw_delayed_StackOverflowError_id,
152 CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
153
154 _throw_AbstractMethodError_blob =
155 generate_throw_exception(StubId::shared_throw_AbstractMethodError_id,
156 CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
157
158 _throw_IncompatibleClassChangeError_blob =
159 generate_throw_exception(StubId::shared_throw_IncompatibleClassChangeError_id,
160 CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
161
162 _throw_NullPointerException_at_call_blob =
163 generate_throw_exception(StubId::shared_throw_NullPointerException_at_call_id,
164 CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
165
166 #ifdef COMPILER2
167 // Vectors are generated only by C2.
168 bool support_wide = is_wide_vector(MaxVectorSize);
169 if (support_wide) {
170 _polling_page_vectors_safepoint_handler_blob =
171 generate_handler_blob(StubId::shared_polling_page_vectors_safepoint_handler_id,
172 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
173 }
174 #endif // COMPILER2
175 _polling_page_safepoint_handler_blob =
176 generate_handler_blob(StubId::shared_polling_page_safepoint_handler_id,
177 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
178 _polling_page_return_handler_blob =
179 generate_handler_blob(StubId::shared_polling_page_return_handler_id,
180 CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception));
181
182 generate_deopt_blob();
183
184 #if INCLUDE_CDS
185 // disallow any further generation of runtime stubs
186 AOTCodeCache::set_shared_stubs_complete();
187 #endif // INCLUDE_CDS
188 }
189
190 void SharedRuntime::init_adapter_library() {
191 AdapterHandlerLibrary::initialize();
192 }
193
194 #if INCLUDE_JFR
195 //------------------------------generate jfr runtime stubs ------
196 void SharedRuntime::generate_jfr_stubs() {
197 ResourceMark rm;
198 const char* timer_msg = "SharedRuntime generate_jfr_stubs";
199 TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime));
200
201 _jfr_write_checkpoint_blob = generate_jfr_write_checkpoint();
202 _jfr_return_lease_blob = generate_jfr_return_lease();
203 }
204
205 #endif // INCLUDE_JFR
206
207 #include <math.h>
208
209 // Implementation of SharedRuntime
210
211 #ifndef PRODUCT
212 // For statistics
213 uint SharedRuntime::_ic_miss_ctr = 0;
214 uint SharedRuntime::_wrong_method_ctr = 0;
215 uint SharedRuntime::_resolve_static_ctr = 0;
216 uint SharedRuntime::_resolve_virtual_ctr = 0;
217 uint SharedRuntime::_resolve_opt_virtual_ctr = 0;
218 uint SharedRuntime::_implicit_null_throws = 0;
219 uint SharedRuntime::_implicit_div0_throws = 0;
220
221 int64_t SharedRuntime::_nof_normal_calls = 0;
222 int64_t SharedRuntime::_nof_inlined_calls = 0;
223 int64_t SharedRuntime::_nof_megamorphic_calls = 0;
224 int64_t SharedRuntime::_nof_static_calls = 0;
225 int64_t SharedRuntime::_nof_inlined_static_calls = 0;
226 int64_t SharedRuntime::_nof_interface_calls = 0;
227 int64_t SharedRuntime::_nof_inlined_interface_calls = 0;
228
229 uint SharedRuntime::_new_instance_ctr=0;
230 uint SharedRuntime::_new_array_ctr=0;
231 uint SharedRuntime::_multi2_ctr=0;
232 uint SharedRuntime::_multi3_ctr=0;
233 uint SharedRuntime::_multi4_ctr=0;
234 uint SharedRuntime::_multi5_ctr=0;
235 uint SharedRuntime::_mon_enter_stub_ctr=0;
236 uint SharedRuntime::_mon_exit_stub_ctr=0;
237 uint SharedRuntime::_mon_enter_ctr=0;
238 uint SharedRuntime::_mon_exit_ctr=0;
239 uint SharedRuntime::_partial_subtype_ctr=0;
240 uint SharedRuntime::_jbyte_array_copy_ctr=0;
241 uint SharedRuntime::_jshort_array_copy_ctr=0;
242 uint SharedRuntime::_jint_array_copy_ctr=0;
243 uint SharedRuntime::_jlong_array_copy_ctr=0;
244 uint SharedRuntime::_oop_array_copy_ctr=0;
245 uint SharedRuntime::_checkcast_array_copy_ctr=0;
246 uint SharedRuntime::_unsafe_array_copy_ctr=0;
247 uint SharedRuntime::_generic_array_copy_ctr=0;
248 uint SharedRuntime::_slow_array_copy_ctr=0;
249 uint SharedRuntime::_find_handler_ctr=0;
250 uint SharedRuntime::_rethrow_ctr=0;
251 uint SharedRuntime::_unsafe_set_memory_ctr=0;
252
253 int SharedRuntime::_ICmiss_index = 0;
254 int SharedRuntime::_ICmiss_count[SharedRuntime::maxICmiss_count];
255 address SharedRuntime::_ICmiss_at[SharedRuntime::maxICmiss_count];
256
257
258 void SharedRuntime::trace_ic_miss(address at) {
259 for (int i = 0; i < _ICmiss_index; i++) {
260 if (_ICmiss_at[i] == at) {
261 _ICmiss_count[i]++;
262 return;
263 }
264 }
265 int index = _ICmiss_index++;
266 if (_ICmiss_index >= maxICmiss_count) _ICmiss_index = maxICmiss_count - 1;
267 _ICmiss_at[index] = at;
268 _ICmiss_count[index] = 1;
269 }
270
271 void SharedRuntime::print_ic_miss_histogram() {
272 if (ICMissHistogram) {
273 tty->print_cr("IC Miss Histogram:");
274 int tot_misses = 0;
275 for (int i = 0; i < _ICmiss_index; i++) {
276 tty->print_cr(" at: " INTPTR_FORMAT " nof: %d", p2i(_ICmiss_at[i]), _ICmiss_count[i]);
277 tot_misses += _ICmiss_count[i];
278 }
279 tty->print_cr("Total IC misses: %7d", tot_misses);
280 }
281 }
282
283 #ifdef COMPILER2
284 // Runtime methods for printf-style debug nodes (same printing format as fieldDescriptor::print_on_for)
285 void SharedRuntime::debug_print_value(jboolean x) {
286 tty->print_cr("boolean %d", x);
287 }
288
289 void SharedRuntime::debug_print_value(jbyte x) {
290 tty->print_cr("byte %d", x);
291 }
292
293 void SharedRuntime::debug_print_value(jshort x) {
294 tty->print_cr("short %d", x);
295 }
296
297 void SharedRuntime::debug_print_value(jchar x) {
298 tty->print_cr("char %c %d", isprint(x) ? x : ' ', x);
299 }
300
301 void SharedRuntime::debug_print_value(jint x) {
302 tty->print_cr("int %d", x);
303 }
304
305 void SharedRuntime::debug_print_value(jlong x) {
306 tty->print_cr("long " JLONG_FORMAT, x);
307 }
308
309 void SharedRuntime::debug_print_value(jfloat x) {
310 tty->print_cr("float %f", x);
311 }
312
313 void SharedRuntime::debug_print_value(jdouble x) {
314 tty->print_cr("double %lf", x);
315 }
316
317 void SharedRuntime::debug_print_value(oopDesc* x) {
318 x->print();
319 }
320 #endif // COMPILER2
321
322 #endif // PRODUCT
323
324
325 JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x))
326 return x * y;
327 JRT_END
328
329
330 JRT_LEAF(jlong, SharedRuntime::ldiv(jlong y, jlong x))
331 if (x == min_jlong && y == CONST64(-1)) {
332 return x;
333 } else {
334 return x / y;
335 }
336 JRT_END
337
338
339 JRT_LEAF(jlong, SharedRuntime::lrem(jlong y, jlong x))
340 if (x == min_jlong && y == CONST64(-1)) {
341 return 0;
342 } else {
343 return x % y;
344 }
345 JRT_END
346
347
348 #ifdef _WIN64
349 const juint float_sign_mask = 0x7FFFFFFF;
350 const juint float_infinity = 0x7F800000;
351 const julong double_sign_mask = CONST64(0x7FFFFFFFFFFFFFFF);
352 const julong double_infinity = CONST64(0x7FF0000000000000);
353 #endif
354
355 #if !defined(X86)
356 JRT_LEAF(jfloat, SharedRuntime::frem(jfloat x, jfloat y))
357 #ifdef _WIN64
358 // 64-bit Windows on amd64 returns the wrong values for
359 // infinity operands.
360 juint xbits = PrimitiveConversions::cast<juint>(x);
361 juint ybits = PrimitiveConversions::cast<juint>(y);
362 // x Mod Infinity == x unless x is infinity
363 if (((xbits & float_sign_mask) != float_infinity) &&
364 ((ybits & float_sign_mask) == float_infinity) ) {
365 return x;
366 }
367 return ((jfloat)fmod_winx64((double)x, (double)y));
368 #else
369 return ((jfloat)fmod((double)x,(double)y));
370 #endif
371 JRT_END
372
373 JRT_LEAF(jdouble, SharedRuntime::drem(jdouble x, jdouble y))
374 #ifdef _WIN64
375 julong xbits = PrimitiveConversions::cast<julong>(x);
376 julong ybits = PrimitiveConversions::cast<julong>(y);
377 // x Mod Infinity == x unless x is infinity
378 if (((xbits & double_sign_mask) != double_infinity) &&
379 ((ybits & double_sign_mask) == double_infinity) ) {
380 return x;
381 }
382 return ((jdouble)fmod_winx64((double)x, (double)y));
383 #else
384 return ((jdouble)fmod((double)x,(double)y));
385 #endif
386 JRT_END
387 #endif // !X86
388
389 JRT_LEAF(jfloat, SharedRuntime::i2f(jint x))
390 return (jfloat)x;
391 JRT_END
392
393 #ifdef __SOFTFP__
394 JRT_LEAF(jfloat, SharedRuntime::fadd(jfloat x, jfloat y))
395 return x + y;
396 JRT_END
397
398 JRT_LEAF(jfloat, SharedRuntime::fsub(jfloat x, jfloat y))
399 return x - y;
400 JRT_END
401
402 JRT_LEAF(jfloat, SharedRuntime::fmul(jfloat x, jfloat y))
403 return x * y;
404 JRT_END
405
406 JRT_LEAF(jfloat, SharedRuntime::fdiv(jfloat x, jfloat y))
407 return x / y;
408 JRT_END
409
410 JRT_LEAF(jdouble, SharedRuntime::dadd(jdouble x, jdouble y))
411 return x + y;
412 JRT_END
413
414 JRT_LEAF(jdouble, SharedRuntime::dsub(jdouble x, jdouble y))
415 return x - y;
416 JRT_END
417
418 JRT_LEAF(jdouble, SharedRuntime::dmul(jdouble x, jdouble y))
419 return x * y;
420 JRT_END
421
422 JRT_LEAF(jdouble, SharedRuntime::ddiv(jdouble x, jdouble y))
423 return x / y;
424 JRT_END
425
426 JRT_LEAF(jdouble, SharedRuntime::i2d(jint x))
427 return (jdouble)x;
428 JRT_END
429
430 JRT_LEAF(jdouble, SharedRuntime::f2d(jfloat x))
431 return (jdouble)x;
432 JRT_END
433
434 JRT_LEAF(int, SharedRuntime::fcmpl(float x, float y))
435 return x>y ? 1 : (x==y ? 0 : -1); /* x<y or is_nan*/
436 JRT_END
437
438 JRT_LEAF(int, SharedRuntime::fcmpg(float x, float y))
439 return x<y ? -1 : (x==y ? 0 : 1); /* x>y or is_nan */
440 JRT_END
441
442 JRT_LEAF(int, SharedRuntime::dcmpl(double x, double y))
443 return x>y ? 1 : (x==y ? 0 : -1); /* x<y or is_nan */
444 JRT_END
445
446 JRT_LEAF(int, SharedRuntime::dcmpg(double x, double y))
447 return x<y ? -1 : (x==y ? 0 : 1); /* x>y or is_nan */
448 JRT_END
449
450 // Functions to return the opposite of the aeabi functions for nan.
451 JRT_LEAF(int, SharedRuntime::unordered_fcmplt(float x, float y))
452 return (x < y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
453 JRT_END
454
455 JRT_LEAF(int, SharedRuntime::unordered_dcmplt(double x, double y))
456 return (x < y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
457 JRT_END
458
459 JRT_LEAF(int, SharedRuntime::unordered_fcmple(float x, float y))
460 return (x <= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
461 JRT_END
462
463 JRT_LEAF(int, SharedRuntime::unordered_dcmple(double x, double y))
464 return (x <= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
465 JRT_END
466
467 JRT_LEAF(int, SharedRuntime::unordered_fcmpge(float x, float y))
468 return (x >= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
469 JRT_END
470
471 JRT_LEAF(int, SharedRuntime::unordered_dcmpge(double x, double y))
472 return (x >= y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
473 JRT_END
474
475 JRT_LEAF(int, SharedRuntime::unordered_fcmpgt(float x, float y))
476 return (x > y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
477 JRT_END
478
479 JRT_LEAF(int, SharedRuntime::unordered_dcmpgt(double x, double y))
480 return (x > y) ? 1 : ((g_isnan(x) || g_isnan(y)) ? 1 : 0);
481 JRT_END
482
483 // Intrinsics make gcc generate code for these.
484 float SharedRuntime::fneg(float f) {
485 return -f;
486 }
487
488 double SharedRuntime::dneg(double f) {
489 return -f;
490 }
491
492 #endif // __SOFTFP__
493
494 #if defined(__SOFTFP__) || defined(E500V2)
495 // Intrinsics make gcc generate code for these.
496 double SharedRuntime::dabs(double f) {
497 return (f <= (double)0.0) ? (double)0.0 - f : f;
498 }
499
500 #endif
501
502 #if defined(__SOFTFP__)
503 double SharedRuntime::dsqrt(double f) {
504 return sqrt(f);
505 }
506 #endif
507
508 JRT_LEAF(jint, SharedRuntime::f2i(jfloat x))
509 if (g_isnan(x))
510 return 0;
511 if (x >= (jfloat) max_jint)
512 return max_jint;
513 if (x <= (jfloat) min_jint)
514 return min_jint;
515 return (jint) x;
516 JRT_END
517
518
519 JRT_LEAF(jlong, SharedRuntime::f2l(jfloat x))
520 if (g_isnan(x))
521 return 0;
522 if (x >= (jfloat) max_jlong)
523 return max_jlong;
524 if (x <= (jfloat) min_jlong)
525 return min_jlong;
526 return (jlong) x;
527 JRT_END
528
529
530 JRT_LEAF(jint, SharedRuntime::d2i(jdouble x))
531 if (g_isnan(x))
532 return 0;
533 if (x >= (jdouble) max_jint)
534 return max_jint;
535 if (x <= (jdouble) min_jint)
536 return min_jint;
537 return (jint) x;
538 JRT_END
539
540
541 JRT_LEAF(jlong, SharedRuntime::d2l(jdouble x))
542 if (g_isnan(x))
543 return 0;
544 if (x >= (jdouble) max_jlong)
545 return max_jlong;
546 if (x <= (jdouble) min_jlong)
547 return min_jlong;
548 return (jlong) x;
549 JRT_END
550
551
552 JRT_LEAF(jfloat, SharedRuntime::d2f(jdouble x))
553 return (jfloat)x;
554 JRT_END
555
556
557 JRT_LEAF(jfloat, SharedRuntime::l2f(jlong x))
558 return (jfloat)x;
559 JRT_END
560
561
562 JRT_LEAF(jdouble, SharedRuntime::l2d(jlong x))
563 return (jdouble)x;
564 JRT_END
565
566
567 // Exception handling across interpreter/compiler boundaries
568 //
569 // exception_handler_for_return_address(...) returns the continuation address.
570 // The continuation address is the entry point of the exception handler of the
571 // previous frame depending on the return address.
572
573 address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* current, address return_address) {
574 // Note: This is called when we have unwound the frame of the callee that did
575 // throw an exception. So far, no check has been performed by the StackWatermarkSet.
576 // Notably, the stack is not walkable at this point, and hence the check must
577 // be deferred until later. Specifically, any of the handlers returned here in
578 // this function, will get dispatched to, and call deferred checks to
579 // StackWatermarkSet::after_unwind at a point where the stack is walkable.
580 assert(frame::verify_return_pc(return_address), "must be a return address: " INTPTR_FORMAT, p2i(return_address));
581 assert(current->frames_to_pop_failed_realloc() == 0 || Interpreter::contains(return_address), "missed frames to pop?");
582
583 if (Continuation::is_return_barrier_entry(return_address)) {
584 return StubRoutines::cont_returnBarrierExc();
585 }
586
587 // The fastest case first
588 CodeBlob* blob = CodeCache::find_blob(return_address);
589 nmethod* nm = (blob != nullptr) ? blob->as_nmethod_or_null() : nullptr;
590 if (nm != nullptr) {
591 // native nmethods don't have exception handlers
592 assert(!nm->is_native_method() || nm->method()->is_continuation_enter_intrinsic(), "no exception handler");
593 assert(nm->header_begin() != nm->exception_begin(), "no exception handler");
594 if (nm->is_deopt_pc(return_address)) {
595 // If we come here because of a stack overflow, the stack may be
596 // unguarded. Reguard the stack otherwise if we return to the
597 // deopt blob and the stack bang causes a stack overflow we
598 // crash.
599 StackOverflow* overflow_state = current->stack_overflow_state();
600 bool guard_pages_enabled = overflow_state->reguard_stack_if_needed();
601 if (overflow_state->reserved_stack_activation() != current->stack_base()) {
602 overflow_state->set_reserved_stack_activation(current->stack_base());
603 }
604 assert(guard_pages_enabled, "stack banging in deopt blob may cause crash");
605 // The deferred StackWatermarkSet::after_unwind check will be performed in
606 // Deoptimization::fetch_unroll_info (with exec_mode == Unpack_exception)
607 return SharedRuntime::deopt_blob()->unpack_with_exception();
608 } else {
609 // The deferred StackWatermarkSet::after_unwind check will be performed in
610 // * OptoRuntime::handle_exception_C_helper for C2 code
611 // * exception_handler_for_pc_helper via Runtime1::handle_exception_from_callee_id for C1 code
612 #ifdef COMPILER2
613 if (nm->compiler_type() == compiler_c2) {
614 return OptoRuntime::exception_blob()->entry_point();
615 }
616 #endif // COMPILER2
617 return nm->exception_begin();
618 }
619 }
620
621 // Entry code
622 if (StubRoutines::returns_to_call_stub(return_address)) {
623 // The deferred StackWatermarkSet::after_unwind check will be performed in
624 // JavaCallWrapper::~JavaCallWrapper
625 assert (StubRoutines::catch_exception_entry() != nullptr, "must be generated before");
626 return StubRoutines::catch_exception_entry();
627 }
628 if (blob != nullptr && blob->is_upcall_stub()) {
629 return StubRoutines::upcall_stub_exception_handler();
630 }
631 // Interpreted code
632 if (Interpreter::contains(return_address)) {
633 // The deferred StackWatermarkSet::after_unwind check will be performed in
634 // InterpreterRuntime::exception_handler_for_exception
635 return Interpreter::rethrow_exception_entry();
636 }
637
638 guarantee(blob == nullptr || !blob->is_runtime_stub(), "caller should have skipped stub");
639 guarantee(!VtableStubs::contains(return_address), "null exceptions in vtables should have been handled already!");
640
641 #ifndef PRODUCT
642 { ResourceMark rm;
643 tty->print_cr("No exception handler found for exception at " INTPTR_FORMAT " - potential problems:", p2i(return_address));
644 os::print_location(tty, (intptr_t)return_address);
645 tty->print_cr("a) exception happened in (new?) code stubs/buffers that is not handled here");
646 tty->print_cr("b) other problem");
647 }
648 #endif // PRODUCT
649 ShouldNotReachHere();
650 return nullptr;
651 }
652
653
654 JRT_LEAF(address, SharedRuntime::exception_handler_for_return_address(JavaThread* current, address return_address))
655 return raw_exception_handler_for_return_address(current, return_address);
656 JRT_END
657
658
659 address SharedRuntime::get_poll_stub(address pc) {
660 address stub;
661 // Look up the code blob
662 CodeBlob *cb = CodeCache::find_blob(pc);
663
664 // Should be an nmethod
665 guarantee(cb != nullptr && cb->is_nmethod(), "safepoint polling: pc must refer to an nmethod");
666
667 // Look up the relocation information
668 assert(cb->as_nmethod()->is_at_poll_or_poll_return(pc),
669 "safepoint polling: type must be poll at pc " INTPTR_FORMAT, p2i(pc));
670
671 #ifdef ASSERT
672 if (!((NativeInstruction*)pc)->is_safepoint_poll()) {
673 tty->print_cr("bad pc: " PTR_FORMAT, p2i(pc));
674 Disassembler::decode(cb);
675 fatal("Only polling locations are used for safepoint");
676 }
677 #endif
678
679 bool at_poll_return = cb->as_nmethod()->is_at_poll_return(pc);
680 bool has_wide_vectors = cb->as_nmethod()->has_wide_vectors();
681 if (at_poll_return) {
682 assert(SharedRuntime::polling_page_return_handler_blob() != nullptr,
683 "polling page return stub not created yet");
684 stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
685 } else if (has_wide_vectors) {
686 assert(SharedRuntime::polling_page_vectors_safepoint_handler_blob() != nullptr,
687 "polling page vectors safepoint stub not created yet");
688 stub = SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point();
689 } else {
690 assert(SharedRuntime::polling_page_safepoint_handler_blob() != nullptr,
691 "polling page safepoint stub not created yet");
692 stub = SharedRuntime::polling_page_safepoint_handler_blob()->entry_point();
693 }
694 log_trace(safepoint)("Polling page exception: thread = " INTPTR_FORMAT " [%d], pc = "
695 INTPTR_FORMAT " (%s), stub = " INTPTR_FORMAT,
696 p2i(Thread::current()),
697 Thread::current()->osthread()->thread_id(),
698 p2i(pc),
699 at_poll_return ? "return" : "loop",
700 p2i(stub));
701 return stub;
702 }
703
704 void SharedRuntime::throw_and_post_jvmti_exception(JavaThread* current, Handle h_exception) {
705 if (JvmtiExport::can_post_on_exceptions()) {
706 vframeStream vfst(current, true);
707 methodHandle method = methodHandle(current, vfst.method());
708 address bcp = method()->bcp_from(vfst.bci());
709 JvmtiExport::post_exception_throw(current, method(), bcp, h_exception());
710 }
711
712 Exceptions::_throw(current, __FILE__, __LINE__, h_exception);
713 }
714
715 void SharedRuntime::throw_and_post_jvmti_exception(JavaThread* current, Symbol* name, const char *message) {
716 Handle h_exception = Exceptions::new_exception(current, name, message);
717 throw_and_post_jvmti_exception(current, h_exception);
718 }
719
720 // The interpreter code to call this tracing function is only
721 // called/generated when UL is on for redefine, class and has the right level
722 // and tags. Since obsolete methods are never compiled, we don't have
723 // to modify the compilers to generate calls to this function.
724 //
725 JRT_LEAF(int, SharedRuntime::rc_trace_method_entry(
726 JavaThread* thread, Method* method))
727 if (method->is_obsolete()) {
728 // We are calling an obsolete method, but this is not necessarily
729 // an error. Our method could have been redefined just after we
730 // fetched the Method* from the constant pool.
731 ResourceMark rm;
732 log_trace(redefine, class, obsolete)("calling obsolete method '%s'", method->name_and_sig_as_C_string());
733 }
734 return 0;
735 JRT_END
736
737 // ret_pc points into caller; we are returning caller's exception handler
738 // for given exception
739 // Note that the implementation of this method assumes it's only called when an exception has actually occured
740 address SharedRuntime::compute_compiled_exc_handler(nmethod* nm, address ret_pc, Handle& exception,
741 bool force_unwind, bool top_frame_only, bool& recursive_exception_occurred) {
742 assert(nm != nullptr, "must exist");
743 ResourceMark rm;
744
745 ScopeDesc* sd = nm->scope_desc_at(ret_pc);
746 // determine handler bci, if any
747 EXCEPTION_MARK;
748
749 Handle orig_exception(THREAD, exception());
750
751 int handler_bci = -1;
752 int scope_depth = 0;
753 if (!force_unwind) {
754 int bci = sd->bci();
755 bool recursive_exception = false;
756 do {
757 bool skip_scope_increment = false;
758 // exception handler lookup
759 Klass* ek = exception->klass();
760 methodHandle mh(THREAD, sd->method());
761 handler_bci = Method::fast_exception_handler_bci_for(mh, ek, bci, THREAD);
762 if (HAS_PENDING_EXCEPTION) {
763 recursive_exception = true;
764 // We threw an exception while trying to find the exception handler.
765 // Transfer the new exception to the exception handle which will
766 // be set into thread local storage, and do another lookup for an
767 // exception handler for this exception, this time starting at the
768 // BCI of the exception handler which caused the exception to be
769 // thrown (bugs 4307310 and 4546590). Set "exception" reference
770 // argument to ensure that the correct exception is thrown (4870175).
771 recursive_exception_occurred = true;
772 exception.replace(PENDING_EXCEPTION);
773 CLEAR_PENDING_EXCEPTION;
774 if (handler_bci >= 0) {
775 bci = handler_bci;
776 handler_bci = -1;
777 skip_scope_increment = true;
778 }
779 }
780 else {
781 recursive_exception = false;
782 }
783 if (!top_frame_only && handler_bci < 0 && !skip_scope_increment) {
784 sd = sd->sender();
785 if (sd != nullptr) {
786 bci = sd->bci();
787 }
788 ++scope_depth;
789 }
790 } while (recursive_exception || (!top_frame_only && handler_bci < 0 && sd != nullptr));
791 }
792
793 // found handling method => lookup exception handler
794 int catch_pco = pointer_delta_as_int(ret_pc, nm->code_begin());
795
796 ExceptionHandlerTable table(nm);
797 HandlerTableEntry *t = table.entry_for(catch_pco, handler_bci, scope_depth);
798
799 // If the compiler did not anticipate a recursive exception, resulting in an exception
800 // thrown from the catch bci, then the compiled exception handler might be missing.
801 // This is rare. Just deoptimize and let the interpreter rethrow the original
802 // exception at the original bci.
803 if (t == nullptr && recursive_exception_occurred) {
804 exception.replace(orig_exception()); // restore original exception
805 bool make_not_entrant = false;
806 return Deoptimization::deoptimize_for_missing_exception_handler(nm, make_not_entrant);
807 }
808
809 if (t == nullptr && (nm->is_compiled_by_c1() || handler_bci != -1)) {
810 // Allow abbreviated catch tables. The idea is to allow a method
811 // to materialize its exceptions without committing to the exact
812 // routing of exceptions. In particular this is needed for adding
813 // a synthetic handler to unlock monitors when inlining
814 // synchronized methods since the unlock path isn't represented in
815 // the bytecodes.
816 t = table.entry_for(catch_pco, -1, 0);
817 }
818
819 #ifdef COMPILER1
820 if (t == nullptr && nm->is_compiled_by_c1()) {
821 assert(nm->unwind_handler_begin() != nullptr, "");
822 return nm->unwind_handler_begin();
823 }
824 #endif
825
826 if (t == nullptr) {
827 ttyLocker ttyl;
828 tty->print_cr("MISSING EXCEPTION HANDLER for pc " INTPTR_FORMAT " and handler bci %d, catch_pco: %d", p2i(ret_pc), handler_bci, catch_pco);
829 tty->print_cr(" Exception:");
830 exception->print();
831 tty->cr();
832 tty->print_cr(" Compiled exception table :");
833 table.print();
834 nm->print();
835 nm->print_code();
836 guarantee(false, "missing exception handler");
837 return nullptr;
838 }
839
840 if (handler_bci != -1) { // did we find a handler in this method?
841 sd->method()->set_exception_handler_entered(handler_bci); // profile
842 }
843 return nm->code_begin() + t->pco();
844 }
845
846 JRT_ENTRY(void, SharedRuntime::throw_AbstractMethodError(JavaThread* current))
847 // These errors occur only at call sites
848 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_AbstractMethodError());
849 JRT_END
850
851 JRT_ENTRY(void, SharedRuntime::throw_IncompatibleClassChangeError(JavaThread* current))
852 // These errors occur only at call sites
853 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IncompatibleClassChangeError(), "vtable stub");
854 JRT_END
855
856 JRT_ENTRY(void, SharedRuntime::throw_ArithmeticException(JavaThread* current))
857 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ArithmeticException(), "/ by zero");
858 JRT_END
859
860 JRT_ENTRY(void, SharedRuntime::throw_NullPointerException(JavaThread* current))
861 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException(), nullptr);
862 JRT_END
863
864 JRT_ENTRY(void, SharedRuntime::throw_NullPointerException_at_call(JavaThread* current))
865 // This entry point is effectively only used for NullPointerExceptions which occur at inline
866 // cache sites (when the callee activation is not yet set up) so we are at a call site
867 throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException(), nullptr);
868 JRT_END
869
870 JRT_ENTRY(void, SharedRuntime::throw_StackOverflowError(JavaThread* current))
871 throw_StackOverflowError_common(current, false);
872 JRT_END
873
874 JRT_ENTRY(void, SharedRuntime::throw_delayed_StackOverflowError(JavaThread* current))
875 throw_StackOverflowError_common(current, true);
876 JRT_END
877
878 void SharedRuntime::throw_StackOverflowError_common(JavaThread* current, bool delayed) {
879 // We avoid using the normal exception construction in this case because
880 // it performs an upcall to Java, and we're already out of stack space.
881 JavaThread* THREAD = current; // For exception macros.
882 InstanceKlass* k = vmClasses::StackOverflowError_klass();
883 oop exception_oop = k->allocate_instance(CHECK);
884 if (delayed) {
885 java_lang_Throwable::set_message(exception_oop,
886 Universe::delayed_stack_overflow_error_message());
887 }
888 Handle exception (current, exception_oop);
889 if (StackTraceInThrowable) {
890 java_lang_Throwable::fill_in_stack_trace(exception);
891 }
892 // Remove the ScopedValue bindings in case we got a
893 // StackOverflowError while we were trying to remove ScopedValue
894 // bindings.
895 current->clear_scopedValueBindings();
896 // Increment counter for hs_err file reporting
897 Exceptions::increment_stack_overflow_errors();
898 throw_and_post_jvmti_exception(current, exception);
899 }
900
901 address SharedRuntime::continuation_for_implicit_exception(JavaThread* current,
902 address pc,
903 ImplicitExceptionKind exception_kind)
904 {
905 address target_pc = nullptr;
906
907 if (Interpreter::contains(pc)) {
908 switch (exception_kind) {
909 case IMPLICIT_NULL: return Interpreter::throw_NullPointerException_entry();
910 case IMPLICIT_DIVIDE_BY_ZERO: return Interpreter::throw_ArithmeticException_entry();
911 case STACK_OVERFLOW: return Interpreter::throw_StackOverflowError_entry();
912 default: ShouldNotReachHere();
913 }
914 } else {
915 switch (exception_kind) {
916 case STACK_OVERFLOW: {
917 // Stack overflow only occurs upon frame setup; the callee is
918 // going to be unwound. Dispatch to a shared runtime stub
919 // which will cause the StackOverflowError to be fabricated
920 // and processed.
921 // Stack overflow should never occur during deoptimization:
922 // the compiled method bangs the stack by as much as the
923 // interpreter would need in case of a deoptimization. The
924 // deoptimization blob and uncommon trap blob bang the stack
925 // in a debug VM to verify the correctness of the compiled
926 // method stack banging.
927 assert(current->deopt_mark() == nullptr, "no stack overflow from deopt blob/uncommon trap");
928 Events::log_exception(current, "StackOverflowError at " INTPTR_FORMAT, p2i(pc));
929 return SharedRuntime::throw_StackOverflowError_entry();
930 }
931
932 case IMPLICIT_NULL: {
933 if (VtableStubs::contains(pc)) {
934 // We haven't yet entered the callee frame. Fabricate an
935 // exception and begin dispatching it in the caller. Since
936 // the caller was at a call site, it's safe to destroy all
937 // caller-saved registers, as these entry points do.
938 VtableStub* vt_stub = VtableStubs::stub_containing(pc);
939
940 // If vt_stub is null, then return null to signal handler to report the SEGV error.
941 if (vt_stub == nullptr) return nullptr;
942
943 if (vt_stub->is_abstract_method_error(pc)) {
944 assert(!vt_stub->is_vtable_stub(), "should never see AbstractMethodErrors from vtable-type VtableStubs");
945 Events::log_exception(current, "AbstractMethodError at " INTPTR_FORMAT, p2i(pc));
946 // Instead of throwing the abstract method error here directly, we re-resolve
947 // and will throw the AbstractMethodError during resolve. As a result, we'll
948 // get a more detailed error message.
949 return SharedRuntime::get_handle_wrong_method_stub();
950 } else {
951 Events::log_exception(current, "NullPointerException at vtable entry " INTPTR_FORMAT, p2i(pc));
952 // Assert that the signal comes from the expected location in stub code.
953 assert(vt_stub->is_null_pointer_exception(pc),
954 "obtained signal from unexpected location in stub code");
955 return SharedRuntime::throw_NullPointerException_at_call_entry();
956 }
957 } else {
958 CodeBlob* cb = CodeCache::find_blob(pc);
959
960 // If code blob is null, then return null to signal handler to report the SEGV error.
961 if (cb == nullptr) return nullptr;
962
963 // Exception happened in CodeCache. Must be either:
964 // 1. Inline-cache check in C2I handler blob,
965 // 2. Inline-cache check in nmethod, or
966 // 3. Implicit null exception in nmethod
967
968 if (!cb->is_nmethod()) {
969 bool is_in_blob = cb->is_adapter_blob() || cb->is_method_handles_adapter_blob();
970 if (!is_in_blob) {
971 // Allow normal crash reporting to handle this
972 return nullptr;
973 }
974 Events::log_exception(current, "NullPointerException in code blob at " INTPTR_FORMAT, p2i(pc));
975 // There is no handler here, so we will simply unwind.
976 return SharedRuntime::throw_NullPointerException_at_call_entry();
977 }
978
979 // Otherwise, it's a compiled method. Consult its exception handlers.
980 nmethod* nm = cb->as_nmethod();
981 if (nm->inlinecache_check_contains(pc)) {
982 // exception happened inside inline-cache check code
983 // => the nmethod is not yet active (i.e., the frame
984 // is not set up yet) => use return address pushed by
985 // caller => don't push another return address
986 Events::log_exception(current, "NullPointerException in IC check " INTPTR_FORMAT, p2i(pc));
987 return SharedRuntime::throw_NullPointerException_at_call_entry();
988 }
989
990 if (nm->method()->is_method_handle_intrinsic()) {
991 // exception happened inside MH dispatch code, similar to a vtable stub
992 Events::log_exception(current, "NullPointerException in MH adapter " INTPTR_FORMAT, p2i(pc));
993 return SharedRuntime::throw_NullPointerException_at_call_entry();
994 }
995
996 #ifndef PRODUCT
997 _implicit_null_throws++;
998 #endif
999 target_pc = nm->continuation_for_implicit_exception(pc);
1000 // If there's an unexpected fault, target_pc might be null,
1001 // in which case we want to fall through into the normal
1002 // error handling code.
1003 }
1004
1005 break; // fall through
1006 }
1007
1008
1009 case IMPLICIT_DIVIDE_BY_ZERO: {
1010 nmethod* nm = CodeCache::find_nmethod(pc);
1011 guarantee(nm != nullptr, "must have containing compiled method for implicit division-by-zero exceptions");
1012 #ifndef PRODUCT
1013 _implicit_div0_throws++;
1014 #endif
1015 target_pc = nm->continuation_for_implicit_exception(pc);
1016 // If there's an unexpected fault, target_pc might be null,
1017 // in which case we want to fall through into the normal
1018 // error handling code.
1019 break; // fall through
1020 }
1021
1022 default: ShouldNotReachHere();
1023 }
1024
1025 assert(exception_kind == IMPLICIT_NULL || exception_kind == IMPLICIT_DIVIDE_BY_ZERO, "wrong implicit exception kind");
1026
1027 if (exception_kind == IMPLICIT_NULL) {
1028 #ifndef PRODUCT
1029 // for AbortVMOnException flag
1030 Exceptions::debug_check_abort("java.lang.NullPointerException");
1031 #endif //PRODUCT
1032 Events::log_exception(current, "Implicit null exception at " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(pc), p2i(target_pc));
1033 } else {
1034 #ifndef PRODUCT
1035 // for AbortVMOnException flag
1036 Exceptions::debug_check_abort("java.lang.ArithmeticException");
1037 #endif //PRODUCT
1038 Events::log_exception(current, "Implicit division by zero exception at " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(pc), p2i(target_pc));
1039 }
1040 return target_pc;
1041 }
1042
1043 ShouldNotReachHere();
1044 return nullptr;
1045 }
1046
1047
1048 /**
1049 * Throws an java/lang/UnsatisfiedLinkError. The address of this method is
1050 * installed in the native function entry of all native Java methods before
1051 * they get linked to their actual native methods.
1052 *
1053 * \note
1054 * This method actually never gets called! The reason is because
1055 * the interpreter's native entries call NativeLookup::lookup() which
1056 * throws the exception when the lookup fails. The exception is then
1057 * caught and forwarded on the return from NativeLookup::lookup() call
1058 * before the call to the native function. This might change in the future.
1059 */
1060 JNI_ENTRY(void*, throw_unsatisfied_link_error(JNIEnv* env, ...))
1061 {
1062 // We return a bad value here to make sure that the exception is
1063 // forwarded before we look at the return value.
1064 THROW_(vmSymbols::java_lang_UnsatisfiedLinkError(), (void*)badAddress);
1065 }
1066 JNI_END
1067
1068 address SharedRuntime::native_method_throw_unsatisfied_link_error_entry() {
1069 return CAST_FROM_FN_PTR(address, &throw_unsatisfied_link_error);
1070 }
1071
1072 JRT_ENTRY_NO_ASYNC(void, SharedRuntime::register_finalizer(JavaThread* current, oopDesc* obj))
1073 assert(oopDesc::is_oop(obj), "must be a valid oop");
1074 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
1075 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
1076 JRT_END
1077
1078 jlong SharedRuntime::get_java_tid(JavaThread* thread) {
1079 assert(thread != nullptr, "No thread");
1080 if (thread == nullptr) {
1081 return 0;
1082 }
1083 guarantee(Thread::current() != thread || thread->is_oop_safe(),
1084 "current cannot touch oops after its GC barrier is detached.");
1085 oop obj = thread->threadObj();
1086 return (obj == nullptr) ? 0 : java_lang_Thread::thread_id(obj);
1087 }
1088
1089 /**
1090 * This function ought to be a void function, but cannot be because
1091 * it gets turned into a tail-call on sparc, which runs into dtrace bug
1092 * 6254741. Once that is fixed we can remove the dummy return value.
1093 */
1094 int SharedRuntime::dtrace_object_alloc(oopDesc* o) {
1095 return dtrace_object_alloc(JavaThread::current(), o, o->size());
1096 }
1097
1098 int SharedRuntime::dtrace_object_alloc(JavaThread* thread, oopDesc* o) {
1099 return dtrace_object_alloc(thread, o, o->size());
1100 }
1101
1102 int SharedRuntime::dtrace_object_alloc(JavaThread* thread, oopDesc* o, size_t size) {
1103 assert(DTraceAllocProbes, "wrong call");
1104 Klass* klass = o->klass();
1105 Symbol* name = klass->name();
1106 HOTSPOT_OBJECT_ALLOC(
1107 get_java_tid(thread),
1108 (char *) name->bytes(), name->utf8_length(), size * HeapWordSize);
1109 return 0;
1110 }
1111
1112 JRT_LEAF(int, SharedRuntime::dtrace_method_entry(
1113 JavaThread* current, Method* method))
1114 assert(current == JavaThread::current(), "pre-condition");
1115
1116 assert(DTraceMethodProbes, "wrong call");
1117 Symbol* kname = method->klass_name();
1118 Symbol* name = method->name();
1119 Symbol* sig = method->signature();
1120 HOTSPOT_METHOD_ENTRY(
1121 get_java_tid(current),
1122 (char *) kname->bytes(), kname->utf8_length(),
1123 (char *) name->bytes(), name->utf8_length(),
1124 (char *) sig->bytes(), sig->utf8_length());
1125 return 0;
1126 JRT_END
1127
1128 JRT_LEAF(int, SharedRuntime::dtrace_method_exit(
1129 JavaThread* current, Method* method))
1130 assert(current == JavaThread::current(), "pre-condition");
1131 assert(DTraceMethodProbes, "wrong call");
1132 Symbol* kname = method->klass_name();
1133 Symbol* name = method->name();
1134 Symbol* sig = method->signature();
1135 HOTSPOT_METHOD_RETURN(
1136 get_java_tid(current),
1137 (char *) kname->bytes(), kname->utf8_length(),
1138 (char *) name->bytes(), name->utf8_length(),
1139 (char *) sig->bytes(), sig->utf8_length());
1140 return 0;
1141 JRT_END
1142
1143
1144 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode)
1145 // for a call current in progress, i.e., arguments has been pushed on stack
1146 // put callee has not been invoked yet. Used by: resolve virtual/static,
1147 // vtable updates, etc. Caller frame must be compiled.
1148 Handle SharedRuntime::find_callee_info(Bytecodes::Code& bc, CallInfo& callinfo, TRAPS) {
1149 JavaThread* current = THREAD;
1150 ResourceMark rm(current);
1151
1152 // last java frame on stack (which includes native call frames)
1153 vframeStream vfst(current, true); // Do not skip and javaCalls
1154
1155 return find_callee_info_helper(vfst, bc, callinfo, THREAD);
1156 }
1157
1158 Method* SharedRuntime::extract_attached_method(vframeStream& vfst) {
1159 nmethod* caller = vfst.nm();
1160
1161 address pc = vfst.frame_pc();
1162 { // Get call instruction under lock because another thread may be busy patching it.
1163 CompiledICLocker ic_locker(caller);
1164 return caller->attached_method_before_pc(pc);
1165 }
1166 return nullptr;
1167 }
1168
1169 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode
1170 // for a call current in progress, i.e., arguments has been pushed on stack
1171 // but callee has not been invoked yet. Caller frame must be compiled.
1172 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1173 CallInfo& callinfo, TRAPS) {
1174 Handle receiver;
1175 Handle nullHandle; // create a handy null handle for exception returns
1176 JavaThread* current = THREAD;
1177
1178 assert(!vfst.at_end(), "Java frame must exist");
1179
1180 // Find caller and bci from vframe
1181 methodHandle caller(current, vfst.method());
1182 int bci = vfst.bci();
1183
1184 if (caller->is_continuation_enter_intrinsic()) {
1185 bc = Bytecodes::_invokestatic;
1186 LinkResolver::resolve_continuation_enter(callinfo, CHECK_NH);
1187 return receiver;
1188 }
1189
1190 // Substitutability test implementation piggy backs on static call resolution
1191 Bytecodes::Code code = caller->java_code_at(bci);
1192 if (code == Bytecodes::_if_acmpeq || code == Bytecodes::_if_acmpne) {
1193 bc = Bytecodes::_invokestatic;
1194 methodHandle attached_method(THREAD, extract_attached_method(vfst));
1195 assert(attached_method.not_null(), "must have attached method");
1196 vmClasses::ValueObjectMethods_klass()->initialize(CHECK_NH);
1197 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, false, CHECK_NH);
1198 #ifdef ASSERT
1199 Symbol* subst_method_name = vmSymbols::isSubstitutable_name();
1200 Method* is_subst = vmClasses::ValueObjectMethods_klass()->find_method(subst_method_name, vmSymbols::object_object_boolean_signature());
1201 assert(callinfo.selected_method() == is_subst, "must be isSubstitutable method");
1202 #endif
1203 return receiver;
1204 }
1205
1206 Bytecode_invoke bytecode(caller, bci);
1207 int bytecode_index = bytecode.index();
1208 bc = bytecode.invoke_code();
1209
1210 methodHandle attached_method(current, extract_attached_method(vfst));
1211 if (attached_method.not_null()) {
1212 Method* callee = bytecode.static_target(CHECK_NH);
1213 vmIntrinsics::ID id = callee->intrinsic_id();
1214 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1215 // it attaches statically resolved method to the call site.
1216 if (MethodHandles::is_signature_polymorphic(id) &&
1217 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1218 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1219
1220 // Adjust invocation mode according to the attached method.
1221 switch (bc) {
1222 case Bytecodes::_invokevirtual:
1223 if (attached_method->method_holder()->is_interface()) {
1224 bc = Bytecodes::_invokeinterface;
1225 }
1226 break;
1227 case Bytecodes::_invokeinterface:
1228 if (!attached_method->method_holder()->is_interface()) {
1229 bc = Bytecodes::_invokevirtual;
1230 }
1231 break;
1232 case Bytecodes::_invokehandle:
1233 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1234 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1235 : Bytecodes::_invokevirtual;
1236 }
1237 break;
1238 default:
1239 break;
1240 }
1241 } else {
1242 assert(attached_method->has_scalarized_args(), "invalid use of attached method");
1243 if (!attached_method->method_holder()->is_inline_klass() || attached_method->is_static()) {
1244 // Ignore the attached method in this case to not confuse below code
1245 attached_method = methodHandle(current, nullptr);
1246 }
1247 }
1248 }
1249
1250 assert(bc != Bytecodes::_illegal, "not initialized");
1251
1252 bool has_receiver = bc != Bytecodes::_invokestatic &&
1253 bc != Bytecodes::_invokedynamic &&
1254 bc != Bytecodes::_invokehandle;
1255 bool check_null_and_abstract = true;
1256
1257 // Find receiver for non-static call
1258 if (has_receiver) {
1259 // This register map must be update since we need to find the receiver for
1260 // compiled frames. The receiver might be in a register.
1261 RegisterMap reg_map2(current,
1262 RegisterMap::UpdateMap::include,
1263 RegisterMap::ProcessFrames::include,
1264 RegisterMap::WalkContinuation::skip);
1265 frame stubFrame = current->last_frame();
1266 // Caller-frame is a compiled frame
1267 frame callerFrame = stubFrame.sender(®_map2);
1268
1269 Method* callee = attached_method();
1270 if (callee == nullptr) {
1271 callee = bytecode.static_target(CHECK_NH);
1272 if (callee == nullptr) {
1273 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1274 }
1275 }
1276 bool caller_is_c1 = callerFrame.is_compiled_frame() && callerFrame.cb()->as_nmethod()->is_compiled_by_c1();
1277 if (!caller_is_c1 && callee->is_scalarized_arg(0)) {
1278 // If the receiver is an inline type that is passed as fields, no oop is available
1279 // Resolve the call without receiver null checking.
1280 assert(!callee->mismatch(), "calls with inline type receivers should never mismatch");
1281 assert(attached_method.not_null() && !attached_method->is_abstract(), "must have non-abstract attached method");
1282 if (bc == Bytecodes::_invokeinterface) {
1283 bc = Bytecodes::_invokevirtual; // C2 optimistically replaces interface calls by virtual calls
1284 }
1285 check_null_and_abstract = false;
1286 } else {
1287 // Retrieve from a compiled argument list
1288 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1289 assert(oopDesc::is_oop_or_null(receiver()), "");
1290 if (receiver.is_null()) {
1291 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1292 }
1293 }
1294 }
1295
1296 // Resolve method
1297 if (attached_method.not_null()) {
1298 // Parameterized by attached method.
1299 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, check_null_and_abstract, CHECK_NH);
1300 } else {
1301 // Parameterized by bytecode.
1302 constantPoolHandle constants(current, caller->constants());
1303 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1304 }
1305
1306 #ifdef ASSERT
1307 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1308 if (has_receiver && check_null_and_abstract) {
1309 assert(receiver.not_null(), "should have thrown exception");
1310 Klass* receiver_klass = receiver->klass();
1311 Klass* rk = nullptr;
1312 if (attached_method.not_null()) {
1313 // In case there's resolved method attached, use its holder during the check.
1314 rk = attached_method->method_holder();
1315 } else {
1316 // Klass is already loaded.
1317 constantPoolHandle constants(current, caller->constants());
1318 rk = constants->klass_ref_at(bytecode_index, bc, CHECK_NH);
1319 }
1320 Klass* static_receiver_klass = rk;
1321 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1322 "actual receiver must be subclass of static receiver klass");
1323 if (receiver_klass->is_instance_klass()) {
1324 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1325 tty->print_cr("ERROR: Klass not yet initialized!!");
1326 receiver_klass->print();
1327 }
1328 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1329 }
1330 }
1331 #endif
1332
1333 return receiver;
1334 }
1335
1336 methodHandle SharedRuntime::find_callee_method(bool& caller_does_not_scalarize, TRAPS) {
1337 JavaThread* current = THREAD;
1338 ResourceMark rm(current);
1339 // We need first to check if any Java activations (compiled, interpreted)
1340 // exist on the stack since last JavaCall. If not, we need
1341 // to get the target method from the JavaCall wrapper.
1342 vframeStream vfst(current, true); // Do not skip any javaCalls
1343 methodHandle callee_method;
1344 if (vfst.at_end()) {
1345 // No Java frames were found on stack since we did the JavaCall.
1346 // Hence the stack can only contain an entry_frame. We need to
1347 // find the target method from the stub frame.
1348 RegisterMap reg_map(current,
1349 RegisterMap::UpdateMap::skip,
1350 RegisterMap::ProcessFrames::include,
1351 RegisterMap::WalkContinuation::skip);
1352 frame fr = current->last_frame();
1353 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1354 fr = fr.sender(®_map);
1355 assert(fr.is_entry_frame(), "must be");
1356 // fr is now pointing to the entry frame.
1357 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1358 } else {
1359 Bytecodes::Code bc;
1360 CallInfo callinfo;
1361 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1362 // Calls via mismatching methods are always non-scalarized
1363 if (callinfo.resolved_method()->mismatch()) {
1364 caller_does_not_scalarize = true;
1365 }
1366 callee_method = methodHandle(current, callinfo.selected_method());
1367 }
1368 assert(callee_method()->is_method(), "must be");
1369 return callee_method;
1370 }
1371
1372 // Resolves a call.
1373 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, bool& caller_does_not_scalarize, TRAPS) {
1374 JavaThread* current = THREAD;
1375 ResourceMark rm(current);
1376 RegisterMap cbl_map(current,
1377 RegisterMap::UpdateMap::skip,
1378 RegisterMap::ProcessFrames::include,
1379 RegisterMap::WalkContinuation::skip);
1380 frame caller_frame = current->last_frame().sender(&cbl_map);
1381
1382 CodeBlob* caller_cb = caller_frame.cb();
1383 guarantee(caller_cb != nullptr && caller_cb->is_nmethod(), "must be called from compiled method");
1384 nmethod* caller_nm = caller_cb->as_nmethod();
1385
1386 // determine call info & receiver
1387 // note: a) receiver is null for static calls
1388 // b) an exception is thrown if receiver is null for non-static calls
1389 CallInfo call_info;
1390 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1391 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1392
1393 NoSafepointVerifier nsv;
1394
1395 methodHandle callee_method(current, call_info.selected_method());
1396 // Calls via mismatching methods are always non-scalarized
1397 bool mismatch = is_optimized ? call_info.selected_method()->mismatch() : call_info.resolved_method()->mismatch();
1398 if (caller_nm->is_compiled_by_c1() || mismatch) {
1399 caller_does_not_scalarize = true;
1400 }
1401
1402 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1403 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1404 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1405 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1406 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1407
1408 assert(!caller_nm->is_unloading(), "It should not be unloading");
1409
1410 #ifndef PRODUCT
1411 // tracing/debugging/statistics
1412 uint *addr = (is_optimized) ? (&_resolve_opt_virtual_ctr) :
1413 (is_virtual) ? (&_resolve_virtual_ctr) :
1414 (&_resolve_static_ctr);
1415 AtomicAccess::inc(addr);
1416
1417 if (TraceCallFixup) {
1418 ResourceMark rm(current);
1419 tty->print("resolving %s%s (%s) %s call to",
1420 (is_optimized) ? "optimized " : "", (is_virtual) ? "virtual" : "static",
1421 Bytecodes::name(invoke_code), (caller_does_not_scalarize) ? "non-scalar" : "");
1422 callee_method->print_short_name(tty);
1423 tty->print_cr(" at pc: " INTPTR_FORMAT " to code: " INTPTR_FORMAT,
1424 p2i(caller_frame.pc()), p2i(callee_method->code()));
1425 }
1426 #endif
1427
1428 if (invoke_code == Bytecodes::_invokestatic) {
1429 assert(callee_method->method_holder()->is_initialized() ||
1430 callee_method->method_holder()->is_reentrant_initialization(current),
1431 "invalid class initialization state for invoke_static");
1432 if (!VM_Version::supports_fast_class_init_checks() && callee_method->needs_clinit_barrier()) {
1433 // In order to keep class initialization check, do not patch call
1434 // site for static call when the class is not fully initialized.
1435 // Proper check is enforced by call site re-resolution on every invocation.
1436 //
1437 // When fast class initialization checks are supported (VM_Version::supports_fast_class_init_checks() == true),
1438 // explicit class initialization check is put in nmethod entry (VEP).
1439 assert(callee_method->method_holder()->is_linked(), "must be");
1440 return callee_method;
1441 }
1442 }
1443
1444
1445 // JSR 292 key invariant:
1446 // If the resolved method is a MethodHandle invoke target, the call
1447 // site must be a MethodHandle call site, because the lambda form might tail-call
1448 // leaving the stack in a state unknown to either caller or callee
1449
1450 // Compute entry points. The computation of the entry points is independent of
1451 // patching the call.
1452
1453 // Make sure the callee nmethod does not get deoptimized and removed before
1454 // we are done patching the code.
1455
1456
1457 CompiledICLocker ml(caller_nm);
1458 if (is_virtual && !is_optimized) {
1459 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1460 inline_cache->update(&call_info, receiver->klass(), caller_does_not_scalarize);
1461 } else {
1462 // Callsite is a direct call - set it to the destination method
1463 CompiledDirectCall* callsite = CompiledDirectCall::before(caller_frame.pc());
1464 callsite->set(callee_method, caller_does_not_scalarize);
1465 }
1466
1467 return callee_method;
1468 }
1469
1470 // Inline caches exist only in compiled code
1471 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1472 #ifdef ASSERT
1473 RegisterMap reg_map(current,
1474 RegisterMap::UpdateMap::skip,
1475 RegisterMap::ProcessFrames::include,
1476 RegisterMap::WalkContinuation::skip);
1477 frame stub_frame = current->last_frame();
1478 assert(stub_frame.is_runtime_frame(), "sanity check");
1479 frame caller_frame = stub_frame.sender(®_map);
1480 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_upcall_stub_frame(), "unexpected frame");
1481 #endif /* ASSERT */
1482
1483 methodHandle callee_method;
1484 bool caller_does_not_scalarize = false;
1485 JRT_BLOCK
1486 callee_method = SharedRuntime::handle_ic_miss_helper(caller_does_not_scalarize, CHECK_NULL);
1487 // Return Method* through TLS
1488 current->set_vm_result_metadata(callee_method());
1489 JRT_BLOCK_END
1490 // return compiled code entry point after potential safepoints
1491 return get_resolved_entry(current, callee_method, false, false, caller_does_not_scalarize);
1492 JRT_END
1493
1494
1495 // Handle call site that has been made non-entrant
1496 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1497 // 6243940 We might end up in here if the callee is deoptimized
1498 // as we race to call it. We don't want to take a safepoint if
1499 // the caller was interpreted because the caller frame will look
1500 // interpreted to the stack walkers and arguments are now
1501 // "compiled" so it is much better to make this transition
1502 // invisible to the stack walking code. The i2c path will
1503 // place the callee method in the callee_target. It is stashed
1504 // there because if we try and find the callee by normal means a
1505 // safepoint is possible and have trouble gc'ing the compiled args.
1506 RegisterMap reg_map(current,
1507 RegisterMap::UpdateMap::skip,
1508 RegisterMap::ProcessFrames::include,
1509 RegisterMap::WalkContinuation::skip);
1510 frame stub_frame = current->last_frame();
1511 assert(stub_frame.is_runtime_frame(), "sanity check");
1512 frame caller_frame = stub_frame.sender(®_map);
1513
1514 if (caller_frame.is_interpreted_frame() ||
1515 caller_frame.is_entry_frame() ||
1516 caller_frame.is_upcall_stub_frame()) {
1517 Method* callee = current->callee_target();
1518 guarantee(callee != nullptr && callee->is_method(), "bad handshake");
1519 current->set_vm_result_metadata(callee);
1520 current->set_callee_target(nullptr);
1521 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1522 // Bypass class initialization checks in c2i when caller is in native.
1523 // JNI calls to static methods don't have class initialization checks.
1524 // Fast class initialization checks are present in c2i adapters and call into
1525 // SharedRuntime::handle_wrong_method() on the slow path.
1526 //
1527 // JVM upcalls may land here as well, but there's a proper check present in
1528 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1529 // so bypassing it in c2i adapter is benign.
1530 return callee->get_c2i_no_clinit_check_entry();
1531 } else {
1532 if (caller_frame.is_interpreted_frame()) {
1533 return callee->get_c2i_inline_entry();
1534 } else {
1535 return callee->get_c2i_entry();
1536 }
1537 }
1538 }
1539
1540 // Must be compiled to compiled path which is safe to stackwalk
1541 methodHandle callee_method;
1542 bool is_static_call = false;
1543 bool is_optimized = false;
1544 bool caller_does_not_scalarize = false;
1545 JRT_BLOCK
1546 // Force resolving of caller (if we called from compiled frame)
1547 callee_method = SharedRuntime::reresolve_call_site(is_optimized, caller_does_not_scalarize, CHECK_NULL);
1548 current->set_vm_result_metadata(callee_method());
1549 JRT_BLOCK_END
1550 // return compiled code entry point after potential safepoints
1551 return get_resolved_entry(current, callee_method, callee_method->is_static(), is_optimized, caller_does_not_scalarize);
1552 JRT_END
1553
1554 // Handle abstract method call
1555 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1556 // Verbose error message for AbstractMethodError.
1557 // Get the called method from the invoke bytecode.
1558 vframeStream vfst(current, true);
1559 assert(!vfst.at_end(), "Java frame must exist");
1560 methodHandle caller(current, vfst.method());
1561 Bytecode_invoke invoke(caller, vfst.bci());
1562 DEBUG_ONLY( invoke.verify(); )
1563
1564 // Find the compiled caller frame.
1565 RegisterMap reg_map(current,
1566 RegisterMap::UpdateMap::include,
1567 RegisterMap::ProcessFrames::include,
1568 RegisterMap::WalkContinuation::skip);
1569 frame stubFrame = current->last_frame();
1570 assert(stubFrame.is_runtime_frame(), "must be");
1571 frame callerFrame = stubFrame.sender(®_map);
1572 assert(callerFrame.is_compiled_frame(), "must be");
1573
1574 // Install exception and return forward entry.
1575 address res = SharedRuntime::throw_AbstractMethodError_entry();
1576 JRT_BLOCK
1577 methodHandle callee(current, invoke.static_target(current));
1578 if (!callee.is_null()) {
1579 oop recv = callerFrame.retrieve_receiver(®_map);
1580 Klass *recv_klass = (recv != nullptr) ? recv->klass() : nullptr;
1581 res = StubRoutines::forward_exception_entry();
1582 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1583 }
1584 JRT_BLOCK_END
1585 return res;
1586 JRT_END
1587
1588 // return verified_code_entry if interp_only_mode is not set for the current thread;
1589 // otherwise return c2i entry.
1590 address SharedRuntime::get_resolved_entry(JavaThread* current, methodHandle callee_method,
1591 bool is_static_call, bool is_optimized, bool caller_does_not_scalarize) {
1592 bool is_interp_only_mode = (StressCallingConvention && (os::random() % (1 << 10)) == 0) || current->is_interp_only_mode();
1593 // In interp_only_mode we need to go to the interpreted entry
1594 // The c2i won't patch in this mode -- see fixup_callers_callsite
1595 bool go_to_interpreter = is_interp_only_mode && !callee_method->is_special_native_intrinsic();
1596
1597 if (caller_does_not_scalarize) {
1598 if (go_to_interpreter) {
1599 return callee_method->get_c2i_inline_entry();
1600 }
1601 assert(callee_method->verified_inline_code_entry() != nullptr, "Jump to zero!");
1602 return callee_method->verified_inline_code_entry();
1603 } else if (is_static_call || is_optimized) {
1604 if (go_to_interpreter) {
1605 return callee_method->get_c2i_entry();
1606 }
1607 assert(callee_method->verified_code_entry() != nullptr, "Jump to zero!");
1608 return callee_method->verified_code_entry();
1609 } else {
1610 if (go_to_interpreter) {
1611 return callee_method->get_c2i_inline_ro_entry();
1612 }
1613 assert(callee_method->verified_inline_ro_code_entry() != nullptr, "Jump to zero!");
1614 return callee_method->verified_inline_ro_code_entry();
1615 }
1616 }
1617
1618 // resolve a static call and patch code
1619 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1620 methodHandle callee_method;
1621 bool caller_does_not_scalarize = false;
1622 bool enter_special = false;
1623 JRT_BLOCK
1624 callee_method = SharedRuntime::resolve_helper(false, false, caller_does_not_scalarize, CHECK_NULL);
1625 current->set_vm_result_metadata(callee_method());
1626 JRT_BLOCK_END
1627 // return compiled code entry point after potential safepoints
1628 return get_resolved_entry(current, callee_method, true, false, caller_does_not_scalarize);
1629 JRT_END
1630
1631 // resolve virtual call and update inline cache to monomorphic
1632 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1633 methodHandle callee_method;
1634 bool caller_does_not_scalarize = false;
1635 JRT_BLOCK
1636 callee_method = SharedRuntime::resolve_helper(true, false, caller_does_not_scalarize, CHECK_NULL);
1637 current->set_vm_result_metadata(callee_method());
1638 JRT_BLOCK_END
1639 // return compiled code entry point after potential safepoints
1640 return get_resolved_entry(current, callee_method, false, false, caller_does_not_scalarize);
1641 JRT_END
1642
1643
1644 // Resolve a virtual call that can be statically bound (e.g., always
1645 // monomorphic, so it has no inline cache). Patch code to resolved target.
1646 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1647 methodHandle callee_method;
1648 bool caller_does_not_scalarize = false;
1649 JRT_BLOCK
1650 callee_method = SharedRuntime::resolve_helper(true, true, caller_does_not_scalarize, CHECK_NULL);
1651 current->set_vm_result_metadata(callee_method());
1652 JRT_BLOCK_END
1653 // return compiled code entry point after potential safepoints
1654 return get_resolved_entry(current, callee_method, false, true, caller_does_not_scalarize);
1655 JRT_END
1656
1657 methodHandle SharedRuntime::handle_ic_miss_helper(bool& caller_does_not_scalarize, TRAPS) {
1658 JavaThread* current = THREAD;
1659 ResourceMark rm(current);
1660 CallInfo call_info;
1661 Bytecodes::Code bc;
1662
1663 // receiver is null for static calls. An exception is thrown for null
1664 // receivers for non-static calls
1665 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1666
1667 methodHandle callee_method(current, call_info.selected_method());
1668
1669 #ifndef PRODUCT
1670 AtomicAccess::inc(&_ic_miss_ctr);
1671
1672 // Statistics & Tracing
1673 if (TraceCallFixup) {
1674 ResourceMark rm(current);
1675 tty->print("IC miss (%s) %s call to", Bytecodes::name(bc), (caller_does_not_scalarize) ? "non-scalar" : "");
1676 callee_method->print_short_name(tty);
1677 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1678 }
1679
1680 if (ICMissHistogram) {
1681 MutexLocker m(VMStatistic_lock);
1682 RegisterMap reg_map(current,
1683 RegisterMap::UpdateMap::skip,
1684 RegisterMap::ProcessFrames::include,
1685 RegisterMap::WalkContinuation::skip);
1686 frame f = current->last_frame().real_sender(®_map);// skip runtime stub
1687 // produce statistics under the lock
1688 trace_ic_miss(f.pc());
1689 }
1690 #endif
1691
1692 // install an event collector so that when a vtable stub is created the
1693 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1694 // event can't be posted when the stub is created as locks are held
1695 // - instead the event will be deferred until the event collector goes
1696 // out of scope.
1697 JvmtiDynamicCodeEventCollector event_collector;
1698
1699 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1700 RegisterMap reg_map(current,
1701 RegisterMap::UpdateMap::skip,
1702 RegisterMap::ProcessFrames::include,
1703 RegisterMap::WalkContinuation::skip);
1704 frame caller_frame = current->last_frame().sender(®_map);
1705 CodeBlob* cb = caller_frame.cb();
1706 nmethod* caller_nm = cb->as_nmethod();
1707 // Calls via mismatching methods are always non-scalarized
1708 if (caller_nm->is_compiled_by_c1() || call_info.resolved_method()->mismatch()) {
1709 caller_does_not_scalarize = true;
1710 }
1711
1712 CompiledICLocker ml(caller_nm);
1713 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1714 inline_cache->update(&call_info, receiver()->klass(), caller_does_not_scalarize);
1715
1716 return callee_method;
1717 }
1718
1719 //
1720 // Resets a call-site in compiled code so it will get resolved again.
1721 // This routines handles both virtual call sites, optimized virtual call
1722 // sites, and static call sites. Typically used to change a call sites
1723 // destination from compiled to interpreted.
1724 //
1725 methodHandle SharedRuntime::reresolve_call_site(bool& is_optimized, bool& caller_does_not_scalarize, TRAPS) {
1726 JavaThread* current = THREAD;
1727 ResourceMark rm(current);
1728 RegisterMap reg_map(current,
1729 RegisterMap::UpdateMap::skip,
1730 RegisterMap::ProcessFrames::include,
1731 RegisterMap::WalkContinuation::skip);
1732 frame stub_frame = current->last_frame();
1733 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1734 frame caller = stub_frame.sender(®_map);
1735 if (caller.is_compiled_frame()) {
1736 caller_does_not_scalarize = caller.cb()->as_nmethod()->is_compiled_by_c1();
1737 }
1738 assert(!caller.is_interpreted_frame(), "must be compiled");
1739
1740 // If the frame isn't a live compiled frame (i.e. deoptimized by the time we get here), no IC clearing must be done
1741 // for the caller. However, when the caller is C2 compiled and the callee a C1 or C2 compiled method, then we still
1742 // need to figure out whether it was an optimized virtual call with an inline type receiver. Otherwise, we end up
1743 // using the wrong method entry point and accidentally skip the buffering of the receiver.
1744 methodHandle callee_method = find_callee_method(caller_does_not_scalarize, CHECK_(methodHandle()));
1745 const bool caller_is_compiled_and_not_deoptimized = caller.is_compiled_frame() && !caller.is_deoptimized_frame();
1746 const bool caller_is_continuation_enter_intrinsic =
1747 caller.is_native_frame() && caller.cb()->as_nmethod()->method()->is_continuation_enter_intrinsic();
1748 const bool do_IC_clearing = caller_is_compiled_and_not_deoptimized || caller_is_continuation_enter_intrinsic;
1749
1750 const bool callee_compiled_with_scalarized_receiver = callee_method->has_compiled_code() &&
1751 !callee_method()->is_static() &&
1752 callee_method()->is_scalarized_arg(0);
1753 const bool compute_is_optimized = !caller_does_not_scalarize && callee_compiled_with_scalarized_receiver;
1754
1755 if (do_IC_clearing || compute_is_optimized) {
1756 address pc = caller.pc();
1757
1758 nmethod* caller_nm = CodeCache::find_nmethod(pc);
1759 assert(caller_nm != nullptr, "did not find caller nmethod");
1760
1761 // Default call_addr is the location of the "basic" call.
1762 // Determine the address of the call we a reresolving. With
1763 // Inline Caches we will always find a recognizable call.
1764 // With Inline Caches disabled we may or may not find a
1765 // recognizable call. We will always find a call for static
1766 // calls and for optimized virtual calls. For vanilla virtual
1767 // calls it depends on the state of the UseInlineCaches switch.
1768 //
1769 // With Inline Caches disabled we can get here for a virtual call
1770 // for two reasons:
1771 // 1 - calling an abstract method. The vtable for abstract methods
1772 // will run us thru handle_wrong_method and we will eventually
1773 // end up in the interpreter to throw the ame.
1774 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1775 // call and between the time we fetch the entry address and
1776 // we jump to it the target gets deoptimized. Similar to 1
1777 // we will wind up in the interprter (thru a c2i with c2).
1778 //
1779 CompiledICLocker ml(caller_nm);
1780 address call_addr = caller_nm->call_instruction_address(pc);
1781
1782 if (call_addr != nullptr) {
1783 // On x86 the logic for finding a call instruction is blindly checking for a call opcode 5
1784 // bytes back in the instruction stream so we must also check for reloc info.
1785 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1786 bool ret = iter.next(); // Get item
1787 if (ret) {
1788 is_optimized = false;
1789 switch (iter.type()) {
1790 case relocInfo::static_call_type:
1791 assert(callee_method->is_static(), "must be");
1792 case relocInfo::opt_virtual_call_type: {
1793 is_optimized = (iter.type() == relocInfo::opt_virtual_call_type);
1794 if (do_IC_clearing) {
1795 CompiledDirectCall* cdc = CompiledDirectCall::at(call_addr);
1796 cdc->set_to_clean();
1797 }
1798 break;
1799 }
1800
1801 case relocInfo::virtual_call_type: {
1802 if (do_IC_clearing) {
1803 // compiled, dispatched call (which used to call an interpreted method)
1804 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1805 inline_cache->set_to_clean();
1806 }
1807 break;
1808 }
1809 default:
1810 break;
1811 }
1812 }
1813 }
1814 }
1815
1816 #ifndef PRODUCT
1817 AtomicAccess::inc(&_wrong_method_ctr);
1818
1819 if (TraceCallFixup) {
1820 ResourceMark rm(current);
1821 tty->print("handle_wrong_method reresolving %s call to", (caller_does_not_scalarize) ? "non-scalar" : "");
1822 callee_method->print_short_name(tty);
1823 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1824 }
1825 #endif
1826
1827 return callee_method;
1828 }
1829
1830 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1831 // The faulting unsafe accesses should be changed to throw the error
1832 // synchronously instead. Meanwhile the faulting instruction will be
1833 // skipped over (effectively turning it into a no-op) and an
1834 // asynchronous exception will be raised which the thread will
1835 // handle at a later point. If the instruction is a load it will
1836 // return garbage.
1837
1838 // Request an async exception.
1839 thread->set_pending_unsafe_access_error();
1840
1841 // Return address of next instruction to execute.
1842 return next_pc;
1843 }
1844
1845 #ifdef ASSERT
1846 void SharedRuntime::check_member_name_argument_is_last_argument(const methodHandle& method,
1847 const BasicType* sig_bt,
1848 const VMRegPair* regs) {
1849 ResourceMark rm;
1850 const int total_args_passed = method->size_of_parameters();
1851 const VMRegPair* regs_with_member_name = regs;
1852 VMRegPair* regs_without_member_name = NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed - 1);
1853
1854 const int member_arg_pos = total_args_passed - 1;
1855 assert(member_arg_pos >= 0 && member_arg_pos < total_args_passed, "oob");
1856 assert(sig_bt[member_arg_pos] == T_OBJECT, "dispatch argument must be an object");
1857
1858 java_calling_convention(sig_bt, regs_without_member_name, total_args_passed - 1);
1859
1860 for (int i = 0; i < member_arg_pos; i++) {
1861 VMReg a = regs_with_member_name[i].first();
1862 VMReg b = regs_without_member_name[i].first();
1863 assert(a->value() == b->value(), "register allocation mismatch: a= %d, b= %d", a->value(), b->value());
1864 }
1865 assert(regs_with_member_name[member_arg_pos].first()->is_valid(), "bad member arg");
1866 }
1867 #endif
1868
1869 // ---------------------------------------------------------------------------
1870 // We are calling the interpreter via a c2i. Normally this would mean that
1871 // we were called by a compiled method. However we could have lost a race
1872 // where we went int -> i2c -> c2i and so the caller could in fact be
1873 // interpreted. If the caller is compiled we attempt to patch the caller
1874 // so he no longer calls into the interpreter.
1875 JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address caller_pc))
1876 AARCH64_PORT_ONLY(assert(pauth_ptr_is_raw(caller_pc), "should be raw"));
1877
1878 // It's possible that deoptimization can occur at a call site which hasn't
1879 // been resolved yet, in which case this function will be called from
1880 // an nmethod that has been patched for deopt and we can ignore the
1881 // request for a fixup.
1882 // Also it is possible that we lost a race in that from_compiled_entry
1883 // is now back to the i2c in that case we don't need to patch and if
1884 // we did we'd leap into space because the callsite needs to use
1885 // "to interpreter" stub in order to load up the Method*. Don't
1886 // ask me how I know this...
1887
1888 // Result from nmethod::is_unloading is not stable across safepoints.
1889 NoSafepointVerifier nsv;
1890
1891 nmethod* callee = method->code();
1892 if (callee == nullptr) {
1893 return;
1894 }
1895
1896 // write lock needed because we might patch call site by set_to_clean()
1897 // and is_unloading() can modify nmethod's state
1898 MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, JavaThread::current()));
1899
1900 CodeBlob* cb = CodeCache::find_blob(caller_pc);
1901 if (cb == nullptr || !cb->is_nmethod() || !callee->is_in_use() || callee->is_unloading()) {
1902 return;
1903 }
1904
1905 // The check above makes sure this is an nmethod.
1906 nmethod* caller = cb->as_nmethod();
1907
1908 // Get the return PC for the passed caller PC.
1909 address return_pc = caller_pc;
1910
1911 if (!caller->is_in_use() || !NativeCall::is_call_before(return_pc)) {
1912 return;
1913 }
1914
1915 // Expect to find a native call there (unless it was no-inline cache vtable dispatch)
1916 CompiledICLocker ic_locker(caller);
1917 ResourceMark rm;
1918
1919 // If we got here through a static call or opt_virtual call, then we know where the
1920 // call address would be; let's peek at it
1921 address callsite_addr = (address)nativeCall_before(return_pc);
1922 RelocIterator iter(caller, callsite_addr, callsite_addr + 1);
1923 if (!iter.next()) {
1924 // No reloc entry found; not a static or optimized virtual call
1925 return;
1926 }
1927
1928 relocInfo::relocType type = iter.reloc()->type();
1929 if (type != relocInfo::static_call_type &&
1930 type != relocInfo::opt_virtual_call_type) {
1931 return;
1932 }
1933
1934 CompiledDirectCall* callsite = CompiledDirectCall::before(return_pc);
1935 callsite->set_to_clean();
1936 JRT_END
1937
1938
1939 // same as JVM_Arraycopy, but called directly from compiled code
1940 JRT_ENTRY(void, SharedRuntime::slow_arraycopy_C(oopDesc* src, jint src_pos,
1941 oopDesc* dest, jint dest_pos,
1942 jint length,
1943 JavaThread* current)) {
1944 #ifndef PRODUCT
1945 _slow_array_copy_ctr++;
1946 #endif
1947 // Check if we have null pointers
1948 if (src == nullptr || dest == nullptr) {
1949 THROW(vmSymbols::java_lang_NullPointerException());
1950 }
1951 // Do the copy. The casts to arrayOop are necessary to the copy_array API,
1952 // even though the copy_array API also performs dynamic checks to ensure
1953 // that src and dest are truly arrays (and are conformable).
1954 // The copy_array mechanism is awkward and could be removed, but
1955 // the compilers don't call this function except as a last resort,
1956 // so it probably doesn't matter.
1957 src->klass()->copy_array((arrayOopDesc*)src, src_pos,
1958 (arrayOopDesc*)dest, dest_pos,
1959 length, current);
1960 }
1961 JRT_END
1962
1963 // The caller of generate_class_cast_message() (or one of its callers)
1964 // must use a ResourceMark in order to correctly free the result.
1965 char* SharedRuntime::generate_class_cast_message(
1966 JavaThread* thread, Klass* caster_klass) {
1967
1968 // Get target class name from the checkcast instruction
1969 vframeStream vfst(thread, true);
1970 assert(!vfst.at_end(), "Java frame must exist");
1971 Bytecode_checkcast cc(vfst.method(), vfst.method()->bcp_from(vfst.bci()));
1972 constantPoolHandle cpool(thread, vfst.method()->constants());
1973 Klass* target_klass = ConstantPool::klass_at_if_loaded(cpool, cc.index());
1974 Symbol* target_klass_name = nullptr;
1975 if (target_klass == nullptr) {
1976 // This klass should be resolved, but just in case, get the name in the klass slot.
1977 target_klass_name = cpool->klass_name_at(cc.index());
1978 }
1979 return generate_class_cast_message(caster_klass, target_klass, target_klass_name);
1980 }
1981
1982
1983 // The caller of generate_class_cast_message() (or one of its callers)
1984 // must use a ResourceMark in order to correctly free the result.
1985 char* SharedRuntime::generate_class_cast_message(
1986 Klass* caster_klass, Klass* target_klass, Symbol* target_klass_name) {
1987 const char* caster_name = caster_klass->external_name();
1988
1989 assert(target_klass != nullptr || target_klass_name != nullptr, "one must be provided");
1990 const char* target_name = target_klass == nullptr ? target_klass_name->as_klass_external_name() :
1991 target_klass->external_name();
1992
1993 size_t msglen = strlen(caster_name) + strlen("class ") + strlen(" cannot be cast to class ") + strlen(target_name) + 1;
1994
1995 const char* caster_klass_description = "";
1996 const char* target_klass_description = "";
1997 const char* klass_separator = "";
1998 if (target_klass != nullptr && caster_klass->module() == target_klass->module()) {
1999 caster_klass_description = caster_klass->joint_in_module_of_loader(target_klass);
2000 } else {
2001 caster_klass_description = caster_klass->class_in_module_of_loader();
2002 target_klass_description = (target_klass != nullptr) ? target_klass->class_in_module_of_loader() : "";
2003 klass_separator = (target_klass != nullptr) ? "; " : "";
2004 }
2005
2006 // add 3 for parenthesis and preceding space
2007 msglen += strlen(caster_klass_description) + strlen(target_klass_description) + strlen(klass_separator) + 3;
2008
2009 char* message = NEW_RESOURCE_ARRAY_RETURN_NULL(char, msglen);
2010 if (message == nullptr) {
2011 // Shouldn't happen, but don't cause even more problems if it does
2012 message = const_cast<char*>(caster_klass->external_name());
2013 } else {
2014 jio_snprintf(message,
2015 msglen,
2016 "class %s cannot be cast to class %s (%s%s%s)",
2017 caster_name,
2018 target_name,
2019 caster_klass_description,
2020 klass_separator,
2021 target_klass_description
2022 );
2023 }
2024 return message;
2025 }
2026
2027 char* SharedRuntime::generate_identity_exception_message(JavaThread* current, Klass* klass) {
2028 assert(klass->is_inline_klass(), "Must be a concrete value class");
2029 const char* desc = "Cannot synchronize on an instance of value class ";
2030 const char* className = klass->external_name();
2031 size_t msglen = strlen(desc) + strlen(className) + 1;
2032 char* message = NEW_RESOURCE_ARRAY(char, msglen);
2033 if (nullptr == message) {
2034 // Out of memory: can't create detailed error message
2035 message = const_cast<char*>(klass->external_name());
2036 } else {
2037 jio_snprintf(message, msglen, "%s%s", desc, className);
2038 }
2039 return message;
2040 }
2041
2042 JRT_LEAF(void, SharedRuntime::reguard_yellow_pages())
2043 (void) JavaThread::current()->stack_overflow_state()->reguard_stack();
2044 JRT_END
2045
2046 void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
2047 if (!SafepointSynchronize::is_synchronizing()) {
2048 // Only try quick_enter() if we're not trying to reach a safepoint
2049 // so that the calling thread reaches the safepoint more quickly.
2050 if (ObjectSynchronizer::quick_enter(obj, lock, current)) {
2051 return;
2052 }
2053 }
2054 // NO_ASYNC required because an async exception on the state transition destructor
2055 // would leave you with the lock held and it would never be released.
2056 // The normal monitorenter NullPointerException is thrown without acquiring a lock
2057 // and the model is that an exception implies the method failed.
2058 JRT_BLOCK_NO_ASYNC
2059 Handle h_obj(THREAD, obj);
2060 ObjectSynchronizer::enter(h_obj, lock, current);
2061 assert(!HAS_PENDING_EXCEPTION, "Should have no exception here");
2062 JRT_BLOCK_END
2063 }
2064
2065 // Handles the uncommon case in locking, i.e., contention or an inflated lock.
2066 JRT_BLOCK_ENTRY(void, SharedRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current))
2067 SharedRuntime::monitor_enter_helper(obj, lock, current);
2068 JRT_END
2069
2070 void SharedRuntime::monitor_exit_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
2071 assert(JavaThread::current() == current, "invariant");
2072 // Exit must be non-blocking, and therefore no exceptions can be thrown.
2073 ExceptionMark em(current);
2074
2075 // Check if C2_MacroAssembler::fast_unlock() or
2076 // C2_MacroAssembler::fast_unlock() unlocked an inflated
2077 // monitor before going slow path. Since there is no safepoint
2078 // polling when calling into the VM, we can be sure that the monitor
2079 // hasn't been deallocated.
2080 ObjectMonitor* m = current->unlocked_inflated_monitor();
2081 if (m != nullptr) {
2082 assert(!m->has_owner(current), "must be");
2083 current->clear_unlocked_inflated_monitor();
2084
2085 // We need to reacquire the lock before we can call ObjectSynchronizer::exit().
2086 if (!m->try_enter(current, /*check_for_recursion*/ false)) {
2087 // Some other thread acquired the lock (or the monitor was
2088 // deflated). Either way we are done.
2089 return;
2090 }
2091 }
2092
2093 // The object could become unlocked through a JNI call, which we have no other checks for.
2094 // Give a fatal message if CheckJNICalls. Otherwise we ignore it.
2095 if (obj->is_unlocked()) {
2096 if (CheckJNICalls) {
2097 fatal("Object has been unlocked by JNI");
2098 }
2099 return;
2100 }
2101 ObjectSynchronizer::exit(obj, lock, current);
2102 }
2103
2104 // Handles the uncommon cases of monitor unlocking in compiled code
2105 JRT_LEAF(void, SharedRuntime::complete_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* current))
2106 assert(current == JavaThread::current(), "pre-condition");
2107 SharedRuntime::monitor_exit_helper(obj, lock, current);
2108 JRT_END
2109
2110 #ifndef PRODUCT
2111
2112 void SharedRuntime::print_statistics() {
2113 ttyLocker ttyl;
2114 if (xtty != nullptr) xtty->head("statistics type='SharedRuntime'");
2115
2116 SharedRuntime::print_ic_miss_histogram();
2117
2118 // Dump the JRT_ENTRY counters
2119 if (_new_instance_ctr) tty->print_cr("%5u new instance requires GC", _new_instance_ctr);
2120 if (_new_array_ctr) tty->print_cr("%5u new array requires GC", _new_array_ctr);
2121 if (_multi2_ctr) tty->print_cr("%5u multianewarray 2 dim", _multi2_ctr);
2122 if (_multi3_ctr) tty->print_cr("%5u multianewarray 3 dim", _multi3_ctr);
2123 if (_multi4_ctr) tty->print_cr("%5u multianewarray 4 dim", _multi4_ctr);
2124 if (_multi5_ctr) tty->print_cr("%5u multianewarray 5 dim", _multi5_ctr);
2125
2126 tty->print_cr("%5u inline cache miss in compiled", _ic_miss_ctr);
2127 tty->print_cr("%5u wrong method", _wrong_method_ctr);
2128 tty->print_cr("%5u unresolved static call site", _resolve_static_ctr);
2129 tty->print_cr("%5u unresolved virtual call site", _resolve_virtual_ctr);
2130 tty->print_cr("%5u unresolved opt virtual call site", _resolve_opt_virtual_ctr);
2131
2132 if (_mon_enter_stub_ctr) tty->print_cr("%5u monitor enter stub", _mon_enter_stub_ctr);
2133 if (_mon_exit_stub_ctr) tty->print_cr("%5u monitor exit stub", _mon_exit_stub_ctr);
2134 if (_mon_enter_ctr) tty->print_cr("%5u monitor enter slow", _mon_enter_ctr);
2135 if (_mon_exit_ctr) tty->print_cr("%5u monitor exit slow", _mon_exit_ctr);
2136 if (_partial_subtype_ctr) tty->print_cr("%5u slow partial subtype", _partial_subtype_ctr);
2137 if (_jbyte_array_copy_ctr) tty->print_cr("%5u byte array copies", _jbyte_array_copy_ctr);
2138 if (_jshort_array_copy_ctr) tty->print_cr("%5u short array copies", _jshort_array_copy_ctr);
2139 if (_jint_array_copy_ctr) tty->print_cr("%5u int array copies", _jint_array_copy_ctr);
2140 if (_jlong_array_copy_ctr) tty->print_cr("%5u long array copies", _jlong_array_copy_ctr);
2141 if (_oop_array_copy_ctr) tty->print_cr("%5u oop array copies", _oop_array_copy_ctr);
2142 if (_checkcast_array_copy_ctr) tty->print_cr("%5u checkcast array copies", _checkcast_array_copy_ctr);
2143 if (_unsafe_array_copy_ctr) tty->print_cr("%5u unsafe array copies", _unsafe_array_copy_ctr);
2144 if (_generic_array_copy_ctr) tty->print_cr("%5u generic array copies", _generic_array_copy_ctr);
2145 if (_slow_array_copy_ctr) tty->print_cr("%5u slow array copies", _slow_array_copy_ctr);
2146 if (_find_handler_ctr) tty->print_cr("%5u find exception handler", _find_handler_ctr);
2147 if (_rethrow_ctr) tty->print_cr("%5u rethrow handler", _rethrow_ctr);
2148 if (_unsafe_set_memory_ctr) tty->print_cr("%5u unsafe set memorys", _unsafe_set_memory_ctr);
2149
2150 AdapterHandlerLibrary::print_statistics();
2151
2152 if (xtty != nullptr) xtty->tail("statistics");
2153 }
2154
2155 inline double percent(int64_t x, int64_t y) {
2156 return 100.0 * (double)x / (double)MAX2(y, (int64_t)1);
2157 }
2158
2159 class MethodArityHistogram {
2160 public:
2161 enum { MAX_ARITY = 256 };
2162 private:
2163 static uint64_t _arity_histogram[MAX_ARITY]; // histogram of #args
2164 static uint64_t _size_histogram[MAX_ARITY]; // histogram of arg size in words
2165 static uint64_t _total_compiled_calls;
2166 static uint64_t _max_compiled_calls_per_method;
2167 static int _max_arity; // max. arity seen
2168 static int _max_size; // max. arg size seen
2169
2170 static void add_method_to_histogram(nmethod* nm) {
2171 Method* method = (nm == nullptr) ? nullptr : nm->method();
2172 if (method != nullptr) {
2173 ArgumentCount args(method->signature());
2174 int arity = args.size() + (method->is_static() ? 0 : 1);
2175 int argsize = method->size_of_parameters();
2176 arity = MIN2(arity, MAX_ARITY-1);
2177 argsize = MIN2(argsize, MAX_ARITY-1);
2178 uint64_t count = (uint64_t)method->compiled_invocation_count();
2179 _max_compiled_calls_per_method = count > _max_compiled_calls_per_method ? count : _max_compiled_calls_per_method;
2180 _total_compiled_calls += count;
2181 _arity_histogram[arity] += count;
2182 _size_histogram[argsize] += count;
2183 _max_arity = MAX2(_max_arity, arity);
2184 _max_size = MAX2(_max_size, argsize);
2185 }
2186 }
2187
2188 void print_histogram_helper(int n, uint64_t* histo, const char* name) {
2189 const int N = MIN2(9, n);
2190 double sum = 0;
2191 double weighted_sum = 0;
2192 for (int i = 0; i <= n; i++) { sum += (double)histo[i]; weighted_sum += (double)(i*histo[i]); }
2193 if (sum >= 1) { // prevent divide by zero or divide overflow
2194 double rest = sum;
2195 double percent = sum / 100;
2196 for (int i = 0; i <= N; i++) {
2197 rest -= (double)histo[i];
2198 tty->print_cr("%4d: " UINT64_FORMAT_W(12) " (%5.1f%%)", i, histo[i], (double)histo[i] / percent);
2199 }
2200 tty->print_cr("rest: " INT64_FORMAT_W(12) " (%5.1f%%)", (int64_t)rest, rest / percent);
2201 tty->print_cr("(avg. %s = %3.1f, max = %d)", name, weighted_sum / sum, n);
2202 tty->print_cr("(total # of compiled calls = " INT64_FORMAT_W(14) ")", _total_compiled_calls);
2203 tty->print_cr("(max # of compiled calls = " INT64_FORMAT_W(14) ")", _max_compiled_calls_per_method);
2204 } else {
2205 tty->print_cr("Histogram generation failed for %s. n = %d, sum = %7.5f", name, n, sum);
2206 }
2207 }
2208
2209 void print_histogram() {
2210 tty->print_cr("\nHistogram of call arity (incl. rcvr, calls to compiled methods only):");
2211 print_histogram_helper(_max_arity, _arity_histogram, "arity");
2212 tty->print_cr("\nHistogram of parameter block size (in words, incl. rcvr):");
2213 print_histogram_helper(_max_size, _size_histogram, "size");
2214 tty->cr();
2215 }
2216
2217 public:
2218 MethodArityHistogram() {
2219 // Take the Compile_lock to protect against changes in the CodeBlob structures
2220 MutexLocker mu1(Compile_lock, Mutex::_safepoint_check_flag);
2221 // Take the CodeCache_lock to protect against changes in the CodeHeap structure
2222 MutexLocker mu2(CodeCache_lock, Mutex::_no_safepoint_check_flag);
2223 _max_arity = _max_size = 0;
2224 _total_compiled_calls = 0;
2225 _max_compiled_calls_per_method = 0;
2226 for (int i = 0; i < MAX_ARITY; i++) _arity_histogram[i] = _size_histogram[i] = 0;
2227 CodeCache::nmethods_do(add_method_to_histogram);
2228 print_histogram();
2229 }
2230 };
2231
2232 uint64_t MethodArityHistogram::_arity_histogram[MethodArityHistogram::MAX_ARITY];
2233 uint64_t MethodArityHistogram::_size_histogram[MethodArityHistogram::MAX_ARITY];
2234 uint64_t MethodArityHistogram::_total_compiled_calls;
2235 uint64_t MethodArityHistogram::_max_compiled_calls_per_method;
2236 int MethodArityHistogram::_max_arity;
2237 int MethodArityHistogram::_max_size;
2238
2239 void SharedRuntime::print_call_statistics(uint64_t comp_total) {
2240 tty->print_cr("Calls from compiled code:");
2241 int64_t total = _nof_normal_calls + _nof_interface_calls + _nof_static_calls;
2242 int64_t mono_c = _nof_normal_calls - _nof_megamorphic_calls;
2243 int64_t mono_i = _nof_interface_calls;
2244 tty->print_cr("\t" INT64_FORMAT_W(12) " (100%%) total non-inlined ", total);
2245 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- virtual calls ", _nof_normal_calls, percent(_nof_normal_calls, total));
2246 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_calls, percent(_nof_inlined_calls, _nof_normal_calls));
2247 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- monomorphic ", mono_c, percent(mono_c, _nof_normal_calls));
2248 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- megamorphic ", _nof_megamorphic_calls, percent(_nof_megamorphic_calls, _nof_normal_calls));
2249 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- interface calls ", _nof_interface_calls, percent(_nof_interface_calls, total));
2250 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_interface_calls, percent(_nof_inlined_interface_calls, _nof_interface_calls));
2251 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- monomorphic ", mono_i, percent(mono_i, _nof_interface_calls));
2252 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.1f%%) |- static/special calls", _nof_static_calls, percent(_nof_static_calls, total));
2253 tty->print_cr("\t" INT64_FORMAT_W(12) " (%4.0f%%) | |- inlined ", _nof_inlined_static_calls, percent(_nof_inlined_static_calls, _nof_static_calls));
2254 tty->cr();
2255 tty->print_cr("Note 1: counter updates are not MT-safe.");
2256 tty->print_cr("Note 2: %% in major categories are relative to total non-inlined calls;");
2257 tty->print_cr(" %% in nested categories are relative to their category");
2258 tty->print_cr(" (and thus add up to more than 100%% with inlining)");
2259 tty->cr();
2260
2261 MethodArityHistogram h;
2262 }
2263 #endif
2264
2265 #ifndef PRODUCT
2266 static int _lookups; // number of calls to lookup
2267 static int _equals; // number of buckets checked with matching hash
2268 static int _archived_hits; // number of successful lookups in archived table
2269 static int _runtime_hits; // number of successful lookups in runtime table
2270 #endif
2271
2272 // A simple wrapper class around the calling convention information
2273 // that allows sharing of adapters for the same calling convention.
2274 class AdapterFingerPrint : public MetaspaceObj {
2275 public:
2276 class Element {
2277 private:
2278 // The highest byte is the type of the argument. The remaining bytes contain the offset of the
2279 // field if it is flattened in the calling convention, -1 otherwise.
2280 juint _payload;
2281
2282 static constexpr int offset_bit_width = 24;
2283 static constexpr juint offset_bit_mask = (1 << offset_bit_width) - 1;
2284 public:
2285 Element(BasicType bt, int offset) : _payload((static_cast<juint>(bt) << offset_bit_width) | (juint(offset) & offset_bit_mask)) {
2286 assert(offset >= -1 && offset < jint(offset_bit_mask), "invalid offset %d", offset);
2287 }
2288
2289 BasicType bt() const {
2290 return static_cast<BasicType>(_payload >> offset_bit_width);
2291 }
2292
2293 int offset() const {
2294 juint res = _payload & offset_bit_mask;
2295 return res == offset_bit_mask ? -1 : res;
2296 }
2297
2298 juint hash() const {
2299 return _payload;
2300 }
2301
2302 bool operator!=(const Element& other) const {
2303 return _payload != other._payload;
2304 }
2305 };
2306
2307 private:
2308 const bool _has_ro_adapter;
2309 const int _length;
2310
2311 static int data_offset() { return sizeof(AdapterFingerPrint); }
2312 Element* data_pointer() {
2313 return reinterpret_cast<Element*>(reinterpret_cast<address>(this) + data_offset());
2314 }
2315
2316 const Element& element_at(int index) {
2317 assert(index < length(), "index %d out of bounds for length %d", index, length());
2318 Element* data = data_pointer();
2319 return data[index];
2320 }
2321
2322 // Private construtor. Use allocate() to get an instance.
2323 AdapterFingerPrint(const GrowableArray<SigEntry>* sig, bool has_ro_adapter)
2324 : _has_ro_adapter(has_ro_adapter), _length(total_args_passed_in_sig(sig)) {
2325 Element* data = data_pointer();
2326 BasicType prev_bt = T_ILLEGAL;
2327 int vt_count = 0;
2328 for (int index = 0; index < _length; index++) {
2329 const SigEntry& sig_entry = sig->at(index);
2330 BasicType bt = sig_entry._bt;
2331 if (bt == T_METADATA) {
2332 // Found start of inline type in signature
2333 assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
2334 vt_count++;
2335 } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
2336 // Found end of inline type in signature
2337 assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
2338 vt_count--;
2339 assert(vt_count >= 0, "invalid vt_count");
2340 } else if (vt_count == 0) {
2341 // Widen fields that are not part of a scalarized inline type argument
2342 assert(sig_entry._offset == -1, "invalid offset for argument that is not a flattened field %d", sig_entry._offset);
2343 bt = adapter_encoding(bt);
2344 }
2345
2346 ::new(&data[index]) Element(bt, sig_entry._offset);
2347 prev_bt = bt;
2348 }
2349 assert(vt_count == 0, "invalid vt_count");
2350 }
2351
2352 // Call deallocate instead
2353 ~AdapterFingerPrint() {
2354 ShouldNotCallThis();
2355 }
2356
2357 static int total_args_passed_in_sig(const GrowableArray<SigEntry>* sig) {
2358 return (sig != nullptr) ? sig->length() : 0;
2359 }
2360
2361 static int compute_size_in_words(int len) {
2362 return (int)heap_word_size(sizeof(AdapterFingerPrint) + (len * sizeof(Element)));
2363 }
2364
2365 // Remap BasicTypes that are handled equivalently by the adapters.
2366 // These are correct for the current system but someday it might be
2367 // necessary to make this mapping platform dependent.
2368 static BasicType adapter_encoding(BasicType in) {
2369 switch (in) {
2370 case T_BOOLEAN:
2371 case T_BYTE:
2372 case T_SHORT:
2373 case T_CHAR:
2374 // They are all promoted to T_INT in the calling convention
2375 return T_INT;
2376
2377 case T_OBJECT:
2378 case T_ARRAY:
2379 // In other words, we assume that any register good enough for
2380 // an int or long is good enough for a managed pointer.
2381 #ifdef _LP64
2382 return T_LONG;
2383 #else
2384 return T_INT;
2385 #endif
2386
2387 case T_INT:
2388 case T_LONG:
2389 case T_FLOAT:
2390 case T_DOUBLE:
2391 case T_VOID:
2392 return in;
2393
2394 default:
2395 ShouldNotReachHere();
2396 return T_CONFLICT;
2397 }
2398 }
2399
2400 void* operator new(size_t size, size_t fp_size) throw() {
2401 assert(fp_size >= size, "sanity check");
2402 void* p = AllocateHeap(fp_size, mtCode);
2403 memset(p, 0, fp_size);
2404 return p;
2405 }
2406
2407 public:
2408 template<typename Function>
2409 void iterate_args(Function function) {
2410 for (int i = 0; i < length(); i++) {
2411 function(element_at(i));
2412 }
2413 }
2414
2415 static AdapterFingerPrint* allocate(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
2416 int len = total_args_passed_in_sig(sig);
2417 int size_in_bytes = BytesPerWord * compute_size_in_words(len);
2418 AdapterFingerPrint* afp = new (size_in_bytes) AdapterFingerPrint(sig, has_ro_adapter);
2419 assert((afp->size() * BytesPerWord) == size_in_bytes, "should match");
2420 return afp;
2421 }
2422
2423 static void deallocate(AdapterFingerPrint* fp) {
2424 FreeHeap(fp);
2425 }
2426
2427 bool has_ro_adapter() const {
2428 return _has_ro_adapter;
2429 }
2430
2431 int length() const {
2432 return _length;
2433 }
2434
2435 unsigned int compute_hash() {
2436 int hash = 0;
2437 for (int i = 0; i < length(); i++) {
2438 const Element& v = element_at(i);
2439 //Add arithmetic operation to the hash, like +3 to improve hashing
2440 hash = ((hash << 8) ^ v.hash() ^ (hash >> 5)) + 3;
2441 }
2442 return (unsigned int)hash;
2443 }
2444
2445 const char* as_string() {
2446 stringStream st;
2447 st.print("{");
2448 if (_has_ro_adapter) {
2449 st.print("has_ro_adapter");
2450 } else {
2451 st.print("no_ro_adapter");
2452 }
2453 for (int i = 0; i < length(); i++) {
2454 st.print(", ");
2455 const Element& elem = element_at(i);
2456 st.print("{%s, %d}", type2name(elem.bt()), elem.offset());
2457 }
2458 st.print("}");
2459 return st.as_string();
2460 }
2461
2462 const char* as_basic_args_string() {
2463 stringStream st;
2464 bool long_prev = false;
2465 iterate_args([&] (const Element& arg) {
2466 if (long_prev) {
2467 long_prev = false;
2468 if (arg.bt() == T_VOID) {
2469 st.print("J");
2470 } else {
2471 st.print("L");
2472 }
2473 }
2474 if (arg.bt() == T_LONG) {
2475 long_prev = true;
2476 } else if (arg.bt() != T_VOID) {
2477 st.print("%c", type2char(arg.bt()));
2478 }
2479 });
2480 if (long_prev) {
2481 st.print("L");
2482 }
2483 return st.as_string();
2484 }
2485
2486 bool equals(AdapterFingerPrint* other) {
2487 if (other->_has_ro_adapter != _has_ro_adapter) {
2488 return false;
2489 } else if (other->_length != _length) {
2490 return false;
2491 } else {
2492 for (int i = 0; i < _length; i++) {
2493 if (element_at(i) != other->element_at(i)) {
2494 return false;
2495 }
2496 }
2497 }
2498 return true;
2499 }
2500
2501 // methods required by virtue of being a MetaspaceObj
2502 void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ }
2503 int size() const { return compute_size_in_words(_length); }
2504 MetaspaceObj::Type type() const { return AdapterFingerPrintType; }
2505
2506 static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) {
2507 NOT_PRODUCT(_equals++);
2508 return fp1->equals(fp2);
2509 }
2510
2511 static unsigned int compute_hash(AdapterFingerPrint* const& fp) {
2512 return fp->compute_hash();
2513 }
2514 };
2515
2516 #if INCLUDE_CDS
2517 static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) {
2518 return AdapterFingerPrint::equals(entry->fingerprint(), fp);
2519 }
2520
2521 class ArchivedAdapterTable : public OffsetCompactHashtable<
2522 AdapterFingerPrint*,
2523 AdapterHandlerEntry*,
2524 adapter_fp_equals_compact_hashtable_entry> {};
2525 #endif // INCLUDE_CDS
2526
2527 // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries
2528 using AdapterHandlerTable = HashTable<AdapterFingerPrint*, AdapterHandlerEntry*, 293,
2529 AnyObj::C_HEAP, mtCode,
2530 AdapterFingerPrint::compute_hash,
2531 AdapterFingerPrint::equals>;
2532 static AdapterHandlerTable* _adapter_handler_table;
2533 static GrowableArray<AdapterHandlerEntry*>* _adapter_handler_list = nullptr;
2534
2535 // Find a entry with the same fingerprint if it exists
2536 AdapterHandlerEntry* AdapterHandlerLibrary::lookup(const GrowableArray<SigEntry>* sig, bool has_ro_adapter) {
2537 NOT_PRODUCT(_lookups++);
2538 assert_lock_strong(AdapterHandlerLibrary_lock);
2539 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(sig, has_ro_adapter);
2540 AdapterHandlerEntry* entry = nullptr;
2541 #if INCLUDE_CDS
2542 // if we are building the archive then the archived adapter table is
2543 // not valid and we need to use the ones added to the runtime table
2544 if (AOTCodeCache::is_using_adapter()) {
2545 // Search archived table first. It is read-only table so can be searched without lock
2546 entry = _aot_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */);
2547 #ifndef PRODUCT
2548 if (entry != nullptr) {
2549 _archived_hits++;
2550 }
2551 #endif
2552 }
2553 #endif // INCLUDE_CDS
2554 if (entry == nullptr) {
2555 assert_lock_strong(AdapterHandlerLibrary_lock);
2556 AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp);
2557 if (entry_p != nullptr) {
2558 entry = *entry_p;
2559 assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)",
2560 entry->fingerprint()->as_basic_args_string(), entry->fingerprint()->as_string(), entry->fingerprint()->compute_hash(),
2561 fp->as_basic_args_string(), fp->as_string(), fp->compute_hash());
2562 #ifndef PRODUCT
2563 _runtime_hits++;
2564 #endif
2565 }
2566 }
2567 AdapterFingerPrint::deallocate(fp);
2568 return entry;
2569 }
2570
2571 #ifndef PRODUCT
2572 static void print_table_statistics() {
2573 auto size = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) {
2574 return sizeof(*key) + sizeof(*a);
2575 };
2576 TableStatistics ts = _adapter_handler_table->statistics_calculate(size);
2577 ts.print(tty, "AdapterHandlerTable");
2578 tty->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)",
2579 _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries());
2580 int total_hits = _archived_hits + _runtime_hits;
2581 tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d)",
2582 _lookups, _equals, total_hits, _archived_hits, _runtime_hits);
2583 }
2584 #endif
2585
2586 // ---------------------------------------------------------------------------
2587 // Implementation of AdapterHandlerLibrary
2588 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = nullptr;
2589 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr;
2590 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr;
2591 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr;
2592 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr;
2593 #if INCLUDE_CDS
2594 ArchivedAdapterTable AdapterHandlerLibrary::_aot_adapter_handler_table;
2595 #endif // INCLUDE_CDS
2596 static const int AdapterHandlerLibrary_size = 48*K;
2597 BufferBlob* AdapterHandlerLibrary::_buffer = nullptr;
2598 volatile uint AdapterHandlerLibrary::_id_counter = 0;
2599
2600 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2601 assert(_buffer != nullptr, "should be initialized");
2602 return _buffer;
2603 }
2604
2605 static void post_adapter_creation(const AdapterHandlerEntry* entry) {
2606 if (Forte::is_enabled() || JvmtiExport::should_post_dynamic_code_generated()) {
2607 AdapterBlob* adapter_blob = entry->adapter_blob();
2608 char blob_id[256];
2609 jio_snprintf(blob_id,
2610 sizeof(blob_id),
2611 "%s(%s)",
2612 adapter_blob->name(),
2613 entry->fingerprint()->as_string());
2614 if (Forte::is_enabled()) {
2615 Forte::register_stub(blob_id, adapter_blob->content_begin(), adapter_blob->content_end());
2616 }
2617
2618 if (JvmtiExport::should_post_dynamic_code_generated()) {
2619 JvmtiExport::post_dynamic_code_generated(blob_id, adapter_blob->content_begin(), adapter_blob->content_end());
2620 }
2621 }
2622 }
2623
2624 void AdapterHandlerLibrary::initialize() {
2625 {
2626 ResourceMark rm;
2627 _adapter_handler_table = new (mtCode) AdapterHandlerTable();
2628 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2629 }
2630
2631 #if INCLUDE_CDS
2632 // Link adapters in AOT Cache to their code in AOT Code Cache
2633 if (AOTCodeCache::is_using_adapter() && !_aot_adapter_handler_table.empty()) {
2634 link_aot_adapters();
2635 lookup_simple_adapters();
2636 return;
2637 }
2638 #endif // INCLUDE_CDS
2639
2640 ResourceMark rm;
2641 {
2642 MutexLocker mu(AdapterHandlerLibrary_lock);
2643
2644 CompiledEntrySignature no_args;
2645 no_args.compute_calling_conventions();
2646 _no_arg_handler = create_adapter(no_args, true);
2647
2648 CompiledEntrySignature obj_args;
2649 SigEntry::add_entry(obj_args.sig(), T_OBJECT);
2650 obj_args.compute_calling_conventions();
2651 _obj_arg_handler = create_adapter(obj_args, true);
2652
2653 CompiledEntrySignature int_args;
2654 SigEntry::add_entry(int_args.sig(), T_INT);
2655 int_args.compute_calling_conventions();
2656 _int_arg_handler = create_adapter(int_args, true);
2657
2658 CompiledEntrySignature obj_int_args;
2659 SigEntry::add_entry(obj_int_args.sig(), T_OBJECT);
2660 SigEntry::add_entry(obj_int_args.sig(), T_INT);
2661 obj_int_args.compute_calling_conventions();
2662 _obj_int_arg_handler = create_adapter(obj_int_args, true);
2663
2664 CompiledEntrySignature obj_obj_args;
2665 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
2666 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
2667 obj_obj_args.compute_calling_conventions();
2668 _obj_obj_arg_handler = create_adapter(obj_obj_args, true);
2669
2670 // we should always get an entry back but we don't have any
2671 // associated blob on Zero
2672 assert(_no_arg_handler != nullptr &&
2673 _obj_arg_handler != nullptr &&
2674 _int_arg_handler != nullptr &&
2675 _obj_int_arg_handler != nullptr &&
2676 _obj_obj_arg_handler != nullptr, "Initial adapter handlers must be properly created");
2677 }
2678
2679 // Outside of the lock
2680 #ifndef ZERO
2681 // no blobs to register when we are on Zero
2682 post_adapter_creation(_no_arg_handler);
2683 post_adapter_creation(_obj_arg_handler);
2684 post_adapter_creation(_int_arg_handler);
2685 post_adapter_creation(_obj_int_arg_handler);
2686 post_adapter_creation(_obj_obj_arg_handler);
2687 #endif // ZERO
2688 }
2689
2690 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint) {
2691 uint id = (uint)AtomicAccess::add((int*)&_id_counter, 1);
2692 assert(id > 0, "we can never overflow because AOT cache cannot contain more than 2^32 methods");
2693 return AdapterHandlerEntry::allocate(id, fingerprint);
2694 }
2695
2696 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2697 int total_args_passed = method->size_of_parameters(); // All args on stack
2698 if (total_args_passed == 0) {
2699 return _no_arg_handler;
2700 } else if (total_args_passed == 1) {
2701 if (!method->is_static()) {
2702 if (InlineTypePassFieldsAsArgs && method->method_holder()->is_inline_klass()) {
2703 return nullptr;
2704 }
2705 return _obj_arg_handler;
2706 }
2707 switch (method->signature()->char_at(1)) {
2708 case JVM_SIGNATURE_CLASS: {
2709 if (InlineTypePassFieldsAsArgs) {
2710 SignatureStream ss(method->signature());
2711 InlineKlass* vk = ss.as_inline_klass(method->method_holder());
2712 if (vk != nullptr) {
2713 return nullptr;
2714 }
2715 }
2716 return _obj_arg_handler;
2717 }
2718 case JVM_SIGNATURE_ARRAY:
2719 return _obj_arg_handler;
2720 case JVM_SIGNATURE_INT:
2721 case JVM_SIGNATURE_BOOLEAN:
2722 case JVM_SIGNATURE_CHAR:
2723 case JVM_SIGNATURE_BYTE:
2724 case JVM_SIGNATURE_SHORT:
2725 return _int_arg_handler;
2726 }
2727 } else if (total_args_passed == 2 &&
2728 !method->is_static() && (!InlineTypePassFieldsAsArgs || !method->method_holder()->is_inline_klass())) {
2729 switch (method->signature()->char_at(1)) {
2730 case JVM_SIGNATURE_CLASS: {
2731 if (InlineTypePassFieldsAsArgs) {
2732 SignatureStream ss(method->signature());
2733 InlineKlass* vk = ss.as_inline_klass(method->method_holder());
2734 if (vk != nullptr) {
2735 return nullptr;
2736 }
2737 }
2738 return _obj_obj_arg_handler;
2739 }
2740 case JVM_SIGNATURE_ARRAY:
2741 return _obj_obj_arg_handler;
2742 case JVM_SIGNATURE_INT:
2743 case JVM_SIGNATURE_BOOLEAN:
2744 case JVM_SIGNATURE_CHAR:
2745 case JVM_SIGNATURE_BYTE:
2746 case JVM_SIGNATURE_SHORT:
2747 return _obj_int_arg_handler;
2748 }
2749 }
2750 return nullptr;
2751 }
2752
2753 CompiledEntrySignature::CompiledEntrySignature(Method* method) :
2754 _method(method), _num_inline_args(0), _has_inline_recv(false),
2755 _regs(nullptr), _regs_cc(nullptr), _regs_cc_ro(nullptr),
2756 _args_on_stack(0), _args_on_stack_cc(0), _args_on_stack_cc_ro(0),
2757 _c1_needs_stack_repair(false), _c2_needs_stack_repair(false), _supers(nullptr) {
2758 _sig = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2759 _sig_cc = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2760 _sig_cc_ro = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2761 }
2762
2763 // See if we can save space by sharing the same entry for VIEP and VIEP(RO),
2764 // or the same entry for VEP and VIEP(RO).
2765 CodeOffsets::Entries CompiledEntrySignature::c1_inline_ro_entry_type() const {
2766 if (!has_scalarized_args()) {
2767 // VEP/VIEP/VIEP(RO) all share the same entry. There's no packing.
2768 return CodeOffsets::Verified_Entry;
2769 }
2770 if (_method->is_static()) {
2771 // Static methods don't need VIEP(RO)
2772 return CodeOffsets::Verified_Entry;
2773 }
2774
2775 if (has_inline_recv()) {
2776 if (num_inline_args() == 1) {
2777 // Share same entry for VIEP and VIEP(RO).
2778 // This is quite common: we have an instance method in an InlineKlass that has
2779 // no inline type args other than <this>.
2780 return CodeOffsets::Verified_Inline_Entry;
2781 } else {
2782 assert(num_inline_args() > 1, "must be");
2783 // No sharing:
2784 // VIEP(RO) -- <this> is passed as object
2785 // VEP -- <this> is passed as fields
2786 return CodeOffsets::Verified_Inline_Entry_RO;
2787 }
2788 }
2789
2790 // Either a static method, or <this> is not an inline type
2791 if (args_on_stack_cc() != args_on_stack_cc_ro()) {
2792 // No sharing:
2793 // Some arguments are passed on the stack, and we have inserted reserved entries
2794 // into the VEP, but we never insert reserved entries into the VIEP(RO).
2795 return CodeOffsets::Verified_Inline_Entry_RO;
2796 } else {
2797 // Share same entry for VEP and VIEP(RO).
2798 return CodeOffsets::Verified_Entry;
2799 }
2800 }
2801
2802 // Returns all super methods (transitive) in classes and interfaces that are overridden by the current method.
2803 GrowableArray<Method*>* CompiledEntrySignature::get_supers() {
2804 if (_supers != nullptr) {
2805 return _supers;
2806 }
2807 _supers = new GrowableArray<Method*>();
2808 // Skip private, static, and <init> methods
2809 if (_method->is_private() || _method->is_static() || _method->is_object_constructor()) {
2810 return _supers;
2811 }
2812 Symbol* name = _method->name();
2813 Symbol* signature = _method->signature();
2814 const Klass* holder = _method->method_holder()->super();
2815 Symbol* holder_name = holder->name();
2816 JavaThread* current = JavaThread::current();
2817 HandleMark hm(current);
2818 Handle loader(current, _method->method_holder()->class_loader());
2819
2820 // Walk up the class hierarchy and search for super methods
2821 while (holder != nullptr) {
2822 Method* super_method = holder->lookup_method(name, signature);
2823 if (super_method == nullptr) {
2824 break;
2825 }
2826 if (!super_method->is_static() && !super_method->is_private() &&
2827 (!super_method->is_package_private() ||
2828 super_method->method_holder()->is_same_class_package(loader(), holder_name))) {
2829 _supers->push(super_method);
2830 }
2831 holder = super_method->method_holder()->super();
2832 }
2833 // Search interfaces for super methods
2834 Array<InstanceKlass*>* interfaces = _method->method_holder()->transitive_interfaces();
2835 for (int i = 0; i < interfaces->length(); ++i) {
2836 Method* m = interfaces->at(i)->lookup_method(name, signature);
2837 if (m != nullptr && !m->is_static() && m->is_public()) {
2838 _supers->push(m);
2839 }
2840 }
2841 return _supers;
2842 }
2843
2844 bool CompiledEntrySignature::check_supers_and_deoptimize(int arg_num) {
2845 assert(JavaThread::current()->thread_state() == _thread_in_vm, "must be in vm state");
2846
2847 bool scalar_super = false;
2848 bool non_scalar_super = false;
2849
2850 GrowableArray<Method*>* supers = get_supers();
2851 for (int i = 0; i < supers->length(); ++i) {
2852 Method* super_method = supers->at(i);
2853 if (super_method->is_scalarized_arg(arg_num)) {
2854 scalar_super = true;
2855 } else {
2856 non_scalar_super = true;
2857 }
2858 }
2859 #ifdef ASSERT
2860 // Randomly enable below code paths for stress testing
2861 bool stress = StressCallingConvention;
2862 if (stress && (os::random() & 1) == 1) {
2863 non_scalar_super = true;
2864 if ((os::random() & 1) == 1) {
2865 scalar_super = true;
2866 }
2867 }
2868 #endif
2869 if (non_scalar_super) {
2870 // Found a super method with a non-scalarized argument. Fall back to the non-scalarized calling convention.
2871 if (scalar_super) {
2872 // Found non-scalar *and* scalar super methods. We can't handle both.
2873 // Mark the scalar method as mismatch and re-compile call sites to use non-scalarized calling convention.
2874 for (int i = 0; i < supers->length(); ++i) {
2875 Method* super_method = supers->at(i);
2876 if (super_method->is_scalarized_arg(arg_num) DEBUG_ONLY(|| (stress && (os::random() & 1) == 1))) {
2877 JavaThread* thread = JavaThread::current();
2878 HandleMark hm(thread);
2879 methodHandle mh(thread, super_method);
2880 DeoptimizationScope deopt_scope;
2881 {
2882 // Keep the lock scope minimal. Prevent interference with other
2883 // dependency checks by setting mismatch and marking within the lock.
2884 MutexLocker ml(Compile_lock, Mutex::_safepoint_check_flag);
2885 super_method->set_mismatch();
2886 CodeCache::mark_for_deoptimization(&deopt_scope, mh());
2887 }
2888 deopt_scope.deoptimize_marked();
2889 }
2890 }
2891 }
2892 }
2893
2894 return non_scalar_super;
2895 }
2896
2897 // Iterate over arguments and compute scalarized and non-scalarized signatures
2898 void CompiledEntrySignature::compute_calling_conventions(bool link_time) {
2899 assert(JavaThread::current()->thread_state() != _thread_in_native, "must not be in native");
2900 assert(link_time || (_method != nullptr && _method->adapter() != nullptr), "invariant");
2901 bool has_scalarized = false;
2902 if (_method != nullptr) {
2903 InstanceKlass* holder = _method->method_holder();
2904 int arg_num = 0;
2905 if (!_method->is_static()) {
2906 // We shouldn't scalarize 'this' in a value class constructor
2907 if (holder->is_inline_klass() && InlineKlass::cast(holder)->can_be_passed_as_fields() &&
2908 !_method->is_object_constructor() && (link_time || _method->is_scalarized_arg(arg_num))) {
2909 _sig_cc->appendAll(InlineKlass::cast(holder)->extended_sig());
2910 _sig_cc->insert_before(1, SigEntry(T_OBJECT, 0, nullptr, false, true)); // buffer argument
2911 has_scalarized = true;
2912 _has_inline_recv = true;
2913 _num_inline_args++;
2914 } else {
2915 SigEntry::add_entry(_sig_cc, T_OBJECT, holder->name());
2916 }
2917 SigEntry::add_entry(_sig, T_OBJECT, holder->name());
2918 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, holder->name());
2919 arg_num++;
2920 }
2921 for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
2922 const BasicType bt = ss.type();
2923 if (InlineTypePassFieldsAsArgs && bt == T_OBJECT) {
2924 InlineKlass* vk = ss.as_inline_klass(holder);
2925 if (vk != nullptr && vk->can_be_passed_as_fields() && (link_time || _method->is_scalarized_arg(arg_num))) {
2926 // Check for a calling convention mismatch with super method(s)
2927 if (link_time && check_supers_and_deoptimize(arg_num)) {
2928 // Fall back to non-scalarized calling convention
2929 SigEntry::add_entry(_sig_cc, T_OBJECT, ss.as_symbol());
2930 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, ss.as_symbol());
2931 } else {
2932 _num_inline_args++;
2933 has_scalarized = true;
2934 int last = _sig_cc->length();
2935 int last_ro = _sig_cc_ro->length();
2936 _sig_cc->appendAll(vk->extended_sig());
2937 _sig_cc_ro->appendAll(vk->extended_sig());
2938 // buffer argument
2939 _sig_cc->insert_before(last + 1, SigEntry(T_OBJECT, 0, nullptr, false, true));
2940 _sig_cc_ro->insert_before(last_ro + 1, SigEntry(T_OBJECT, 0, nullptr, false, true));
2941 // Insert InlineTypeNode::NullMarker field right after T_METADATA delimiter
2942 _sig_cc->insert_before(last + 2, SigEntry(T_BOOLEAN, -1, nullptr, true, false));
2943 _sig_cc_ro->insert_before(last_ro + 2, SigEntry(T_BOOLEAN, -1, nullptr, true, false));
2944 }
2945 } else {
2946 SigEntry::add_entry(_sig_cc, T_OBJECT, ss.as_symbol());
2947 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, ss.as_symbol());
2948 }
2949 } else {
2950 SigEntry::add_entry(_sig_cc, ss.type(), ss.as_symbol());
2951 SigEntry::add_entry(_sig_cc_ro, ss.type(), ss.as_symbol());
2952 }
2953 SigEntry::add_entry(_sig, bt, ss.as_symbol());
2954 if (bt != T_VOID) {
2955 arg_num++;
2956 }
2957 }
2958 }
2959
2960 // Compute the non-scalarized calling convention
2961 _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
2962 _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
2963
2964 // Compute the scalarized calling conventions if there are scalarized inline types in the signature
2965 if (has_scalarized && !_method->is_native()) {
2966 _regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc->length());
2967 _args_on_stack_cc = SharedRuntime::java_calling_convention(_sig_cc, _regs_cc);
2968
2969 _regs_cc_ro = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc_ro->length());
2970 _args_on_stack_cc_ro = SharedRuntime::java_calling_convention(_sig_cc_ro, _regs_cc_ro);
2971
2972 _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
2973 _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
2974
2975 // Upper bound on stack arguments to avoid hitting the argument limit and
2976 // bailing out of compilation ("unsupported incoming calling sequence").
2977 // TODO 8281260 We need a reasonable limit (flag?) here
2978 if (MAX2(_args_on_stack_cc, _args_on_stack_cc_ro) <= 75) {
2979 return; // Success
2980 }
2981 }
2982
2983 // No scalarized args
2984 _sig_cc = _sig;
2985 _regs_cc = _regs;
2986 _args_on_stack_cc = _args_on_stack;
2987
2988 _sig_cc_ro = _sig;
2989 _regs_cc_ro = _regs;
2990 _args_on_stack_cc_ro = _args_on_stack;
2991 }
2992
2993 void CompiledEntrySignature::initialize_from_fingerprint(AdapterFingerPrint* fingerprint) {
2994 _has_inline_recv = fingerprint->has_ro_adapter();
2995
2996 int value_object_count = 0;
2997 BasicType prev_bt = T_ILLEGAL;
2998 bool has_scalarized_arguments = false;
2999 bool long_prev = false;
3000 int long_prev_offset = -1;
3001 bool skipping_inline_recv = false;
3002 bool receiver_handled = false;
3003
3004 fingerprint->iterate_args([&] (const AdapterFingerPrint::Element& arg) {
3005 BasicType bt = arg.bt();
3006 int offset = arg.offset();
3007
3008 if (long_prev) {
3009 long_prev = false;
3010 BasicType bt_to_add;
3011 if (bt == T_VOID) {
3012 bt_to_add = T_LONG;
3013 } else {
3014 bt_to_add = T_OBJECT;
3015 }
3016 if (value_object_count == 0) {
3017 SigEntry::add_entry(_sig, bt_to_add);
3018 }
3019 assert(long_prev_offset != 0, "no buffer argument here");
3020 SigEntry::add_entry(_sig_cc, bt_to_add, nullptr, long_prev_offset);
3021 if (!skipping_inline_recv) {
3022 SigEntry::add_entry(_sig_cc_ro, bt_to_add, nullptr, long_prev_offset);
3023 }
3024 }
3025
3026 switch (bt) {
3027 case T_VOID:
3028 if (prev_bt != T_LONG && prev_bt != T_DOUBLE) {
3029 assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
3030 value_object_count--;
3031 SigEntry::add_entry(_sig_cc, T_VOID, nullptr, offset);
3032 if (!skipping_inline_recv) {
3033 SigEntry::add_entry(_sig_cc_ro, T_VOID, nullptr, offset);
3034 } else if (value_object_count == 0) {
3035 skipping_inline_recv = false;
3036 }
3037 assert(value_object_count >= 0, "invalid value object count");
3038 } else {
3039 // Nothing to add for _sig: We already added an addition T_VOID in add_entry() when adding T_LONG or T_DOUBLE.
3040 }
3041 break;
3042 case T_INT:
3043 case T_FLOAT:
3044 case T_DOUBLE:
3045 if (value_object_count == 0) {
3046 SigEntry::add_entry(_sig, bt);
3047 }
3048 SigEntry::add_entry(_sig_cc, bt, nullptr, offset);
3049 if (!skipping_inline_recv) {
3050 SigEntry::add_entry(_sig_cc_ro, bt, nullptr, offset);
3051 }
3052 break;
3053 case T_LONG:
3054 long_prev = true;
3055 long_prev_offset = offset;
3056 break;
3057 case T_BOOLEAN:
3058 case T_CHAR:
3059 case T_BYTE:
3060 case T_SHORT:
3061 case T_OBJECT:
3062 case T_ARRAY:
3063 assert(value_object_count > 0, "must be value object field");
3064 assert(offset != 0 || (bt == T_OBJECT && prev_bt == T_METADATA), "buffer input expected here");
3065 SigEntry::add_entry(_sig_cc, bt, nullptr, offset, offset == -1, offset == 0);
3066 if (!skipping_inline_recv) {
3067 SigEntry::add_entry(_sig_cc_ro, bt, nullptr, offset, offset == -1, offset == 0);
3068 }
3069 break;
3070 case T_METADATA:
3071 assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
3072 if (value_object_count == 0) {
3073 SigEntry::add_entry(_sig, T_OBJECT);
3074 }
3075 SigEntry::add_entry(_sig_cc, T_METADATA, nullptr, offset);
3076 if (!skipping_inline_recv) {
3077 if (!receiver_handled && _has_inline_recv && value_object_count == 0) {
3078 SigEntry::add_entry(_sig_cc_ro, T_OBJECT);
3079 skipping_inline_recv = true;
3080 receiver_handled = true;
3081 } else {
3082 SigEntry::add_entry(_sig_cc_ro, T_METADATA, nullptr, offset);
3083 }
3084 }
3085 value_object_count++;
3086 has_scalarized_arguments = true;
3087 break;
3088 default: {
3089 fatal("Unexpected BasicType: %s", basictype_to_str(bt));
3090 }
3091 }
3092 prev_bt = bt;
3093 });
3094
3095 if (long_prev) {
3096 // If previous bt was T_LONG and we reached the end of the signature, we know that it must be a T_OBJECT.
3097 SigEntry::add_entry(_sig, T_OBJECT);
3098 SigEntry::add_entry(_sig_cc, T_OBJECT);
3099 SigEntry::add_entry(_sig_cc_ro, T_OBJECT);
3100 }
3101 assert(value_object_count == 0, "invalid value object count");
3102
3103 #ifdef ASSERT
3104 if (_has_inline_recv) {
3105 // In RO signatures, inline receivers must be represented as a single T_OBJECT
3106 assert(_sig_cc_ro->length() >= 1, "sig_cc_ro must include receiver");
3107 assert(_sig_cc_ro->at(0)._bt == T_OBJECT,
3108 "sig_cc_ro must represent inline receiver as T_OBJECT");
3109 assert(_sig_cc_ro->length() <= _sig_cc->length(),
3110 "sig_cc_ro must not be longer than sig_cc");
3111 }
3112 #endif
3113
3114 _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
3115 _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
3116
3117 // Compute the scalarized calling conventions if there are scalarized inline types in the signature
3118 if (has_scalarized_arguments) {
3119 _regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc->length());
3120 _args_on_stack_cc = SharedRuntime::java_calling_convention(_sig_cc, _regs_cc);
3121
3122 _regs_cc_ro = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc_ro->length());
3123 _args_on_stack_cc_ro = SharedRuntime::java_calling_convention(_sig_cc_ro, _regs_cc_ro);
3124
3125 _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
3126 _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
3127 } else {
3128 // No scalarized args
3129 _sig_cc = _sig;
3130 _regs_cc = _regs;
3131 _args_on_stack_cc = _args_on_stack;
3132
3133 _sig_cc_ro = _sig;
3134 _regs_cc_ro = _regs;
3135 _args_on_stack_cc_ro = _args_on_stack;
3136 }
3137
3138 #ifdef ASSERT
3139 {
3140 AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(_sig_cc, _has_inline_recv);
3141 assert(fingerprint->equals(compare_fp), "%s - %s", fingerprint->as_string(), compare_fp->as_string());
3142 AdapterFingerPrint::deallocate(compare_fp);
3143 }
3144 #endif
3145 }
3146
3147 const char* AdapterHandlerEntry::_entry_names[] = {
3148 "i2c", "c2i", "c2i_unverified", "c2i_no_clinit_check"
3149 };
3150
3151 #ifdef ASSERT
3152 void AdapterHandlerLibrary::verify_adapter_sharing(CompiledEntrySignature& ces, AdapterHandlerEntry* cached_entry) {
3153 // we can only check for the same code if there is any
3154 #ifndef ZERO
3155 AdapterHandlerEntry* comparison_entry = create_adapter(ces, false, true);
3156 assert(comparison_entry->adapter_blob() == nullptr, "no blob should be created when creating an adapter for comparison");
3157 assert(comparison_entry->compare_code(cached_entry), "code must match");
3158 // Release the one just created
3159 AdapterHandlerEntry::deallocate(comparison_entry);
3160 # endif // ZERO
3161 }
3162 #endif /* ASSERT*/
3163
3164 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
3165 assert(!method->is_abstract() || InlineTypePassFieldsAsArgs, "abstract methods do not have adapters");
3166 // Use customized signature handler. Need to lock around updates to
3167 // the _adapter_handler_table (it is not safe for concurrent readers
3168 // and a single writer: this could be fixed if it becomes a
3169 // problem).
3170
3171 // Fast-path for trivial adapters
3172 AdapterHandlerEntry* entry = get_simple_adapter(method);
3173 if (entry != nullptr) {
3174 return entry;
3175 }
3176
3177 ResourceMark rm;
3178 bool new_entry = false;
3179
3180 CompiledEntrySignature ces(method());
3181 ces.compute_calling_conventions();
3182 if (ces.has_scalarized_args()) {
3183 if (!method->has_scalarized_args()) {
3184 method->set_has_scalarized_args();
3185 }
3186 if (ces.c1_needs_stack_repair()) {
3187 method->set_c1_needs_stack_repair();
3188 }
3189 if (ces.c2_needs_stack_repair() && !method->c2_needs_stack_repair()) {
3190 method->set_c2_needs_stack_repair();
3191 }
3192 }
3193
3194 {
3195 MutexLocker mu(AdapterHandlerLibrary_lock);
3196
3197 // Lookup method signature's fingerprint
3198 entry = lookup(ces.sig_cc(), ces.has_inline_recv());
3199
3200 if (entry != nullptr) {
3201 #ifndef ZERO
3202 assert(entry->is_linked(), "AdapterHandlerEntry must have been linked");
3203 #endif
3204 #ifdef ASSERT
3205 if (!entry->in_aot_cache() && VerifyAdapterSharing) {
3206 verify_adapter_sharing(ces, entry);
3207 }
3208 #endif
3209 } else {
3210 entry = create_adapter(ces, /* allocate_code_blob */ true);
3211 if (entry != nullptr) {
3212 new_entry = true;
3213 }
3214 }
3215 }
3216
3217 // Outside of the lock
3218 if (new_entry) {
3219 post_adapter_creation(entry);
3220 }
3221 return entry;
3222 }
3223
3224 void AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler) {
3225 ResourceMark rm;
3226 const char* name = AdapterHandlerLibrary::name(handler);
3227 const uint32_t id = AdapterHandlerLibrary::id(handler);
3228
3229 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::Adapter, id, name);
3230 if (blob != nullptr) {
3231 handler->set_adapter_blob(blob->as_adapter_blob());
3232 }
3233 }
3234
3235 #ifndef PRODUCT
3236 void AdapterHandlerLibrary::print_adapter_handler_info(outputStream* st, AdapterHandlerEntry* handler) {
3237 ttyLocker ttyl;
3238 ResourceMark rm;
3239 int insts_size;
3240 // on Zero the blob may be null
3241 handler->print_adapter_on(tty);
3242 AdapterBlob* adapter_blob = handler->adapter_blob();
3243 if (adapter_blob == nullptr) {
3244 return;
3245 }
3246 insts_size = adapter_blob->code_size();
3247 st->print_cr("i2c argument handler for: %s %s (%d bytes generated)",
3248 handler->fingerprint()->as_basic_args_string(),
3249 handler->fingerprint()->as_string(), insts_size);
3250 st->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry()));
3251 if (Verbose || PrintStubCode) {
3252 address first_pc = adapter_blob->content_begin();
3253 if (first_pc != nullptr) {
3254 Disassembler::decode(first_pc, first_pc + insts_size, st, &adapter_blob->asm_remarks());
3255 st->cr();
3256 }
3257 }
3258 }
3259 #endif // PRODUCT
3260
3261 void AdapterHandlerLibrary::address_to_offset(address entry_address[AdapterBlob::ENTRY_COUNT],
3262 int entry_offset[AdapterBlob::ENTRY_COUNT]) {
3263 entry_offset[AdapterBlob::I2C] = 0;
3264 entry_offset[AdapterBlob::C2I] = entry_address[AdapterBlob::C2I] - entry_address[AdapterBlob::I2C];
3265 entry_offset[AdapterBlob::C2I_Inline] = entry_address[AdapterBlob::C2I_Inline] - entry_address[AdapterBlob::I2C];
3266 entry_offset[AdapterBlob::C2I_Inline_RO] = entry_address[AdapterBlob::C2I_Inline_RO] - entry_address[AdapterBlob::I2C];
3267 entry_offset[AdapterBlob::C2I_Unverified] = entry_address[AdapterBlob::C2I_Unverified] - entry_address[AdapterBlob::I2C];
3268 entry_offset[AdapterBlob::C2I_Unverified_Inline] = entry_address[AdapterBlob::C2I_Unverified_Inline] - entry_address[AdapterBlob::I2C];
3269 if (entry_address[AdapterBlob::C2I_No_Clinit_Check] == nullptr) {
3270 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = -1;
3271 } else {
3272 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = entry_address[AdapterBlob::C2I_No_Clinit_Check] - entry_address[AdapterBlob::I2C];
3273 }
3274 }
3275
3276 bool AdapterHandlerLibrary::generate_adapter_code(AdapterHandlerEntry* handler,
3277 CompiledEntrySignature& ces,
3278 bool allocate_code_blob,
3279 bool is_transient) {
3280 if (log_is_enabled(Info, perf, class, link)) {
3281 ClassLoader::perf_method_adapters_count()->inc();
3282 }
3283
3284 #ifndef ZERO
3285 AdapterBlob* adapter_blob = nullptr;
3286 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3287 CodeBuffer buffer(buf);
3288 short buffer_locs[20];
3289 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3290 sizeof(buffer_locs)/sizeof(relocInfo));
3291 MacroAssembler masm(&buffer);
3292 address entry_address[AdapterBlob::ENTRY_COUNT];
3293
3294 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
3295 SharedRuntime::generate_i2c2i_adapters(&masm,
3296 ces.args_on_stack(),
3297 ces.sig(),
3298 ces.regs(),
3299 ces.sig_cc(),
3300 ces.regs_cc(),
3301 ces.sig_cc_ro(),
3302 ces.regs_cc_ro(),
3303 entry_address,
3304 adapter_blob,
3305 allocate_code_blob);
3306
3307 if (ces.has_scalarized_args()) {
3308 // Save a C heap allocated version of the scalarized signature and store it in the adapter
3309 GrowableArray<SigEntry>* heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc()->length(), mtCode);
3310 heap_sig->appendAll(ces.sig_cc());
3311 handler->set_sig_cc(heap_sig);
3312 heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc_ro()->length(), mtCode);
3313 heap_sig->appendAll(ces.sig_cc_ro());
3314 handler->set_sig_cc_ro(heap_sig);
3315 }
3316 // On zero there is no code to save and no need to create a blob and
3317 // or relocate the handler.
3318 int entry_offset[AdapterBlob::ENTRY_COUNT];
3319 address_to_offset(entry_address, entry_offset);
3320 #ifdef ASSERT
3321 if (VerifyAdapterSharing) {
3322 handler->save_code(buf->code_begin(), buffer.insts_size());
3323 if (is_transient) {
3324 return true;
3325 }
3326 }
3327 #endif
3328 if (adapter_blob == nullptr) {
3329 // CodeCache is full, disable compilation
3330 // Ought to log this but compile log is only per compile thread
3331 // and we're some non descript Java thread.
3332 return false;
3333 }
3334 handler->set_adapter_blob(adapter_blob);
3335 if (!is_transient && AOTCodeCache::is_dumping_adapter()) {
3336 // try to save generated code
3337 const char* name = AdapterHandlerLibrary::name(handler);
3338 const uint32_t id = AdapterHandlerLibrary::id(handler);
3339 bool success = AOTCodeCache::store_code_blob(*adapter_blob, AOTCodeEntry::Adapter, id, name);
3340 assert(success || !AOTCodeCache::is_dumping_adapter(), "caching of adapter must be disabled");
3341 }
3342 #endif // ZERO
3343
3344 #ifndef PRODUCT
3345 // debugging support
3346 if (PrintAdapterHandlers || PrintStubCode) {
3347 print_adapter_handler_info(tty, handler);
3348 }
3349 #endif
3350
3351 return true;
3352 }
3353
3354 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(CompiledEntrySignature& ces,
3355 bool allocate_code_blob,
3356 bool is_transient) {
3357 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(ces.sig_cc(), ces.has_inline_recv());
3358 #ifdef ASSERT
3359 // Verify that we can successfully restore the compiled entry signature object.
3360 CompiledEntrySignature ces_verify;
3361 ces_verify.initialize_from_fingerprint(fp);
3362 #endif
3363 AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fp);
3364 if (!generate_adapter_code(handler, ces, allocate_code_blob, is_transient)) {
3365 AdapterHandlerEntry::deallocate(handler);
3366 return nullptr;
3367 }
3368 if (!is_transient) {
3369 assert_lock_strong(AdapterHandlerLibrary_lock);
3370 _adapter_handler_table->put(fp, handler);
3371 }
3372 return handler;
3373 }
3374
3375 #if INCLUDE_CDS
3376 void AdapterHandlerEntry::remove_unshareable_info() {
3377 #ifdef ASSERT
3378 _saved_code = nullptr;
3379 _saved_code_length = 0;
3380 #endif // ASSERT
3381 _adapter_blob = nullptr;
3382 _linked = false;
3383 _sig_cc = nullptr;
3384 _sig_cc_ro = nullptr;
3385 }
3386
3387 class CopyAdapterTableToArchive : StackObj {
3388 private:
3389 CompactHashtableWriter* _writer;
3390 ArchiveBuilder* _builder;
3391 public:
3392 CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer),
3393 _builder(ArchiveBuilder::current())
3394 {}
3395
3396 bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) {
3397 LogStreamHandle(Trace, aot) lsh;
3398 if (ArchiveBuilder::current()->has_been_archived((address)entry)) {
3399 assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be");
3400 AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp);
3401 assert(buffered_fp != nullptr,"sanity check");
3402 AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry);
3403 assert(buffered_entry != nullptr,"sanity check");
3404
3405 uint hash = fp->compute_hash();
3406 _writer->add(hash, AOTCompressedPointers::encode_not_null(buffered_entry));
3407 if (lsh.is_enabled()) {
3408 address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta();
3409 address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta();
3410 log_trace(aot)("Added fp=%p (%s), entry=%p to the archived adater table", buffered_fp, buffered_fp->as_basic_args_string(), buffered_entry);
3411 }
3412 } else {
3413 if (lsh.is_enabled()) {
3414 log_trace(aot)("Skipping adapter handler %p (fp=%s) as it is not archived", entry, fp->as_basic_args_string());
3415 }
3416 }
3417 return true;
3418 }
3419 };
3420
3421 void AdapterHandlerLibrary::dump_aot_adapter_table() {
3422 CompactHashtableStats stats;
3423 CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats);
3424 CopyAdapterTableToArchive copy(&writer);
3425 _adapter_handler_table->iterate(©);
3426 writer.dump(&_aot_adapter_handler_table, "archived adapter table");
3427 }
3428
3429 void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) {
3430 _aot_adapter_handler_table.serialize_header(soc);
3431 }
3432
3433 void AdapterHandlerLibrary::link_aot_adapter_handler(AdapterHandlerEntry* handler) {
3434 #ifdef ASSERT
3435 if (TestAOTAdapterLinkFailure) {
3436 return;
3437 }
3438 #endif
3439 lookup_aot_cache(handler);
3440 #ifndef PRODUCT
3441 // debugging support
3442 if (PrintAdapterHandlers || PrintStubCode) {
3443 print_adapter_handler_info(tty, handler);
3444 }
3445 #endif
3446 }
3447
3448 // This method is used during production run to link archived adapters (stored in AOT Cache)
3449 // to their code in AOT Code Cache
3450 void AdapterHandlerEntry::link() {
3451 ResourceMark rm;
3452 assert(_fingerprint != nullptr, "_fingerprint must not be null");
3453 bool generate_code = false;
3454 // Generate code only if AOTCodeCache is not available, or
3455 // caching adapters is disabled, or we fail to link
3456 // the AdapterHandlerEntry to its code in the AOTCodeCache
3457 if (AOTCodeCache::is_using_adapter()) {
3458 AdapterHandlerLibrary::link_aot_adapter_handler(this);
3459 // If link_aot_adapter_handler() succeeds, _adapter_blob will be non-null
3460 if (_adapter_blob == nullptr) {
3461 log_warning(aot)("Failed to link AdapterHandlerEntry (fp=%s) to its code in the AOT code cache", _fingerprint->as_basic_args_string());
3462 generate_code = true;
3463 }
3464
3465 if (get_sig_cc() == nullptr) {
3466 // Calling conventions have to be regenerated at runtime and are accessed through method adapters,
3467 // which are archived in the AOT code cache. If the adapters are not regenerated, the
3468 // calling conventions should be regenerated here.
3469 CompiledEntrySignature ces;
3470 ces.initialize_from_fingerprint(_fingerprint);
3471 if (ces.has_scalarized_args()) {
3472 // Save a C heap allocated version of the scalarized signature and store it in the adapter
3473 GrowableArray<SigEntry>* heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc()->length(), mtCode);
3474 heap_sig->appendAll(ces.sig_cc());
3475 set_sig_cc(heap_sig);
3476 heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc_ro()->length(), mtCode);
3477 heap_sig->appendAll(ces.sig_cc_ro());
3478 set_sig_cc_ro(heap_sig);
3479 }
3480 }
3481 } else {
3482 generate_code = true;
3483 }
3484 if (generate_code) {
3485 CompiledEntrySignature ces;
3486 ces.initialize_from_fingerprint(_fingerprint);
3487 if (!AdapterHandlerLibrary::generate_adapter_code(this, ces, true, false)) {
3488 // Don't throw exceptions during VM initialization because java.lang.* classes
3489 // might not have been initialized, causing problems when constructing the
3490 // Java exception object.
3491 vm_exit_during_initialization("Out of space in CodeCache for adapters");
3492 }
3493 }
3494 if (_adapter_blob != nullptr) {
3495 post_adapter_creation(this);
3496 }
3497 assert(_linked, "AdapterHandlerEntry must now be linked");
3498 }
3499
3500 void AdapterHandlerLibrary::link_aot_adapters() {
3501 uint max_id = 0;
3502 assert(AOTCodeCache::is_using_adapter(), "AOT adapters code should be available");
3503 /* It is possible that some adapters generated in assembly phase are not stored in the cache.
3504 * That implies adapter ids of the adapters in the cache may not be contiguous.
3505 * If the size of the _aot_adapter_handler_table is used to initialize _id_counter, then it may
3506 * result in collision of adapter ids between AOT stored handlers and runtime generated handlers.
3507 * To avoid such situation, initialize the _id_counter with the largest adapter id among the AOT stored handlers.
3508 */
3509 _aot_adapter_handler_table.iterate_all([&](AdapterHandlerEntry* entry) {
3510 assert(!entry->is_linked(), "AdapterHandlerEntry is already linked!");
3511 entry->link();
3512 max_id = MAX2(max_id, entry->id());
3513 });
3514 // Set adapter id to the maximum id found in the AOTCache
3515 assert(_id_counter == 0, "Did not expect new AdapterHandlerEntry to be created at this stage");
3516 _id_counter = max_id;
3517 }
3518
3519 // This method is called during production run to lookup simple adapters
3520 // in the archived adapter handler table
3521 void AdapterHandlerLibrary::lookup_simple_adapters() {
3522 assert(!_aot_adapter_handler_table.empty(), "archived adapter handler table is empty");
3523
3524 MutexLocker mu(AdapterHandlerLibrary_lock);
3525 ResourceMark rm;
3526 CompiledEntrySignature no_args;
3527 no_args.compute_calling_conventions();
3528 _no_arg_handler = lookup(no_args.sig_cc(), no_args.has_inline_recv());
3529
3530 CompiledEntrySignature obj_args;
3531 SigEntry::add_entry(obj_args.sig(), T_OBJECT);
3532 obj_args.compute_calling_conventions();
3533 _obj_arg_handler = lookup(obj_args.sig_cc(), obj_args.has_inline_recv());
3534
3535 CompiledEntrySignature int_args;
3536 SigEntry::add_entry(int_args.sig(), T_INT);
3537 int_args.compute_calling_conventions();
3538 _int_arg_handler = lookup(int_args.sig_cc(), int_args.has_inline_recv());
3539
3540 CompiledEntrySignature obj_int_args;
3541 SigEntry::add_entry(obj_int_args.sig(), T_OBJECT);
3542 SigEntry::add_entry(obj_int_args.sig(), T_INT);
3543 obj_int_args.compute_calling_conventions();
3544 _obj_int_arg_handler = lookup(obj_int_args.sig_cc(), obj_int_args.has_inline_recv());
3545
3546 CompiledEntrySignature obj_obj_args;
3547 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
3548 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
3549 obj_obj_args.compute_calling_conventions();
3550 _obj_obj_arg_handler = lookup(obj_obj_args.sig_cc(), obj_obj_args.has_inline_recv());
3551
3552 assert(_no_arg_handler != nullptr &&
3553 _obj_arg_handler != nullptr &&
3554 _int_arg_handler != nullptr &&
3555 _obj_int_arg_handler != nullptr &&
3556 _obj_obj_arg_handler != nullptr, "Initial adapters not found in archived adapter handler table");
3557 assert(_no_arg_handler->is_linked() &&
3558 _obj_arg_handler->is_linked() &&
3559 _int_arg_handler->is_linked() &&
3560 _obj_int_arg_handler->is_linked() &&
3561 _obj_obj_arg_handler->is_linked(), "Initial adapters not in linked state");
3562 }
3563 #endif // INCLUDE_CDS
3564
3565 void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) {
3566 LogStreamHandle(Trace, aot) lsh;
3567 if (lsh.is_enabled()) {
3568 lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string());
3569 lsh.cr();
3570 }
3571 it->push(&_fingerprint);
3572 }
3573
3574 AdapterHandlerEntry::~AdapterHandlerEntry() {
3575 if (_fingerprint != nullptr) {
3576 AdapterFingerPrint::deallocate(_fingerprint);
3577 _fingerprint = nullptr;
3578 }
3579 if (_sig_cc != nullptr) {
3580 delete _sig_cc;
3581 }
3582 if (_sig_cc_ro != nullptr) {
3583 delete _sig_cc_ro;
3584 }
3585 #ifdef ASSERT
3586 FREE_C_HEAP_ARRAY(_saved_code);
3587 #endif
3588 FreeHeap(this);
3589 }
3590
3591
3592 #ifdef ASSERT
3593 // Capture the code before relocation so that it can be compared
3594 // against other versions. If the code is captured after relocation
3595 // then relative instructions won't be equivalent.
3596 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
3597 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
3598 _saved_code_length = length;
3599 memcpy(_saved_code, buffer, length);
3600 }
3601
3602
3603 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
3604 assert(_saved_code != nullptr && other->_saved_code != nullptr, "code not saved");
3605
3606 if (other->_saved_code_length != _saved_code_length) {
3607 return false;
3608 }
3609
3610 return memcmp(other->_saved_code, _saved_code, _saved_code_length) == 0;
3611 }
3612 #endif
3613
3614
3615 /**
3616 * Create a native wrapper for this native method. The wrapper converts the
3617 * Java-compiled calling convention to the native convention, handles
3618 * arguments, and transitions to native. On return from the native we transition
3619 * back to java blocking if a safepoint is in progress.
3620 */
3621 void AdapterHandlerLibrary::create_native_wrapper(const methodHandle& method) {
3622 ResourceMark rm;
3623 nmethod* nm = nullptr;
3624
3625 // Check if memory should be freed before allocation
3626 CodeCache::gc_on_allocation();
3627
3628 assert(method->is_native(), "must be native");
3629 assert(method->is_special_native_intrinsic() ||
3630 method->has_native_function(), "must have something valid to call!");
3631
3632 {
3633 // Perform the work while holding the lock, but perform any printing outside the lock
3634 MutexLocker mu(AdapterHandlerLibrary_lock);
3635 // See if somebody beat us to it
3636 if (method->code() != nullptr) {
3637 return;
3638 }
3639
3640 const int compile_id = CompileBroker::assign_compile_id(method, CompileBroker::standard_entry_bci);
3641 assert(compile_id > 0, "Must generate native wrapper");
3642
3643
3644 ResourceMark rm;
3645 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3646 if (buf != nullptr) {
3647 CodeBuffer buffer(buf);
3648
3649 if (method->is_continuation_enter_intrinsic()) {
3650 buffer.initialize_stubs_size(192);
3651 }
3652
3653 struct { double data[20]; } locs_buf;
3654 struct { double data[20]; } stubs_locs_buf;
3655 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
3656 #if defined(AARCH64)
3657 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
3658 // in the constant pool to ensure ordering between the barrier and oops
3659 // accesses. For native_wrappers we need a constant.
3660 buffer.initialize_consts_size(8);
3661 #elif defined(PPC64) || defined(S390)
3662 // On PPC64/S390 the continuation enter intrinsic needs the constant pool for the compiled
3663 // static java call that is resolved in the runtime.
3664 if (method->is_continuation_enter_intrinsic()) {
3665 buffer.initialize_consts_size(8 PPC64_ONLY(+ 24) S390_ONLY(+ 17));
3666 }
3667 #endif
3668 buffer.stubs()->initialize_shared_locs((relocInfo*)&stubs_locs_buf, sizeof(stubs_locs_buf) / sizeof(relocInfo));
3669 MacroAssembler _masm(&buffer);
3670
3671 // Fill in the signature array, for the calling-convention call.
3672 const int total_args_passed = method->size_of_parameters();
3673
3674 BasicType stack_sig_bt[16];
3675 VMRegPair stack_regs[16];
3676 BasicType* sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
3677 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
3678
3679 int i = 0;
3680 if (!method->is_static()) { // Pass in receiver first
3681 sig_bt[i++] = T_OBJECT;
3682 }
3683 SignatureStream ss(method->signature());
3684 for (; !ss.at_return_type(); ss.next()) {
3685 sig_bt[i++] = ss.type(); // Collect remaining bits of signature
3686 if (ss.type() == T_LONG || ss.type() == T_DOUBLE) {
3687 sig_bt[i++] = T_VOID; // Longs & doubles take 2 Java slots
3688 }
3689 }
3690 assert(i == total_args_passed, "");
3691 BasicType ret_type = ss.type();
3692
3693 // Now get the compiled-Java arguments layout.
3694 SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
3695
3696 // Generate the compiled-to-native wrapper code
3697 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type);
3698
3699 if (nm != nullptr) {
3700 {
3701 MutexLocker pl(NMethodState_lock, Mutex::_no_safepoint_check_flag);
3702 if (nm->make_in_use()) {
3703 method->set_code(method, nm);
3704 }
3705 }
3706
3707 CompilerDirectiveMatcher matcher(method, CompLevel_simple);
3708 if (matcher.directive_set()->PrintAssemblyOption) {
3709 nm->print_code();
3710 }
3711 }
3712 }
3713 } // Unlock AdapterHandlerLibrary_lock
3714
3715
3716 // Install the generated code.
3717 if (nm != nullptr) {
3718 const char *msg = method->is_static() ? "(static)" : "";
3719 CompileTask::print_ul(nm, msg);
3720 if (PrintCompilation) {
3721 ttyLocker ttyl;
3722 CompileTask::print(tty, nm, msg);
3723 }
3724 nm->post_compiled_method_load_event();
3725 }
3726 }
3727
3728 // -------------------------------------------------------------------------
3729 // Java-Java calling convention
3730 // (what you use when Java calls Java)
3731
3732 //------------------------------name_for_receiver----------------------------------
3733 // For a given signature, return the VMReg for parameter 0.
3734 VMReg SharedRuntime::name_for_receiver() {
3735 VMRegPair regs;
3736 BasicType sig_bt = T_OBJECT;
3737 (void) java_calling_convention(&sig_bt, ®s, 1);
3738 // Return argument 0 register. In the LP64 build pointers
3739 // take 2 registers, but the VM wants only the 'main' name.
3740 return regs.first();
3741 }
3742
3743 VMRegPair *SharedRuntime::find_callee_arguments(Symbol* sig, bool has_receiver, bool has_appendix, int* arg_size) {
3744 // This method is returning a data structure allocating as a
3745 // ResourceObject, so do not put any ResourceMarks in here.
3746
3747 BasicType *sig_bt = NEW_RESOURCE_ARRAY(BasicType, 256);
3748 VMRegPair *regs = NEW_RESOURCE_ARRAY(VMRegPair, 256);
3749 int cnt = 0;
3750 if (has_receiver) {
3751 sig_bt[cnt++] = T_OBJECT; // Receiver is argument 0; not in signature
3752 }
3753
3754 for (SignatureStream ss(sig); !ss.at_return_type(); ss.next()) {
3755 BasicType type = ss.type();
3756 sig_bt[cnt++] = type;
3757 if (is_double_word_type(type))
3758 sig_bt[cnt++] = T_VOID;
3759 }
3760
3761 if (has_appendix) {
3762 sig_bt[cnt++] = T_OBJECT;
3763 }
3764
3765 assert(cnt < 256, "grow table size");
3766
3767 int comp_args_on_stack;
3768 comp_args_on_stack = java_calling_convention(sig_bt, regs, cnt);
3769
3770 // the calling convention doesn't count out_preserve_stack_slots so
3771 // we must add that in to get "true" stack offsets.
3772
3773 if (comp_args_on_stack) {
3774 for (int i = 0; i < cnt; i++) {
3775 VMReg reg1 = regs[i].first();
3776 if (reg1->is_stack()) {
3777 // Yuck
3778 reg1 = reg1->bias(out_preserve_stack_slots());
3779 }
3780 VMReg reg2 = regs[i].second();
3781 if (reg2->is_stack()) {
3782 // Yuck
3783 reg2 = reg2->bias(out_preserve_stack_slots());
3784 }
3785 regs[i].set_pair(reg2, reg1);
3786 }
3787 }
3788
3789 // results
3790 *arg_size = cnt;
3791 return regs;
3792 }
3793
3794 // OSR Migration Code
3795 //
3796 // This code is used convert interpreter frames into compiled frames. It is
3797 // called from very start of a compiled OSR nmethod. A temp array is
3798 // allocated to hold the interesting bits of the interpreter frame. All
3799 // active locks are inflated to allow them to move. The displaced headers and
3800 // active interpreter locals are copied into the temp buffer. Then we return
3801 // back to the compiled code. The compiled code then pops the current
3802 // interpreter frame off the stack and pushes a new compiled frame. Then it
3803 // copies the interpreter locals and displaced headers where it wants.
3804 // Finally it calls back to free the temp buffer.
3805 //
3806 // All of this is done NOT at any Safepoint, nor is any safepoint or GC allowed.
3807
3808 JRT_LEAF(intptr_t*, SharedRuntime::OSR_migration_begin( JavaThread *current) )
3809 assert(current == JavaThread::current(), "pre-condition");
3810 JFR_ONLY(Jfr::check_and_process_sample_request(current);)
3811 // During OSR migration, we unwind the interpreted frame and replace it with a compiled
3812 // frame. The stack watermark code below ensures that the interpreted frame is processed
3813 // before it gets unwound. This is helpful as the size of the compiled frame could be
3814 // larger than the interpreted frame, which could result in the new frame not being
3815 // processed correctly.
3816 StackWatermarkSet::before_unwind(current);
3817
3818 //
3819 // This code is dependent on the memory layout of the interpreter local
3820 // array and the monitors. On all of our platforms the layout is identical
3821 // so this code is shared. If some platform lays the their arrays out
3822 // differently then this code could move to platform specific code or
3823 // the code here could be modified to copy items one at a time using
3824 // frame accessor methods and be platform independent.
3825
3826 frame fr = current->last_frame();
3827 assert(fr.is_interpreted_frame(), "");
3828 assert(fr.interpreter_frame_expression_stack_size()==0, "only handle empty stacks");
3829
3830 // Figure out how many monitors are active.
3831 int active_monitor_count = 0;
3832 for (BasicObjectLock *kptr = fr.interpreter_frame_monitor_end();
3833 kptr < fr.interpreter_frame_monitor_begin();
3834 kptr = fr.next_monitor_in_interpreter_frame(kptr) ) {
3835 if (kptr->obj() != nullptr) active_monitor_count++;
3836 }
3837
3838 // QQQ we could place number of active monitors in the array so that compiled code
3839 // could double check it.
3840
3841 Method* moop = fr.interpreter_frame_method();
3842 int max_locals = moop->max_locals();
3843 // Allocate temp buffer, 1 word per local & 2 per active monitor
3844 int buf_size_words = max_locals + active_monitor_count * BasicObjectLock::size();
3845 intptr_t *buf = NEW_C_HEAP_ARRAY(intptr_t,buf_size_words, mtCode);
3846
3847 // Copy the locals. Order is preserved so that loading of longs works.
3848 // Since there's no GC I can copy the oops blindly.
3849 assert(sizeof(HeapWord)==sizeof(intptr_t), "fix this code");
3850 Copy::disjoint_words((HeapWord*)fr.interpreter_frame_local_at(max_locals-1),
3851 (HeapWord*)&buf[0],
3852 max_locals);
3853
3854 // Inflate locks. Copy the displaced headers. Be careful, there can be holes.
3855 int i = max_locals;
3856 for (BasicObjectLock *kptr2 = fr.interpreter_frame_monitor_end();
3857 kptr2 < fr.interpreter_frame_monitor_begin();
3858 kptr2 = fr.next_monitor_in_interpreter_frame(kptr2) ) {
3859 if (kptr2->obj() != nullptr) { // Avoid 'holes' in the monitor array
3860 BasicLock *lock = kptr2->lock();
3861 if (UseObjectMonitorTable) {
3862 buf[i] = (intptr_t)lock->object_monitor_cache();
3863 }
3864 #ifdef ASSERT
3865 else {
3866 buf[i] = badDispHeaderOSR;
3867 }
3868 #endif
3869 i++;
3870 buf[i++] = cast_from_oop<intptr_t>(kptr2->obj());
3871 }
3872 }
3873 assert(i - max_locals == active_monitor_count*2, "found the expected number of monitors");
3874
3875 RegisterMap map(current,
3876 RegisterMap::UpdateMap::skip,
3877 RegisterMap::ProcessFrames::include,
3878 RegisterMap::WalkContinuation::skip);
3879 frame sender = fr.sender(&map);
3880 if (sender.is_interpreted_frame()) {
3881 current->push_cont_fastpath(sender.unextended_sp());
3882 }
3883
3884 return buf;
3885 JRT_END
3886
3887 JRT_LEAF(void, SharedRuntime::OSR_migration_end( intptr_t* buf) )
3888 FREE_C_HEAP_ARRAY(buf);
3889 JRT_END
3890
3891 const char* AdapterHandlerLibrary::name(AdapterHandlerEntry* handler) {
3892 return handler->fingerprint()->as_basic_args_string();
3893 }
3894
3895 uint32_t AdapterHandlerLibrary::id(AdapterHandlerEntry* handler) {
3896 return handler->id();
3897 }
3898
3899 void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) {
3900 bool found = false;
3901 #if INCLUDE_CDS
3902 if (AOTCodeCache::is_using_adapter()) {
3903 auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) {
3904 if (b == handler->adapter_blob()) {
3905 found = true;
3906 st->print("Adapter for signature: ");
3907 handler->print_adapter_on(st);
3908 return false; // abort iteration
3909 } else {
3910 return true; // keep looking
3911 }
3912 };
3913 _aot_adapter_handler_table.iterate(findblob_archived_table);
3914 }
3915 #endif // INCLUDE_CDS
3916 if (!found) {
3917 auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* handler) {
3918 if (b == handler->adapter_blob()) {
3919 found = true;
3920 st->print("Adapter for signature: ");
3921 handler->print_adapter_on(st);
3922 return false; // abort iteration
3923 } else {
3924 return true; // keep looking
3925 }
3926 };
3927 assert_locked_or_safepoint(AdapterHandlerLibrary_lock);
3928 _adapter_handler_table->iterate(findblob_runtime_table);
3929 }
3930 assert(found, "Should have found handler");
3931 }
3932
3933 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3934 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3935 if (adapter_blob() != nullptr) {
3936 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3937 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3938 st->print(" c2iVE: " INTPTR_FORMAT, p2i(get_c2i_inline_entry()));
3939 st->print(" c2iVROE: " INTPTR_FORMAT, p2i(get_c2i_inline_ro_entry()));
3940 st->print(" c2iUE: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3941 st->print(" c2iUVE: " INTPTR_FORMAT, p2i(get_c2i_unverified_inline_entry()));
3942 if (get_c2i_no_clinit_check_entry() != nullptr) {
3943 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3944 }
3945 }
3946 st->cr();
3947 }
3948
3949 #ifndef PRODUCT
3950
3951 void AdapterHandlerLibrary::print_statistics() {
3952 print_table_statistics();
3953 }
3954
3955 #endif /* PRODUCT */
3956
3957 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3958 assert(current == JavaThread::current(), "pre-condition");
3959 StackOverflow* overflow_state = current->stack_overflow_state();
3960 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3961 overflow_state->set_reserved_stack_activation(current->stack_base());
3962 JRT_END
3963
3964 frame SharedRuntime::look_for_reserved_stack_annotated_method(JavaThread* current, frame fr) {
3965 ResourceMark rm(current);
3966 frame activation;
3967 nmethod* nm = nullptr;
3968 int count = 1;
3969
3970 assert(fr.is_java_frame(), "Must start on Java frame");
3971
3972 RegisterMap map(JavaThread::current(),
3973 RegisterMap::UpdateMap::skip,
3974 RegisterMap::ProcessFrames::skip,
3975 RegisterMap::WalkContinuation::skip); // don't walk continuations
3976 for (; !fr.is_first_frame(); fr = fr.sender(&map)) {
3977 if (!fr.is_java_frame()) {
3978 continue;
3979 }
3980
3981 Method* method = nullptr;
3982 bool found = false;
3983 if (fr.is_interpreted_frame()) {
3984 method = fr.interpreter_frame_method();
3985 if (method != nullptr && method->has_reserved_stack_access()) {
3986 found = true;
3987 }
3988 } else {
3989 CodeBlob* cb = fr.cb();
3990 if (cb != nullptr && cb->is_nmethod()) {
3991 nm = cb->as_nmethod();
3992 method = nm->method();
3993 for (ScopeDesc *sd = nm->scope_desc_near(fr.pc()); sd != nullptr; sd = sd->sender()) {
3994 method = sd->method();
3995 if (method != nullptr && method->has_reserved_stack_access()) {
3996 found = true;
3997 }
3998 }
3999 }
4000 }
4001 if (found) {
4002 activation = fr;
4003 warning("Potentially dangerous stack overflow in "
4004 "ReservedStackAccess annotated method %s [%d]",
4005 method->name_and_sig_as_C_string(), count++);
4006 EventReservedStackActivation event;
4007 if (event.should_commit()) {
4008 event.set_method(method);
4009 event.commit();
4010 }
4011 }
4012 }
4013 return activation;
4014 }
4015
4016 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
4017 // After any safepoint, just before going back to compiled code,
4018 // we inform the GC that we will be doing initializing writes to
4019 // this object in the future without emitting card-marks, so
4020 // GC may take any compensating steps.
4021
4022 oop new_obj = current->vm_result_oop();
4023 if (new_obj == nullptr) return;
4024
4025 BarrierSet *bs = BarrierSet::barrier_set();
4026 bs->on_slowpath_allocation_exit(current, new_obj);
4027 }
4028
4029 // We are at a compiled code to interpreter call. We need backing
4030 // buffers for all inline type arguments. Allocate an object array to
4031 // hold them (convenient because once we're done with it we don't have
4032 // to worry about freeing it).
4033 oop SharedRuntime::allocate_inline_types_impl(JavaThread* current, methodHandle callee, bool allocate_receiver, bool from_c1, TRAPS) {
4034 assert(InlineTypePassFieldsAsArgs, "no reason to call this");
4035 ResourceMark rm;
4036
4037 // Retrieve arguments passed at the call
4038 RegisterMap reg_map2(THREAD,
4039 RegisterMap::UpdateMap::include,
4040 RegisterMap::ProcessFrames::include,
4041 RegisterMap::WalkContinuation::skip);
4042 frame stubFrame = THREAD->last_frame();
4043 frame callerFrame = stubFrame.sender(®_map2);
4044 if (from_c1) {
4045 callerFrame = callerFrame.sender(®_map2);
4046 }
4047 int arg_size;
4048 const GrowableArray<SigEntry>* sig = allocate_receiver ? callee->adapter()->get_sig_cc() : callee->adapter()->get_sig_cc_ro();
4049 assert(sig != nullptr, "sig should never be null");
4050 TempNewSymbol tmp_sig = SigEntry::create_symbol(sig);
4051 VMRegPair* reg_pairs = find_callee_arguments(tmp_sig, false, false, &arg_size);
4052
4053 int nb_slots = 0;
4054 InstanceKlass* holder = callee->method_holder();
4055 allocate_receiver &= !callee->is_static() && holder->is_inline_klass() && callee->is_scalarized_arg(0);
4056 if (allocate_receiver) {
4057 nb_slots++;
4058 }
4059 int arg_num = callee->is_static() ? 0 : 1;
4060 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
4061 BasicType bt = ss.type();
4062 if (bt == T_OBJECT && callee->is_scalarized_arg(arg_num)) {
4063 nb_slots++;
4064 }
4065 if (bt != T_VOID) {
4066 arg_num++;
4067 }
4068 }
4069 objArrayOop array_oop = nullptr;
4070 objArrayHandle array;
4071 arg_num = callee->is_static() ? 0 : 1;
4072 int i = 0;
4073 uint pos = 0;
4074 uint depth = 0;
4075 uint ignored = 0;
4076 if (allocate_receiver) {
4077 assert(sig->at(pos)._bt == T_METADATA, "scalarized value expected");
4078 pos++;
4079 ignored++;
4080 depth++;
4081 assert(sig->at(pos)._bt == T_OBJECT, "buffer argument");
4082 uint reg_pos = 0;
4083 assert(reg_pos < (uint)arg_size, "");
4084 VMRegPair reg_pair = reg_pairs[reg_pos];
4085 oop* buffer = callerFrame.oopmapreg_to_oop_location(reg_pair.first(), ®_map2);
4086 instanceHandle h_buffer(THREAD, (instanceOop)*buffer);
4087 InlineKlass* vk = InlineKlass::cast(holder);
4088 if (h_buffer.not_null()) {
4089 assert(h_buffer->klass() == vk, "buffer not of expected class");
4090 } else {
4091 // Only allocate if buffer passed at the call is null
4092 if (array_oop == nullptr) {
4093 array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
4094 array = objArrayHandle(THREAD, array_oop);
4095 }
4096 oop res = vk->allocate_instance(CHECK_NULL);
4097 array->obj_at_put(i, res);
4098 }
4099 i++;
4100 }
4101 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
4102 BasicType bt = ss.type();
4103 if (bt == T_OBJECT && callee->is_scalarized_arg(arg_num)) {
4104 while (true) {
4105 BasicType bt = sig->at(pos)._bt;
4106 if (bt == T_METADATA) {
4107 depth++;
4108 ignored++;
4109 if (depth == 1) {
4110 break;
4111 }
4112 } else if (bt == T_VOID && sig->at(pos - 1)._bt != T_LONG && sig->at(pos - 1)._bt != T_DOUBLE) {
4113 ignored++;
4114 depth--;
4115 }
4116 pos++;
4117 }
4118 pos++;
4119 assert(sig->at(pos)._bt == T_OBJECT, "buffer argument expected");
4120 uint reg_pos = pos - ignored;
4121 assert(reg_pos < (uint)arg_size, "out of bound register?");
4122 VMRegPair reg_pair = reg_pairs[reg_pos];
4123 oop* buffer = callerFrame.oopmapreg_to_oop_location(reg_pair.first(), ®_map2);
4124 instanceHandle h_buffer(THREAD, (instanceOop)*buffer);
4125 InlineKlass* vk = ss.as_inline_klass(holder);
4126 assert(vk != nullptr, "Unexpected klass");
4127 if (h_buffer.not_null()) {
4128 assert(h_buffer->klass() == vk, "buffer not of expected class");
4129 } else {
4130 // Only allocate if buffer passed at the call is null
4131 if (array_oop == nullptr) {
4132 array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
4133 array = objArrayHandle(THREAD, array_oop);
4134 }
4135 oop res = vk->allocate_instance(CHECK_NULL);
4136 array->obj_at_put(i, res);
4137 }
4138 i++;
4139 }
4140 if (bt != T_VOID) {
4141 arg_num++;
4142 }
4143 }
4144 return array();
4145 }
4146
4147 JRT_ENTRY(void, SharedRuntime::allocate_inline_types(JavaThread* current, Method* callee_method, bool allocate_receiver))
4148 methodHandle callee(current, callee_method);
4149 oop array = SharedRuntime::allocate_inline_types_impl(current, callee, allocate_receiver, false, CHECK);
4150 current->set_vm_result_oop(array);
4151 JRT_END
4152
4153 // We've returned to an interpreted method, the interpreter needs a
4154 // reference to an inline type instance. Allocate it and initialize it
4155 // from field's values in registers.
4156 JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread* current, intptr_t res))
4157 {
4158 if (!is_set_nth_bit(res, 0)) {
4159 // We're not returning with inline type fields in registers (the
4160 // calling convention didn't allow it for this inline klass)
4161 assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area");
4162 current->set_vm_result_oop((oopDesc*)res);
4163 current->set_vm_result_metadata(nullptr);
4164 return;
4165 }
4166
4167 clear_nth_bit(res, 0);
4168 InlineKlass* vk = (InlineKlass*)res;
4169 assert(Metaspace::contains((void*)res), "should be klass");
4170
4171 if (!vk->contains_oops()) {
4172 // No oop fields. Initialize the fields by calling the pack handler from
4173 // the stub which is much faster (see 'generate_return_value_stub').
4174 // Signal this by setting the metadata result to the value klass.
4175 JRT_BLOCK;
4176 {
4177 oop vt = vk->allocate_instance(CHECK);
4178 current->set_vm_result_oop(vt);
4179 current->set_vm_result_metadata(vk);
4180 }
4181 JRT_BLOCK_END;
4182 return;
4183 }
4184
4185 ResourceMark rm;
4186 RegisterMap reg_map(current,
4187 RegisterMap::UpdateMap::include,
4188 RegisterMap::ProcessFrames::include,
4189 RegisterMap::WalkContinuation::skip);
4190 frame stubFrame = current->last_frame();
4191 stubFrame.sender(®_map);
4192
4193 assert(vk == InlineKlass::returned_inline_klass(reg_map), "broken calling convention");
4194
4195 // Allocate handles for every oop field so they are safe in case of
4196 // a safepoint when allocating
4197 GrowableArray<Handle> handles;
4198 vk->save_oop_fields(reg_map, handles);
4199
4200 // It's unsafe to safepoint until we are here
4201 JRT_BLOCK;
4202 {
4203 oop vt = vk->realloc_result(reg_map, handles, CHECK);
4204 current->set_vm_result_oop(vt);
4205 current->set_vm_result_metadata(nullptr);
4206 }
4207 JRT_BLOCK_END;
4208 }
4209 JRT_END