30 #include "classfile/javaClasses.inline.hpp"
31 #include "classfile/stringTable.hpp"
32 #include "classfile/vmClasses.hpp"
33 #include "classfile/vmSymbols.hpp"
34 #include "code/aotCodeCache.hpp"
35 #include "code/codeCache.hpp"
36 #include "code/compiledIC.hpp"
37 #include "code/nmethod.inline.hpp"
38 #include "code/scopeDesc.hpp"
39 #include "code/vtableStubs.hpp"
40 #include "compiler/abstractCompiler.hpp"
41 #include "compiler/compileBroker.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "gc/shared/barrierSet.hpp"
44 #include "gc/shared/collectedHeap.hpp"
45 #include "interpreter/interpreter.hpp"
46 #include "interpreter/interpreterRuntime.hpp"
47 #include "jfr/jfrEvents.hpp"
48 #include "jvm.h"
49 #include "logging/log.hpp"
50 #include "memory/resourceArea.hpp"
51 #include "memory/universe.hpp"
52 #include "metaprogramming/primitiveConversions.hpp"
53 #include "oops/klass.hpp"
54 #include "oops/method.inline.hpp"
55 #include "oops/objArrayKlass.hpp"
56 #include "oops/oop.inline.hpp"
57 #include "prims/forte.hpp"
58 #include "prims/jvmtiExport.hpp"
59 #include "prims/jvmtiThreadState.hpp"
60 #include "prims/methodHandles.hpp"
61 #include "prims/nativeLookup.hpp"
62 #include "runtime/arguments.hpp"
63 #include "runtime/atomicAccess.hpp"
64 #include "runtime/basicLock.inline.hpp"
65 #include "runtime/frame.inline.hpp"
66 #include "runtime/handles.inline.hpp"
67 #include "runtime/init.hpp"
68 #include "runtime/interfaceSupport.inline.hpp"
69 #include "runtime/java.hpp"
70 #include "runtime/javaCalls.hpp"
71 #include "runtime/jniHandles.inline.hpp"
72 #include "runtime/osThread.hpp"
73 #include "runtime/perfData.hpp"
74 #include "runtime/sharedRuntime.hpp"
75 #include "runtime/stackWatermarkSet.hpp"
76 #include "runtime/stubRoutines.hpp"
77 #include "runtime/synchronizer.hpp"
78 #include "runtime/timerTrace.hpp"
79 #include "runtime/vframe.inline.hpp"
80 #include "runtime/vframeArray.hpp"
81 #include "runtime/vm_version.hpp"
82 #include "utilities/copy.hpp"
83 #include "utilities/dtrace.hpp"
84 #include "utilities/events.hpp"
85 #include "utilities/exceptions.hpp"
86 #include "utilities/globalDefinitions.hpp"
87 #include "utilities/hashTable.hpp"
88 #include "utilities/macros.hpp"
89 #include "utilities/xmlstream.hpp"
90 #ifdef COMPILER1
91 #include "c1/c1_Runtime1.hpp"
92 #endif
93 #ifdef COMPILER2
94 #include "opto/runtime.hpp"
97 #include "jfr/jfr.inline.hpp"
98 #endif
99
100 // Shared runtime stub routines reside in their own unique blob with a
101 // single entry point
102
103
104 #define SHARED_STUB_FIELD_DEFINE(name, type) \
105 type* SharedRuntime::BLOB_FIELD_NAME(name);
106 SHARED_STUBS_DO(SHARED_STUB_FIELD_DEFINE)
107 #undef SHARED_STUB_FIELD_DEFINE
108
109 nmethod* SharedRuntime::_cont_doYield_stub;
110
111 //----------------------------generate_stubs-----------------------------------
112 void SharedRuntime::generate_initial_stubs() {
113 // Build this early so it's available for the interpreter.
114 _throw_StackOverflowError_blob =
115 generate_throw_exception(StubId::shared_throw_StackOverflowError_id,
116 CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
117 }
118
119 void SharedRuntime::generate_stubs() {
120 _wrong_method_blob =
121 generate_resolve_blob(StubId::shared_wrong_method_id,
122 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method));
123 _wrong_method_abstract_blob =
124 generate_resolve_blob(StubId::shared_wrong_method_abstract_id,
125 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract));
126 _ic_miss_blob =
127 generate_resolve_blob(StubId::shared_ic_miss_id,
128 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss));
129 _resolve_opt_virtual_call_blob =
130 generate_resolve_blob(StubId::shared_resolve_opt_virtual_call_id,
131 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C));
132 _resolve_virtual_call_blob =
133 generate_resolve_blob(StubId::shared_resolve_virtual_call_id,
134 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C));
135 _resolve_static_call_blob =
136 generate_resolve_blob(StubId::shared_resolve_static_call_id,
1159 // for a call current in progress, i.e., arguments has been pushed on stack
1160 // but callee has not been invoked yet. Caller frame must be compiled.
1161 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1162 CallInfo& callinfo, TRAPS) {
1163 Handle receiver;
1164 Handle nullHandle; // create a handy null handle for exception returns
1165 JavaThread* current = THREAD;
1166
1167 assert(!vfst.at_end(), "Java frame must exist");
1168
1169 // Find caller and bci from vframe
1170 methodHandle caller(current, vfst.method());
1171 int bci = vfst.bci();
1172
1173 if (caller->is_continuation_enter_intrinsic()) {
1174 bc = Bytecodes::_invokestatic;
1175 LinkResolver::resolve_continuation_enter(callinfo, CHECK_NH);
1176 return receiver;
1177 }
1178
1179 Bytecode_invoke bytecode(caller, bci);
1180 int bytecode_index = bytecode.index();
1181 bc = bytecode.invoke_code();
1182
1183 methodHandle attached_method(current, extract_attached_method(vfst));
1184 if (attached_method.not_null()) {
1185 Method* callee = bytecode.static_target(CHECK_NH);
1186 vmIntrinsics::ID id = callee->intrinsic_id();
1187 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1188 // it attaches statically resolved method to the call site.
1189 if (MethodHandles::is_signature_polymorphic(id) &&
1190 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1191 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1192
1193 // Adjust invocation mode according to the attached method.
1194 switch (bc) {
1195 case Bytecodes::_invokevirtual:
1196 if (attached_method->method_holder()->is_interface()) {
1197 bc = Bytecodes::_invokeinterface;
1198 }
1199 break;
1200 case Bytecodes::_invokeinterface:
1201 if (!attached_method->method_holder()->is_interface()) {
1202 bc = Bytecodes::_invokevirtual;
1203 }
1204 break;
1205 case Bytecodes::_invokehandle:
1206 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1207 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1208 : Bytecodes::_invokevirtual;
1209 }
1210 break;
1211 default:
1212 break;
1213 }
1214 }
1215 }
1216
1217 assert(bc != Bytecodes::_illegal, "not initialized");
1218
1219 bool has_receiver = bc != Bytecodes::_invokestatic &&
1220 bc != Bytecodes::_invokedynamic &&
1221 bc != Bytecodes::_invokehandle;
1222
1223 // Find receiver for non-static call
1224 if (has_receiver) {
1225 // This register map must be update since we need to find the receiver for
1226 // compiled frames. The receiver might be in a register.
1227 RegisterMap reg_map2(current,
1228 RegisterMap::UpdateMap::include,
1229 RegisterMap::ProcessFrames::include,
1230 RegisterMap::WalkContinuation::skip);
1231 frame stubFrame = current->last_frame();
1232 // Caller-frame is a compiled frame
1233 frame callerFrame = stubFrame.sender(®_map2);
1234
1235 if (attached_method.is_null()) {
1236 Method* callee = bytecode.static_target(CHECK_NH);
1237 if (callee == nullptr) {
1238 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1239 }
1240 }
1241
1242 // Retrieve from a compiled argument list
1243 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1244 assert(oopDesc::is_oop_or_null(receiver()), "");
1245
1246 if (receiver.is_null()) {
1247 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1248 }
1249 }
1250
1251 // Resolve method
1252 if (attached_method.not_null()) {
1253 // Parameterized by attached method.
1254 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, CHECK_NH);
1255 } else {
1256 // Parameterized by bytecode.
1257 constantPoolHandle constants(current, caller->constants());
1258 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1259 }
1260
1261 #ifdef ASSERT
1262 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1263 if (has_receiver) {
1264 assert(receiver.not_null(), "should have thrown exception");
1265 Klass* receiver_klass = receiver->klass();
1266 Klass* rk = nullptr;
1267 if (attached_method.not_null()) {
1268 // In case there's resolved method attached, use its holder during the check.
1269 rk = attached_method->method_holder();
1270 } else {
1271 // Klass is already loaded.
1272 constantPoolHandle constants(current, caller->constants());
1273 rk = constants->klass_ref_at(bytecode_index, bc, CHECK_NH);
1274 }
1275 Klass* static_receiver_klass = rk;
1276 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1277 "actual receiver must be subclass of static receiver klass");
1278 if (receiver_klass->is_instance_klass()) {
1279 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1280 tty->print_cr("ERROR: Klass not yet initialized!!");
1281 receiver_klass->print();
1282 }
1283 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1284 }
1285 }
1286 #endif
1287
1288 return receiver;
1289 }
1290
1291 methodHandle SharedRuntime::find_callee_method(TRAPS) {
1292 JavaThread* current = THREAD;
1293 ResourceMark rm(current);
1294 // We need first to check if any Java activations (compiled, interpreted)
1295 // exist on the stack since last JavaCall. If not, we need
1296 // to get the target method from the JavaCall wrapper.
1297 vframeStream vfst(current, true); // Do not skip any javaCalls
1298 methodHandle callee_method;
1299 if (vfst.at_end()) {
1300 // No Java frames were found on stack since we did the JavaCall.
1301 // Hence the stack can only contain an entry_frame. We need to
1302 // find the target method from the stub frame.
1303 RegisterMap reg_map(current,
1304 RegisterMap::UpdateMap::skip,
1305 RegisterMap::ProcessFrames::include,
1306 RegisterMap::WalkContinuation::skip);
1307 frame fr = current->last_frame();
1308 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1309 fr = fr.sender(®_map);
1310 assert(fr.is_entry_frame(), "must be");
1311 // fr is now pointing to the entry frame.
1312 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1313 } else {
1314 Bytecodes::Code bc;
1315 CallInfo callinfo;
1316 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1317 callee_method = methodHandle(current, callinfo.selected_method());
1318 }
1319 assert(callee_method()->is_method(), "must be");
1320 return callee_method;
1321 }
1322
1323 // Resolves a call.
1324 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, TRAPS) {
1325 JavaThread* current = THREAD;
1326 ResourceMark rm(current);
1327 RegisterMap cbl_map(current,
1328 RegisterMap::UpdateMap::skip,
1329 RegisterMap::ProcessFrames::include,
1330 RegisterMap::WalkContinuation::skip);
1331 frame caller_frame = current->last_frame().sender(&cbl_map);
1332
1333 CodeBlob* caller_cb = caller_frame.cb();
1334 guarantee(caller_cb != nullptr && caller_cb->is_nmethod(), "must be called from compiled method");
1335 nmethod* caller_nm = caller_cb->as_nmethod();
1336
1337 // determine call info & receiver
1338 // note: a) receiver is null for static calls
1339 // b) an exception is thrown if receiver is null for non-static calls
1340 CallInfo call_info;
1341 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1342 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1343
1344 NoSafepointVerifier nsv;
1345
1346 methodHandle callee_method(current, call_info.selected_method());
1347
1348 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1349 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1350 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1351 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1352 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1353
1354 assert(!caller_nm->is_unloading(), "It should not be unloading");
1355
1356 #ifndef PRODUCT
1357 // tracing/debugging/statistics
1358 uint *addr = (is_optimized) ? (&_resolve_opt_virtual_ctr) :
1359 (is_virtual) ? (&_resolve_virtual_ctr) :
1360 (&_resolve_static_ctr);
1361 AtomicAccess::inc(addr);
1362
1363 if (TraceCallFixup) {
1364 ResourceMark rm(current);
1365 tty->print("resolving %s%s (%s) call to",
1366 (is_optimized) ? "optimized " : "", (is_virtual) ? "virtual" : "static",
1367 Bytecodes::name(invoke_code));
1368 callee_method->print_short_name(tty);
1369 tty->print_cr(" at pc: " INTPTR_FORMAT " to code: " INTPTR_FORMAT,
1370 p2i(caller_frame.pc()), p2i(callee_method->code()));
1371 }
1372 #endif
1373
1374 if (invoke_code == Bytecodes::_invokestatic) {
1375 assert(callee_method->method_holder()->is_initialized() ||
1376 callee_method->method_holder()->is_reentrant_initialization(current),
1377 "invalid class initialization state for invoke_static");
1378 if (!VM_Version::supports_fast_class_init_checks() && callee_method->needs_clinit_barrier()) {
1379 // In order to keep class initialization check, do not patch call
1380 // site for static call when the class is not fully initialized.
1381 // Proper check is enforced by call site re-resolution on every invocation.
1382 //
1383 // When fast class initialization checks are supported (VM_Version::supports_fast_class_init_checks() == true),
1384 // explicit class initialization check is put in nmethod entry (VEP).
1385 assert(callee_method->method_holder()->is_linked(), "must be");
1386 return callee_method;
1387 }
1388 }
1389
1390
1391 // JSR 292 key invariant:
1392 // If the resolved method is a MethodHandle invoke target, the call
1393 // site must be a MethodHandle call site, because the lambda form might tail-call
1394 // leaving the stack in a state unknown to either caller or callee
1395
1396 // Compute entry points. The computation of the entry points is independent of
1397 // patching the call.
1398
1399 // Make sure the callee nmethod does not get deoptimized and removed before
1400 // we are done patching the code.
1401
1402
1403 CompiledICLocker ml(caller_nm);
1404 if (is_virtual && !is_optimized) {
1405 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1406 inline_cache->update(&call_info, receiver->klass());
1407 } else {
1408 // Callsite is a direct call - set it to the destination method
1409 CompiledDirectCall* callsite = CompiledDirectCall::before(caller_frame.pc());
1410 callsite->set(callee_method);
1411 }
1412
1413 return callee_method;
1414 }
1415
1416 // Inline caches exist only in compiled code
1417 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1418 #ifdef ASSERT
1419 RegisterMap reg_map(current,
1420 RegisterMap::UpdateMap::skip,
1421 RegisterMap::ProcessFrames::include,
1422 RegisterMap::WalkContinuation::skip);
1423 frame stub_frame = current->last_frame();
1424 assert(stub_frame.is_runtime_frame(), "sanity check");
1425 frame caller_frame = stub_frame.sender(®_map);
1426 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_upcall_stub_frame(), "unexpected frame");
1427 #endif /* ASSERT */
1428
1429 methodHandle callee_method;
1430 JRT_BLOCK
1431 callee_method = SharedRuntime::handle_ic_miss_helper(CHECK_NULL);
1432 // Return Method* through TLS
1433 current->set_vm_result_metadata(callee_method());
1434 JRT_BLOCK_END
1435 // return compiled code entry point after potential safepoints
1436 return get_resolved_entry(current, callee_method);
1437 JRT_END
1438
1439
1440 // Handle call site that has been made non-entrant
1441 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1442 // 6243940 We might end up in here if the callee is deoptimized
1443 // as we race to call it. We don't want to take a safepoint if
1444 // the caller was interpreted because the caller frame will look
1445 // interpreted to the stack walkers and arguments are now
1446 // "compiled" so it is much better to make this transition
1447 // invisible to the stack walking code. The i2c path will
1448 // place the callee method in the callee_target. It is stashed
1449 // there because if we try and find the callee by normal means a
1450 // safepoint is possible and have trouble gc'ing the compiled args.
1451 RegisterMap reg_map(current,
1452 RegisterMap::UpdateMap::skip,
1453 RegisterMap::ProcessFrames::include,
1454 RegisterMap::WalkContinuation::skip);
1455 frame stub_frame = current->last_frame();
1456 assert(stub_frame.is_runtime_frame(), "sanity check");
1457 frame caller_frame = stub_frame.sender(®_map);
1458
1459 if (caller_frame.is_interpreted_frame() ||
1460 caller_frame.is_entry_frame() ||
1461 caller_frame.is_upcall_stub_frame()) {
1462 Method* callee = current->callee_target();
1463 guarantee(callee != nullptr && callee->is_method(), "bad handshake");
1464 current->set_vm_result_metadata(callee);
1465 current->set_callee_target(nullptr);
1466 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1467 // Bypass class initialization checks in c2i when caller is in native.
1468 // JNI calls to static methods don't have class initialization checks.
1469 // Fast class initialization checks are present in c2i adapters and call into
1470 // SharedRuntime::handle_wrong_method() on the slow path.
1471 //
1472 // JVM upcalls may land here as well, but there's a proper check present in
1473 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1474 // so bypassing it in c2i adapter is benign.
1475 return callee->get_c2i_no_clinit_check_entry();
1476 } else {
1477 return callee->get_c2i_entry();
1478 }
1479 }
1480
1481 // Must be compiled to compiled path which is safe to stackwalk
1482 methodHandle callee_method;
1483 JRT_BLOCK
1484 // Force resolving of caller (if we called from compiled frame)
1485 callee_method = SharedRuntime::reresolve_call_site(CHECK_NULL);
1486 current->set_vm_result_metadata(callee_method());
1487 JRT_BLOCK_END
1488 // return compiled code entry point after potential safepoints
1489 return get_resolved_entry(current, callee_method);
1490 JRT_END
1491
1492 // Handle abstract method call
1493 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1494 // Verbose error message for AbstractMethodError.
1495 // Get the called method from the invoke bytecode.
1496 vframeStream vfst(current, true);
1497 assert(!vfst.at_end(), "Java frame must exist");
1498 methodHandle caller(current, vfst.method());
1499 Bytecode_invoke invoke(caller, vfst.bci());
1500 DEBUG_ONLY( invoke.verify(); )
1501
1502 // Find the compiled caller frame.
1503 RegisterMap reg_map(current,
1504 RegisterMap::UpdateMap::include,
1505 RegisterMap::ProcessFrames::include,
1506 RegisterMap::WalkContinuation::skip);
1507 frame stubFrame = current->last_frame();
1508 assert(stubFrame.is_runtime_frame(), "must be");
1509 frame callerFrame = stubFrame.sender(®_map);
1510 assert(callerFrame.is_compiled_frame(), "must be");
1511
1512 // Install exception and return forward entry.
1513 address res = SharedRuntime::throw_AbstractMethodError_entry();
1514 JRT_BLOCK
1515 methodHandle callee(current, invoke.static_target(current));
1516 if (!callee.is_null()) {
1517 oop recv = callerFrame.retrieve_receiver(®_map);
1518 Klass *recv_klass = (recv != nullptr) ? recv->klass() : nullptr;
1519 res = StubRoutines::forward_exception_entry();
1520 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1521 }
1522 JRT_BLOCK_END
1523 return res;
1524 JRT_END
1525
1526 // return verified_code_entry if interp_only_mode is not set for the current thread;
1527 // otherwise return c2i entry.
1528 address SharedRuntime::get_resolved_entry(JavaThread* current, methodHandle callee_method) {
1529 if (current->is_interp_only_mode() && !callee_method->is_special_native_intrinsic()) {
1530 // In interp_only_mode we need to go to the interpreted entry
1531 // The c2i won't patch in this mode -- see fixup_callers_callsite
1532 return callee_method->get_c2i_entry();
1533 }
1534 assert(callee_method->verified_code_entry() != nullptr, " Jump to zero!");
1535 return callee_method->verified_code_entry();
1536 }
1537
1538 // resolve a static call and patch code
1539 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1540 methodHandle callee_method;
1541 bool enter_special = false;
1542 JRT_BLOCK
1543 callee_method = SharedRuntime::resolve_helper(false, false, CHECK_NULL);
1544 current->set_vm_result_metadata(callee_method());
1545 JRT_BLOCK_END
1546 // return compiled code entry point after potential safepoints
1547 return get_resolved_entry(current, callee_method);
1548 JRT_END
1549
1550 // resolve virtual call and update inline cache to monomorphic
1551 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1552 methodHandle callee_method;
1553 JRT_BLOCK
1554 callee_method = SharedRuntime::resolve_helper(true, false, CHECK_NULL);
1555 current->set_vm_result_metadata(callee_method());
1556 JRT_BLOCK_END
1557 // return compiled code entry point after potential safepoints
1558 return get_resolved_entry(current, callee_method);
1559 JRT_END
1560
1561
1562 // Resolve a virtual call that can be statically bound (e.g., always
1563 // monomorphic, so it has no inline cache). Patch code to resolved target.
1564 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1565 methodHandle callee_method;
1566 JRT_BLOCK
1567 callee_method = SharedRuntime::resolve_helper(true, true, CHECK_NULL);
1568 current->set_vm_result_metadata(callee_method());
1569 JRT_BLOCK_END
1570 // return compiled code entry point after potential safepoints
1571 return get_resolved_entry(current, callee_method);
1572 JRT_END
1573
1574 methodHandle SharedRuntime::handle_ic_miss_helper(TRAPS) {
1575 JavaThread* current = THREAD;
1576 ResourceMark rm(current);
1577 CallInfo call_info;
1578 Bytecodes::Code bc;
1579
1580 // receiver is null for static calls. An exception is thrown for null
1581 // receivers for non-static calls
1582 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1583
1584 methodHandle callee_method(current, call_info.selected_method());
1585
1586 #ifndef PRODUCT
1587 AtomicAccess::inc(&_ic_miss_ctr);
1588
1589 // Statistics & Tracing
1590 if (TraceCallFixup) {
1591 ResourceMark rm(current);
1592 tty->print("IC miss (%s) call to", Bytecodes::name(bc));
1593 callee_method->print_short_name(tty);
1594 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1595 }
1596
1597 if (ICMissHistogram) {
1598 MutexLocker m(VMStatistic_lock);
1599 RegisterMap reg_map(current,
1600 RegisterMap::UpdateMap::skip,
1601 RegisterMap::ProcessFrames::include,
1602 RegisterMap::WalkContinuation::skip);
1603 frame f = current->last_frame().real_sender(®_map);// skip runtime stub
1604 // produce statistics under the lock
1605 trace_ic_miss(f.pc());
1606 }
1607 #endif
1608
1609 // install an event collector so that when a vtable stub is created the
1610 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1611 // event can't be posted when the stub is created as locks are held
1612 // - instead the event will be deferred until the event collector goes
1613 // out of scope.
1614 JvmtiDynamicCodeEventCollector event_collector;
1615
1616 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1617 RegisterMap reg_map(current,
1618 RegisterMap::UpdateMap::skip,
1619 RegisterMap::ProcessFrames::include,
1620 RegisterMap::WalkContinuation::skip);
1621 frame caller_frame = current->last_frame().sender(®_map);
1622 CodeBlob* cb = caller_frame.cb();
1623 nmethod* caller_nm = cb->as_nmethod();
1624
1625 CompiledICLocker ml(caller_nm);
1626 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1627 inline_cache->update(&call_info, receiver()->klass());
1628
1629 return callee_method;
1630 }
1631
1632 //
1633 // Resets a call-site in compiled code so it will get resolved again.
1634 // This routines handles both virtual call sites, optimized virtual call
1635 // sites, and static call sites. Typically used to change a call sites
1636 // destination from compiled to interpreted.
1637 //
1638 methodHandle SharedRuntime::reresolve_call_site(TRAPS) {
1639 JavaThread* current = THREAD;
1640 ResourceMark rm(current);
1641 RegisterMap reg_map(current,
1642 RegisterMap::UpdateMap::skip,
1643 RegisterMap::ProcessFrames::include,
1644 RegisterMap::WalkContinuation::skip);
1645 frame stub_frame = current->last_frame();
1646 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1647 frame caller = stub_frame.sender(®_map);
1648
1649 // Do nothing if the frame isn't a live compiled frame.
1650 // nmethod could be deoptimized by the time we get here
1651 // so no update to the caller is needed.
1652
1653 if ((caller.is_compiled_frame() && !caller.is_deoptimized_frame()) ||
1654 (caller.is_native_frame() && caller.cb()->as_nmethod()->method()->is_continuation_enter_intrinsic())) {
1655
1656 address pc = caller.pc();
1657
1658 nmethod* caller_nm = CodeCache::find_nmethod(pc);
1659 assert(caller_nm != nullptr, "did not find caller nmethod");
1660
1661 // Default call_addr is the location of the "basic" call.
1662 // Determine the address of the call we a reresolving. With
1663 // Inline Caches we will always find a recognizable call.
1664 // With Inline Caches disabled we may or may not find a
1665 // recognizable call. We will always find a call for static
1666 // calls and for optimized virtual calls. For vanilla virtual
1667 // calls it depends on the state of the UseInlineCaches switch.
1668 //
1669 // With Inline Caches disabled we can get here for a virtual call
1670 // for two reasons:
1671 // 1 - calling an abstract method. The vtable for abstract methods
1672 // will run us thru handle_wrong_method and we will eventually
1673 // end up in the interpreter to throw the ame.
1674 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1675 // call and between the time we fetch the entry address and
1676 // we jump to it the target gets deoptimized. Similar to 1
1677 // we will wind up in the interprter (thru a c2i with c2).
1678 //
1679 CompiledICLocker ml(caller_nm);
1680 address call_addr = caller_nm->call_instruction_address(pc);
1681
1682 if (call_addr != nullptr) {
1683 // On x86 the logic for finding a call instruction is blindly checking for a call opcode 5
1684 // bytes back in the instruction stream so we must also check for reloc info.
1685 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1686 bool ret = iter.next(); // Get item
1687 if (ret) {
1688 switch (iter.type()) {
1689 case relocInfo::static_call_type:
1690 case relocInfo::opt_virtual_call_type: {
1691 CompiledDirectCall* cdc = CompiledDirectCall::at(call_addr);
1692 cdc->set_to_clean();
1693 break;
1694 }
1695
1696 case relocInfo::virtual_call_type: {
1697 // compiled, dispatched call (which used to call an interpreted method)
1698 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1699 inline_cache->set_to_clean();
1700 break;
1701 }
1702 default:
1703 break;
1704 }
1705 }
1706 }
1707 }
1708
1709 methodHandle callee_method = find_callee_method(CHECK_(methodHandle()));
1710
1711
1712 #ifndef PRODUCT
1713 AtomicAccess::inc(&_wrong_method_ctr);
1714
1715 if (TraceCallFixup) {
1716 ResourceMark rm(current);
1717 tty->print("handle_wrong_method reresolving call to");
1718 callee_method->print_short_name(tty);
1719 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1720 }
1721 #endif
1722
1723 return callee_method;
1724 }
1725
1726 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1727 // The faulting unsafe accesses should be changed to throw the error
1728 // synchronously instead. Meanwhile the faulting instruction will be
1729 // skipped over (effectively turning it into a no-op) and an
1730 // asynchronous exception will be raised which the thread will
1731 // handle at a later point. If the instruction is a load it will
1732 // return garbage.
1733
1734 // Request an async exception.
1735 thread->set_pending_unsafe_access_error();
1736
1737 // Return address of next instruction to execute.
1903 msglen += strlen(caster_klass_description) + strlen(target_klass_description) + strlen(klass_separator) + 3;
1904
1905 char* message = NEW_RESOURCE_ARRAY_RETURN_NULL(char, msglen);
1906 if (message == nullptr) {
1907 // Shouldn't happen, but don't cause even more problems if it does
1908 message = const_cast<char*>(caster_klass->external_name());
1909 } else {
1910 jio_snprintf(message,
1911 msglen,
1912 "class %s cannot be cast to class %s (%s%s%s)",
1913 caster_name,
1914 target_name,
1915 caster_klass_description,
1916 klass_separator,
1917 target_klass_description
1918 );
1919 }
1920 return message;
1921 }
1922
1923 JRT_LEAF(void, SharedRuntime::reguard_yellow_pages())
1924 (void) JavaThread::current()->stack_overflow_state()->reguard_stack();
1925 JRT_END
1926
1927 void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
1928 if (!SafepointSynchronize::is_synchronizing()) {
1929 // Only try quick_enter() if we're not trying to reach a safepoint
1930 // so that the calling thread reaches the safepoint more quickly.
1931 if (ObjectSynchronizer::quick_enter(obj, lock, current)) {
1932 return;
1933 }
1934 }
1935 // NO_ASYNC required because an async exception on the state transition destructor
1936 // would leave you with the lock held and it would never be released.
1937 // The normal monitorenter NullPointerException is thrown without acquiring a lock
1938 // and the model is that an exception implies the method failed.
1939 JRT_BLOCK_NO_ASYNC
1940 Handle h_obj(THREAD, obj);
1941 ObjectSynchronizer::enter(h_obj, lock, current);
1942 assert(!HAS_PENDING_EXCEPTION, "Should have no exception here");
2136 tty->print_cr("Note 1: counter updates are not MT-safe.");
2137 tty->print_cr("Note 2: %% in major categories are relative to total non-inlined calls;");
2138 tty->print_cr(" %% in nested categories are relative to their category");
2139 tty->print_cr(" (and thus add up to more than 100%% with inlining)");
2140 tty->cr();
2141
2142 MethodArityHistogram h;
2143 }
2144 #endif
2145
2146 #ifndef PRODUCT
2147 static int _lookups; // number of calls to lookup
2148 static int _equals; // number of buckets checked with matching hash
2149 static int _archived_hits; // number of successful lookups in archived table
2150 static int _runtime_hits; // number of successful lookups in runtime table
2151 #endif
2152
2153 // A simple wrapper class around the calling convention information
2154 // that allows sharing of adapters for the same calling convention.
2155 class AdapterFingerPrint : public MetaspaceObj {
2156 private:
2157 enum {
2158 _basic_type_bits = 4,
2159 _basic_type_mask = right_n_bits(_basic_type_bits),
2160 _basic_types_per_int = BitsPerInt / _basic_type_bits,
2161 };
2162 // TO DO: Consider integrating this with a more global scheme for compressing signatures.
2163 // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive.
2164
2165 int _length;
2166
2167 static int data_offset() { return sizeof(AdapterFingerPrint); }
2168 int* data_pointer() {
2169 return (int*)((address)this + data_offset());
2170 }
2171
2172 // Private construtor. Use allocate() to get an instance.
2173 AdapterFingerPrint(int total_args_passed, BasicType* sig_bt, int len) {
2174 int* data = data_pointer();
2175 // Pack the BasicTypes with 8 per int
2176 assert(len == length(total_args_passed), "sanity");
2177 _length = len;
2178 int sig_index = 0;
2179 for (int index = 0; index < _length; index++) {
2180 int value = 0;
2181 for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) {
2182 int bt = adapter_encoding(sig_bt[sig_index++]);
2183 assert((bt & _basic_type_mask) == bt, "must fit in 4 bits");
2184 value = (value << _basic_type_bits) | bt;
2185 }
2186 data[index] = value;
2187 }
2188 }
2189
2190 // Call deallocate instead
2191 ~AdapterFingerPrint() {
2192 ShouldNotCallThis();
2193 }
2194
2195 static int length(int total_args) {
2196 return (total_args + (_basic_types_per_int-1)) / _basic_types_per_int;
2197 }
2198
2199 static int compute_size_in_words(int len) {
2200 return (int)heap_word_size(sizeof(AdapterFingerPrint) + (len * sizeof(int)));
2201 }
2202
2203 // Remap BasicTypes that are handled equivalently by the adapters.
2204 // These are correct for the current system but someday it might be
2205 // necessary to make this mapping platform dependent.
2206 static int adapter_encoding(BasicType in) {
2207 switch (in) {
2208 case T_BOOLEAN:
2209 case T_BYTE:
2210 case T_SHORT:
2211 case T_CHAR:
2212 // There are all promoted to T_INT in the calling convention
2213 return T_INT;
2214
2215 case T_OBJECT:
2216 case T_ARRAY:
2217 // In other words, we assume that any register good enough for
2218 // an int or long is good enough for a managed pointer.
2219 #ifdef _LP64
2220 return T_LONG;
2221 #else
2222 return T_INT;
2223 #endif
2224
2225 case T_INT:
2226 case T_LONG:
2227 case T_FLOAT:
2228 case T_DOUBLE:
2229 case T_VOID:
2230 return in;
2231
2232 default:
2233 ShouldNotReachHere();
2234 return T_CONFLICT;
2235 }
2236 }
2237
2238 void* operator new(size_t size, size_t fp_size) throw() {
2239 assert(fp_size >= size, "sanity check");
2240 void* p = AllocateHeap(fp_size, mtCode);
2241 memset(p, 0, fp_size);
2242 return p;
2243 }
2244
2245 template<typename Function>
2246 void iterate_args(Function function) {
2247 for (int i = 0; i < length(); i++) {
2248 unsigned val = (unsigned)value(i);
2249 // args are packed so that first/lower arguments are in the highest
2250 // bits of each int value, so iterate from highest to the lowest
2251 for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) {
2252 unsigned v = (val >> j) & _basic_type_mask;
2253 if (v == 0) {
2254 continue;
2255 }
2256 function(v);
2257 }
2258 }
2259 }
2260
2261 public:
2262 static AdapterFingerPrint* allocate(int total_args_passed, BasicType* sig_bt) {
2263 int len = length(total_args_passed);
2264 int size_in_bytes = BytesPerWord * compute_size_in_words(len);
2265 AdapterFingerPrint* afp = new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt, len);
2266 assert((afp->size() * BytesPerWord) == size_in_bytes, "should match");
2267 return afp;
2268 }
2269
2270 static void deallocate(AdapterFingerPrint* fp) {
2271 FreeHeap(fp);
2272 }
2273
2274 int value(int index) {
2275 int* data = data_pointer();
2276 return data[index];
2277 }
2278
2279 int length() {
2280 return _length;
2281 }
2282
2283 unsigned int compute_hash() {
2284 int hash = 0;
2285 for (int i = 0; i < length(); i++) {
2286 int v = value(i);
2287 //Add arithmetic operation to the hash, like +3 to improve hashing
2288 hash = ((hash << 8) ^ v ^ (hash >> 5)) + 3;
2289 }
2290 return (unsigned int)hash;
2291 }
2292
2293 const char* as_string() {
2294 stringStream st;
2295 st.print("0x");
2296 for (int i = 0; i < length(); i++) {
2297 st.print("%x", value(i));
2298 }
2299 return st.as_string();
2300 }
2301
2302 const char* as_basic_args_string() {
2303 stringStream st;
2304 bool long_prev = false;
2305 iterate_args([&] (int arg) {
2306 if (long_prev) {
2307 long_prev = false;
2308 if (arg == T_VOID) {
2309 st.print("J");
2310 } else {
2311 st.print("L");
2312 }
2313 }
2314 switch (arg) {
2315 case T_INT: st.print("I"); break;
2316 case T_LONG: long_prev = true; break;
2317 case T_FLOAT: st.print("F"); break;
2318 case T_DOUBLE: st.print("D"); break;
2319 case T_VOID: break;
2320 default: ShouldNotReachHere();
2321 }
2322 });
2323 if (long_prev) {
2324 st.print("L");
2325 }
2326 return st.as_string();
2327 }
2328
2329 BasicType* as_basic_type(int& nargs) {
2330 nargs = 0;
2331 GrowableArray<BasicType> btarray;
2332 bool long_prev = false;
2333
2334 iterate_args([&] (int arg) {
2335 if (long_prev) {
2336 long_prev = false;
2337 if (arg == T_VOID) {
2338 btarray.append(T_LONG);
2339 } else {
2340 btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter
2341 }
2342 }
2343 switch (arg) {
2344 case T_INT: // fallthrough
2345 case T_FLOAT: // fallthrough
2346 case T_DOUBLE:
2347 case T_VOID:
2348 btarray.append((BasicType)arg);
2349 break;
2350 case T_LONG:
2351 long_prev = true;
2352 break;
2353 default: ShouldNotReachHere();
2354 }
2355 });
2356
2357 if (long_prev) {
2358 btarray.append(T_OBJECT);
2359 }
2360
2361 nargs = btarray.length();
2362 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nargs);
2363 int index = 0;
2364 GrowableArrayIterator<BasicType> iter = btarray.begin();
2365 while (iter != btarray.end()) {
2366 sig_bt[index++] = *iter;
2367 ++iter;
2368 }
2369 assert(index == btarray.length(), "sanity check");
2370 #ifdef ASSERT
2371 {
2372 AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(nargs, sig_bt);
2373 assert(this->equals(compare_fp), "sanity check");
2374 AdapterFingerPrint::deallocate(compare_fp);
2375 }
2376 #endif
2377 return sig_bt;
2378 }
2379
2380 bool equals(AdapterFingerPrint* other) {
2381 if (other->_length != _length) {
2382 return false;
2383 } else {
2384 for (int i = 0; i < _length; i++) {
2385 if (value(i) != other->value(i)) {
2386 return false;
2387 }
2388 }
2389 }
2390 return true;
2391 }
2392
2393 // methods required by virtue of being a MetaspaceObj
2394 void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ }
2395 int size() const { return compute_size_in_words(_length); }
2396 MetaspaceObj::Type type() const { return AdapterFingerPrintType; }
2397
2398 static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) {
2399 NOT_PRODUCT(_equals++);
2400 return fp1->equals(fp2);
2401 }
2402
2403 static unsigned int compute_hash(AdapterFingerPrint* const& fp) {
2404 return fp->compute_hash();
2405 }
2408 #if INCLUDE_CDS
2409 static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) {
2410 return AdapterFingerPrint::equals(entry->fingerprint(), fp);
2411 }
2412
2413 class ArchivedAdapterTable : public OffsetCompactHashtable<
2414 AdapterFingerPrint*,
2415 AdapterHandlerEntry*,
2416 adapter_fp_equals_compact_hashtable_entry> {};
2417 #endif // INCLUDE_CDS
2418
2419 // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries
2420 using AdapterHandlerTable = HashTable<AdapterFingerPrint*, AdapterHandlerEntry*, 293,
2421 AnyObj::C_HEAP, mtCode,
2422 AdapterFingerPrint::compute_hash,
2423 AdapterFingerPrint::equals>;
2424 static AdapterHandlerTable* _adapter_handler_table;
2425 static GrowableArray<AdapterHandlerEntry*>* _adapter_handler_list = nullptr;
2426
2427 // Find a entry with the same fingerprint if it exists
2428 AdapterHandlerEntry* AdapterHandlerLibrary::lookup(int total_args_passed, BasicType* sig_bt) {
2429 NOT_PRODUCT(_lookups++);
2430 assert_lock_strong(AdapterHandlerLibrary_lock);
2431 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt);
2432 AdapterHandlerEntry* entry = nullptr;
2433 #if INCLUDE_CDS
2434 // if we are building the archive then the archived adapter table is
2435 // not valid and we need to use the ones added to the runtime table
2436 if (AOTCodeCache::is_using_adapter()) {
2437 // Search archived table first. It is read-only table so can be searched without lock
2438 entry = _aot_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */);
2439 #ifndef PRODUCT
2440 if (entry != nullptr) {
2441 _archived_hits++;
2442 }
2443 #endif
2444 }
2445 #endif // INCLUDE_CDS
2446 if (entry == nullptr) {
2447 assert_lock_strong(AdapterHandlerLibrary_lock);
2448 AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp);
2449 if (entry_p != nullptr) {
2450 entry = *entry_p;
2451 assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)",
2468 TableStatistics ts = _adapter_handler_table->statistics_calculate(size);
2469 ts.print(tty, "AdapterHandlerTable");
2470 tty->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)",
2471 _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries());
2472 int total_hits = _archived_hits + _runtime_hits;
2473 tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d)",
2474 _lookups, _equals, total_hits, _archived_hits, _runtime_hits);
2475 }
2476 #endif
2477
2478 // ---------------------------------------------------------------------------
2479 // Implementation of AdapterHandlerLibrary
2480 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = nullptr;
2481 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr;
2482 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr;
2483 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr;
2484 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr;
2485 #if INCLUDE_CDS
2486 ArchivedAdapterTable AdapterHandlerLibrary::_aot_adapter_handler_table;
2487 #endif // INCLUDE_CDS
2488 static const int AdapterHandlerLibrary_size = 16*K;
2489 BufferBlob* AdapterHandlerLibrary::_buffer = nullptr;
2490 volatile uint AdapterHandlerLibrary::_id_counter = 0;
2491
2492 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2493 assert(_buffer != nullptr, "should be initialized");
2494 return _buffer;
2495 }
2496
2497 static void post_adapter_creation(const AdapterHandlerEntry* entry) {
2498 if (Forte::is_enabled() || JvmtiExport::should_post_dynamic_code_generated()) {
2499 AdapterBlob* adapter_blob = entry->adapter_blob();
2500 char blob_id[256];
2501 jio_snprintf(blob_id,
2502 sizeof(blob_id),
2503 "%s(%s)",
2504 adapter_blob->name(),
2505 entry->fingerprint()->as_string());
2506 if (Forte::is_enabled()) {
2507 Forte::register_stub(blob_id, adapter_blob->content_begin(), adapter_blob->content_end());
2508 }
2516 void AdapterHandlerLibrary::initialize() {
2517 {
2518 ResourceMark rm;
2519 _adapter_handler_table = new (mtCode) AdapterHandlerTable();
2520 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2521 }
2522
2523 #if INCLUDE_CDS
2524 // Link adapters in AOT Cache to their code in AOT Code Cache
2525 if (AOTCodeCache::is_using_adapter() && !_aot_adapter_handler_table.empty()) {
2526 link_aot_adapters();
2527 lookup_simple_adapters();
2528 return;
2529 }
2530 #endif // INCLUDE_CDS
2531
2532 ResourceMark rm;
2533 {
2534 MutexLocker mu(AdapterHandlerLibrary_lock);
2535
2536 _no_arg_handler = create_adapter(0, nullptr);
2537
2538 BasicType obj_args[] = { T_OBJECT };
2539 _obj_arg_handler = create_adapter(1, obj_args);
2540
2541 BasicType int_args[] = { T_INT };
2542 _int_arg_handler = create_adapter(1, int_args);
2543
2544 BasicType obj_int_args[] = { T_OBJECT, T_INT };
2545 _obj_int_arg_handler = create_adapter(2, obj_int_args);
2546
2547 BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT };
2548 _obj_obj_arg_handler = create_adapter(2, obj_obj_args);
2549
2550 // we should always get an entry back but we don't have any
2551 // associated blob on Zero
2552 assert(_no_arg_handler != nullptr &&
2553 _obj_arg_handler != nullptr &&
2554 _int_arg_handler != nullptr &&
2555 _obj_int_arg_handler != nullptr &&
2556 _obj_obj_arg_handler != nullptr, "Initial adapter handlers must be properly created");
2557 }
2558
2559 // Outside of the lock
2560 #ifndef ZERO
2561 // no blobs to register when we are on Zero
2562 post_adapter_creation(_no_arg_handler);
2563 post_adapter_creation(_obj_arg_handler);
2564 post_adapter_creation(_int_arg_handler);
2565 post_adapter_creation(_obj_int_arg_handler);
2566 post_adapter_creation(_obj_obj_arg_handler);
2567 #endif // ZERO
2568 }
2569
2570 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint) {
2571 uint id = (uint)AtomicAccess::add((int*)&_id_counter, 1);
2572 assert(id > 0, "we can never overflow because AOT cache cannot contain more than 2^32 methods");
2573 return AdapterHandlerEntry::allocate(id, fingerprint);
2574 }
2575
2576 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2577 int total_args_passed = method->size_of_parameters(); // All args on stack
2578 if (total_args_passed == 0) {
2579 return _no_arg_handler;
2580 } else if (total_args_passed == 1) {
2581 if (!method->is_static()) {
2582 return _obj_arg_handler;
2583 }
2584 switch (method->signature()->char_at(1)) {
2585 case JVM_SIGNATURE_CLASS:
2586 case JVM_SIGNATURE_ARRAY:
2587 return _obj_arg_handler;
2588 case JVM_SIGNATURE_INT:
2589 case JVM_SIGNATURE_BOOLEAN:
2590 case JVM_SIGNATURE_CHAR:
2591 case JVM_SIGNATURE_BYTE:
2592 case JVM_SIGNATURE_SHORT:
2593 return _int_arg_handler;
2594 }
2595 } else if (total_args_passed == 2 &&
2596 !method->is_static()) {
2597 switch (method->signature()->char_at(1)) {
2598 case JVM_SIGNATURE_CLASS:
2599 case JVM_SIGNATURE_ARRAY:
2600 return _obj_obj_arg_handler;
2601 case JVM_SIGNATURE_INT:
2602 case JVM_SIGNATURE_BOOLEAN:
2603 case JVM_SIGNATURE_CHAR:
2604 case JVM_SIGNATURE_BYTE:
2605 case JVM_SIGNATURE_SHORT:
2606 return _obj_int_arg_handler;
2607 }
2608 }
2609 return nullptr;
2610 }
2611
2612 class AdapterSignatureIterator : public SignatureIterator {
2613 private:
2614 BasicType stack_sig_bt[16];
2615 BasicType* sig_bt;
2616 int index;
2617
2618 public:
2619 AdapterSignatureIterator(Symbol* signature,
2620 fingerprint_t fingerprint,
2621 bool is_static,
2622 int total_args_passed) :
2623 SignatureIterator(signature, fingerprint),
2624 index(0)
2625 {
2626 sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
2627 if (!is_static) { // Pass in receiver first
2628 sig_bt[index++] = T_OBJECT;
2629 }
2630 do_parameters_on(this);
2631 }
2632
2633 BasicType* basic_types() {
2634 return sig_bt;
2635 }
2636
2637 #ifdef ASSERT
2638 int slots() {
2639 return index;
2640 }
2641 #endif
2642
2643 private:
2644
2645 friend class SignatureIterator; // so do_parameters_on can call do_type
2646 void do_type(BasicType type) {
2647 sig_bt[index++] = type;
2648 if (type == T_LONG || type == T_DOUBLE) {
2649 sig_bt[index++] = T_VOID; // Longs & doubles take 2 Java slots
2650 }
2651 }
2652 };
2653
2654
2655 const char* AdapterHandlerEntry::_entry_names[] = {
2656 "i2c", "c2i", "c2i_unverified", "c2i_no_clinit_check"
2657 };
2658
2659 #ifdef ASSERT
2660 void AdapterHandlerLibrary::verify_adapter_sharing(int total_args_passed, BasicType* sig_bt, AdapterHandlerEntry* cached_entry) {
2661 // we can only check for the same code if there is any
2662 #ifndef ZERO
2663 AdapterHandlerEntry* comparison_entry = create_adapter(total_args_passed, sig_bt, true);
2664 assert(comparison_entry->adapter_blob() == nullptr, "no blob should be created when creating an adapter for comparison");
2665 assert(comparison_entry->compare_code(cached_entry), "code must match");
2666 // Release the one just created
2667 AdapterHandlerEntry::deallocate(comparison_entry);
2668 # endif // ZERO
2669 }
2670 #endif /* ASSERT*/
2671
2672 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
2673 assert(!method->is_abstract(), "abstract methods do not have adapters");
2674 // Use customized signature handler. Need to lock around updates to
2675 // the _adapter_handler_table (it is not safe for concurrent readers
2676 // and a single writer: this could be fixed if it becomes a
2677 // problem).
2678
2679 // Fast-path for trivial adapters
2680 AdapterHandlerEntry* entry = get_simple_adapter(method);
2681 if (entry != nullptr) {
2682 return entry;
2683 }
2684
2685 ResourceMark rm;
2686 bool new_entry = false;
2687
2688 // Fill in the signature array, for the calling-convention call.
2689 int total_args_passed = method->size_of_parameters(); // All args on stack
2690
2691 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
2692 method->is_static(), total_args_passed);
2693 assert(si.slots() == total_args_passed, "");
2694 BasicType* sig_bt = si.basic_types();
2695 {
2696 MutexLocker mu(AdapterHandlerLibrary_lock);
2697
2698 // Lookup method signature's fingerprint
2699 entry = lookup(total_args_passed, sig_bt);
2700
2701 if (entry != nullptr) {
2702 #ifndef ZERO
2703 assert(entry->is_linked(), "AdapterHandlerEntry must have been linked");
2704 #endif
2705 #ifdef ASSERT
2706 if (!entry->in_aot_cache() && VerifyAdapterSharing) {
2707 verify_adapter_sharing(total_args_passed, sig_bt, entry);
2708 }
2709 #endif
2710 } else {
2711 entry = create_adapter(total_args_passed, sig_bt);
2712 if (entry != nullptr) {
2713 new_entry = true;
2714 }
2715 }
2716 }
2717
2718 // Outside of the lock
2719 if (new_entry) {
2720 post_adapter_creation(entry);
2721 }
2722 return entry;
2723 }
2724
2725 void AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler) {
2726 ResourceMark rm;
2727 const char* name = AdapterHandlerLibrary::name(handler);
2728 const uint32_t id = AdapterHandlerLibrary::id(handler);
2729
2730 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::Adapter, id, name);
2731 if (blob != nullptr) {
2746 }
2747 insts_size = adapter_blob->code_size();
2748 st->print_cr("i2c argument handler for: %s %s (%d bytes generated)",
2749 handler->fingerprint()->as_basic_args_string(),
2750 handler->fingerprint()->as_string(), insts_size);
2751 st->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry()));
2752 if (Verbose || PrintStubCode) {
2753 address first_pc = adapter_blob->content_begin();
2754 if (first_pc != nullptr) {
2755 Disassembler::decode(first_pc, first_pc + insts_size, st, &adapter_blob->asm_remarks());
2756 st->cr();
2757 }
2758 }
2759 }
2760 #endif // PRODUCT
2761
2762 void AdapterHandlerLibrary::address_to_offset(address entry_address[AdapterBlob::ENTRY_COUNT],
2763 int entry_offset[AdapterBlob::ENTRY_COUNT]) {
2764 entry_offset[AdapterBlob::I2C] = 0;
2765 entry_offset[AdapterBlob::C2I] = entry_address[AdapterBlob::C2I] - entry_address[AdapterBlob::I2C];
2766 entry_offset[AdapterBlob::C2I_Unverified] = entry_address[AdapterBlob::C2I_Unverified] - entry_address[AdapterBlob::I2C];
2767 if (entry_address[AdapterBlob::C2I_No_Clinit_Check] == nullptr) {
2768 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = -1;
2769 } else {
2770 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = entry_address[AdapterBlob::C2I_No_Clinit_Check] - entry_address[AdapterBlob::I2C];
2771 }
2772 }
2773
2774 bool AdapterHandlerLibrary::generate_adapter_code(AdapterHandlerEntry* handler,
2775 int total_args_passed,
2776 BasicType* sig_bt,
2777 bool is_transient) {
2778 if (log_is_enabled(Info, perf, class, link)) {
2779 ClassLoader::perf_method_adapters_count()->inc();
2780 }
2781
2782 #ifndef ZERO
2783 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
2784 CodeBuffer buffer(buf);
2785 short buffer_locs[20];
2786 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
2787 sizeof(buffer_locs)/sizeof(relocInfo));
2788 MacroAssembler masm(&buffer);
2789 VMRegPair stack_regs[16];
2790 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
2791
2792 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
2793 int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
2794 address entry_address[AdapterBlob::ENTRY_COUNT];
2795 SharedRuntime::generate_i2c2i_adapters(&masm,
2796 total_args_passed,
2797 comp_args_on_stack,
2798 sig_bt,
2799 regs,
2800 entry_address);
2801 // On zero there is no code to save and no need to create a blob and
2802 // or relocate the handler.
2803 int entry_offset[AdapterBlob::ENTRY_COUNT];
2804 address_to_offset(entry_address, entry_offset);
2805 #ifdef ASSERT
2806 if (VerifyAdapterSharing) {
2807 handler->save_code(buf->code_begin(), buffer.insts_size());
2808 if (is_transient) {
2809 return true;
2810 }
2811 }
2812 #endif
2813 AdapterBlob* adapter_blob = AdapterBlob::create(&buffer, entry_offset);
2814 if (adapter_blob == nullptr) {
2815 // CodeCache is full, disable compilation
2816 // Ought to log this but compile log is only per compile thread
2817 // and we're some non descript Java thread.
2818 return false;
2819 }
2820 handler->set_adapter_blob(adapter_blob);
2821 if (!is_transient && AOTCodeCache::is_dumping_adapter()) {
2822 // try to save generated code
2823 const char* name = AdapterHandlerLibrary::name(handler);
2824 const uint32_t id = AdapterHandlerLibrary::id(handler);
2825 bool success = AOTCodeCache::store_code_blob(*adapter_blob, AOTCodeEntry::Adapter, id, name);
2826 assert(success || !AOTCodeCache::is_dumping_adapter(), "caching of adapter must be disabled");
2827 }
2828 #endif // ZERO
2829
2830 #ifndef PRODUCT
2831 // debugging support
2832 if (PrintAdapterHandlers || PrintStubCode) {
2833 print_adapter_handler_info(tty, handler);
2834 }
2835 #endif
2836
2837 return true;
2838 }
2839
2840 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(int total_args_passed,
2841 BasicType* sig_bt,
2842 bool is_transient) {
2843 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt);
2844 AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fp);
2845 if (!generate_adapter_code(handler, total_args_passed, sig_bt, is_transient)) {
2846 AdapterHandlerEntry::deallocate(handler);
2847 return nullptr;
2848 }
2849 if (!is_transient) {
2850 assert_lock_strong(AdapterHandlerLibrary_lock);
2851 _adapter_handler_table->put(fp, handler);
2852 }
2853 return handler;
2854 }
2855
2856 #if INCLUDE_CDS
2857 void AdapterHandlerEntry::remove_unshareable_info() {
2858 #ifdef ASSERT
2859 _saved_code = nullptr;
2860 _saved_code_length = 0;
2861 #endif // ASSERT
2862 _adapter_blob = nullptr;
2863 _linked = false;
2864 }
2865
2866 class CopyAdapterTableToArchive : StackObj {
2867 private:
2868 CompactHashtableWriter* _writer;
2869 ArchiveBuilder* _builder;
2870 public:
2871 CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer),
2872 _builder(ArchiveBuilder::current())
2873 {}
2874
2875 bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) {
2876 LogStreamHandle(Trace, aot) lsh;
2877 if (ArchiveBuilder::current()->has_been_archived((address)entry)) {
2878 assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be");
2879 AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp);
2880 assert(buffered_fp != nullptr,"sanity check");
2881 AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry);
2882 assert(buffered_entry != nullptr,"sanity check");
2883
2923 }
2924 #endif
2925 }
2926
2927 // This method is used during production run to link archived adapters (stored in AOT Cache)
2928 // to their code in AOT Code Cache
2929 void AdapterHandlerEntry::link() {
2930 ResourceMark rm;
2931 assert(_fingerprint != nullptr, "_fingerprint must not be null");
2932 bool generate_code = false;
2933 // Generate code only if AOTCodeCache is not available, or
2934 // caching adapters is disabled, or we fail to link
2935 // the AdapterHandlerEntry to its code in the AOTCodeCache
2936 if (AOTCodeCache::is_using_adapter()) {
2937 AdapterHandlerLibrary::link_aot_adapter_handler(this);
2938 // If link_aot_adapter_handler() succeeds, _adapter_blob will be non-null
2939 if (_adapter_blob == nullptr) {
2940 log_warning(aot)("Failed to link AdapterHandlerEntry (fp=%s) to its code in the AOT code cache", _fingerprint->as_basic_args_string());
2941 generate_code = true;
2942 }
2943 } else {
2944 generate_code = true;
2945 }
2946 if (generate_code) {
2947 int nargs;
2948 BasicType* bt = _fingerprint->as_basic_type(nargs);
2949 if (!AdapterHandlerLibrary::generate_adapter_code(this, nargs, bt, /* is_transient */ false)) {
2950 // Don't throw exceptions during VM initialization because java.lang.* classes
2951 // might not have been initialized, causing problems when constructing the
2952 // Java exception object.
2953 vm_exit_during_initialization("Out of space in CodeCache for adapters");
2954 }
2955 }
2956 if (_adapter_blob != nullptr) {
2957 post_adapter_creation(this);
2958 }
2959 assert(_linked, "AdapterHandlerEntry must now be linked");
2960 }
2961
2962 void AdapterHandlerLibrary::link_aot_adapters() {
2963 uint max_id = 0;
2964 assert(AOTCodeCache::is_using_adapter(), "AOT adapters code should be available");
2965 /* It is possible that some adapters generated in assembly phase are not stored in the cache.
2966 * That implies adapter ids of the adapters in the cache may not be contiguous.
2967 * If the size of the _aot_adapter_handler_table is used to initialize _id_counter, then it may
2968 * result in collision of adapter ids between AOT stored handlers and runtime generated handlers.
2969 * To avoid such situation, initialize the _id_counter with the largest adapter id among the AOT stored handlers.
2970 */
2971 _aot_adapter_handler_table.iterate_all([&](AdapterHandlerEntry* entry) {
2972 assert(!entry->is_linked(), "AdapterHandlerEntry is already linked!");
2973 entry->link();
2974 max_id = MAX2(max_id, entry->id());
2975 });
2976 // Set adapter id to the maximum id found in the AOTCache
2977 assert(_id_counter == 0, "Did not expect new AdapterHandlerEntry to be created at this stage");
2978 _id_counter = max_id;
2979 }
2980
2981 // This method is called during production run to lookup simple adapters
2982 // in the archived adapter handler table
2983 void AdapterHandlerLibrary::lookup_simple_adapters() {
2984 assert(!_aot_adapter_handler_table.empty(), "archived adapter handler table is empty");
2985
2986 MutexLocker mu(AdapterHandlerLibrary_lock);
2987 _no_arg_handler = lookup(0, nullptr);
2988
2989 BasicType obj_args[] = { T_OBJECT };
2990 _obj_arg_handler = lookup(1, obj_args);
2991
2992 BasicType int_args[] = { T_INT };
2993 _int_arg_handler = lookup(1, int_args);
2994
2995 BasicType obj_int_args[] = { T_OBJECT, T_INT };
2996 _obj_int_arg_handler = lookup(2, obj_int_args);
2997
2998 BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT };
2999 _obj_obj_arg_handler = lookup(2, obj_obj_args);
3000
3001 assert(_no_arg_handler != nullptr &&
3002 _obj_arg_handler != nullptr &&
3003 _int_arg_handler != nullptr &&
3004 _obj_int_arg_handler != nullptr &&
3005 _obj_obj_arg_handler != nullptr, "Initial adapters not found in archived adapter handler table");
3006 assert(_no_arg_handler->is_linked() &&
3007 _obj_arg_handler->is_linked() &&
3008 _int_arg_handler->is_linked() &&
3009 _obj_int_arg_handler->is_linked() &&
3010 _obj_obj_arg_handler->is_linked(), "Initial adapters not in linked state");
3011 }
3012 #endif // INCLUDE_CDS
3013
3014 void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) {
3015 LogStreamHandle(Trace, aot) lsh;
3016 if (lsh.is_enabled()) {
3017 lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string());
3018 lsh.cr();
3019 }
3020 it->push(&_fingerprint);
3021 }
3022
3023 AdapterHandlerEntry::~AdapterHandlerEntry() {
3024 if (_fingerprint != nullptr) {
3025 AdapterFingerPrint::deallocate(_fingerprint);
3026 _fingerprint = nullptr;
3027 }
3028 #ifdef ASSERT
3029 FREE_C_HEAP_ARRAY(_saved_code);
3030 #endif
3031 FreeHeap(this);
3032 }
3033
3034
3035 #ifdef ASSERT
3036 // Capture the code before relocation so that it can be compared
3037 // against other versions. If the code is captured after relocation
3038 // then relative instructions won't be equivalent.
3039 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
3040 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
3041 _saved_code_length = length;
3042 memcpy(_saved_code, buffer, length);
3043 }
3044
3045
3046 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
3047 assert(_saved_code != nullptr && other->_saved_code != nullptr, "code not saved");
3097 struct { double data[20]; } stubs_locs_buf;
3098 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
3099 #if defined(AARCH64)
3100 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
3101 // in the constant pool to ensure ordering between the barrier and oops
3102 // accesses. For native_wrappers we need a constant.
3103 buffer.initialize_consts_size(8);
3104 #elif defined(PPC64) || defined(S390)
3105 // On PPC64/S390 the continuation enter intrinsic needs the constant pool for the compiled
3106 // static java call that is resolved in the runtime.
3107 if (method->is_continuation_enter_intrinsic()) {
3108 buffer.initialize_consts_size(8 PPC64_ONLY(+ 24) S390_ONLY(+ 17));
3109 }
3110 #endif
3111 buffer.stubs()->initialize_shared_locs((relocInfo*)&stubs_locs_buf, sizeof(stubs_locs_buf) / sizeof(relocInfo));
3112 MacroAssembler _masm(&buffer);
3113
3114 // Fill in the signature array, for the calling-convention call.
3115 const int total_args_passed = method->size_of_parameters();
3116
3117 VMRegPair stack_regs[16];
3118 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
3119
3120 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
3121 method->is_static(), total_args_passed);
3122 BasicType* sig_bt = si.basic_types();
3123 assert(si.slots() == total_args_passed, "");
3124 BasicType ret_type = si.return_type();
3125
3126 // Now get the compiled-Java arguments layout.
3127 SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
3128
3129 // Generate the compiled-to-native wrapper code
3130 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type);
3131
3132 if (nm != nullptr) {
3133 {
3134 MutexLocker pl(NMethodState_lock, Mutex::_no_safepoint_check_flag);
3135 if (nm->make_in_use()) {
3136 method->set_code(method, nm);
3137 }
3138 }
3139
3140 CompilerDirectiveMatcher matcher(method, CompLevel_simple);
3141 if (matcher.directive_set()->PrintAssemblyOption) {
3142 nm->print_code();
3143 }
3144 }
3351 if (b == handler->adapter_blob()) {
3352 found = true;
3353 st->print("Adapter for signature: ");
3354 handler->print_adapter_on(st);
3355 return false; // abort iteration
3356 } else {
3357 return true; // keep looking
3358 }
3359 };
3360 assert_locked_or_safepoint(AdapterHandlerLibrary_lock);
3361 _adapter_handler_table->iterate(findblob_runtime_table);
3362 }
3363 assert(found, "Should have found handler");
3364 }
3365
3366 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3367 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3368 if (adapter_blob() != nullptr) {
3369 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3370 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3371 st->print(" c2iUV: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3372 if (get_c2i_no_clinit_check_entry() != nullptr) {
3373 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3374 }
3375 }
3376 st->cr();
3377 }
3378
3379 #ifndef PRODUCT
3380
3381 void AdapterHandlerLibrary::print_statistics() {
3382 print_table_statistics();
3383 }
3384
3385 #endif /* PRODUCT */
3386
3387 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3388 assert(current == JavaThread::current(), "pre-condition");
3389 StackOverflow* overflow_state = current->stack_overflow_state();
3390 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3391 overflow_state->set_reserved_stack_activation(current->stack_base());
3438 event.set_method(method);
3439 event.commit();
3440 }
3441 }
3442 }
3443 return activation;
3444 }
3445
3446 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
3447 // After any safepoint, just before going back to compiled code,
3448 // we inform the GC that we will be doing initializing writes to
3449 // this object in the future without emitting card-marks, so
3450 // GC may take any compensating steps.
3451
3452 oop new_obj = current->vm_result_oop();
3453 if (new_obj == nullptr) return;
3454
3455 BarrierSet *bs = BarrierSet::barrier_set();
3456 bs->on_slowpath_allocation_exit(current, new_obj);
3457 }
|
30 #include "classfile/javaClasses.inline.hpp"
31 #include "classfile/stringTable.hpp"
32 #include "classfile/vmClasses.hpp"
33 #include "classfile/vmSymbols.hpp"
34 #include "code/aotCodeCache.hpp"
35 #include "code/codeCache.hpp"
36 #include "code/compiledIC.hpp"
37 #include "code/nmethod.inline.hpp"
38 #include "code/scopeDesc.hpp"
39 #include "code/vtableStubs.hpp"
40 #include "compiler/abstractCompiler.hpp"
41 #include "compiler/compileBroker.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "gc/shared/barrierSet.hpp"
44 #include "gc/shared/collectedHeap.hpp"
45 #include "interpreter/interpreter.hpp"
46 #include "interpreter/interpreterRuntime.hpp"
47 #include "jfr/jfrEvents.hpp"
48 #include "jvm.h"
49 #include "logging/log.hpp"
50 #include "memory/oopFactory.hpp"
51 #include "memory/resourceArea.hpp"
52 #include "memory/universe.hpp"
53 #include "metaprogramming/primitiveConversions.hpp"
54 #include "oops/access.hpp"
55 #include "oops/fieldStreams.inline.hpp"
56 #include "oops/inlineKlass.inline.hpp"
57 #include "oops/klass.hpp"
58 #include "oops/method.inline.hpp"
59 #include "oops/objArrayKlass.hpp"
60 #include "oops/objArrayOop.inline.hpp"
61 #include "oops/oop.inline.hpp"
62 #include "prims/forte.hpp"
63 #include "prims/jvmtiExport.hpp"
64 #include "prims/jvmtiThreadState.hpp"
65 #include "prims/methodHandles.hpp"
66 #include "prims/nativeLookup.hpp"
67 #include "runtime/arguments.hpp"
68 #include "runtime/atomicAccess.hpp"
69 #include "runtime/basicLock.inline.hpp"
70 #include "runtime/frame.inline.hpp"
71 #include "runtime/handles.inline.hpp"
72 #include "runtime/init.hpp"
73 #include "runtime/interfaceSupport.inline.hpp"
74 #include "runtime/java.hpp"
75 #include "runtime/javaCalls.hpp"
76 #include "runtime/jniHandles.inline.hpp"
77 #include "runtime/osThread.hpp"
78 #include "runtime/perfData.hpp"
79 #include "runtime/sharedRuntime.hpp"
80 #include "runtime/signature.hpp"
81 #include "runtime/stackWatermarkSet.hpp"
82 #include "runtime/stubRoutines.hpp"
83 #include "runtime/synchronizer.hpp"
84 #include "runtime/timerTrace.hpp"
85 #include "runtime/vframe.inline.hpp"
86 #include "runtime/vframeArray.hpp"
87 #include "runtime/vm_version.hpp"
88 #include "utilities/copy.hpp"
89 #include "utilities/dtrace.hpp"
90 #include "utilities/events.hpp"
91 #include "utilities/exceptions.hpp"
92 #include "utilities/globalDefinitions.hpp"
93 #include "utilities/hashTable.hpp"
94 #include "utilities/macros.hpp"
95 #include "utilities/xmlstream.hpp"
96 #ifdef COMPILER1
97 #include "c1/c1_Runtime1.hpp"
98 #endif
99 #ifdef COMPILER2
100 #include "opto/runtime.hpp"
103 #include "jfr/jfr.inline.hpp"
104 #endif
105
106 // Shared runtime stub routines reside in their own unique blob with a
107 // single entry point
108
109
110 #define SHARED_STUB_FIELD_DEFINE(name, type) \
111 type* SharedRuntime::BLOB_FIELD_NAME(name);
112 SHARED_STUBS_DO(SHARED_STUB_FIELD_DEFINE)
113 #undef SHARED_STUB_FIELD_DEFINE
114
115 nmethod* SharedRuntime::_cont_doYield_stub;
116
117 //----------------------------generate_stubs-----------------------------------
118 void SharedRuntime::generate_initial_stubs() {
119 // Build this early so it's available for the interpreter.
120 _throw_StackOverflowError_blob =
121 generate_throw_exception(StubId::shared_throw_StackOverflowError_id,
122 CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
123
124 if (InlineTypeReturnedAsFields) {
125 _store_inline_type_fields_to_buf_blob =
126 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf));
127 }
128 }
129
130 void SharedRuntime::generate_stubs() {
131 _wrong_method_blob =
132 generate_resolve_blob(StubId::shared_wrong_method_id,
133 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method));
134 _wrong_method_abstract_blob =
135 generate_resolve_blob(StubId::shared_wrong_method_abstract_id,
136 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract));
137 _ic_miss_blob =
138 generate_resolve_blob(StubId::shared_ic_miss_id,
139 CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss));
140 _resolve_opt_virtual_call_blob =
141 generate_resolve_blob(StubId::shared_resolve_opt_virtual_call_id,
142 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C));
143 _resolve_virtual_call_blob =
144 generate_resolve_blob(StubId::shared_resolve_virtual_call_id,
145 CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C));
146 _resolve_static_call_blob =
147 generate_resolve_blob(StubId::shared_resolve_static_call_id,
1170 // for a call current in progress, i.e., arguments has been pushed on stack
1171 // but callee has not been invoked yet. Caller frame must be compiled.
1172 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1173 CallInfo& callinfo, TRAPS) {
1174 Handle receiver;
1175 Handle nullHandle; // create a handy null handle for exception returns
1176 JavaThread* current = THREAD;
1177
1178 assert(!vfst.at_end(), "Java frame must exist");
1179
1180 // Find caller and bci from vframe
1181 methodHandle caller(current, vfst.method());
1182 int bci = vfst.bci();
1183
1184 if (caller->is_continuation_enter_intrinsic()) {
1185 bc = Bytecodes::_invokestatic;
1186 LinkResolver::resolve_continuation_enter(callinfo, CHECK_NH);
1187 return receiver;
1188 }
1189
1190 // Substitutability test implementation piggy backs on static call resolution
1191 Bytecodes::Code code = caller->java_code_at(bci);
1192 if (code == Bytecodes::_if_acmpeq || code == Bytecodes::_if_acmpne) {
1193 bc = Bytecodes::_invokestatic;
1194 methodHandle attached_method(THREAD, extract_attached_method(vfst));
1195 assert(attached_method.not_null(), "must have attached method");
1196 vmClasses::ValueObjectMethods_klass()->initialize(CHECK_NH);
1197 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, false, CHECK_NH);
1198 #ifdef ASSERT
1199 Symbol* subst_method_name = vmSymbols::isSubstitutable_name();
1200 Method* is_subst = vmClasses::ValueObjectMethods_klass()->find_method(subst_method_name, vmSymbols::object_object_boolean_signature());
1201 assert(callinfo.selected_method() == is_subst, "must be isSubstitutable method");
1202 #endif
1203 return receiver;
1204 }
1205
1206 Bytecode_invoke bytecode(caller, bci);
1207 int bytecode_index = bytecode.index();
1208 bc = bytecode.invoke_code();
1209
1210 methodHandle attached_method(current, extract_attached_method(vfst));
1211 if (attached_method.not_null()) {
1212 Method* callee = bytecode.static_target(CHECK_NH);
1213 vmIntrinsics::ID id = callee->intrinsic_id();
1214 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1215 // it attaches statically resolved method to the call site.
1216 if (MethodHandles::is_signature_polymorphic(id) &&
1217 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1218 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1219
1220 // Adjust invocation mode according to the attached method.
1221 switch (bc) {
1222 case Bytecodes::_invokevirtual:
1223 if (attached_method->method_holder()->is_interface()) {
1224 bc = Bytecodes::_invokeinterface;
1225 }
1226 break;
1227 case Bytecodes::_invokeinterface:
1228 if (!attached_method->method_holder()->is_interface()) {
1229 bc = Bytecodes::_invokevirtual;
1230 }
1231 break;
1232 case Bytecodes::_invokehandle:
1233 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1234 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1235 : Bytecodes::_invokevirtual;
1236 }
1237 break;
1238 default:
1239 break;
1240 }
1241 } else {
1242 assert(attached_method->has_scalarized_args(), "invalid use of attached method");
1243 if (!attached_method->method_holder()->is_inline_klass() || attached_method->is_static()) {
1244 // Ignore the attached method in this case to not confuse below code
1245 attached_method = methodHandle(current, nullptr);
1246 }
1247 }
1248 }
1249
1250 assert(bc != Bytecodes::_illegal, "not initialized");
1251
1252 bool has_receiver = bc != Bytecodes::_invokestatic &&
1253 bc != Bytecodes::_invokedynamic &&
1254 bc != Bytecodes::_invokehandle;
1255 bool check_null_and_abstract = true;
1256
1257 // Find receiver for non-static call
1258 if (has_receiver) {
1259 // This register map must be update since we need to find the receiver for
1260 // compiled frames. The receiver might be in a register.
1261 RegisterMap reg_map2(current,
1262 RegisterMap::UpdateMap::include,
1263 RegisterMap::ProcessFrames::include,
1264 RegisterMap::WalkContinuation::skip);
1265 frame stubFrame = current->last_frame();
1266 // Caller-frame is a compiled frame
1267 frame callerFrame = stubFrame.sender(®_map2);
1268
1269 Method* callee = attached_method();
1270 if (callee == nullptr) {
1271 callee = bytecode.static_target(CHECK_NH);
1272 if (callee == nullptr) {
1273 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1274 }
1275 }
1276 bool caller_is_c1 = callerFrame.is_compiled_frame() && callerFrame.cb()->as_nmethod()->is_compiled_by_c1();
1277 if (!caller_is_c1 && callee->is_scalarized_arg(0)) {
1278 // If the receiver is an inline type that is passed as fields, no oop is available
1279 // Resolve the call without receiver null checking.
1280 assert(!callee->mismatch(), "calls with inline type receivers should never mismatch");
1281 assert(attached_method.not_null() && !attached_method->is_abstract(), "must have non-abstract attached method");
1282 if (bc == Bytecodes::_invokeinterface) {
1283 bc = Bytecodes::_invokevirtual; // C2 optimistically replaces interface calls by virtual calls
1284 }
1285 check_null_and_abstract = false;
1286 } else {
1287 // Retrieve from a compiled argument list
1288 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1289 assert(oopDesc::is_oop_or_null(receiver()), "");
1290 if (receiver.is_null()) {
1291 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1292 }
1293 }
1294 }
1295
1296 // Resolve method
1297 if (attached_method.not_null()) {
1298 // Parameterized by attached method.
1299 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, check_null_and_abstract, CHECK_NH);
1300 } else {
1301 // Parameterized by bytecode.
1302 constantPoolHandle constants(current, caller->constants());
1303 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1304 }
1305
1306 #ifdef ASSERT
1307 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1308 if (has_receiver && check_null_and_abstract) {
1309 assert(receiver.not_null(), "should have thrown exception");
1310 Klass* receiver_klass = receiver->klass();
1311 Klass* rk = nullptr;
1312 if (attached_method.not_null()) {
1313 // In case there's resolved method attached, use its holder during the check.
1314 rk = attached_method->method_holder();
1315 } else {
1316 // Klass is already loaded.
1317 constantPoolHandle constants(current, caller->constants());
1318 rk = constants->klass_ref_at(bytecode_index, bc, CHECK_NH);
1319 }
1320 Klass* static_receiver_klass = rk;
1321 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1322 "actual receiver must be subclass of static receiver klass");
1323 if (receiver_klass->is_instance_klass()) {
1324 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1325 tty->print_cr("ERROR: Klass not yet initialized!!");
1326 receiver_klass->print();
1327 }
1328 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1329 }
1330 }
1331 #endif
1332
1333 return receiver;
1334 }
1335
1336 methodHandle SharedRuntime::find_callee_method(bool& caller_does_not_scalarize, TRAPS) {
1337 JavaThread* current = THREAD;
1338 ResourceMark rm(current);
1339 // We need first to check if any Java activations (compiled, interpreted)
1340 // exist on the stack since last JavaCall. If not, we need
1341 // to get the target method from the JavaCall wrapper.
1342 vframeStream vfst(current, true); // Do not skip any javaCalls
1343 methodHandle callee_method;
1344 if (vfst.at_end()) {
1345 // No Java frames were found on stack since we did the JavaCall.
1346 // Hence the stack can only contain an entry_frame. We need to
1347 // find the target method from the stub frame.
1348 RegisterMap reg_map(current,
1349 RegisterMap::UpdateMap::skip,
1350 RegisterMap::ProcessFrames::include,
1351 RegisterMap::WalkContinuation::skip);
1352 frame fr = current->last_frame();
1353 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1354 fr = fr.sender(®_map);
1355 assert(fr.is_entry_frame(), "must be");
1356 // fr is now pointing to the entry frame.
1357 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1358 } else {
1359 Bytecodes::Code bc;
1360 CallInfo callinfo;
1361 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1362 // Calls via mismatching methods are always non-scalarized
1363 if (callinfo.resolved_method()->mismatch()) {
1364 caller_does_not_scalarize = true;
1365 }
1366 callee_method = methodHandle(current, callinfo.selected_method());
1367 }
1368 assert(callee_method()->is_method(), "must be");
1369 return callee_method;
1370 }
1371
1372 // Resolves a call.
1373 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, bool& caller_does_not_scalarize, TRAPS) {
1374 JavaThread* current = THREAD;
1375 ResourceMark rm(current);
1376 RegisterMap cbl_map(current,
1377 RegisterMap::UpdateMap::skip,
1378 RegisterMap::ProcessFrames::include,
1379 RegisterMap::WalkContinuation::skip);
1380 frame caller_frame = current->last_frame().sender(&cbl_map);
1381
1382 CodeBlob* caller_cb = caller_frame.cb();
1383 guarantee(caller_cb != nullptr && caller_cb->is_nmethod(), "must be called from compiled method");
1384 nmethod* caller_nm = caller_cb->as_nmethod();
1385
1386 // determine call info & receiver
1387 // note: a) receiver is null for static calls
1388 // b) an exception is thrown if receiver is null for non-static calls
1389 CallInfo call_info;
1390 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1391 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1392
1393 NoSafepointVerifier nsv;
1394
1395 methodHandle callee_method(current, call_info.selected_method());
1396 // Calls via mismatching methods are always non-scalarized
1397 bool mismatch = is_optimized ? call_info.selected_method()->mismatch() : call_info.resolved_method()->mismatch();
1398 if (caller_nm->is_compiled_by_c1() || mismatch) {
1399 caller_does_not_scalarize = true;
1400 }
1401
1402 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1403 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1404 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1405 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1406 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1407
1408 assert(!caller_nm->is_unloading(), "It should not be unloading");
1409
1410 #ifndef PRODUCT
1411 // tracing/debugging/statistics
1412 uint *addr = (is_optimized) ? (&_resolve_opt_virtual_ctr) :
1413 (is_virtual) ? (&_resolve_virtual_ctr) :
1414 (&_resolve_static_ctr);
1415 AtomicAccess::inc(addr);
1416
1417 if (TraceCallFixup) {
1418 ResourceMark rm(current);
1419 tty->print("resolving %s%s (%s) %s call to",
1420 (is_optimized) ? "optimized " : "", (is_virtual) ? "virtual" : "static",
1421 Bytecodes::name(invoke_code), (caller_does_not_scalarize) ? "non-scalar" : "");
1422 callee_method->print_short_name(tty);
1423 tty->print_cr(" at pc: " INTPTR_FORMAT " to code: " INTPTR_FORMAT,
1424 p2i(caller_frame.pc()), p2i(callee_method->code()));
1425 }
1426 #endif
1427
1428 if (invoke_code == Bytecodes::_invokestatic) {
1429 assert(callee_method->method_holder()->is_initialized() ||
1430 callee_method->method_holder()->is_reentrant_initialization(current),
1431 "invalid class initialization state for invoke_static");
1432 if (!VM_Version::supports_fast_class_init_checks() && callee_method->needs_clinit_barrier()) {
1433 // In order to keep class initialization check, do not patch call
1434 // site for static call when the class is not fully initialized.
1435 // Proper check is enforced by call site re-resolution on every invocation.
1436 //
1437 // When fast class initialization checks are supported (VM_Version::supports_fast_class_init_checks() == true),
1438 // explicit class initialization check is put in nmethod entry (VEP).
1439 assert(callee_method->method_holder()->is_linked(), "must be");
1440 return callee_method;
1441 }
1442 }
1443
1444
1445 // JSR 292 key invariant:
1446 // If the resolved method is a MethodHandle invoke target, the call
1447 // site must be a MethodHandle call site, because the lambda form might tail-call
1448 // leaving the stack in a state unknown to either caller or callee
1449
1450 // Compute entry points. The computation of the entry points is independent of
1451 // patching the call.
1452
1453 // Make sure the callee nmethod does not get deoptimized and removed before
1454 // we are done patching the code.
1455
1456
1457 CompiledICLocker ml(caller_nm);
1458 if (is_virtual && !is_optimized) {
1459 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1460 inline_cache->update(&call_info, receiver->klass(), caller_does_not_scalarize);
1461 } else {
1462 // Callsite is a direct call - set it to the destination method
1463 CompiledDirectCall* callsite = CompiledDirectCall::before(caller_frame.pc());
1464 callsite->set(callee_method, caller_does_not_scalarize);
1465 }
1466
1467 return callee_method;
1468 }
1469
1470 // Inline caches exist only in compiled code
1471 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1472 #ifdef ASSERT
1473 RegisterMap reg_map(current,
1474 RegisterMap::UpdateMap::skip,
1475 RegisterMap::ProcessFrames::include,
1476 RegisterMap::WalkContinuation::skip);
1477 frame stub_frame = current->last_frame();
1478 assert(stub_frame.is_runtime_frame(), "sanity check");
1479 frame caller_frame = stub_frame.sender(®_map);
1480 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_upcall_stub_frame(), "unexpected frame");
1481 #endif /* ASSERT */
1482
1483 methodHandle callee_method;
1484 bool caller_does_not_scalarize = false;
1485 JRT_BLOCK
1486 callee_method = SharedRuntime::handle_ic_miss_helper(caller_does_not_scalarize, CHECK_NULL);
1487 // Return Method* through TLS
1488 current->set_vm_result_metadata(callee_method());
1489 JRT_BLOCK_END
1490 // return compiled code entry point after potential safepoints
1491 return get_resolved_entry(current, callee_method, false, false, caller_does_not_scalarize);
1492 JRT_END
1493
1494
1495 // Handle call site that has been made non-entrant
1496 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1497 // 6243940 We might end up in here if the callee is deoptimized
1498 // as we race to call it. We don't want to take a safepoint if
1499 // the caller was interpreted because the caller frame will look
1500 // interpreted to the stack walkers and arguments are now
1501 // "compiled" so it is much better to make this transition
1502 // invisible to the stack walking code. The i2c path will
1503 // place the callee method in the callee_target. It is stashed
1504 // there because if we try and find the callee by normal means a
1505 // safepoint is possible and have trouble gc'ing the compiled args.
1506 RegisterMap reg_map(current,
1507 RegisterMap::UpdateMap::skip,
1508 RegisterMap::ProcessFrames::include,
1509 RegisterMap::WalkContinuation::skip);
1510 frame stub_frame = current->last_frame();
1511 assert(stub_frame.is_runtime_frame(), "sanity check");
1512 frame caller_frame = stub_frame.sender(®_map);
1513
1514 if (caller_frame.is_interpreted_frame() ||
1515 caller_frame.is_entry_frame() ||
1516 caller_frame.is_upcall_stub_frame()) {
1517 Method* callee = current->callee_target();
1518 guarantee(callee != nullptr && callee->is_method(), "bad handshake");
1519 current->set_vm_result_metadata(callee);
1520 current->set_callee_target(nullptr);
1521 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1522 // Bypass class initialization checks in c2i when caller is in native.
1523 // JNI calls to static methods don't have class initialization checks.
1524 // Fast class initialization checks are present in c2i adapters and call into
1525 // SharedRuntime::handle_wrong_method() on the slow path.
1526 //
1527 // JVM upcalls may land here as well, but there's a proper check present in
1528 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1529 // so bypassing it in c2i adapter is benign.
1530 return callee->get_c2i_no_clinit_check_entry();
1531 } else {
1532 if (caller_frame.is_interpreted_frame()) {
1533 return callee->get_c2i_inline_entry();
1534 } else {
1535 return callee->get_c2i_entry();
1536 }
1537 }
1538 }
1539
1540 // Must be compiled to compiled path which is safe to stackwalk
1541 methodHandle callee_method;
1542 bool is_static_call = false;
1543 bool is_optimized = false;
1544 bool caller_does_not_scalarize = false;
1545 JRT_BLOCK
1546 // Force resolving of caller (if we called from compiled frame)
1547 callee_method = SharedRuntime::reresolve_call_site(is_optimized, caller_does_not_scalarize, CHECK_NULL);
1548 current->set_vm_result_metadata(callee_method());
1549 JRT_BLOCK_END
1550 // return compiled code entry point after potential safepoints
1551 return get_resolved_entry(current, callee_method, callee_method->is_static(), is_optimized, caller_does_not_scalarize);
1552 JRT_END
1553
1554 // Handle abstract method call
1555 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1556 // Verbose error message for AbstractMethodError.
1557 // Get the called method from the invoke bytecode.
1558 vframeStream vfst(current, true);
1559 assert(!vfst.at_end(), "Java frame must exist");
1560 methodHandle caller(current, vfst.method());
1561 Bytecode_invoke invoke(caller, vfst.bci());
1562 DEBUG_ONLY( invoke.verify(); )
1563
1564 // Find the compiled caller frame.
1565 RegisterMap reg_map(current,
1566 RegisterMap::UpdateMap::include,
1567 RegisterMap::ProcessFrames::include,
1568 RegisterMap::WalkContinuation::skip);
1569 frame stubFrame = current->last_frame();
1570 assert(stubFrame.is_runtime_frame(), "must be");
1571 frame callerFrame = stubFrame.sender(®_map);
1572 assert(callerFrame.is_compiled_frame(), "must be");
1573
1574 // Install exception and return forward entry.
1575 address res = SharedRuntime::throw_AbstractMethodError_entry();
1576 JRT_BLOCK
1577 methodHandle callee(current, invoke.static_target(current));
1578 if (!callee.is_null()) {
1579 oop recv = callerFrame.retrieve_receiver(®_map);
1580 Klass *recv_klass = (recv != nullptr) ? recv->klass() : nullptr;
1581 res = StubRoutines::forward_exception_entry();
1582 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1583 }
1584 JRT_BLOCK_END
1585 return res;
1586 JRT_END
1587
1588 // return verified_code_entry if interp_only_mode is not set for the current thread;
1589 // otherwise return c2i entry.
1590 address SharedRuntime::get_resolved_entry(JavaThread* current, methodHandle callee_method,
1591 bool is_static_call, bool is_optimized, bool caller_does_not_scalarize) {
1592 bool is_interp_only_mode = (StressCallingConvention && (os::random() % (1 << 10)) == 0) || current->is_interp_only_mode();
1593 // In interp_only_mode we need to go to the interpreted entry
1594 // The c2i won't patch in this mode -- see fixup_callers_callsite
1595 bool go_to_interpreter = is_interp_only_mode && !callee_method->is_special_native_intrinsic();
1596
1597 if (caller_does_not_scalarize) {
1598 if (go_to_interpreter) {
1599 return callee_method->get_c2i_inline_entry();
1600 }
1601 assert(callee_method->verified_inline_code_entry() != nullptr, "Jump to zero!");
1602 return callee_method->verified_inline_code_entry();
1603 } else if (is_static_call || is_optimized) {
1604 if (go_to_interpreter) {
1605 return callee_method->get_c2i_entry();
1606 }
1607 assert(callee_method->verified_code_entry() != nullptr, "Jump to zero!");
1608 return callee_method->verified_code_entry();
1609 } else {
1610 if (go_to_interpreter) {
1611 return callee_method->get_c2i_inline_ro_entry();
1612 }
1613 assert(callee_method->verified_inline_ro_code_entry() != nullptr, "Jump to zero!");
1614 return callee_method->verified_inline_ro_code_entry();
1615 }
1616 }
1617
1618 // resolve a static call and patch code
1619 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1620 methodHandle callee_method;
1621 bool caller_does_not_scalarize = false;
1622 bool enter_special = false;
1623 JRT_BLOCK
1624 callee_method = SharedRuntime::resolve_helper(false, false, caller_does_not_scalarize, CHECK_NULL);
1625 current->set_vm_result_metadata(callee_method());
1626 JRT_BLOCK_END
1627 // return compiled code entry point after potential safepoints
1628 return get_resolved_entry(current, callee_method, true, false, caller_does_not_scalarize);
1629 JRT_END
1630
1631 // resolve virtual call and update inline cache to monomorphic
1632 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1633 methodHandle callee_method;
1634 bool caller_does_not_scalarize = false;
1635 JRT_BLOCK
1636 callee_method = SharedRuntime::resolve_helper(true, false, caller_does_not_scalarize, CHECK_NULL);
1637 current->set_vm_result_metadata(callee_method());
1638 JRT_BLOCK_END
1639 // return compiled code entry point after potential safepoints
1640 return get_resolved_entry(current, callee_method, false, false, caller_does_not_scalarize);
1641 JRT_END
1642
1643
1644 // Resolve a virtual call that can be statically bound (e.g., always
1645 // monomorphic, so it has no inline cache). Patch code to resolved target.
1646 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1647 methodHandle callee_method;
1648 bool caller_does_not_scalarize = false;
1649 JRT_BLOCK
1650 callee_method = SharedRuntime::resolve_helper(true, true, caller_does_not_scalarize, CHECK_NULL);
1651 current->set_vm_result_metadata(callee_method());
1652 JRT_BLOCK_END
1653 // return compiled code entry point after potential safepoints
1654 return get_resolved_entry(current, callee_method, false, true, caller_does_not_scalarize);
1655 JRT_END
1656
1657 methodHandle SharedRuntime::handle_ic_miss_helper(bool& caller_does_not_scalarize, TRAPS) {
1658 JavaThread* current = THREAD;
1659 ResourceMark rm(current);
1660 CallInfo call_info;
1661 Bytecodes::Code bc;
1662
1663 // receiver is null for static calls. An exception is thrown for null
1664 // receivers for non-static calls
1665 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1666
1667 methodHandle callee_method(current, call_info.selected_method());
1668
1669 #ifndef PRODUCT
1670 AtomicAccess::inc(&_ic_miss_ctr);
1671
1672 // Statistics & Tracing
1673 if (TraceCallFixup) {
1674 ResourceMark rm(current);
1675 tty->print("IC miss (%s) %s call to", Bytecodes::name(bc), (caller_does_not_scalarize) ? "non-scalar" : "");
1676 callee_method->print_short_name(tty);
1677 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1678 }
1679
1680 if (ICMissHistogram) {
1681 MutexLocker m(VMStatistic_lock);
1682 RegisterMap reg_map(current,
1683 RegisterMap::UpdateMap::skip,
1684 RegisterMap::ProcessFrames::include,
1685 RegisterMap::WalkContinuation::skip);
1686 frame f = current->last_frame().real_sender(®_map);// skip runtime stub
1687 // produce statistics under the lock
1688 trace_ic_miss(f.pc());
1689 }
1690 #endif
1691
1692 // install an event collector so that when a vtable stub is created the
1693 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1694 // event can't be posted when the stub is created as locks are held
1695 // - instead the event will be deferred until the event collector goes
1696 // out of scope.
1697 JvmtiDynamicCodeEventCollector event_collector;
1698
1699 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1700 RegisterMap reg_map(current,
1701 RegisterMap::UpdateMap::skip,
1702 RegisterMap::ProcessFrames::include,
1703 RegisterMap::WalkContinuation::skip);
1704 frame caller_frame = current->last_frame().sender(®_map);
1705 CodeBlob* cb = caller_frame.cb();
1706 nmethod* caller_nm = cb->as_nmethod();
1707 // Calls via mismatching methods are always non-scalarized
1708 if (caller_nm->is_compiled_by_c1() || call_info.resolved_method()->mismatch()) {
1709 caller_does_not_scalarize = true;
1710 }
1711
1712 CompiledICLocker ml(caller_nm);
1713 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1714 inline_cache->update(&call_info, receiver()->klass(), caller_does_not_scalarize);
1715
1716 return callee_method;
1717 }
1718
1719 //
1720 // Resets a call-site in compiled code so it will get resolved again.
1721 // This routines handles both virtual call sites, optimized virtual call
1722 // sites, and static call sites. Typically used to change a call sites
1723 // destination from compiled to interpreted.
1724 //
1725 methodHandle SharedRuntime::reresolve_call_site(bool& is_optimized, bool& caller_does_not_scalarize, TRAPS) {
1726 JavaThread* current = THREAD;
1727 ResourceMark rm(current);
1728 RegisterMap reg_map(current,
1729 RegisterMap::UpdateMap::skip,
1730 RegisterMap::ProcessFrames::include,
1731 RegisterMap::WalkContinuation::skip);
1732 frame stub_frame = current->last_frame();
1733 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1734 frame caller = stub_frame.sender(®_map);
1735 if (caller.is_compiled_frame()) {
1736 caller_does_not_scalarize = caller.cb()->as_nmethod()->is_compiled_by_c1();
1737 }
1738 assert(!caller.is_interpreted_frame(), "must be compiled");
1739
1740 // If the frame isn't a live compiled frame (i.e. deoptimized by the time we get here), no IC clearing must be done
1741 // for the caller. However, when the caller is C2 compiled and the callee a C1 or C2 compiled method, then we still
1742 // need to figure out whether it was an optimized virtual call with an inline type receiver. Otherwise, we end up
1743 // using the wrong method entry point and accidentally skip the buffering of the receiver.
1744 methodHandle callee_method = find_callee_method(caller_does_not_scalarize, CHECK_(methodHandle()));
1745 const bool caller_is_compiled_and_not_deoptimized = caller.is_compiled_frame() && !caller.is_deoptimized_frame();
1746 const bool caller_is_continuation_enter_intrinsic =
1747 caller.is_native_frame() && caller.cb()->as_nmethod()->method()->is_continuation_enter_intrinsic();
1748 const bool do_IC_clearing = caller_is_compiled_and_not_deoptimized || caller_is_continuation_enter_intrinsic;
1749
1750 const bool callee_compiled_with_scalarized_receiver = callee_method->has_compiled_code() &&
1751 !callee_method()->is_static() &&
1752 callee_method()->is_scalarized_arg(0);
1753 const bool compute_is_optimized = !caller_does_not_scalarize && callee_compiled_with_scalarized_receiver;
1754
1755 if (do_IC_clearing || compute_is_optimized) {
1756 address pc = caller.pc();
1757
1758 nmethod* caller_nm = CodeCache::find_nmethod(pc);
1759 assert(caller_nm != nullptr, "did not find caller nmethod");
1760
1761 // Default call_addr is the location of the "basic" call.
1762 // Determine the address of the call we a reresolving. With
1763 // Inline Caches we will always find a recognizable call.
1764 // With Inline Caches disabled we may or may not find a
1765 // recognizable call. We will always find a call for static
1766 // calls and for optimized virtual calls. For vanilla virtual
1767 // calls it depends on the state of the UseInlineCaches switch.
1768 //
1769 // With Inline Caches disabled we can get here for a virtual call
1770 // for two reasons:
1771 // 1 - calling an abstract method. The vtable for abstract methods
1772 // will run us thru handle_wrong_method and we will eventually
1773 // end up in the interpreter to throw the ame.
1774 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1775 // call and between the time we fetch the entry address and
1776 // we jump to it the target gets deoptimized. Similar to 1
1777 // we will wind up in the interprter (thru a c2i with c2).
1778 //
1779 CompiledICLocker ml(caller_nm);
1780 address call_addr = caller_nm->call_instruction_address(pc);
1781
1782 if (call_addr != nullptr) {
1783 // On x86 the logic for finding a call instruction is blindly checking for a call opcode 5
1784 // bytes back in the instruction stream so we must also check for reloc info.
1785 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1786 bool ret = iter.next(); // Get item
1787 if (ret) {
1788 is_optimized = false;
1789 switch (iter.type()) {
1790 case relocInfo::static_call_type:
1791 assert(callee_method->is_static(), "must be");
1792 case relocInfo::opt_virtual_call_type: {
1793 is_optimized = (iter.type() == relocInfo::opt_virtual_call_type);
1794 if (do_IC_clearing) {
1795 CompiledDirectCall* cdc = CompiledDirectCall::at(call_addr);
1796 cdc->set_to_clean();
1797 }
1798 break;
1799 }
1800
1801 case relocInfo::virtual_call_type: {
1802 if (do_IC_clearing) {
1803 // compiled, dispatched call (which used to call an interpreted method)
1804 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1805 inline_cache->set_to_clean();
1806 }
1807 break;
1808 }
1809 default:
1810 break;
1811 }
1812 }
1813 }
1814 }
1815
1816 #ifndef PRODUCT
1817 AtomicAccess::inc(&_wrong_method_ctr);
1818
1819 if (TraceCallFixup) {
1820 ResourceMark rm(current);
1821 tty->print("handle_wrong_method reresolving %s call to", (caller_does_not_scalarize) ? "non-scalar" : "");
1822 callee_method->print_short_name(tty);
1823 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1824 }
1825 #endif
1826
1827 return callee_method;
1828 }
1829
1830 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1831 // The faulting unsafe accesses should be changed to throw the error
1832 // synchronously instead. Meanwhile the faulting instruction will be
1833 // skipped over (effectively turning it into a no-op) and an
1834 // asynchronous exception will be raised which the thread will
1835 // handle at a later point. If the instruction is a load it will
1836 // return garbage.
1837
1838 // Request an async exception.
1839 thread->set_pending_unsafe_access_error();
1840
1841 // Return address of next instruction to execute.
2007 msglen += strlen(caster_klass_description) + strlen(target_klass_description) + strlen(klass_separator) + 3;
2008
2009 char* message = NEW_RESOURCE_ARRAY_RETURN_NULL(char, msglen);
2010 if (message == nullptr) {
2011 // Shouldn't happen, but don't cause even more problems if it does
2012 message = const_cast<char*>(caster_klass->external_name());
2013 } else {
2014 jio_snprintf(message,
2015 msglen,
2016 "class %s cannot be cast to class %s (%s%s%s)",
2017 caster_name,
2018 target_name,
2019 caster_klass_description,
2020 klass_separator,
2021 target_klass_description
2022 );
2023 }
2024 return message;
2025 }
2026
2027 char* SharedRuntime::generate_identity_exception_message(JavaThread* current, Klass* klass) {
2028 assert(klass->is_inline_klass(), "Must be a concrete value class");
2029 const char* desc = "Cannot synchronize on an instance of value class ";
2030 const char* className = klass->external_name();
2031 size_t msglen = strlen(desc) + strlen(className) + 1;
2032 char* message = NEW_RESOURCE_ARRAY(char, msglen);
2033 if (nullptr == message) {
2034 // Out of memory: can't create detailed error message
2035 message = const_cast<char*>(klass->external_name());
2036 } else {
2037 jio_snprintf(message, msglen, "%s%s", desc, className);
2038 }
2039 return message;
2040 }
2041
2042 JRT_LEAF(void, SharedRuntime::reguard_yellow_pages())
2043 (void) JavaThread::current()->stack_overflow_state()->reguard_stack();
2044 JRT_END
2045
2046 void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) {
2047 if (!SafepointSynchronize::is_synchronizing()) {
2048 // Only try quick_enter() if we're not trying to reach a safepoint
2049 // so that the calling thread reaches the safepoint more quickly.
2050 if (ObjectSynchronizer::quick_enter(obj, lock, current)) {
2051 return;
2052 }
2053 }
2054 // NO_ASYNC required because an async exception on the state transition destructor
2055 // would leave you with the lock held and it would never be released.
2056 // The normal monitorenter NullPointerException is thrown without acquiring a lock
2057 // and the model is that an exception implies the method failed.
2058 JRT_BLOCK_NO_ASYNC
2059 Handle h_obj(THREAD, obj);
2060 ObjectSynchronizer::enter(h_obj, lock, current);
2061 assert(!HAS_PENDING_EXCEPTION, "Should have no exception here");
2255 tty->print_cr("Note 1: counter updates are not MT-safe.");
2256 tty->print_cr("Note 2: %% in major categories are relative to total non-inlined calls;");
2257 tty->print_cr(" %% in nested categories are relative to their category");
2258 tty->print_cr(" (and thus add up to more than 100%% with inlining)");
2259 tty->cr();
2260
2261 MethodArityHistogram h;
2262 }
2263 #endif
2264
2265 #ifndef PRODUCT
2266 static int _lookups; // number of calls to lookup
2267 static int _equals; // number of buckets checked with matching hash
2268 static int _archived_hits; // number of successful lookups in archived table
2269 static int _runtime_hits; // number of successful lookups in runtime table
2270 #endif
2271
2272 // A simple wrapper class around the calling convention information
2273 // that allows sharing of adapters for the same calling convention.
2274 class AdapterFingerPrint : public MetaspaceObj {
2275 public:
2276 class Element {
2277 private:
2278 // The highest byte is the type of the argument. The remaining bytes contain the offset of the
2279 // field if it is flattened in the calling convention, -1 otherwise.
2280 juint _payload;
2281
2282 static constexpr int offset_bit_width = 24;
2283 static constexpr juint offset_bit_mask = (1 << offset_bit_width) - 1;
2284 public:
2285 Element(BasicType bt, int offset) : _payload((static_cast<juint>(bt) << offset_bit_width) | (juint(offset) & offset_bit_mask)) {
2286 assert(offset >= -1 && offset < jint(offset_bit_mask), "invalid offset %d", offset);
2287 }
2288
2289 BasicType bt() const {
2290 return static_cast<BasicType>(_payload >> offset_bit_width);
2291 }
2292
2293 int offset() const {
2294 juint res = _payload & offset_bit_mask;
2295 return res == offset_bit_mask ? -1 : res;
2296 }
2297
2298 juint hash() const {
2299 return _payload;
2300 }
2301
2302 bool operator!=(const Element& other) const {
2303 return _payload != other._payload;
2304 }
2305 };
2306
2307 private:
2308 const bool _has_ro_adapter;
2309 const int _length;
2310
2311 static int data_offset() { return sizeof(AdapterFingerPrint); }
2312 Element* data_pointer() {
2313 return reinterpret_cast<Element*>(reinterpret_cast<address>(this) + data_offset());
2314 }
2315
2316 const Element& element_at(int index) {
2317 assert(index < length(), "index %d out of bounds for length %d", index, length());
2318 Element* data = data_pointer();
2319 return data[index];
2320 }
2321
2322 // Private construtor. Use allocate() to get an instance.
2323 AdapterFingerPrint(const GrowableArray<SigEntry>* sig, bool has_ro_adapter)
2324 : _has_ro_adapter(has_ro_adapter), _length(total_args_passed_in_sig(sig)) {
2325 Element* data = data_pointer();
2326 BasicType prev_bt = T_ILLEGAL;
2327 int vt_count = 0;
2328 for (int index = 0; index < _length; index++) {
2329 const SigEntry& sig_entry = sig->at(index);
2330 BasicType bt = sig_entry._bt;
2331 if (bt == T_METADATA) {
2332 // Found start of inline type in signature
2333 assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
2334 vt_count++;
2335 } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
2336 // Found end of inline type in signature
2337 assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
2338 vt_count--;
2339 assert(vt_count >= 0, "invalid vt_count");
2340 } else if (vt_count == 0) {
2341 // Widen fields that are not part of a scalarized inline type argument
2342 assert(sig_entry._offset == -1, "invalid offset for argument that is not a flattened field %d", sig_entry._offset);
2343 bt = adapter_encoding(bt);
2344 }
2345
2346 ::new(&data[index]) Element(bt, sig_entry._offset);
2347 prev_bt = bt;
2348 }
2349 assert(vt_count == 0, "invalid vt_count");
2350 }
2351
2352 // Call deallocate instead
2353 ~AdapterFingerPrint() {
2354 ShouldNotCallThis();
2355 }
2356
2357 static int total_args_passed_in_sig(const GrowableArray<SigEntry>* sig) {
2358 return (sig != nullptr) ? sig->length() : 0;
2359 }
2360
2361 static int compute_size_in_words(int len) {
2362 return (int)heap_word_size(sizeof(AdapterFingerPrint) + (len * sizeof(Element)));
2363 }
2364
2365 // Remap BasicTypes that are handled equivalently by the adapters.
2366 // These are correct for the current system but someday it might be
2367 // necessary to make this mapping platform dependent.
2368 static BasicType adapter_encoding(BasicType in) {
2369 switch (in) {
2370 case T_BOOLEAN:
2371 case T_BYTE:
2372 case T_SHORT:
2373 case T_CHAR:
2374 // They are all promoted to T_INT in the calling convention
2375 return T_INT;
2376
2377 case T_OBJECT:
2378 case T_ARRAY:
2379 // In other words, we assume that any register good enough for
2380 // an int or long is good enough for a managed pointer.
2381 #ifdef _LP64
2382 return T_LONG;
2383 #else
2384 return T_INT;
2385 #endif
2386
2387 case T_INT:
2388 case T_LONG:
2389 case T_FLOAT:
2390 case T_DOUBLE:
2391 case T_VOID:
2392 return in;
2393
2394 default:
2395 ShouldNotReachHere();
2396 return T_CONFLICT;
2397 }
2398 }
2399
2400 void* operator new(size_t size, size_t fp_size) throw() {
2401 assert(fp_size >= size, "sanity check");
2402 void* p = AllocateHeap(fp_size, mtCode);
2403 memset(p, 0, fp_size);
2404 return p;
2405 }
2406
2407 public:
2408 template<typename Function>
2409 void iterate_args(Function function) {
2410 for (int i = 0; i < length(); i++) {
2411 function(element_at(i));
2412 }
2413 }
2414
2415 static AdapterFingerPrint* allocate(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
2416 int len = total_args_passed_in_sig(sig);
2417 int size_in_bytes = BytesPerWord * compute_size_in_words(len);
2418 AdapterFingerPrint* afp = new (size_in_bytes) AdapterFingerPrint(sig, has_ro_adapter);
2419 assert((afp->size() * BytesPerWord) == size_in_bytes, "should match");
2420 return afp;
2421 }
2422
2423 static void deallocate(AdapterFingerPrint* fp) {
2424 FreeHeap(fp);
2425 }
2426
2427 bool has_ro_adapter() const {
2428 return _has_ro_adapter;
2429 }
2430
2431 int length() const {
2432 return _length;
2433 }
2434
2435 unsigned int compute_hash() {
2436 int hash = 0;
2437 for (int i = 0; i < length(); i++) {
2438 const Element& v = element_at(i);
2439 //Add arithmetic operation to the hash, like +3 to improve hashing
2440 hash = ((hash << 8) ^ v.hash() ^ (hash >> 5)) + 3;
2441 }
2442 return (unsigned int)hash;
2443 }
2444
2445 const char* as_string() {
2446 stringStream st;
2447 st.print("{");
2448 if (_has_ro_adapter) {
2449 st.print("has_ro_adapter");
2450 } else {
2451 st.print("no_ro_adapter");
2452 }
2453 for (int i = 0; i < length(); i++) {
2454 st.print(", ");
2455 const Element& elem = element_at(i);
2456 st.print("{%s, %d}", type2name(elem.bt()), elem.offset());
2457 }
2458 st.print("}");
2459 return st.as_string();
2460 }
2461
2462 const char* as_basic_args_string() {
2463 stringStream st;
2464 bool long_prev = false;
2465 iterate_args([&] (const Element& arg) {
2466 if (long_prev) {
2467 long_prev = false;
2468 if (arg.bt() == T_VOID) {
2469 st.print("J");
2470 } else {
2471 st.print("L");
2472 }
2473 }
2474 if (arg.bt() == T_LONG) {
2475 long_prev = true;
2476 } else if (arg.bt() != T_VOID) {
2477 st.print("%c", type2char(arg.bt()));
2478 }
2479 });
2480 if (long_prev) {
2481 st.print("L");
2482 }
2483 return st.as_string();
2484 }
2485
2486 bool equals(AdapterFingerPrint* other) {
2487 if (other->_has_ro_adapter != _has_ro_adapter) {
2488 return false;
2489 } else if (other->_length != _length) {
2490 return false;
2491 } else {
2492 for (int i = 0; i < _length; i++) {
2493 if (element_at(i) != other->element_at(i)) {
2494 return false;
2495 }
2496 }
2497 }
2498 return true;
2499 }
2500
2501 // methods required by virtue of being a MetaspaceObj
2502 void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ }
2503 int size() const { return compute_size_in_words(_length); }
2504 MetaspaceObj::Type type() const { return AdapterFingerPrintType; }
2505
2506 static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) {
2507 NOT_PRODUCT(_equals++);
2508 return fp1->equals(fp2);
2509 }
2510
2511 static unsigned int compute_hash(AdapterFingerPrint* const& fp) {
2512 return fp->compute_hash();
2513 }
2516 #if INCLUDE_CDS
2517 static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) {
2518 return AdapterFingerPrint::equals(entry->fingerprint(), fp);
2519 }
2520
2521 class ArchivedAdapterTable : public OffsetCompactHashtable<
2522 AdapterFingerPrint*,
2523 AdapterHandlerEntry*,
2524 adapter_fp_equals_compact_hashtable_entry> {};
2525 #endif // INCLUDE_CDS
2526
2527 // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries
2528 using AdapterHandlerTable = HashTable<AdapterFingerPrint*, AdapterHandlerEntry*, 293,
2529 AnyObj::C_HEAP, mtCode,
2530 AdapterFingerPrint::compute_hash,
2531 AdapterFingerPrint::equals>;
2532 static AdapterHandlerTable* _adapter_handler_table;
2533 static GrowableArray<AdapterHandlerEntry*>* _adapter_handler_list = nullptr;
2534
2535 // Find a entry with the same fingerprint if it exists
2536 AdapterHandlerEntry* AdapterHandlerLibrary::lookup(const GrowableArray<SigEntry>* sig, bool has_ro_adapter) {
2537 NOT_PRODUCT(_lookups++);
2538 assert_lock_strong(AdapterHandlerLibrary_lock);
2539 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(sig, has_ro_adapter);
2540 AdapterHandlerEntry* entry = nullptr;
2541 #if INCLUDE_CDS
2542 // if we are building the archive then the archived adapter table is
2543 // not valid and we need to use the ones added to the runtime table
2544 if (AOTCodeCache::is_using_adapter()) {
2545 // Search archived table first. It is read-only table so can be searched without lock
2546 entry = _aot_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */);
2547 #ifndef PRODUCT
2548 if (entry != nullptr) {
2549 _archived_hits++;
2550 }
2551 #endif
2552 }
2553 #endif // INCLUDE_CDS
2554 if (entry == nullptr) {
2555 assert_lock_strong(AdapterHandlerLibrary_lock);
2556 AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp);
2557 if (entry_p != nullptr) {
2558 entry = *entry_p;
2559 assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)",
2576 TableStatistics ts = _adapter_handler_table->statistics_calculate(size);
2577 ts.print(tty, "AdapterHandlerTable");
2578 tty->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)",
2579 _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries());
2580 int total_hits = _archived_hits + _runtime_hits;
2581 tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d)",
2582 _lookups, _equals, total_hits, _archived_hits, _runtime_hits);
2583 }
2584 #endif
2585
2586 // ---------------------------------------------------------------------------
2587 // Implementation of AdapterHandlerLibrary
2588 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = nullptr;
2589 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr;
2590 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr;
2591 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr;
2592 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr;
2593 #if INCLUDE_CDS
2594 ArchivedAdapterTable AdapterHandlerLibrary::_aot_adapter_handler_table;
2595 #endif // INCLUDE_CDS
2596 static const int AdapterHandlerLibrary_size = 48*K;
2597 BufferBlob* AdapterHandlerLibrary::_buffer = nullptr;
2598 volatile uint AdapterHandlerLibrary::_id_counter = 0;
2599
2600 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2601 assert(_buffer != nullptr, "should be initialized");
2602 return _buffer;
2603 }
2604
2605 static void post_adapter_creation(const AdapterHandlerEntry* entry) {
2606 if (Forte::is_enabled() || JvmtiExport::should_post_dynamic_code_generated()) {
2607 AdapterBlob* adapter_blob = entry->adapter_blob();
2608 char blob_id[256];
2609 jio_snprintf(blob_id,
2610 sizeof(blob_id),
2611 "%s(%s)",
2612 adapter_blob->name(),
2613 entry->fingerprint()->as_string());
2614 if (Forte::is_enabled()) {
2615 Forte::register_stub(blob_id, adapter_blob->content_begin(), adapter_blob->content_end());
2616 }
2624 void AdapterHandlerLibrary::initialize() {
2625 {
2626 ResourceMark rm;
2627 _adapter_handler_table = new (mtCode) AdapterHandlerTable();
2628 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2629 }
2630
2631 #if INCLUDE_CDS
2632 // Link adapters in AOT Cache to their code in AOT Code Cache
2633 if (AOTCodeCache::is_using_adapter() && !_aot_adapter_handler_table.empty()) {
2634 link_aot_adapters();
2635 lookup_simple_adapters();
2636 return;
2637 }
2638 #endif // INCLUDE_CDS
2639
2640 ResourceMark rm;
2641 {
2642 MutexLocker mu(AdapterHandlerLibrary_lock);
2643
2644 CompiledEntrySignature no_args;
2645 no_args.compute_calling_conventions();
2646 _no_arg_handler = create_adapter(no_args, true);
2647
2648 CompiledEntrySignature obj_args;
2649 SigEntry::add_entry(obj_args.sig(), T_OBJECT);
2650 obj_args.compute_calling_conventions();
2651 _obj_arg_handler = create_adapter(obj_args, true);
2652
2653 CompiledEntrySignature int_args;
2654 SigEntry::add_entry(int_args.sig(), T_INT);
2655 int_args.compute_calling_conventions();
2656 _int_arg_handler = create_adapter(int_args, true);
2657
2658 CompiledEntrySignature obj_int_args;
2659 SigEntry::add_entry(obj_int_args.sig(), T_OBJECT);
2660 SigEntry::add_entry(obj_int_args.sig(), T_INT);
2661 obj_int_args.compute_calling_conventions();
2662 _obj_int_arg_handler = create_adapter(obj_int_args, true);
2663
2664 CompiledEntrySignature obj_obj_args;
2665 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
2666 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
2667 obj_obj_args.compute_calling_conventions();
2668 _obj_obj_arg_handler = create_adapter(obj_obj_args, true);
2669
2670 // we should always get an entry back but we don't have any
2671 // associated blob on Zero
2672 assert(_no_arg_handler != nullptr &&
2673 _obj_arg_handler != nullptr &&
2674 _int_arg_handler != nullptr &&
2675 _obj_int_arg_handler != nullptr &&
2676 _obj_obj_arg_handler != nullptr, "Initial adapter handlers must be properly created");
2677 }
2678
2679 // Outside of the lock
2680 #ifndef ZERO
2681 // no blobs to register when we are on Zero
2682 post_adapter_creation(_no_arg_handler);
2683 post_adapter_creation(_obj_arg_handler);
2684 post_adapter_creation(_int_arg_handler);
2685 post_adapter_creation(_obj_int_arg_handler);
2686 post_adapter_creation(_obj_obj_arg_handler);
2687 #endif // ZERO
2688 }
2689
2690 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint) {
2691 uint id = (uint)AtomicAccess::add((int*)&_id_counter, 1);
2692 assert(id > 0, "we can never overflow because AOT cache cannot contain more than 2^32 methods");
2693 return AdapterHandlerEntry::allocate(id, fingerprint);
2694 }
2695
2696 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2697 int total_args_passed = method->size_of_parameters(); // All args on stack
2698 if (total_args_passed == 0) {
2699 return _no_arg_handler;
2700 } else if (total_args_passed == 1) {
2701 if (!method->is_static()) {
2702 if (InlineTypePassFieldsAsArgs && method->method_holder()->is_inline_klass()) {
2703 return nullptr;
2704 }
2705 return _obj_arg_handler;
2706 }
2707 switch (method->signature()->char_at(1)) {
2708 case JVM_SIGNATURE_CLASS: {
2709 if (InlineTypePassFieldsAsArgs) {
2710 SignatureStream ss(method->signature());
2711 InlineKlass* vk = ss.as_inline_klass(method->method_holder());
2712 if (vk != nullptr) {
2713 return nullptr;
2714 }
2715 }
2716 return _obj_arg_handler;
2717 }
2718 case JVM_SIGNATURE_ARRAY:
2719 return _obj_arg_handler;
2720 case JVM_SIGNATURE_INT:
2721 case JVM_SIGNATURE_BOOLEAN:
2722 case JVM_SIGNATURE_CHAR:
2723 case JVM_SIGNATURE_BYTE:
2724 case JVM_SIGNATURE_SHORT:
2725 return _int_arg_handler;
2726 }
2727 } else if (total_args_passed == 2 &&
2728 !method->is_static() && (!InlineTypePassFieldsAsArgs || !method->method_holder()->is_inline_klass())) {
2729 switch (method->signature()->char_at(1)) {
2730 case JVM_SIGNATURE_CLASS: {
2731 if (InlineTypePassFieldsAsArgs) {
2732 SignatureStream ss(method->signature());
2733 InlineKlass* vk = ss.as_inline_klass(method->method_holder());
2734 if (vk != nullptr) {
2735 return nullptr;
2736 }
2737 }
2738 return _obj_obj_arg_handler;
2739 }
2740 case JVM_SIGNATURE_ARRAY:
2741 return _obj_obj_arg_handler;
2742 case JVM_SIGNATURE_INT:
2743 case JVM_SIGNATURE_BOOLEAN:
2744 case JVM_SIGNATURE_CHAR:
2745 case JVM_SIGNATURE_BYTE:
2746 case JVM_SIGNATURE_SHORT:
2747 return _obj_int_arg_handler;
2748 }
2749 }
2750 return nullptr;
2751 }
2752
2753 CompiledEntrySignature::CompiledEntrySignature(Method* method) :
2754 _method(method), _num_inline_args(0), _has_inline_recv(false),
2755 _regs(nullptr), _regs_cc(nullptr), _regs_cc_ro(nullptr),
2756 _args_on_stack(0), _args_on_stack_cc(0), _args_on_stack_cc_ro(0),
2757 _c1_needs_stack_repair(false), _c2_needs_stack_repair(false), _supers(nullptr) {
2758 _sig = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2759 _sig_cc = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2760 _sig_cc_ro = new GrowableArray<SigEntry>((method != nullptr) ? method->size_of_parameters() : 1);
2761 }
2762
2763 // See if we can save space by sharing the same entry for VIEP and VIEP(RO),
2764 // or the same entry for VEP and VIEP(RO).
2765 CodeOffsets::Entries CompiledEntrySignature::c1_inline_ro_entry_type() const {
2766 if (!has_scalarized_args()) {
2767 // VEP/VIEP/VIEP(RO) all share the same entry. There's no packing.
2768 return CodeOffsets::Verified_Entry;
2769 }
2770 if (_method->is_static()) {
2771 // Static methods don't need VIEP(RO)
2772 return CodeOffsets::Verified_Entry;
2773 }
2774
2775 if (has_inline_recv()) {
2776 if (num_inline_args() == 1) {
2777 // Share same entry for VIEP and VIEP(RO).
2778 // This is quite common: we have an instance method in an InlineKlass that has
2779 // no inline type args other than <this>.
2780 return CodeOffsets::Verified_Inline_Entry;
2781 } else {
2782 assert(num_inline_args() > 1, "must be");
2783 // No sharing:
2784 // VIEP(RO) -- <this> is passed as object
2785 // VEP -- <this> is passed as fields
2786 return CodeOffsets::Verified_Inline_Entry_RO;
2787 }
2788 }
2789
2790 // Either a static method, or <this> is not an inline type
2791 if (args_on_stack_cc() != args_on_stack_cc_ro()) {
2792 // No sharing:
2793 // Some arguments are passed on the stack, and we have inserted reserved entries
2794 // into the VEP, but we never insert reserved entries into the VIEP(RO).
2795 return CodeOffsets::Verified_Inline_Entry_RO;
2796 } else {
2797 // Share same entry for VEP and VIEP(RO).
2798 return CodeOffsets::Verified_Entry;
2799 }
2800 }
2801
2802 // Returns all super methods (transitive) in classes and interfaces that are overridden by the current method.
2803 GrowableArray<Method*>* CompiledEntrySignature::get_supers() {
2804 if (_supers != nullptr) {
2805 return _supers;
2806 }
2807 _supers = new GrowableArray<Method*>();
2808 // Skip private, static, and <init> methods
2809 if (_method->is_private() || _method->is_static() || _method->is_object_constructor()) {
2810 return _supers;
2811 }
2812 Symbol* name = _method->name();
2813 Symbol* signature = _method->signature();
2814 const Klass* holder = _method->method_holder()->super();
2815 Symbol* holder_name = holder->name();
2816 JavaThread* current = JavaThread::current();
2817 HandleMark hm(current);
2818 Handle loader(current, _method->method_holder()->class_loader());
2819
2820 // Walk up the class hierarchy and search for super methods
2821 while (holder != nullptr) {
2822 Method* super_method = holder->lookup_method(name, signature);
2823 if (super_method == nullptr) {
2824 break;
2825 }
2826 if (!super_method->is_static() && !super_method->is_private() &&
2827 (!super_method->is_package_private() ||
2828 super_method->method_holder()->is_same_class_package(loader(), holder_name))) {
2829 _supers->push(super_method);
2830 }
2831 holder = super_method->method_holder()->super();
2832 }
2833 // Search interfaces for super methods
2834 Array<InstanceKlass*>* interfaces = _method->method_holder()->transitive_interfaces();
2835 for (int i = 0; i < interfaces->length(); ++i) {
2836 Method* m = interfaces->at(i)->lookup_method(name, signature);
2837 if (m != nullptr && !m->is_static() && m->is_public()) {
2838 _supers->push(m);
2839 }
2840 }
2841 return _supers;
2842 }
2843
2844 bool CompiledEntrySignature::check_supers_and_deoptimize(int arg_num) {
2845 assert(JavaThread::current()->thread_state() == _thread_in_vm, "must be in vm state");
2846
2847 bool scalar_super = false;
2848 bool non_scalar_super = false;
2849
2850 GrowableArray<Method*>* supers = get_supers();
2851 for (int i = 0; i < supers->length(); ++i) {
2852 Method* super_method = supers->at(i);
2853 if (super_method->is_scalarized_arg(arg_num)) {
2854 scalar_super = true;
2855 } else {
2856 non_scalar_super = true;
2857 }
2858 }
2859 #ifdef ASSERT
2860 // Randomly enable below code paths for stress testing
2861 bool stress = StressCallingConvention;
2862 if (stress && (os::random() & 1) == 1) {
2863 non_scalar_super = true;
2864 if ((os::random() & 1) == 1) {
2865 scalar_super = true;
2866 }
2867 }
2868 #endif
2869 if (non_scalar_super) {
2870 // Found a super method with a non-scalarized argument. Fall back to the non-scalarized calling convention.
2871 if (scalar_super) {
2872 // Found non-scalar *and* scalar super methods. We can't handle both.
2873 // Mark the scalar method as mismatch and re-compile call sites to use non-scalarized calling convention.
2874 for (int i = 0; i < supers->length(); ++i) {
2875 Method* super_method = supers->at(i);
2876 if (super_method->is_scalarized_arg(arg_num) DEBUG_ONLY(|| (stress && (os::random() & 1) == 1))) {
2877 JavaThread* thread = JavaThread::current();
2878 HandleMark hm(thread);
2879 methodHandle mh(thread, super_method);
2880 DeoptimizationScope deopt_scope;
2881 {
2882 // Keep the lock scope minimal. Prevent interference with other
2883 // dependency checks by setting mismatch and marking within the lock.
2884 MutexLocker ml(Compile_lock, Mutex::_safepoint_check_flag);
2885 super_method->set_mismatch();
2886 CodeCache::mark_for_deoptimization(&deopt_scope, mh());
2887 }
2888 deopt_scope.deoptimize_marked();
2889 }
2890 }
2891 }
2892 }
2893
2894 return non_scalar_super;
2895 }
2896
2897 // Iterate over arguments and compute scalarized and non-scalarized signatures
2898 void CompiledEntrySignature::compute_calling_conventions(bool link_time) {
2899 assert(JavaThread::current()->thread_state() != _thread_in_native, "must not be in native");
2900 assert(link_time || (_method != nullptr && _method->adapter() != nullptr), "invariant");
2901 bool has_scalarized = false;
2902 if (_method != nullptr) {
2903 InstanceKlass* holder = _method->method_holder();
2904 int arg_num = 0;
2905 if (!_method->is_static()) {
2906 // We shouldn't scalarize 'this' in a value class constructor
2907 if (holder->is_inline_klass() && InlineKlass::cast(holder)->can_be_passed_as_fields() &&
2908 !_method->is_object_constructor() && (link_time || _method->is_scalarized_arg(arg_num))) {
2909 _sig_cc->appendAll(InlineKlass::cast(holder)->extended_sig());
2910 _sig_cc->insert_before(1, SigEntry(T_OBJECT, 0, nullptr, false, true)); // buffer argument
2911 has_scalarized = true;
2912 _has_inline_recv = true;
2913 _num_inline_args++;
2914 } else {
2915 SigEntry::add_entry(_sig_cc, T_OBJECT, holder->name());
2916 }
2917 SigEntry::add_entry(_sig, T_OBJECT, holder->name());
2918 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, holder->name());
2919 arg_num++;
2920 }
2921 for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
2922 const BasicType bt = ss.type();
2923 if (InlineTypePassFieldsAsArgs && bt == T_OBJECT) {
2924 InlineKlass* vk = ss.as_inline_klass(holder);
2925 if (vk != nullptr && vk->can_be_passed_as_fields() && (link_time || _method->is_scalarized_arg(arg_num))) {
2926 // Check for a calling convention mismatch with super method(s)
2927 if (link_time && check_supers_and_deoptimize(arg_num)) {
2928 // Fall back to non-scalarized calling convention
2929 SigEntry::add_entry(_sig_cc, T_OBJECT, ss.as_symbol());
2930 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, ss.as_symbol());
2931 } else {
2932 _num_inline_args++;
2933 has_scalarized = true;
2934 int last = _sig_cc->length();
2935 int last_ro = _sig_cc_ro->length();
2936 _sig_cc->appendAll(vk->extended_sig());
2937 _sig_cc_ro->appendAll(vk->extended_sig());
2938 // buffer argument
2939 _sig_cc->insert_before(last + 1, SigEntry(T_OBJECT, 0, nullptr, false, true));
2940 _sig_cc_ro->insert_before(last_ro + 1, SigEntry(T_OBJECT, 0, nullptr, false, true));
2941 // Insert InlineTypeNode::NullMarker field right after T_METADATA delimiter
2942 _sig_cc->insert_before(last + 2, SigEntry(T_BOOLEAN, -1, nullptr, true, false));
2943 _sig_cc_ro->insert_before(last_ro + 2, SigEntry(T_BOOLEAN, -1, nullptr, true, false));
2944 }
2945 } else {
2946 SigEntry::add_entry(_sig_cc, T_OBJECT, ss.as_symbol());
2947 SigEntry::add_entry(_sig_cc_ro, T_OBJECT, ss.as_symbol());
2948 }
2949 } else {
2950 SigEntry::add_entry(_sig_cc, ss.type(), ss.as_symbol());
2951 SigEntry::add_entry(_sig_cc_ro, ss.type(), ss.as_symbol());
2952 }
2953 SigEntry::add_entry(_sig, bt, ss.as_symbol());
2954 if (bt != T_VOID) {
2955 arg_num++;
2956 }
2957 }
2958 }
2959
2960 // Compute the non-scalarized calling convention
2961 _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
2962 _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
2963
2964 // Compute the scalarized calling conventions if there are scalarized inline types in the signature
2965 if (has_scalarized && !_method->is_native()) {
2966 _regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc->length());
2967 _args_on_stack_cc = SharedRuntime::java_calling_convention(_sig_cc, _regs_cc);
2968
2969 _regs_cc_ro = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc_ro->length());
2970 _args_on_stack_cc_ro = SharedRuntime::java_calling_convention(_sig_cc_ro, _regs_cc_ro);
2971
2972 _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
2973 _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
2974
2975 // Upper bound on stack arguments to avoid hitting the argument limit and
2976 // bailing out of compilation ("unsupported incoming calling sequence").
2977 // TODO 8281260 We need a reasonable limit (flag?) here
2978 if (MAX2(_args_on_stack_cc, _args_on_stack_cc_ro) <= 75) {
2979 return; // Success
2980 }
2981 }
2982
2983 // No scalarized args
2984 _sig_cc = _sig;
2985 _regs_cc = _regs;
2986 _args_on_stack_cc = _args_on_stack;
2987
2988 _sig_cc_ro = _sig;
2989 _regs_cc_ro = _regs;
2990 _args_on_stack_cc_ro = _args_on_stack;
2991 }
2992
2993 void CompiledEntrySignature::initialize_from_fingerprint(AdapterFingerPrint* fingerprint) {
2994 _has_inline_recv = fingerprint->has_ro_adapter();
2995
2996 int value_object_count = 0;
2997 BasicType prev_bt = T_ILLEGAL;
2998 bool has_scalarized_arguments = false;
2999 bool long_prev = false;
3000 int long_prev_offset = -1;
3001 bool skipping_inline_recv = false;
3002 bool receiver_handled = false;
3003
3004 fingerprint->iterate_args([&] (const AdapterFingerPrint::Element& arg) {
3005 BasicType bt = arg.bt();
3006 int offset = arg.offset();
3007
3008 if (long_prev) {
3009 long_prev = false;
3010 BasicType bt_to_add;
3011 if (bt == T_VOID) {
3012 bt_to_add = T_LONG;
3013 } else {
3014 bt_to_add = T_OBJECT;
3015 }
3016 if (value_object_count == 0) {
3017 SigEntry::add_entry(_sig, bt_to_add);
3018 }
3019 assert(long_prev_offset != 0, "no buffer argument here");
3020 SigEntry::add_entry(_sig_cc, bt_to_add, nullptr, long_prev_offset);
3021 if (!skipping_inline_recv) {
3022 SigEntry::add_entry(_sig_cc_ro, bt_to_add, nullptr, long_prev_offset);
3023 }
3024 }
3025
3026 switch (bt) {
3027 case T_VOID:
3028 if (prev_bt != T_LONG && prev_bt != T_DOUBLE) {
3029 assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
3030 value_object_count--;
3031 SigEntry::add_entry(_sig_cc, T_VOID, nullptr, offset);
3032 if (!skipping_inline_recv) {
3033 SigEntry::add_entry(_sig_cc_ro, T_VOID, nullptr, offset);
3034 } else if (value_object_count == 0) {
3035 skipping_inline_recv = false;
3036 }
3037 assert(value_object_count >= 0, "invalid value object count");
3038 } else {
3039 // Nothing to add for _sig: We already added an addition T_VOID in add_entry() when adding T_LONG or T_DOUBLE.
3040 }
3041 break;
3042 case T_INT:
3043 case T_FLOAT:
3044 case T_DOUBLE:
3045 if (value_object_count == 0) {
3046 SigEntry::add_entry(_sig, bt);
3047 }
3048 SigEntry::add_entry(_sig_cc, bt, nullptr, offset);
3049 if (!skipping_inline_recv) {
3050 SigEntry::add_entry(_sig_cc_ro, bt, nullptr, offset);
3051 }
3052 break;
3053 case T_LONG:
3054 long_prev = true;
3055 long_prev_offset = offset;
3056 break;
3057 case T_BOOLEAN:
3058 case T_CHAR:
3059 case T_BYTE:
3060 case T_SHORT:
3061 case T_OBJECT:
3062 case T_ARRAY:
3063 assert(value_object_count > 0, "must be value object field");
3064 assert(offset != 0 || (bt == T_OBJECT && prev_bt == T_METADATA), "buffer input expected here");
3065 SigEntry::add_entry(_sig_cc, bt, nullptr, offset, offset == -1, offset == 0);
3066 if (!skipping_inline_recv) {
3067 SigEntry::add_entry(_sig_cc_ro, bt, nullptr, offset, offset == -1, offset == 0);
3068 }
3069 break;
3070 case T_METADATA:
3071 assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
3072 if (value_object_count == 0) {
3073 SigEntry::add_entry(_sig, T_OBJECT);
3074 }
3075 SigEntry::add_entry(_sig_cc, T_METADATA, nullptr, offset);
3076 if (!skipping_inline_recv) {
3077 if (!receiver_handled && _has_inline_recv && value_object_count == 0) {
3078 SigEntry::add_entry(_sig_cc_ro, T_OBJECT);
3079 skipping_inline_recv = true;
3080 receiver_handled = true;
3081 } else {
3082 SigEntry::add_entry(_sig_cc_ro, T_METADATA, nullptr, offset);
3083 }
3084 }
3085 value_object_count++;
3086 has_scalarized_arguments = true;
3087 break;
3088 default: {
3089 fatal("Unexpected BasicType: %s", basictype_to_str(bt));
3090 }
3091 }
3092 prev_bt = bt;
3093 });
3094
3095 if (long_prev) {
3096 // If previous bt was T_LONG and we reached the end of the signature, we know that it must be a T_OBJECT.
3097 SigEntry::add_entry(_sig, T_OBJECT);
3098 SigEntry::add_entry(_sig_cc, T_OBJECT);
3099 SigEntry::add_entry(_sig_cc_ro, T_OBJECT);
3100 }
3101 assert(value_object_count == 0, "invalid value object count");
3102
3103 #ifdef ASSERT
3104 if (_has_inline_recv) {
3105 // In RO signatures, inline receivers must be represented as a single T_OBJECT
3106 assert(_sig_cc_ro->length() >= 1, "sig_cc_ro must include receiver");
3107 assert(_sig_cc_ro->at(0)._bt == T_OBJECT,
3108 "sig_cc_ro must represent inline receiver as T_OBJECT");
3109 assert(_sig_cc_ro->length() <= _sig_cc->length(),
3110 "sig_cc_ro must not be longer than sig_cc");
3111 }
3112 #endif
3113
3114 _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
3115 _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
3116
3117 // Compute the scalarized calling conventions if there are scalarized inline types in the signature
3118 if (has_scalarized_arguments) {
3119 _regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc->length());
3120 _args_on_stack_cc = SharedRuntime::java_calling_convention(_sig_cc, _regs_cc);
3121
3122 _regs_cc_ro = NEW_RESOURCE_ARRAY(VMRegPair, _sig_cc_ro->length());
3123 _args_on_stack_cc_ro = SharedRuntime::java_calling_convention(_sig_cc_ro, _regs_cc_ro);
3124
3125 _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
3126 _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
3127 } else {
3128 // No scalarized args
3129 _sig_cc = _sig;
3130 _regs_cc = _regs;
3131 _args_on_stack_cc = _args_on_stack;
3132
3133 _sig_cc_ro = _sig;
3134 _regs_cc_ro = _regs;
3135 _args_on_stack_cc_ro = _args_on_stack;
3136 }
3137
3138 #ifdef ASSERT
3139 {
3140 AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(_sig_cc, _has_inline_recv);
3141 assert(fingerprint->equals(compare_fp), "%s - %s", fingerprint->as_string(), compare_fp->as_string());
3142 AdapterFingerPrint::deallocate(compare_fp);
3143 }
3144 #endif
3145 }
3146
3147 const char* AdapterHandlerEntry::_entry_names[] = {
3148 "i2c", "c2i", "c2i_unverified", "c2i_no_clinit_check"
3149 };
3150
3151 #ifdef ASSERT
3152 void AdapterHandlerLibrary::verify_adapter_sharing(CompiledEntrySignature& ces, AdapterHandlerEntry* cached_entry) {
3153 // we can only check for the same code if there is any
3154 #ifndef ZERO
3155 AdapterHandlerEntry* comparison_entry = create_adapter(ces, false, true);
3156 assert(comparison_entry->adapter_blob() == nullptr, "no blob should be created when creating an adapter for comparison");
3157 assert(comparison_entry->compare_code(cached_entry), "code must match");
3158 // Release the one just created
3159 AdapterHandlerEntry::deallocate(comparison_entry);
3160 # endif // ZERO
3161 }
3162 #endif /* ASSERT*/
3163
3164 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
3165 assert(!method->is_abstract() || InlineTypePassFieldsAsArgs, "abstract methods do not have adapters");
3166 // Use customized signature handler. Need to lock around updates to
3167 // the _adapter_handler_table (it is not safe for concurrent readers
3168 // and a single writer: this could be fixed if it becomes a
3169 // problem).
3170
3171 // Fast-path for trivial adapters
3172 AdapterHandlerEntry* entry = get_simple_adapter(method);
3173 if (entry != nullptr) {
3174 return entry;
3175 }
3176
3177 ResourceMark rm;
3178 bool new_entry = false;
3179
3180 CompiledEntrySignature ces(method());
3181 ces.compute_calling_conventions();
3182 if (ces.has_scalarized_args()) {
3183 if (!method->has_scalarized_args()) {
3184 method->set_has_scalarized_args();
3185 }
3186 if (ces.c1_needs_stack_repair()) {
3187 method->set_c1_needs_stack_repair();
3188 }
3189 if (ces.c2_needs_stack_repair() && !method->c2_needs_stack_repair()) {
3190 method->set_c2_needs_stack_repair();
3191 }
3192 }
3193
3194 {
3195 MutexLocker mu(AdapterHandlerLibrary_lock);
3196
3197 // Lookup method signature's fingerprint
3198 entry = lookup(ces.sig_cc(), ces.has_inline_recv());
3199
3200 if (entry != nullptr) {
3201 #ifndef ZERO
3202 assert(entry->is_linked(), "AdapterHandlerEntry must have been linked");
3203 #endif
3204 #ifdef ASSERT
3205 if (!entry->in_aot_cache() && VerifyAdapterSharing) {
3206 verify_adapter_sharing(ces, entry);
3207 }
3208 #endif
3209 } else {
3210 entry = create_adapter(ces, /* allocate_code_blob */ true);
3211 if (entry != nullptr) {
3212 new_entry = true;
3213 }
3214 }
3215 }
3216
3217 // Outside of the lock
3218 if (new_entry) {
3219 post_adapter_creation(entry);
3220 }
3221 return entry;
3222 }
3223
3224 void AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler) {
3225 ResourceMark rm;
3226 const char* name = AdapterHandlerLibrary::name(handler);
3227 const uint32_t id = AdapterHandlerLibrary::id(handler);
3228
3229 CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::Adapter, id, name);
3230 if (blob != nullptr) {
3245 }
3246 insts_size = adapter_blob->code_size();
3247 st->print_cr("i2c argument handler for: %s %s (%d bytes generated)",
3248 handler->fingerprint()->as_basic_args_string(),
3249 handler->fingerprint()->as_string(), insts_size);
3250 st->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry()));
3251 if (Verbose || PrintStubCode) {
3252 address first_pc = adapter_blob->content_begin();
3253 if (first_pc != nullptr) {
3254 Disassembler::decode(first_pc, first_pc + insts_size, st, &adapter_blob->asm_remarks());
3255 st->cr();
3256 }
3257 }
3258 }
3259 #endif // PRODUCT
3260
3261 void AdapterHandlerLibrary::address_to_offset(address entry_address[AdapterBlob::ENTRY_COUNT],
3262 int entry_offset[AdapterBlob::ENTRY_COUNT]) {
3263 entry_offset[AdapterBlob::I2C] = 0;
3264 entry_offset[AdapterBlob::C2I] = entry_address[AdapterBlob::C2I] - entry_address[AdapterBlob::I2C];
3265 entry_offset[AdapterBlob::C2I_Inline] = entry_address[AdapterBlob::C2I_Inline] - entry_address[AdapterBlob::I2C];
3266 entry_offset[AdapterBlob::C2I_Inline_RO] = entry_address[AdapterBlob::C2I_Inline_RO] - entry_address[AdapterBlob::I2C];
3267 entry_offset[AdapterBlob::C2I_Unverified] = entry_address[AdapterBlob::C2I_Unverified] - entry_address[AdapterBlob::I2C];
3268 entry_offset[AdapterBlob::C2I_Unverified_Inline] = entry_address[AdapterBlob::C2I_Unverified_Inline] - entry_address[AdapterBlob::I2C];
3269 if (entry_address[AdapterBlob::C2I_No_Clinit_Check] == nullptr) {
3270 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = -1;
3271 } else {
3272 entry_offset[AdapterBlob::C2I_No_Clinit_Check] = entry_address[AdapterBlob::C2I_No_Clinit_Check] - entry_address[AdapterBlob::I2C];
3273 }
3274 }
3275
3276 bool AdapterHandlerLibrary::generate_adapter_code(AdapterHandlerEntry* handler,
3277 CompiledEntrySignature& ces,
3278 bool allocate_code_blob,
3279 bool is_transient) {
3280 if (log_is_enabled(Info, perf, class, link)) {
3281 ClassLoader::perf_method_adapters_count()->inc();
3282 }
3283
3284 #ifndef ZERO
3285 AdapterBlob* adapter_blob = nullptr;
3286 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3287 CodeBuffer buffer(buf);
3288 short buffer_locs[20];
3289 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3290 sizeof(buffer_locs)/sizeof(relocInfo));
3291 MacroAssembler masm(&buffer);
3292 address entry_address[AdapterBlob::ENTRY_COUNT];
3293
3294 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
3295 SharedRuntime::generate_i2c2i_adapters(&masm,
3296 ces.args_on_stack(),
3297 ces.sig(),
3298 ces.regs(),
3299 ces.sig_cc(),
3300 ces.regs_cc(),
3301 ces.sig_cc_ro(),
3302 ces.regs_cc_ro(),
3303 entry_address,
3304 adapter_blob,
3305 allocate_code_blob);
3306
3307 if (ces.has_scalarized_args()) {
3308 // Save a C heap allocated version of the scalarized signature and store it in the adapter
3309 GrowableArray<SigEntry>* heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc()->length(), mtCode);
3310 heap_sig->appendAll(ces.sig_cc());
3311 handler->set_sig_cc(heap_sig);
3312 heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc_ro()->length(), mtCode);
3313 heap_sig->appendAll(ces.sig_cc_ro());
3314 handler->set_sig_cc_ro(heap_sig);
3315 }
3316 // On zero there is no code to save and no need to create a blob and
3317 // or relocate the handler.
3318 int entry_offset[AdapterBlob::ENTRY_COUNT];
3319 address_to_offset(entry_address, entry_offset);
3320 #ifdef ASSERT
3321 if (VerifyAdapterSharing) {
3322 handler->save_code(buf->code_begin(), buffer.insts_size());
3323 if (is_transient) {
3324 return true;
3325 }
3326 }
3327 #endif
3328 if (adapter_blob == nullptr) {
3329 // CodeCache is full, disable compilation
3330 // Ought to log this but compile log is only per compile thread
3331 // and we're some non descript Java thread.
3332 return false;
3333 }
3334 handler->set_adapter_blob(adapter_blob);
3335 if (!is_transient && AOTCodeCache::is_dumping_adapter()) {
3336 // try to save generated code
3337 const char* name = AdapterHandlerLibrary::name(handler);
3338 const uint32_t id = AdapterHandlerLibrary::id(handler);
3339 bool success = AOTCodeCache::store_code_blob(*adapter_blob, AOTCodeEntry::Adapter, id, name);
3340 assert(success || !AOTCodeCache::is_dumping_adapter(), "caching of adapter must be disabled");
3341 }
3342 #endif // ZERO
3343
3344 #ifndef PRODUCT
3345 // debugging support
3346 if (PrintAdapterHandlers || PrintStubCode) {
3347 print_adapter_handler_info(tty, handler);
3348 }
3349 #endif
3350
3351 return true;
3352 }
3353
3354 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(CompiledEntrySignature& ces,
3355 bool allocate_code_blob,
3356 bool is_transient) {
3357 AdapterFingerPrint* fp = AdapterFingerPrint::allocate(ces.sig_cc(), ces.has_inline_recv());
3358 #ifdef ASSERT
3359 // Verify that we can successfully restore the compiled entry signature object.
3360 CompiledEntrySignature ces_verify;
3361 ces_verify.initialize_from_fingerprint(fp);
3362 #endif
3363 AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fp);
3364 if (!generate_adapter_code(handler, ces, allocate_code_blob, is_transient)) {
3365 AdapterHandlerEntry::deallocate(handler);
3366 return nullptr;
3367 }
3368 if (!is_transient) {
3369 assert_lock_strong(AdapterHandlerLibrary_lock);
3370 _adapter_handler_table->put(fp, handler);
3371 }
3372 return handler;
3373 }
3374
3375 #if INCLUDE_CDS
3376 void AdapterHandlerEntry::remove_unshareable_info() {
3377 #ifdef ASSERT
3378 _saved_code = nullptr;
3379 _saved_code_length = 0;
3380 #endif // ASSERT
3381 _adapter_blob = nullptr;
3382 _linked = false;
3383 _sig_cc = nullptr;
3384 _sig_cc_ro = nullptr;
3385 }
3386
3387 class CopyAdapterTableToArchive : StackObj {
3388 private:
3389 CompactHashtableWriter* _writer;
3390 ArchiveBuilder* _builder;
3391 public:
3392 CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer),
3393 _builder(ArchiveBuilder::current())
3394 {}
3395
3396 bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) {
3397 LogStreamHandle(Trace, aot) lsh;
3398 if (ArchiveBuilder::current()->has_been_archived((address)entry)) {
3399 assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be");
3400 AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp);
3401 assert(buffered_fp != nullptr,"sanity check");
3402 AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry);
3403 assert(buffered_entry != nullptr,"sanity check");
3404
3444 }
3445 #endif
3446 }
3447
3448 // This method is used during production run to link archived adapters (stored in AOT Cache)
3449 // to their code in AOT Code Cache
3450 void AdapterHandlerEntry::link() {
3451 ResourceMark rm;
3452 assert(_fingerprint != nullptr, "_fingerprint must not be null");
3453 bool generate_code = false;
3454 // Generate code only if AOTCodeCache is not available, or
3455 // caching adapters is disabled, or we fail to link
3456 // the AdapterHandlerEntry to its code in the AOTCodeCache
3457 if (AOTCodeCache::is_using_adapter()) {
3458 AdapterHandlerLibrary::link_aot_adapter_handler(this);
3459 // If link_aot_adapter_handler() succeeds, _adapter_blob will be non-null
3460 if (_adapter_blob == nullptr) {
3461 log_warning(aot)("Failed to link AdapterHandlerEntry (fp=%s) to its code in the AOT code cache", _fingerprint->as_basic_args_string());
3462 generate_code = true;
3463 }
3464
3465 if (get_sig_cc() == nullptr) {
3466 // Calling conventions have to be regenerated at runtime and are accessed through method adapters,
3467 // which are archived in the AOT code cache. If the adapters are not regenerated, the
3468 // calling conventions should be regenerated here.
3469 CompiledEntrySignature ces;
3470 ces.initialize_from_fingerprint(_fingerprint);
3471 if (ces.has_scalarized_args()) {
3472 // Save a C heap allocated version of the scalarized signature and store it in the adapter
3473 GrowableArray<SigEntry>* heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc()->length(), mtCode);
3474 heap_sig->appendAll(ces.sig_cc());
3475 set_sig_cc(heap_sig);
3476 heap_sig = new (mtCode) GrowableArray<SigEntry>(ces.sig_cc_ro()->length(), mtCode);
3477 heap_sig->appendAll(ces.sig_cc_ro());
3478 set_sig_cc_ro(heap_sig);
3479 }
3480 }
3481 } else {
3482 generate_code = true;
3483 }
3484 if (generate_code) {
3485 CompiledEntrySignature ces;
3486 ces.initialize_from_fingerprint(_fingerprint);
3487 if (!AdapterHandlerLibrary::generate_adapter_code(this, ces, true, false)) {
3488 // Don't throw exceptions during VM initialization because java.lang.* classes
3489 // might not have been initialized, causing problems when constructing the
3490 // Java exception object.
3491 vm_exit_during_initialization("Out of space in CodeCache for adapters");
3492 }
3493 }
3494 if (_adapter_blob != nullptr) {
3495 post_adapter_creation(this);
3496 }
3497 assert(_linked, "AdapterHandlerEntry must now be linked");
3498 }
3499
3500 void AdapterHandlerLibrary::link_aot_adapters() {
3501 uint max_id = 0;
3502 assert(AOTCodeCache::is_using_adapter(), "AOT adapters code should be available");
3503 /* It is possible that some adapters generated in assembly phase are not stored in the cache.
3504 * That implies adapter ids of the adapters in the cache may not be contiguous.
3505 * If the size of the _aot_adapter_handler_table is used to initialize _id_counter, then it may
3506 * result in collision of adapter ids between AOT stored handlers and runtime generated handlers.
3507 * To avoid such situation, initialize the _id_counter with the largest adapter id among the AOT stored handlers.
3508 */
3509 _aot_adapter_handler_table.iterate_all([&](AdapterHandlerEntry* entry) {
3510 assert(!entry->is_linked(), "AdapterHandlerEntry is already linked!");
3511 entry->link();
3512 max_id = MAX2(max_id, entry->id());
3513 });
3514 // Set adapter id to the maximum id found in the AOTCache
3515 assert(_id_counter == 0, "Did not expect new AdapterHandlerEntry to be created at this stage");
3516 _id_counter = max_id;
3517 }
3518
3519 // This method is called during production run to lookup simple adapters
3520 // in the archived adapter handler table
3521 void AdapterHandlerLibrary::lookup_simple_adapters() {
3522 assert(!_aot_adapter_handler_table.empty(), "archived adapter handler table is empty");
3523
3524 MutexLocker mu(AdapterHandlerLibrary_lock);
3525 ResourceMark rm;
3526 CompiledEntrySignature no_args;
3527 no_args.compute_calling_conventions();
3528 _no_arg_handler = lookup(no_args.sig_cc(), no_args.has_inline_recv());
3529
3530 CompiledEntrySignature obj_args;
3531 SigEntry::add_entry(obj_args.sig(), T_OBJECT);
3532 obj_args.compute_calling_conventions();
3533 _obj_arg_handler = lookup(obj_args.sig_cc(), obj_args.has_inline_recv());
3534
3535 CompiledEntrySignature int_args;
3536 SigEntry::add_entry(int_args.sig(), T_INT);
3537 int_args.compute_calling_conventions();
3538 _int_arg_handler = lookup(int_args.sig_cc(), int_args.has_inline_recv());
3539
3540 CompiledEntrySignature obj_int_args;
3541 SigEntry::add_entry(obj_int_args.sig(), T_OBJECT);
3542 SigEntry::add_entry(obj_int_args.sig(), T_INT);
3543 obj_int_args.compute_calling_conventions();
3544 _obj_int_arg_handler = lookup(obj_int_args.sig_cc(), obj_int_args.has_inline_recv());
3545
3546 CompiledEntrySignature obj_obj_args;
3547 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
3548 SigEntry::add_entry(obj_obj_args.sig(), T_OBJECT);
3549 obj_obj_args.compute_calling_conventions();
3550 _obj_obj_arg_handler = lookup(obj_obj_args.sig_cc(), obj_obj_args.has_inline_recv());
3551
3552 assert(_no_arg_handler != nullptr &&
3553 _obj_arg_handler != nullptr &&
3554 _int_arg_handler != nullptr &&
3555 _obj_int_arg_handler != nullptr &&
3556 _obj_obj_arg_handler != nullptr, "Initial adapters not found in archived adapter handler table");
3557 assert(_no_arg_handler->is_linked() &&
3558 _obj_arg_handler->is_linked() &&
3559 _int_arg_handler->is_linked() &&
3560 _obj_int_arg_handler->is_linked() &&
3561 _obj_obj_arg_handler->is_linked(), "Initial adapters not in linked state");
3562 }
3563 #endif // INCLUDE_CDS
3564
3565 void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) {
3566 LogStreamHandle(Trace, aot) lsh;
3567 if (lsh.is_enabled()) {
3568 lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string());
3569 lsh.cr();
3570 }
3571 it->push(&_fingerprint);
3572 }
3573
3574 AdapterHandlerEntry::~AdapterHandlerEntry() {
3575 if (_fingerprint != nullptr) {
3576 AdapterFingerPrint::deallocate(_fingerprint);
3577 _fingerprint = nullptr;
3578 }
3579 if (_sig_cc != nullptr) {
3580 delete _sig_cc;
3581 }
3582 if (_sig_cc_ro != nullptr) {
3583 delete _sig_cc_ro;
3584 }
3585 #ifdef ASSERT
3586 FREE_C_HEAP_ARRAY(_saved_code);
3587 #endif
3588 FreeHeap(this);
3589 }
3590
3591
3592 #ifdef ASSERT
3593 // Capture the code before relocation so that it can be compared
3594 // against other versions. If the code is captured after relocation
3595 // then relative instructions won't be equivalent.
3596 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
3597 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
3598 _saved_code_length = length;
3599 memcpy(_saved_code, buffer, length);
3600 }
3601
3602
3603 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
3604 assert(_saved_code != nullptr && other->_saved_code != nullptr, "code not saved");
3654 struct { double data[20]; } stubs_locs_buf;
3655 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
3656 #if defined(AARCH64)
3657 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
3658 // in the constant pool to ensure ordering between the barrier and oops
3659 // accesses. For native_wrappers we need a constant.
3660 buffer.initialize_consts_size(8);
3661 #elif defined(PPC64) || defined(S390)
3662 // On PPC64/S390 the continuation enter intrinsic needs the constant pool for the compiled
3663 // static java call that is resolved in the runtime.
3664 if (method->is_continuation_enter_intrinsic()) {
3665 buffer.initialize_consts_size(8 PPC64_ONLY(+ 24) S390_ONLY(+ 17));
3666 }
3667 #endif
3668 buffer.stubs()->initialize_shared_locs((relocInfo*)&stubs_locs_buf, sizeof(stubs_locs_buf) / sizeof(relocInfo));
3669 MacroAssembler _masm(&buffer);
3670
3671 // Fill in the signature array, for the calling-convention call.
3672 const int total_args_passed = method->size_of_parameters();
3673
3674 BasicType stack_sig_bt[16];
3675 VMRegPair stack_regs[16];
3676 BasicType* sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
3677 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
3678
3679 int i = 0;
3680 if (!method->is_static()) { // Pass in receiver first
3681 sig_bt[i++] = T_OBJECT;
3682 }
3683 SignatureStream ss(method->signature());
3684 for (; !ss.at_return_type(); ss.next()) {
3685 sig_bt[i++] = ss.type(); // Collect remaining bits of signature
3686 if (ss.type() == T_LONG || ss.type() == T_DOUBLE) {
3687 sig_bt[i++] = T_VOID; // Longs & doubles take 2 Java slots
3688 }
3689 }
3690 assert(i == total_args_passed, "");
3691 BasicType ret_type = ss.type();
3692
3693 // Now get the compiled-Java arguments layout.
3694 SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
3695
3696 // Generate the compiled-to-native wrapper code
3697 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type);
3698
3699 if (nm != nullptr) {
3700 {
3701 MutexLocker pl(NMethodState_lock, Mutex::_no_safepoint_check_flag);
3702 if (nm->make_in_use()) {
3703 method->set_code(method, nm);
3704 }
3705 }
3706
3707 CompilerDirectiveMatcher matcher(method, CompLevel_simple);
3708 if (matcher.directive_set()->PrintAssemblyOption) {
3709 nm->print_code();
3710 }
3711 }
3918 if (b == handler->adapter_blob()) {
3919 found = true;
3920 st->print("Adapter for signature: ");
3921 handler->print_adapter_on(st);
3922 return false; // abort iteration
3923 } else {
3924 return true; // keep looking
3925 }
3926 };
3927 assert_locked_or_safepoint(AdapterHandlerLibrary_lock);
3928 _adapter_handler_table->iterate(findblob_runtime_table);
3929 }
3930 assert(found, "Should have found handler");
3931 }
3932
3933 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3934 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3935 if (adapter_blob() != nullptr) {
3936 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3937 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3938 st->print(" c2iVE: " INTPTR_FORMAT, p2i(get_c2i_inline_entry()));
3939 st->print(" c2iVROE: " INTPTR_FORMAT, p2i(get_c2i_inline_ro_entry()));
3940 st->print(" c2iUE: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3941 st->print(" c2iUVE: " INTPTR_FORMAT, p2i(get_c2i_unverified_inline_entry()));
3942 if (get_c2i_no_clinit_check_entry() != nullptr) {
3943 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3944 }
3945 }
3946 st->cr();
3947 }
3948
3949 #ifndef PRODUCT
3950
3951 void AdapterHandlerLibrary::print_statistics() {
3952 print_table_statistics();
3953 }
3954
3955 #endif /* PRODUCT */
3956
3957 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3958 assert(current == JavaThread::current(), "pre-condition");
3959 StackOverflow* overflow_state = current->stack_overflow_state();
3960 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3961 overflow_state->set_reserved_stack_activation(current->stack_base());
4008 event.set_method(method);
4009 event.commit();
4010 }
4011 }
4012 }
4013 return activation;
4014 }
4015
4016 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
4017 // After any safepoint, just before going back to compiled code,
4018 // we inform the GC that we will be doing initializing writes to
4019 // this object in the future without emitting card-marks, so
4020 // GC may take any compensating steps.
4021
4022 oop new_obj = current->vm_result_oop();
4023 if (new_obj == nullptr) return;
4024
4025 BarrierSet *bs = BarrierSet::barrier_set();
4026 bs->on_slowpath_allocation_exit(current, new_obj);
4027 }
4028
4029 // We are at a compiled code to interpreter call. We need backing
4030 // buffers for all inline type arguments. Allocate an object array to
4031 // hold them (convenient because once we're done with it we don't have
4032 // to worry about freeing it).
4033 oop SharedRuntime::allocate_inline_types_impl(JavaThread* current, methodHandle callee, bool allocate_receiver, bool from_c1, TRAPS) {
4034 assert(InlineTypePassFieldsAsArgs, "no reason to call this");
4035 ResourceMark rm;
4036
4037 // Retrieve arguments passed at the call
4038 RegisterMap reg_map2(THREAD,
4039 RegisterMap::UpdateMap::include,
4040 RegisterMap::ProcessFrames::include,
4041 RegisterMap::WalkContinuation::skip);
4042 frame stubFrame = THREAD->last_frame();
4043 frame callerFrame = stubFrame.sender(®_map2);
4044 if (from_c1) {
4045 callerFrame = callerFrame.sender(®_map2);
4046 }
4047 int arg_size;
4048 const GrowableArray<SigEntry>* sig = allocate_receiver ? callee->adapter()->get_sig_cc() : callee->adapter()->get_sig_cc_ro();
4049 assert(sig != nullptr, "sig should never be null");
4050 TempNewSymbol tmp_sig = SigEntry::create_symbol(sig);
4051 VMRegPair* reg_pairs = find_callee_arguments(tmp_sig, false, false, &arg_size);
4052
4053 int nb_slots = 0;
4054 InstanceKlass* holder = callee->method_holder();
4055 allocate_receiver &= !callee->is_static() && holder->is_inline_klass() && callee->is_scalarized_arg(0);
4056 if (allocate_receiver) {
4057 nb_slots++;
4058 }
4059 int arg_num = callee->is_static() ? 0 : 1;
4060 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
4061 BasicType bt = ss.type();
4062 if (bt == T_OBJECT && callee->is_scalarized_arg(arg_num)) {
4063 nb_slots++;
4064 }
4065 if (bt != T_VOID) {
4066 arg_num++;
4067 }
4068 }
4069 objArrayOop array_oop = nullptr;
4070 objArrayHandle array;
4071 arg_num = callee->is_static() ? 0 : 1;
4072 int i = 0;
4073 uint pos = 0;
4074 uint depth = 0;
4075 uint ignored = 0;
4076 if (allocate_receiver) {
4077 assert(sig->at(pos)._bt == T_METADATA, "scalarized value expected");
4078 pos++;
4079 ignored++;
4080 depth++;
4081 assert(sig->at(pos)._bt == T_OBJECT, "buffer argument");
4082 uint reg_pos = 0;
4083 assert(reg_pos < (uint)arg_size, "");
4084 VMRegPair reg_pair = reg_pairs[reg_pos];
4085 oop* buffer = callerFrame.oopmapreg_to_oop_location(reg_pair.first(), ®_map2);
4086 instanceHandle h_buffer(THREAD, (instanceOop)*buffer);
4087 InlineKlass* vk = InlineKlass::cast(holder);
4088 if (h_buffer.not_null()) {
4089 assert(h_buffer->klass() == vk, "buffer not of expected class");
4090 } else {
4091 // Only allocate if buffer passed at the call is null
4092 if (array_oop == nullptr) {
4093 array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
4094 array = objArrayHandle(THREAD, array_oop);
4095 }
4096 oop res = vk->allocate_instance(CHECK_NULL);
4097 array->obj_at_put(i, res);
4098 }
4099 i++;
4100 }
4101 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
4102 BasicType bt = ss.type();
4103 if (bt == T_OBJECT && callee->is_scalarized_arg(arg_num)) {
4104 while (true) {
4105 BasicType bt = sig->at(pos)._bt;
4106 if (bt == T_METADATA) {
4107 depth++;
4108 ignored++;
4109 if (depth == 1) {
4110 break;
4111 }
4112 } else if (bt == T_VOID && sig->at(pos - 1)._bt != T_LONG && sig->at(pos - 1)._bt != T_DOUBLE) {
4113 ignored++;
4114 depth--;
4115 }
4116 pos++;
4117 }
4118 pos++;
4119 assert(sig->at(pos)._bt == T_OBJECT, "buffer argument expected");
4120 uint reg_pos = pos - ignored;
4121 assert(reg_pos < (uint)arg_size, "out of bound register?");
4122 VMRegPair reg_pair = reg_pairs[reg_pos];
4123 oop* buffer = callerFrame.oopmapreg_to_oop_location(reg_pair.first(), ®_map2);
4124 instanceHandle h_buffer(THREAD, (instanceOop)*buffer);
4125 InlineKlass* vk = ss.as_inline_klass(holder);
4126 assert(vk != nullptr, "Unexpected klass");
4127 if (h_buffer.not_null()) {
4128 assert(h_buffer->klass() == vk, "buffer not of expected class");
4129 } else {
4130 // Only allocate if buffer passed at the call is null
4131 if (array_oop == nullptr) {
4132 array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
4133 array = objArrayHandle(THREAD, array_oop);
4134 }
4135 oop res = vk->allocate_instance(CHECK_NULL);
4136 array->obj_at_put(i, res);
4137 }
4138 i++;
4139 }
4140 if (bt != T_VOID) {
4141 arg_num++;
4142 }
4143 }
4144 return array();
4145 }
4146
4147 JRT_ENTRY(void, SharedRuntime::allocate_inline_types(JavaThread* current, Method* callee_method, bool allocate_receiver))
4148 methodHandle callee(current, callee_method);
4149 oop array = SharedRuntime::allocate_inline_types_impl(current, callee, allocate_receiver, false, CHECK);
4150 current->set_vm_result_oop(array);
4151 JRT_END
4152
4153 // We've returned to an interpreted method, the interpreter needs a
4154 // reference to an inline type instance. Allocate it and initialize it
4155 // from field's values in registers.
4156 JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread* current, intptr_t res))
4157 {
4158 if (!is_set_nth_bit(res, 0)) {
4159 // We're not returning with inline type fields in registers (the
4160 // calling convention didn't allow it for this inline klass)
4161 assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area");
4162 current->set_vm_result_oop((oopDesc*)res);
4163 current->set_vm_result_metadata(nullptr);
4164 return;
4165 }
4166
4167 clear_nth_bit(res, 0);
4168 InlineKlass* vk = (InlineKlass*)res;
4169 assert(Metaspace::contains((void*)res), "should be klass");
4170
4171 if (!vk->contains_oops()) {
4172 // No oop fields. Initialize the fields by calling the pack handler from
4173 // the stub which is much faster (see 'generate_return_value_stub').
4174 // Signal this by setting the metadata result to the value klass.
4175 JRT_BLOCK;
4176 {
4177 oop vt = vk->allocate_instance(CHECK);
4178 current->set_vm_result_oop(vt);
4179 current->set_vm_result_metadata(vk);
4180 }
4181 JRT_BLOCK_END;
4182 return;
4183 }
4184
4185 ResourceMark rm;
4186 RegisterMap reg_map(current,
4187 RegisterMap::UpdateMap::include,
4188 RegisterMap::ProcessFrames::include,
4189 RegisterMap::WalkContinuation::skip);
4190 frame stubFrame = current->last_frame();
4191 stubFrame.sender(®_map);
4192
4193 assert(vk == InlineKlass::returned_inline_klass(reg_map), "broken calling convention");
4194
4195 // Allocate handles for every oop field so they are safe in case of
4196 // a safepoint when allocating
4197 GrowableArray<Handle> handles;
4198 vk->save_oop_fields(reg_map, handles);
4199
4200 // It's unsafe to safepoint until we are here
4201 JRT_BLOCK;
4202 {
4203 oop vt = vk->realloc_result(reg_map, handles, CHECK);
4204 current->set_vm_result_oop(vt);
4205 current->set_vm_result_metadata(nullptr);
4206 }
4207 JRT_BLOCK_END;
4208 }
4209 JRT_END
|