27 #include "jvm.h"
28 #include "classfile/stringTable.hpp"
29 #include "classfile/vmClasses.hpp"
30 #include "classfile/vmSymbols.hpp"
31 #include "code/codeCache.hpp"
32 #include "code/compiledIC.hpp"
33 #include "code/icBuffer.hpp"
34 #include "code/compiledMethod.inline.hpp"
35 #include "code/scopeDesc.hpp"
36 #include "code/vtableStubs.hpp"
37 #include "compiler/abstractCompiler.hpp"
38 #include "compiler/compileBroker.hpp"
39 #include "compiler/disassembler.hpp"
40 #include "gc/shared/barrierSet.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "gc/shared/gcLocker.inline.hpp"
43 #include "interpreter/interpreter.hpp"
44 #include "interpreter/interpreterRuntime.hpp"
45 #include "jfr/jfrEvents.hpp"
46 #include "logging/log.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "memory/universe.hpp"
49 #include "oops/compiledICHolder.inline.hpp"
50 #include "oops/klass.hpp"
51 #include "oops/method.inline.hpp"
52 #include "oops/objArrayKlass.hpp"
53 #include "oops/oop.inline.hpp"
54 #include "prims/forte.hpp"
55 #include "prims/jvmtiExport.hpp"
56 #include "prims/methodHandles.hpp"
57 #include "prims/nativeLookup.hpp"
58 #include "runtime/atomic.hpp"
59 #include "runtime/frame.inline.hpp"
60 #include "runtime/handles.inline.hpp"
61 #include "runtime/init.hpp"
62 #include "runtime/interfaceSupport.inline.hpp"
63 #include "runtime/java.hpp"
64 #include "runtime/javaCalls.hpp"
65 #include "runtime/sharedRuntime.hpp"
66 #include "runtime/stackWatermarkSet.hpp"
67 #include "runtime/stubRoutines.hpp"
68 #include "runtime/synchronizer.hpp"
69 #include "runtime/vframe.inline.hpp"
70 #include "runtime/vframeArray.hpp"
71 #include "runtime/vm_version.hpp"
72 #include "utilities/copy.hpp"
73 #include "utilities/dtrace.hpp"
74 #include "utilities/events.hpp"
75 #include "utilities/hashtable.inline.hpp"
76 #include "utilities/macros.hpp"
77 #include "utilities/xmlstream.hpp"
78 #ifdef COMPILER1
79 #include "c1/c1_Runtime1.hpp"
80 #endif
81
82 // Shared stub locations
83 RuntimeStub* SharedRuntime::_wrong_method_blob;
84 RuntimeStub* SharedRuntime::_wrong_method_abstract_blob;
85 RuntimeStub* SharedRuntime::_ic_miss_blob;
86 RuntimeStub* SharedRuntime::_resolve_opt_virtual_call_blob;
87 RuntimeStub* SharedRuntime::_resolve_virtual_call_blob;
88 RuntimeStub* SharedRuntime::_resolve_static_call_blob;
89 address SharedRuntime::_resolve_static_call_entry;
90
91 DeoptimizationBlob* SharedRuntime::_deopt_blob;
92 SafepointBlob* SharedRuntime::_polling_page_vectors_safepoint_handler_blob;
93 SafepointBlob* SharedRuntime::_polling_page_safepoint_handler_blob;
94 SafepointBlob* SharedRuntime::_polling_page_return_handler_blob;
95
96 #ifdef COMPILER2
97 UncommonTrapBlob* SharedRuntime::_uncommon_trap_blob;
98 #endif // COMPILER2
99
100
101 //----------------------------generate_stubs-----------------------------------
102 void SharedRuntime::generate_stubs() {
103 _wrong_method_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method), "wrong_method_stub");
104 _wrong_method_abstract_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract), "wrong_method_abstract_stub");
105 _ic_miss_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss), "ic_miss_stub");
106 _resolve_opt_virtual_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C), "resolve_opt_virtual_call");
107 _resolve_virtual_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C), "resolve_virtual_call");
108 _resolve_static_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C), "resolve_static_call");
109 _resolve_static_call_entry = _resolve_static_call_blob->entry_point();
110
111 AdapterHandlerLibrary::initialize();
112
113 #if COMPILER2_OR_JVMCI
114 // Vectors are generated only by C2 and JVMCI.
115 bool support_wide = is_wide_vector(MaxVectorSize);
116 if (support_wide) {
117 _polling_page_vectors_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_VECTOR_LOOP);
118 }
119 #endif // COMPILER2_OR_JVMCI
120 _polling_page_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_LOOP);
121 _polling_page_return_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_RETURN);
122
123 generate_deopt_blob();
124
125 #ifdef COMPILER2
126 generate_uncommon_trap_blob();
127 #endif // COMPILER2
128 }
129
969 // forwarded before we look at the return value.
970 THROW_(vmSymbols::java_lang_UnsatisfiedLinkError(), (void*)badAddress);
971 }
972 JNI_END
973
974 address SharedRuntime::native_method_throw_unsatisfied_link_error_entry() {
975 return CAST_FROM_FN_PTR(address, &throw_unsatisfied_link_error);
976 }
977
978 JRT_ENTRY_NO_ASYNC(void, SharedRuntime::register_finalizer(JavaThread* current, oopDesc* obj))
979 #if INCLUDE_JVMCI
980 if (!obj->klass()->has_finalizer()) {
981 return;
982 }
983 #endif // INCLUDE_JVMCI
984 assert(oopDesc::is_oop(obj), "must be a valid oop");
985 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
986 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
987 JRT_END
988
989 jlong SharedRuntime::get_java_tid(Thread* thread) {
990 if (thread != NULL) {
991 if (thread->is_Java_thread()) {
992 oop obj = JavaThread::cast(thread)->threadObj();
993 return (obj == NULL) ? 0 : java_lang_Thread::thread_id(obj);
994 }
995 }
996 return 0;
997 }
998
999 /**
1000 * This function ought to be a void function, but cannot be because
1001 * it gets turned into a tail-call on sparc, which runs into dtrace bug
1002 * 6254741. Once that is fixed we can remove the dummy return value.
1003 */
1004 int SharedRuntime::dtrace_object_alloc(oopDesc* o, int size) {
1005 return dtrace_object_alloc_base(Thread::current(), o, size);
1006 }
1007
1008 int SharedRuntime::dtrace_object_alloc_base(Thread* thread, oopDesc* o, int size) {
1069 return caller->attached_method_before_pc(pc);
1070 }
1071 return NULL;
1072 }
1073
1074 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode
1075 // for a call current in progress, i.e., arguments has been pushed on stack
1076 // but callee has not been invoked yet. Caller frame must be compiled.
1077 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1078 CallInfo& callinfo, TRAPS) {
1079 Handle receiver;
1080 Handle nullHandle; // create a handy null handle for exception returns
1081 JavaThread* current = THREAD;
1082
1083 assert(!vfst.at_end(), "Java frame must exist");
1084
1085 // Find caller and bci from vframe
1086 methodHandle caller(current, vfst.method());
1087 int bci = vfst.bci();
1088
1089 Bytecode_invoke bytecode(caller, bci);
1090 int bytecode_index = bytecode.index();
1091 bc = bytecode.invoke_code();
1092
1093 methodHandle attached_method(current, extract_attached_method(vfst));
1094 if (attached_method.not_null()) {
1095 Method* callee = bytecode.static_target(CHECK_NH);
1096 vmIntrinsics::ID id = callee->intrinsic_id();
1097 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1098 // it attaches statically resolved method to the call site.
1099 if (MethodHandles::is_signature_polymorphic(id) &&
1100 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1101 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1102
1103 // Adjust invocation mode according to the attached method.
1104 switch (bc) {
1105 case Bytecodes::_invokevirtual:
1106 if (attached_method->method_holder()->is_interface()) {
1107 bc = Bytecodes::_invokeinterface;
1108 }
1109 break;
1110 case Bytecodes::_invokeinterface:
1111 if (!attached_method->method_holder()->is_interface()) {
1112 bc = Bytecodes::_invokevirtual;
1113 }
1114 break;
1115 case Bytecodes::_invokehandle:
1116 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1117 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1118 : Bytecodes::_invokevirtual;
1119 }
1120 break;
1121 default:
1122 break;
1123 }
1124 }
1125 }
1126
1127 assert(bc != Bytecodes::_illegal, "not initialized");
1128
1129 bool has_receiver = bc != Bytecodes::_invokestatic &&
1130 bc != Bytecodes::_invokedynamic &&
1131 bc != Bytecodes::_invokehandle;
1132
1133 // Find receiver for non-static call
1134 if (has_receiver) {
1135 // This register map must be update since we need to find the receiver for
1136 // compiled frames. The receiver might be in a register.
1137 RegisterMap reg_map2(current);
1138 frame stubFrame = current->last_frame();
1139 // Caller-frame is a compiled frame
1140 frame callerFrame = stubFrame.sender(®_map2);
1141
1142 if (attached_method.is_null()) {
1143 Method* callee = bytecode.static_target(CHECK_NH);
1144 if (callee == NULL) {
1145 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1146 }
1147 }
1148
1149 // Retrieve from a compiled argument list
1150 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1151
1152 if (receiver.is_null()) {
1153 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1154 }
1155 }
1156
1157 // Resolve method
1158 if (attached_method.not_null()) {
1159 // Parameterized by attached method.
1160 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, CHECK_NH);
1161 } else {
1162 // Parameterized by bytecode.
1163 constantPoolHandle constants(current, caller->constants());
1164 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1165 }
1166
1167 #ifdef ASSERT
1168 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1169 if (has_receiver) {
1170 assert(receiver.not_null(), "should have thrown exception");
1171 Klass* receiver_klass = receiver->klass();
1172 Klass* rk = NULL;
1173 if (attached_method.not_null()) {
1174 // In case there's resolved method attached, use its holder during the check.
1175 rk = attached_method->method_holder();
1176 } else {
1177 // Klass is already loaded.
1178 constantPoolHandle constants(current, caller->constants());
1179 rk = constants->klass_ref_at(bytecode_index, CHECK_NH);
1180 }
1181 Klass* static_receiver_klass = rk;
1182 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1183 "actual receiver must be subclass of static receiver klass");
1184 if (receiver_klass->is_instance_klass()) {
1185 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1186 tty->print_cr("ERROR: Klass not yet initialized!!");
1187 receiver_klass->print();
1188 }
1189 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1207 // Hence the stack can only contain an entry_frame. We need to
1208 // find the target method from the stub frame.
1209 RegisterMap reg_map(current, false);
1210 frame fr = current->last_frame();
1211 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1212 fr = fr.sender(®_map);
1213 assert(fr.is_entry_frame(), "must be");
1214 // fr is now pointing to the entry frame.
1215 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1216 } else {
1217 Bytecodes::Code bc;
1218 CallInfo callinfo;
1219 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1220 callee_method = methodHandle(current, callinfo.selected_method());
1221 }
1222 assert(callee_method()->is_method(), "must be");
1223 return callee_method;
1224 }
1225
1226 // Resolves a call.
1227 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, TRAPS) {
1228 methodHandle callee_method;
1229 callee_method = resolve_sub_helper(is_virtual, is_optimized, THREAD);
1230 if (JvmtiExport::can_hotswap_or_post_breakpoint()) {
1231 int retry_count = 0;
1232 while (!HAS_PENDING_EXCEPTION && callee_method->is_old() &&
1233 callee_method->method_holder() != vmClasses::Object_klass()) {
1234 // If has a pending exception then there is no need to re-try to
1235 // resolve this method.
1236 // If the method has been redefined, we need to try again.
1237 // Hack: we have no way to update the vtables of arrays, so don't
1238 // require that java.lang.Object has been updated.
1239
1240 // It is very unlikely that method is redefined more than 100 times
1241 // in the middle of resolve. If it is looping here more than 100 times
1242 // means then there could be a bug here.
1243 guarantee((retry_count++ < 100),
1244 "Could not resolve to latest version of redefined method");
1245 // method is redefined in the middle of resolve so re-try.
1246 callee_method = resolve_sub_helper(is_virtual, is_optimized, THREAD);
1247 }
1248 }
1249 return callee_method;
1250 }
1251
1252 // This fails if resolution required refilling of IC stubs
1253 bool SharedRuntime::resolve_sub_helper_internal(methodHandle callee_method, const frame& caller_frame,
1254 CompiledMethod* caller_nm, bool is_virtual, bool is_optimized,
1255 Handle receiver, CallInfo& call_info, Bytecodes::Code invoke_code, TRAPS) {
1256 StaticCallInfo static_call_info;
1257 CompiledICInfo virtual_call_info;
1258
1259 // Make sure the callee nmethod does not get deoptimized and removed before
1260 // we are done patching the code.
1261 CompiledMethod* callee = callee_method->code();
1262
1263 if (callee != NULL) {
1264 assert(callee->is_compiled(), "must be nmethod for patching");
1265 }
1266
1267 if (callee != NULL && !callee->is_in_use()) {
1268 // Patch call site to C2I adapter if callee nmethod is deoptimized or unloaded.
1269 callee = NULL;
1270 }
1271 nmethodLocker nl_callee(callee);
1272 #ifdef ASSERT
1273 address dest_entry_point = callee == NULL ? 0 : callee->entry_point(); // used below
1274 #endif
1275
1276 bool is_nmethod = caller_nm->is_nmethod();
1277
1278 if (is_virtual) {
1279 assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check");
1280 bool static_bound = call_info.resolved_method()->can_be_statically_bound();
1281 Klass* klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass();
1282 CompiledIC::compute_monomorphic_entry(callee_method, klass,
1283 is_optimized, static_bound, is_nmethod, virtual_call_info,
1284 CHECK_false);
1285 } else {
1286 // static call
1287 CompiledStaticCall::compute_entry(callee_method, is_nmethod, static_call_info);
1288 }
1289
1290 // grab lock, check for deoptimization and potentially patch caller
1291 {
1292 CompiledICLocker ml(caller_nm);
1293
1294 // Lock blocks for safepoint during which both nmethods can change state.
1295
1296 // Now that we are ready to patch if the Method* was redefined then
1297 // don't update call site and let the caller retry.
1298 // Don't update call site if callee nmethod was unloaded or deoptimized.
1299 // Don't update call site if callee nmethod was replaced by an other nmethod
1300 // which may happen when multiply alive nmethod (tiered compilation)
1301 // will be supported.
1302 if (!callee_method->is_old() &&
1303 (callee == NULL || (callee->is_in_use() && callee_method->code() == callee))) {
1304 NoSafepointVerifier nsv;
1305 #ifdef ASSERT
1306 // We must not try to patch to jump to an already unloaded method.
1307 if (dest_entry_point != 0) {
1317 return false;
1318 }
1319 }
1320 } else {
1321 if (VM_Version::supports_fast_class_init_checks() &&
1322 invoke_code == Bytecodes::_invokestatic &&
1323 callee_method->needs_clinit_barrier() &&
1324 callee != NULL && callee->is_compiled_by_jvmci()) {
1325 return true; // skip patching for JVMCI
1326 }
1327 CompiledStaticCall* ssc = caller_nm->compiledStaticCall_before(caller_frame.pc());
1328 if (ssc->is_clean()) ssc->set(static_call_info);
1329 }
1330 }
1331 } // unlock CompiledICLocker
1332 return true;
1333 }
1334
1335 // Resolves a call. The compilers generate code for calls that go here
1336 // and are patched with the real destination of the call.
1337 methodHandle SharedRuntime::resolve_sub_helper(bool is_virtual, bool is_optimized, TRAPS) {
1338 JavaThread* current = THREAD;
1339 ResourceMark rm(current);
1340 RegisterMap cbl_map(current, false);
1341 frame caller_frame = current->last_frame().sender(&cbl_map);
1342
1343 CodeBlob* caller_cb = caller_frame.cb();
1344 guarantee(caller_cb != NULL && caller_cb->is_compiled(), "must be called from compiled method");
1345 CompiledMethod* caller_nm = caller_cb->as_compiled_method_or_null();
1346
1347 // make sure caller is not getting deoptimized
1348 // and removed before we are done with it.
1349 // CLEANUP - with lazy deopt shouldn't need this lock
1350 nmethodLocker caller_lock(caller_nm);
1351
1352 // determine call info & receiver
1353 // note: a) receiver is NULL for static calls
1354 // b) an exception is thrown if receiver is NULL for non-static calls
1355 CallInfo call_info;
1356 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1357 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1358 methodHandle callee_method(current, call_info.selected_method());
1359
1360 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1361 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1362 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1363 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1364 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1365
1422 bool successful = resolve_sub_helper_internal(callee_method, caller_frame, caller_nm,
1423 is_virtual, is_optimized, receiver,
1424 call_info, invoke_code, CHECK_(methodHandle()));
1425 if (successful) {
1426 return callee_method;
1427 } else {
1428 InlineCacheBuffer::refill_ic_stubs();
1429 }
1430 }
1431
1432 }
1433
1434
1435 // Inline caches exist only in compiled code
1436 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1437 #ifdef ASSERT
1438 RegisterMap reg_map(current, false);
1439 frame stub_frame = current->last_frame();
1440 assert(stub_frame.is_runtime_frame(), "sanity check");
1441 frame caller_frame = stub_frame.sender(®_map);
1442 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_optimized_entry_frame(), "unexpected frame");
1443 #endif /* ASSERT */
1444
1445 methodHandle callee_method;
1446 JRT_BLOCK
1447 callee_method = SharedRuntime::handle_ic_miss_helper(CHECK_NULL);
1448 // Return Method* through TLS
1449 current->set_vm_result_2(callee_method());
1450 JRT_BLOCK_END
1451 // return compiled code entry point after potential safepoints
1452 assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
1453 return callee_method->verified_code_entry();
1454 JRT_END
1455
1456
1457 // Handle call site that has been made non-entrant
1458 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1459 // 6243940 We might end up in here if the callee is deoptimized
1460 // as we race to call it. We don't want to take a safepoint if
1461 // the caller was interpreted because the caller frame will look
1462 // interpreted to the stack walkers and arguments are now
1463 // "compiled" so it is much better to make this transition
1464 // invisible to the stack walking code. The i2c path will
1465 // place the callee method in the callee_target. It is stashed
1466 // there because if we try and find the callee by normal means a
1467 // safepoint is possible and have trouble gc'ing the compiled args.
1468 RegisterMap reg_map(current, false);
1469 frame stub_frame = current->last_frame();
1470 assert(stub_frame.is_runtime_frame(), "sanity check");
1471 frame caller_frame = stub_frame.sender(®_map);
1472
1473 if (caller_frame.is_interpreted_frame() ||
1477 guarantee(callee != NULL && callee->is_method(), "bad handshake");
1478 current->set_vm_result_2(callee);
1479 current->set_callee_target(NULL);
1480 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1481 // Bypass class initialization checks in c2i when caller is in native.
1482 // JNI calls to static methods don't have class initialization checks.
1483 // Fast class initialization checks are present in c2i adapters and call into
1484 // SharedRuntime::handle_wrong_method() on the slow path.
1485 //
1486 // JVM upcalls may land here as well, but there's a proper check present in
1487 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1488 // so bypassing it in c2i adapter is benign.
1489 return callee->get_c2i_no_clinit_check_entry();
1490 } else {
1491 return callee->get_c2i_entry();
1492 }
1493 }
1494
1495 // Must be compiled to compiled path which is safe to stackwalk
1496 methodHandle callee_method;
1497 JRT_BLOCK
1498 // Force resolving of caller (if we called from compiled frame)
1499 callee_method = SharedRuntime::reresolve_call_site(CHECK_NULL);
1500 current->set_vm_result_2(callee_method());
1501 JRT_BLOCK_END
1502 // return compiled code entry point after potential safepoints
1503 assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
1504 return callee_method->verified_code_entry();
1505 JRT_END
1506
1507 // Handle abstract method call
1508 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1509 // Verbose error message for AbstractMethodError.
1510 // Get the called method from the invoke bytecode.
1511 vframeStream vfst(current, true);
1512 assert(!vfst.at_end(), "Java frame must exist");
1513 methodHandle caller(current, vfst.method());
1514 Bytecode_invoke invoke(caller, vfst.bci());
1515 DEBUG_ONLY( invoke.verify(); )
1516
1517 // Find the compiled caller frame.
1518 RegisterMap reg_map(current);
1519 frame stubFrame = current->last_frame();
1520 assert(stubFrame.is_runtime_frame(), "must be");
1521 frame callerFrame = stubFrame.sender(®_map);
1522 assert(callerFrame.is_compiled_frame(), "must be");
1523
1524 // Install exception and return forward entry.
1525 address res = StubRoutines::throw_AbstractMethodError_entry();
1526 JRT_BLOCK
1527 methodHandle callee(current, invoke.static_target(current));
1528 if (!callee.is_null()) {
1529 oop recv = callerFrame.retrieve_receiver(®_map);
1530 Klass *recv_klass = (recv != NULL) ? recv->klass() : NULL;
1531 res = StubRoutines::forward_exception_entry();
1532 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1533 }
1534 JRT_BLOCK_END
1535 return res;
1536 JRT_END
1537
1538
1539 // resolve a static call and patch code
1540 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1541 methodHandle callee_method;
1542 JRT_BLOCK
1543 callee_method = SharedRuntime::resolve_helper(false, false, CHECK_NULL);
1544 current->set_vm_result_2(callee_method());
1545 JRT_BLOCK_END
1546 // return compiled code entry point after potential safepoints
1547 assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
1548 return callee_method->verified_code_entry();
1549 JRT_END
1550
1551
1552 // resolve virtual call and update inline cache to monomorphic
1553 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1554 methodHandle callee_method;
1555 JRT_BLOCK
1556 callee_method = SharedRuntime::resolve_helper(true, false, CHECK_NULL);
1557 current->set_vm_result_2(callee_method());
1558 JRT_BLOCK_END
1559 // return compiled code entry point after potential safepoints
1560 assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
1561 return callee_method->verified_code_entry();
1562 JRT_END
1563
1564
1565 // Resolve a virtual call that can be statically bound (e.g., always
1566 // monomorphic, so it has no inline cache). Patch code to resolved target.
1567 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1568 methodHandle callee_method;
1569 JRT_BLOCK
1570 callee_method = SharedRuntime::resolve_helper(true, true, CHECK_NULL);
1571 current->set_vm_result_2(callee_method());
1572 JRT_BLOCK_END
1573 // return compiled code entry point after potential safepoints
1574 assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
1575 return callee_method->verified_code_entry();
1576 JRT_END
1577
1578 // The handle_ic_miss_helper_internal function returns false if it failed due
1579 // to either running out of vtable stubs or ic stubs due to IC transitions
1580 // to transitional states. The needs_ic_stub_refill value will be set if
1581 // the failure was due to running out of IC stubs, in which case handle_ic_miss_helper
1582 // refills the IC stubs and tries again.
1583 bool SharedRuntime::handle_ic_miss_helper_internal(Handle receiver, CompiledMethod* caller_nm,
1584 const frame& caller_frame, methodHandle callee_method,
1585 Bytecodes::Code bc, CallInfo& call_info,
1586 bool& needs_ic_stub_refill, TRAPS) {
1587 CompiledICLocker ml(caller_nm);
1588 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1589 bool should_be_mono = false;
1590 if (inline_cache->is_optimized()) {
1591 if (TraceCallFixup) {
1592 ResourceMark rm(THREAD);
1593 tty->print("OPTIMIZED IC miss (%s) call to", Bytecodes::name(bc));
1594 callee_method->print_short_name(tty);
1595 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1596 }
1597 should_be_mono = true;
1598 } else if (inline_cache->is_icholder_call()) {
1599 CompiledICHolder* ic_oop = inline_cache->cached_icholder();
1600 if (ic_oop != NULL) {
1601 if (!ic_oop->is_loader_alive()) {
1602 // Deferred IC cleaning due to concurrent class unloading
1603 if (!inline_cache->set_to_clean()) {
1604 needs_ic_stub_refill = true;
1605 return false;
1606 }
1607 } else if (receiver()->klass() == ic_oop->holder_klass()) {
1608 // This isn't a real miss. We must have seen that compiled code
1609 // is now available and we want the call site converted to a
1610 // monomorphic compiled call site.
1611 // We can't assert for callee_method->code() != NULL because it
1612 // could have been deoptimized in the meantime
1613 if (TraceCallFixup) {
1614 ResourceMark rm(THREAD);
1615 tty->print("FALSE IC miss (%s) converting to compiled call to", Bytecodes::name(bc));
1616 callee_method->print_short_name(tty);
1617 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1618 }
1619 should_be_mono = true;
1620 }
1621 }
1622 }
1623
1624 if (should_be_mono) {
1625 // We have a path that was monomorphic but was going interpreted
1626 // and now we have (or had) a compiled entry. We correct the IC
1627 // by using a new icBuffer.
1628 CompiledICInfo info;
1629 Klass* receiver_klass = receiver()->klass();
1630 inline_cache->compute_monomorphic_entry(callee_method,
1631 receiver_klass,
1632 inline_cache->is_optimized(),
1633 false, caller_nm->is_nmethod(),
1634 info, CHECK_false);
1635 if (!inline_cache->set_to_monomorphic(info)) {
1636 needs_ic_stub_refill = true;
1637 return false;
1638 }
1639 } else if (!inline_cache->is_megamorphic() && !inline_cache->is_clean()) {
1640 // Potential change to megamorphic
1641
1642 bool successful = inline_cache->set_to_megamorphic(&call_info, bc, needs_ic_stub_refill, CHECK_false);
1643 if (needs_ic_stub_refill) {
1644 return false;
1645 }
1646 if (!successful) {
1647 if (!inline_cache->set_to_clean()) {
1648 needs_ic_stub_refill = true;
1649 return false;
1650 }
1651 }
1652 } else {
1653 // Either clean or megamorphic
1654 }
1655 return true;
1656 }
1657
1658 methodHandle SharedRuntime::handle_ic_miss_helper(TRAPS) {
1659 JavaThread* current = THREAD;
1660 ResourceMark rm(current);
1661 CallInfo call_info;
1662 Bytecodes::Code bc;
1663
1664 // receiver is NULL for static calls. An exception is thrown for NULL
1665 // receivers for non-static calls
1666 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1667 // Compiler1 can produce virtual call sites that can actually be statically bound
1668 // If we fell thru to below we would think that the site was going megamorphic
1669 // when in fact the site can never miss. Worse because we'd think it was megamorphic
1670 // we'd try and do a vtable dispatch however methods that can be statically bound
1671 // don't have vtable entries (vtable_index < 0) and we'd blow up. So we force a
1672 // reresolution of the call site (as if we did a handle_wrong_method and not an
1673 // plain ic_miss) and the site will be converted to an optimized virtual call site
1674 // never to miss again. I don't believe C2 will produce code like this but if it
1675 // did this would still be the correct thing to do for it too, hence no ifdef.
1676 //
1677 if (call_info.resolved_method()->can_be_statically_bound()) {
1678 methodHandle callee_method = SharedRuntime::reresolve_call_site(CHECK_(methodHandle()));
1679 if (TraceCallFixup) {
1680 RegisterMap reg_map(current, false);
1681 frame caller_frame = current->last_frame().sender(®_map);
1682 ResourceMark rm(current);
1683 tty->print("converting IC miss to reresolve (%s) call to", Bytecodes::name(bc));
1684 callee_method->print_short_name(tty);
1685 tty->print_cr(" from pc: " INTPTR_FORMAT, p2i(caller_frame.pc()));
1686 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1687 }
1688 return callee_method;
1689 }
1690
1691 methodHandle callee_method(current, call_info.selected_method());
1692
1693 #ifndef PRODUCT
1694 Atomic::inc(&_ic_miss_ctr);
1695
1696 // Statistics & Tracing
1697 if (TraceCallFixup) {
1698 ResourceMark rm(current);
1708 // produce statistics under the lock
1709 trace_ic_miss(f.pc());
1710 }
1711 #endif
1712
1713 // install an event collector so that when a vtable stub is created the
1714 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1715 // event can't be posted when the stub is created as locks are held
1716 // - instead the event will be deferred until the event collector goes
1717 // out of scope.
1718 JvmtiDynamicCodeEventCollector event_collector;
1719
1720 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1721 // Transitioning IC caches may require transition stubs. If we run out
1722 // of transition stubs, we have to drop locks and perform a safepoint
1723 // that refills them.
1724 RegisterMap reg_map(current, false);
1725 frame caller_frame = current->last_frame().sender(®_map);
1726 CodeBlob* cb = caller_frame.cb();
1727 CompiledMethod* caller_nm = cb->as_compiled_method();
1728
1729 for (;;) {
1730 ICRefillVerifier ic_refill_verifier;
1731 bool needs_ic_stub_refill = false;
1732 bool successful = handle_ic_miss_helper_internal(receiver, caller_nm, caller_frame, callee_method,
1733 bc, call_info, needs_ic_stub_refill, CHECK_(methodHandle()));
1734 if (successful || !needs_ic_stub_refill) {
1735 return callee_method;
1736 } else {
1737 InlineCacheBuffer::refill_ic_stubs();
1738 }
1739 }
1740 }
1741
1742 static bool clear_ic_at_addr(CompiledMethod* caller_nm, address call_addr, bool is_static_call) {
1743 CompiledICLocker ml(caller_nm);
1744 if (is_static_call) {
1745 CompiledStaticCall* ssc = caller_nm->compiledStaticCall_at(call_addr);
1746 if (!ssc->is_clean()) {
1747 return ssc->set_to_clean();
1748 }
1749 } else {
1750 // compiled, dispatched call (which used to call an interpreted method)
1751 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1752 if (!inline_cache->is_clean()) {
1753 return inline_cache->set_to_clean();
1754 }
1755 }
1756 return true;
1757 }
1758
1759 //
1760 // Resets a call-site in compiled code so it will get resolved again.
1761 // This routines handles both virtual call sites, optimized virtual call
1762 // sites, and static call sites. Typically used to change a call sites
1763 // destination from compiled to interpreted.
1764 //
1765 methodHandle SharedRuntime::reresolve_call_site(TRAPS) {
1766 JavaThread* current = THREAD;
1767 ResourceMark rm(current);
1768 RegisterMap reg_map(current, false);
1769 frame stub_frame = current->last_frame();
1770 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1771 frame caller = stub_frame.sender(®_map);
1772
1773 // Do nothing if the frame isn't a live compiled frame.
1774 // nmethod could be deoptimized by the time we get here
1775 // so no update to the caller is needed.
1776
1777 if (caller.is_compiled_frame() && !caller.is_deoptimized_frame()) {
1778
1779 address pc = caller.pc();
1780
1781 // Check for static or virtual call
1782 bool is_static_call = false;
1783 CompiledMethod* caller_nm = CodeCache::find_compiled(pc);
1784
1785 // Default call_addr is the location of the "basic" call.
1786 // Determine the address of the call we a reresolving. With
1787 // Inline Caches we will always find a recognizable call.
1788 // With Inline Caches disabled we may or may not find a
1789 // recognizable call. We will always find a call for static
1790 // calls and for optimized virtual calls. For vanilla virtual
1791 // calls it depends on the state of the UseInlineCaches switch.
1792 //
1793 // With Inline Caches disabled we can get here for a virtual call
1794 // for two reasons:
1795 // 1 - calling an abstract method. The vtable for abstract methods
1796 // will run us thru handle_wrong_method and we will eventually
1797 // end up in the interpreter to throw the ame.
1798 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1799 // call and between the time we fetch the entry address and
1800 // we jump to it the target gets deoptimized. Similar to 1
1801 // we will wind up in the interprter (thru a c2i with c2).
1802 //
1803 address call_addr = NULL;
1807 CompiledICLocker ml(caller_nm);
1808 // Location of call instruction
1809 call_addr = caller_nm->call_instruction_address(pc);
1810 }
1811 // Make sure nmethod doesn't get deoptimized and removed until
1812 // this is done with it.
1813 // CLEANUP - with lazy deopt shouldn't need this lock
1814 nmethodLocker nmlock(caller_nm);
1815
1816 if (call_addr != NULL) {
1817 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1818 int ret = iter.next(); // Get item
1819 if (ret) {
1820 assert(iter.addr() == call_addr, "must find call");
1821 if (iter.type() == relocInfo::static_call_type) {
1822 is_static_call = true;
1823 } else {
1824 assert(iter.type() == relocInfo::virtual_call_type ||
1825 iter.type() == relocInfo::opt_virtual_call_type
1826 , "unexpected relocInfo. type");
1827 }
1828 } else {
1829 assert(!UseInlineCaches, "relocation info. must exist for this address");
1830 }
1831
1832 // Cleaning the inline cache will force a new resolve. This is more robust
1833 // than directly setting it to the new destination, since resolving of calls
1834 // is always done through the same code path. (experience shows that it
1835 // leads to very hard to track down bugs, if an inline cache gets updated
1836 // to a wrong method). It should not be performance critical, since the
1837 // resolve is only done once.
1838
1839 for (;;) {
1840 ICRefillVerifier ic_refill_verifier;
1841 if (!clear_ic_at_addr(caller_nm, call_addr, is_static_call)) {
1842 InlineCacheBuffer::refill_ic_stubs();
1843 } else {
1844 break;
1845 }
1846 }
1847 }
1848 }
1849
1850 methodHandle callee_method = find_callee_method(CHECK_(methodHandle()));
1851
1852
1853 #ifndef PRODUCT
1854 Atomic::inc(&_wrong_method_ctr);
1855
1856 if (TraceCallFixup) {
1857 ResourceMark rm(current);
1858 tty->print("handle_wrong_method reresolving call to");
1859 callee_method->print_short_name(tty);
1860 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1861 }
1862 #endif
1863
1864 return callee_method;
1865 }
1866
1867 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1868 // The faulting unsafe accesses should be changed to throw the error
1869 // synchronously instead. Meanwhile the faulting instruction will be
1870 // skipped over (effectively turning it into a no-op) and an
1871 // asynchronous exception will be raised which the thread will
1872 // handle at a later point. If the instruction is a load it will
1926 }
1927 } else {
1928 if (TraceCallFixup) {
1929 tty->print("already patched callsite at " INTPTR_FORMAT " to compiled code for", p2i(caller_pc));
1930 moop->print_short_name(tty);
1931 tty->print_cr(" to " INTPTR_FORMAT, p2i(entry_point));
1932 }
1933 }
1934 return false;
1935 }
1936
1937 // ---------------------------------------------------------------------------
1938 // We are calling the interpreter via a c2i. Normally this would mean that
1939 // we were called by a compiled method. However we could have lost a race
1940 // where we went int -> i2c -> c2i and so the caller could in fact be
1941 // interpreted. If the caller is compiled we attempt to patch the caller
1942 // so he no longer calls into the interpreter.
1943 JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address caller_pc))
1944 Method* moop(method);
1945
1946 address entry_point = moop->from_compiled_entry_no_trampoline();
1947
1948 // It's possible that deoptimization can occur at a call site which hasn't
1949 // been resolved yet, in which case this function will be called from
1950 // an nmethod that has been patched for deopt and we can ignore the
1951 // request for a fixup.
1952 // Also it is possible that we lost a race in that from_compiled_entry
1953 // is now back to the i2c in that case we don't need to patch and if
1954 // we did we'd leap into space because the callsite needs to use
1955 // "to interpreter" stub in order to load up the Method*. Don't
1956 // ask me how I know this...
1957
1958 CodeBlob* cb = CodeCache::find_blob(caller_pc);
1959 if (cb == NULL || !cb->is_compiled() || entry_point == moop->get_c2i_entry()) {
1960 return;
1961 }
1962
1963 // The check above makes sure this is a nmethod.
1964 CompiledMethod* nm = cb->as_compiled_method_or_null();
1965 assert(nm, "must be");
1966
1967 // Get the return PC for the passed caller PC.
1968 address return_pc = caller_pc + frame::pc_return_offset;
1969
1970 // There is a benign race here. We could be attempting to patch to a compiled
1971 // entry point at the same time the callee is being deoptimized. If that is
1972 // the case then entry_point may in fact point to a c2i and we'd patch the
1973 // call site with the same old data. clear_code will set code() to NULL
1974 // at the end of it. If we happen to see that NULL then we can skip trying
1975 // to patch. If we hit the window where the callee has a c2i in the
1976 // from_compiled_entry and the NULL isn't present yet then we lose the race
1977 // and patch the code with the same old data. Asi es la vida.
1978
1979 if (moop->code() == NULL) return;
2314 private:
2315 enum {
2316 _basic_type_bits = 4,
2317 _basic_type_mask = right_n_bits(_basic_type_bits),
2318 _basic_types_per_int = BitsPerInt / _basic_type_bits,
2319 _compact_int_count = 3
2320 };
2321 // TO DO: Consider integrating this with a more global scheme for compressing signatures.
2322 // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive.
2323
2324 union {
2325 int _compact[_compact_int_count];
2326 int* _fingerprint;
2327 } _value;
2328 int _length; // A negative length indicates the fingerprint is in the compact form,
2329 // Otherwise _value._fingerprint is the array.
2330
2331 // Remap BasicTypes that are handled equivalently by the adapters.
2332 // These are correct for the current system but someday it might be
2333 // necessary to make this mapping platform dependent.
2334 static int adapter_encoding(BasicType in) {
2335 switch (in) {
2336 case T_BOOLEAN:
2337 case T_BYTE:
2338 case T_SHORT:
2339 case T_CHAR:
2340 // There are all promoted to T_INT in the calling convention
2341 return T_INT;
2342
2343 case T_OBJECT:
2344 case T_ARRAY:
2345 // In other words, we assume that any register good enough for
2346 // an int or long is good enough for a managed pointer.
2347 #ifdef _LP64
2348 return T_LONG;
2349 #else
2350 return T_INT;
2351 #endif
2352
2353 case T_INT:
2354 case T_LONG:
2355 case T_FLOAT:
2356 case T_DOUBLE:
2357 case T_VOID:
2358 return in;
2359
2360 default:
2361 ShouldNotReachHere();
2362 return T_CONFLICT;
2363 }
2364 }
2365
2366 public:
2367 AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) {
2368 // The fingerprint is based on the BasicType signature encoded
2369 // into an array of ints with eight entries per int.
2370 int* ptr;
2371 int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int;
2372 if (len <= _compact_int_count) {
2373 assert(_compact_int_count == 3, "else change next line");
2374 _value._compact[0] = _value._compact[1] = _value._compact[2] = 0;
2375 // Storing the signature encoded as signed chars hits about 98%
2376 // of the time.
2377 _length = -len;
2378 ptr = _value._compact;
2379 } else {
2380 _length = len;
2381 _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode);
2382 ptr = _value._fingerprint;
2383 }
2384
2385 // Now pack the BasicTypes with 8 per int
2386 int sig_index = 0;
2387 for (int index = 0; index < len; index++) {
2388 int value = 0;
2389 for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) {
2390 int bt = adapter_encoding(sig_bt[sig_index++]);
2391 assert((bt & _basic_type_mask) == bt, "must fit in 4 bits");
2392 value = (value << _basic_type_bits) | bt;
2393 }
2394 ptr[index] = value;
2395 }
2396 }
2397
2398 ~AdapterFingerPrint() {
2399 if (_length > 0) {
2400 FREE_C_HEAP_ARRAY(int, _value._fingerprint);
2401 }
2402 }
2403
2404 int value(int index) {
2405 if (_length < 0) {
2406 return _value._compact[index];
2407 }
2408 return _value._fingerprint[index];
2409 }
2410 int length() {
2411 if (_length < 0) return -_length;
2412 return _length;
2413 }
2414
2415 bool is_compact() {
2502
2503 private:
2504
2505 #ifndef PRODUCT
2506 static int _lookups; // number of calls to lookup
2507 static int _buckets; // number of buckets checked
2508 static int _equals; // number of buckets checked with matching hash
2509 static int _hits; // number of successful lookups
2510 static int _compact; // number of equals calls with compact signature
2511 #endif
2512
2513 AdapterHandlerEntry* bucket(int i) {
2514 return (AdapterHandlerEntry*)BasicHashtable<mtCode>::bucket(i);
2515 }
2516
2517 public:
2518 AdapterHandlerTable()
2519 : BasicHashtable<mtCode>(293, (sizeof(AdapterHandlerEntry))) { }
2520
2521 // Create a new entry suitable for insertion in the table
2522 AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) {
2523 AdapterHandlerEntry* entry = (AdapterHandlerEntry*)BasicHashtable<mtCode>::new_entry(fingerprint->compute_hash());
2524 entry->init(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
2525 return entry;
2526 }
2527
2528 // Insert an entry into the table
2529 void add(AdapterHandlerEntry* entry) {
2530 int index = hash_to_index(entry->hash());
2531 add_entry(index, entry);
2532 }
2533
2534 void free_entry(AdapterHandlerEntry* entry) {
2535 entry->deallocate();
2536 BasicHashtable<mtCode>::free_entry(entry);
2537 }
2538
2539 // Find a entry with the same fingerprint if it exists
2540 AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) {
2541 NOT_PRODUCT(_lookups++);
2542 AdapterFingerPrint fp(total_args_passed, sig_bt);
2543 unsigned int hash = fp.compute_hash();
2544 int index = hash_to_index(hash);
2545 for (AdapterHandlerEntry* e = bucket(index); e != NULL; e = e->next()) {
2546 NOT_PRODUCT(_buckets++);
2547 if (e->hash() == hash) {
2548 NOT_PRODUCT(_equals++);
2549 if (fp.equals(e->fingerprint())) {
2550 #ifndef PRODUCT
2551 if (fp.is_compact()) _compact++;
2552 _hits++;
2553 #endif
2554 return e;
2555 }
2556 }
2557 }
2558 return NULL;
2559 }
2560
2561 #ifndef PRODUCT
2562 void print_statistics() {
2623 AdapterHandlerEntry* result = _current;
2624 _current = _current->next();
2625 if (_current == NULL) scan();
2626 return result;
2627 } else {
2628 return NULL;
2629 }
2630 }
2631 };
2632
2633
2634 // ---------------------------------------------------------------------------
2635 // Implementation of AdapterHandlerLibrary
2636 AdapterHandlerTable* AdapterHandlerLibrary::_adapters = NULL;
2637 AdapterHandlerEntry* AdapterHandlerLibrary::_abstract_method_handler = NULL;
2638 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = NULL;
2639 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = NULL;
2640 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = NULL;
2641 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = NULL;
2642 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = NULL;
2643 const int AdapterHandlerLibrary_size = 16*K;
2644 BufferBlob* AdapterHandlerLibrary::_buffer = NULL;
2645
2646 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2647 return _buffer;
2648 }
2649
2650 extern "C" void unexpected_adapter_call() {
2651 ShouldNotCallThis();
2652 }
2653
2654 static void post_adapter_creation(const AdapterBlob* new_adapter, const AdapterHandlerEntry* entry) {
2655 char blob_id[256];
2656 jio_snprintf(blob_id,
2657 sizeof(blob_id),
2658 "%s(%s)",
2659 new_adapter->name(),
2660 entry->fingerprint()->as_string());
2661 Forte::register_stub(blob_id, new_adapter->content_begin(), new_adapter->content_end());
2662
2663 if (JvmtiExport::should_post_dynamic_code_generated()) {
2667
2668 void AdapterHandlerLibrary::initialize() {
2669 ResourceMark rm;
2670 AdapterBlob* no_arg_blob = NULL;
2671 AdapterBlob* int_arg_blob = NULL;
2672 AdapterBlob* obj_arg_blob = NULL;
2673 AdapterBlob* obj_int_arg_blob = NULL;
2674 AdapterBlob* obj_obj_arg_blob = NULL;
2675 {
2676 MutexLocker mu(AdapterHandlerLibrary_lock);
2677 assert(_adapters == NULL, "Initializing more than once");
2678
2679 _adapters = new AdapterHandlerTable();
2680
2681 // Create a special handler for abstract methods. Abstract methods
2682 // are never compiled so an i2c entry is somewhat meaningless, but
2683 // throw AbstractMethodError just in case.
2684 // Pass wrong_method_abstract for the c2i transitions to return
2685 // AbstractMethodError for invalid invocations.
2686 address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
2687 _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, NULL),
2688 StubRoutines::throw_AbstractMethodError_entry(),
2689 wrong_method_abstract, wrong_method_abstract);
2690
2691 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2692
2693 _no_arg_handler = create_adapter(no_arg_blob, 0, NULL, true);
2694
2695 BasicType obj_args[] = { T_OBJECT };
2696 _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true);
2697
2698 BasicType int_args[] = { T_INT };
2699 _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true);
2700
2701 BasicType obj_int_args[] = { T_OBJECT, T_INT };
2702 _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true);
2703
2704 BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT };
2705 _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true);
2706
2707 assert(no_arg_blob != NULL &&
2708 obj_arg_blob != NULL &&
2709 int_arg_blob != NULL &&
2710 obj_int_arg_blob != NULL &&
2711 obj_obj_arg_blob != NULL, "Initial adapters must be properly created");
2712 }
2713
2714 // Outside of the lock
2715 post_adapter_creation(no_arg_blob, _no_arg_handler);
2716 post_adapter_creation(obj_arg_blob, _obj_arg_handler);
2717 post_adapter_creation(int_arg_blob, _int_arg_handler);
2718 post_adapter_creation(obj_int_arg_blob, _obj_int_arg_handler);
2719 post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler);
2720 }
2721
2722 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint,
2723 address i2c_entry,
2724 address c2i_entry,
2725 address c2i_unverified_entry,
2726 address c2i_no_clinit_check_entry) {
2727 return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
2728 }
2729
2730 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2731 if (method->is_abstract()) {
2732 return _abstract_method_handler;
2733 }
2734 int total_args_passed = method->size_of_parameters(); // All args on stack
2735 if (total_args_passed == 0) {
2736 return _no_arg_handler;
2737 } else if (total_args_passed == 1) {
2738 if (!method->is_static()) {
2739 return _obj_arg_handler;
2740 }
2741 switch (method->signature()->char_at(1)) {
2742 case JVM_SIGNATURE_CLASS:
2743 case JVM_SIGNATURE_ARRAY:
2744 return _obj_arg_handler;
2745 case JVM_SIGNATURE_INT:
2746 case JVM_SIGNATURE_BOOLEAN:
2747 case JVM_SIGNATURE_CHAR:
2748 case JVM_SIGNATURE_BYTE:
2749 case JVM_SIGNATURE_SHORT:
2750 return _int_arg_handler;
2751 }
2752 } else if (total_args_passed == 2 &&
2753 !method->is_static()) {
2754 switch (method->signature()->char_at(1)) {
2755 case JVM_SIGNATURE_CLASS:
2756 case JVM_SIGNATURE_ARRAY:
2757 return _obj_obj_arg_handler;
2758 case JVM_SIGNATURE_INT:
2759 case JVM_SIGNATURE_BOOLEAN:
2760 case JVM_SIGNATURE_CHAR:
2761 case JVM_SIGNATURE_BYTE:
2762 case JVM_SIGNATURE_SHORT:
2763 return _obj_int_arg_handler;
2764 }
2765 }
2766 return NULL;
2767 }
2768
2769 class AdapterSignatureIterator : public SignatureIterator {
2770 private:
2771 BasicType stack_sig_bt[16];
2772 BasicType* sig_bt;
2773 int index;
2774
2775 public:
2776 AdapterSignatureIterator(Symbol* signature,
2777 fingerprint_t fingerprint,
2778 bool is_static,
2779 int total_args_passed) :
2780 SignatureIterator(signature, fingerprint),
2781 index(0)
2782 {
2783 sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
2784 if (!is_static) { // Pass in receiver first
2785 sig_bt[index++] = T_OBJECT;
2786 }
2787 do_parameters_on(this);
2788 }
2789
2790 BasicType* basic_types() {
2791 return sig_bt;
2792 }
2793
2794 #ifdef ASSERT
2795 int slots() {
2796 return index;
2797 }
2798 #endif
2799
2800 private:
2801
2802 friend class SignatureIterator; // so do_parameters_on can call do_type
2803 void do_type(BasicType type) {
2804 sig_bt[index++] = type;
2805 if (type == T_LONG || type == T_DOUBLE) {
2806 sig_bt[index++] = T_VOID; // Longs & doubles take 2 Java slots
2807 }
2808 }
2809 };
2810
2811 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
2812 // Use customized signature handler. Need to lock around updates to
2813 // the AdapterHandlerTable (it is not safe for concurrent readers
2814 // and a single writer: this could be fixed if it becomes a
2815 // problem).
2816 assert(_adapters != NULL, "Uninitialized");
2817
2818 // Fast-path for trivial adapters
2819 AdapterHandlerEntry* entry = get_simple_adapter(method);
2820 if (entry != NULL) {
2821 return entry;
2822 }
2823
2824 ResourceMark rm;
2825 AdapterBlob* new_adapter = NULL;
2826
2827 // Fill in the signature array, for the calling-convention call.
2828 int total_args_passed = method->size_of_parameters(); // All args on stack
2829
2830 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
2831 method->is_static(), total_args_passed);
2832 assert(si.slots() == total_args_passed, "");
2833 BasicType* sig_bt = si.basic_types();
2834 {
2835 MutexLocker mu(AdapterHandlerLibrary_lock);
2836
2837 // Lookup method signature's fingerprint
2838 entry = _adapters->lookup(total_args_passed, sig_bt);
2839
2840 if (entry != NULL) {
2841 #ifdef ASSERT
2842 if (VerifyAdapterSharing) {
2843 AdapterBlob* comparison_blob = NULL;
2844 AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false);
2845 assert(comparison_blob == NULL, "no blob should be created when creating an adapter for comparison");
2846 assert(comparison_entry->compare_code(entry), "code must match");
2847 // Release the one just created and return the original
2848 _adapters->free_entry(comparison_entry);
2849 }
2850 #endif
2851 return entry;
2852 }
2853
2854 entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true);
2855 }
2856
2857 // Outside of the lock
2858 if (new_adapter != NULL) {
2859 post_adapter_creation(new_adapter, entry);
2860 }
2861 return entry;
2862 }
2863
2864 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter,
2865 int total_args_passed,
2866 BasicType* sig_bt,
2867 bool allocate_code_blob) {
2868
2869 // StubRoutines::code2() is initialized after this function can be called. As a result,
2870 // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated
2871 // prior to StubRoutines::code2() being set. Checks refer to checks generated in an I2C
2872 // stub that ensure that an I2C stub is called from an interpreter frame.
2873 bool contains_all_checks = StubRoutines::code2() != NULL;
2874
2875 VMRegPair stack_regs[16];
2876 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
2877
2878 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
2879 int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
2880 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
2881 CodeBuffer buffer(buf);
2882 short buffer_locs[20];
2883 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
2884 sizeof(buffer_locs)/sizeof(relocInfo));
2885
2886 // Make a C heap allocated version of the fingerprint to store in the adapter
2887 AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt);
2888 MacroAssembler _masm(&buffer);
2889 AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm,
2890 total_args_passed,
2891 comp_args_on_stack,
2892 sig_bt,
2893 regs,
2894 fingerprint);
2895
2896 #ifdef ASSERT
2897 if (VerifyAdapterSharing) {
2898 entry->save_code(buf->code_begin(), buffer.insts_size());
2899 if (!allocate_code_blob) {
2900 return entry;
2901 }
2902 }
2903 #endif
2904
2905 new_adapter = AdapterBlob::create(&buffer);
2906 NOT_PRODUCT(int insts_size = buffer.insts_size());
2907 if (new_adapter == NULL) {
2908 // CodeCache is full, disable compilation
2909 // Ought to log this but compile log is only per compile thread
2910 // and we're some non descript Java thread.
2911 return NULL;
2912 }
2913 entry->relocate(new_adapter->content_begin());
2914 #ifndef PRODUCT
2915 // debugging suppport
2916 if (PrintAdapterHandlers || PrintStubCode) {
2917 ttyLocker ttyl;
2918 entry->print_adapter_on(tty);
2919 tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)",
2920 _adapters->number_of_entries(), fingerprint->as_basic_args_string(),
2921 fingerprint->as_string(), insts_size);
2922 tty->print_cr("c2i argument handler starts at %p", entry->get_c2i_entry());
2923 if (Verbose || PrintStubCode) {
2924 address first_pc = entry->base_address();
2925 if (first_pc != NULL) {
2926 Disassembler::decode(first_pc, first_pc + insts_size, tty
2927 NOT_PRODUCT(COMMA &new_adapter->asm_remarks()));
2928 tty->cr();
2929 }
2930 }
2931 }
2932 #endif
2933
2934 // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp)
2935 // The checks are inserted only if -XX:+VerifyAdapterCalls is specified.
2936 if (contains_all_checks || !VerifyAdapterCalls) {
2937 _adapters->add(entry);
2938 }
2939 return entry;
2940 }
2941
2942 address AdapterHandlerEntry::base_address() {
2943 address base = _i2c_entry;
2944 if (base == NULL) base = _c2i_entry;
2945 assert(base <= _c2i_entry || _c2i_entry == NULL, "");
2946 assert(base <= _c2i_unverified_entry || _c2i_unverified_entry == NULL, "");
2947 assert(base <= _c2i_no_clinit_check_entry || _c2i_no_clinit_check_entry == NULL, "");
2948 return base;
2949 }
2950
2951 void AdapterHandlerEntry::relocate(address new_base) {
2952 address old_base = base_address();
2953 assert(old_base != NULL, "");
2954 ptrdiff_t delta = new_base - old_base;
2955 if (_i2c_entry != NULL)
2956 _i2c_entry += delta;
2957 if (_c2i_entry != NULL)
2958 _c2i_entry += delta;
2959 if (_c2i_unverified_entry != NULL)
2960 _c2i_unverified_entry += delta;
2961 if (_c2i_no_clinit_check_entry != NULL)
2962 _c2i_no_clinit_check_entry += delta;
2963 assert(base_address() == new_base, "");
2964 }
2965
2966
2967 void AdapterHandlerEntry::deallocate() {
2968 delete _fingerprint;
2969 #ifdef ASSERT
2970 FREE_C_HEAP_ARRAY(unsigned char, _saved_code);
2971 #endif
2972 }
2973
2974
2975 #ifdef ASSERT
2976 // Capture the code before relocation so that it can be compared
2977 // against other versions. If the code is captured after relocation
2978 // then relative instructions won't be equivalent.
2979 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
2980 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
2981 _saved_code_length = length;
2982 memcpy(_saved_code, buffer, length);
2983 }
2984
2985
2986 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
2987 assert(_saved_code != NULL && other->_saved_code != NULL, "code not saved");
2988
3027 assert(compile_id > 0, "Must generate native wrapper");
3028
3029
3030 ResourceMark rm;
3031 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3032 if (buf != NULL) {
3033 CodeBuffer buffer(buf);
3034 struct { double data[20]; } locs_buf;
3035 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
3036 #if defined(AARCH64)
3037 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
3038 // in the constant pool to ensure ordering between the barrier and oops
3039 // accesses. For native_wrappers we need a constant.
3040 buffer.initialize_consts_size(8);
3041 #endif
3042 MacroAssembler _masm(&buffer);
3043
3044 // Fill in the signature array, for the calling-convention call.
3045 const int total_args_passed = method->size_of_parameters();
3046
3047 VMRegPair stack_regs[16];
3048 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
3049
3050 AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
3051 method->is_static(), total_args_passed);
3052 BasicType* sig_bt = si.basic_types();
3053 assert(si.slots() == total_args_passed, "");
3054 BasicType ret_type = si.return_type();
3055
3056 // Now get the compiled-Java arguments layout.
3057 int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
3058
3059 // Generate the compiled-to-native wrapper code
3060 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type, critical_entry);
3061
3062 if (nm != NULL) {
3063 {
3064 MutexLocker pl(CompiledMethod_lock, Mutex::_no_safepoint_check_flag);
3065 if (nm->make_in_use()) {
3066 method->set_code(method, nm);
3067 }
3068 }
3069
3070 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_simple));
3071 if (directive->PrintAssemblyOption) {
3072 nm->print_code();
3073 }
3074 DirectivesStack::release(directive);
3255 AdapterHandlerTableIterator iter(_adapters);
3256 while (iter.has_next()) {
3257 AdapterHandlerEntry* a = iter.next();
3258 if (b == CodeCache::find_blob(a->get_i2c_entry())) {
3259 st->print("Adapter for signature: ");
3260 a->print_adapter_on(tty);
3261 return;
3262 }
3263 }
3264 assert(false, "Should have found handler");
3265 }
3266
3267 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3268 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3269 if (get_i2c_entry() != NULL) {
3270 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3271 }
3272 if (get_c2i_entry() != NULL) {
3273 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3274 }
3275 if (get_c2i_unverified_entry() != NULL) {
3276 st->print(" c2iUV: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3277 }
3278 if (get_c2i_no_clinit_check_entry() != NULL) {
3279 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3280 }
3281 st->cr();
3282 }
3283
3284 #ifndef PRODUCT
3285
3286 void AdapterHandlerLibrary::print_statistics() {
3287 _adapters->print_statistics();
3288 }
3289
3290 #endif /* PRODUCT */
3291
3292 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3293 StackOverflow* overflow_state = current->stack_overflow_state();
3294 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3295 overflow_state->set_reserved_stack_activation(current->stack_base());
3296 JRT_END
3341 break;
3342 } else {
3343 fr = fr.java_sender();
3344 }
3345 }
3346 return activation;
3347 }
3348
3349 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
3350 // After any safepoint, just before going back to compiled code,
3351 // we inform the GC that we will be doing initializing writes to
3352 // this object in the future without emitting card-marks, so
3353 // GC may take any compensating steps.
3354
3355 oop new_obj = current->vm_result();
3356 if (new_obj == NULL) return;
3357
3358 BarrierSet *bs = BarrierSet::barrier_set();
3359 bs->on_slowpath_allocation_exit(current, new_obj);
3360 }
|
27 #include "jvm.h"
28 #include "classfile/stringTable.hpp"
29 #include "classfile/vmClasses.hpp"
30 #include "classfile/vmSymbols.hpp"
31 #include "code/codeCache.hpp"
32 #include "code/compiledIC.hpp"
33 #include "code/icBuffer.hpp"
34 #include "code/compiledMethod.inline.hpp"
35 #include "code/scopeDesc.hpp"
36 #include "code/vtableStubs.hpp"
37 #include "compiler/abstractCompiler.hpp"
38 #include "compiler/compileBroker.hpp"
39 #include "compiler/disassembler.hpp"
40 #include "gc/shared/barrierSet.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "gc/shared/gcLocker.inline.hpp"
43 #include "interpreter/interpreter.hpp"
44 #include "interpreter/interpreterRuntime.hpp"
45 #include "jfr/jfrEvents.hpp"
46 #include "logging/log.hpp"
47 #include "memory/oopFactory.hpp"
48 #include "memory/resourceArea.hpp"
49 #include "memory/universe.hpp"
50 #include "oops/access.hpp"
51 #include "oops/fieldStreams.inline.hpp"
52 #include "oops/compiledICHolder.inline.hpp"
53 #include "oops/klass.hpp"
54 #include "oops/method.inline.hpp"
55 #include "oops/objArrayKlass.hpp"
56 #include "oops/objArrayOop.inline.hpp"
57 #include "oops/oop.inline.hpp"
58 #include "oops/inlineKlass.inline.hpp"
59 #include "prims/forte.hpp"
60 #include "prims/jvmtiExport.hpp"
61 #include "prims/methodHandles.hpp"
62 #include "prims/nativeLookup.hpp"
63 #include "runtime/atomic.hpp"
64 #include "runtime/frame.inline.hpp"
65 #include "runtime/handles.inline.hpp"
66 #include "runtime/init.hpp"
67 #include "runtime/interfaceSupport.inline.hpp"
68 #include "runtime/java.hpp"
69 #include "runtime/javaCalls.hpp"
70 #include "runtime/sharedRuntime.hpp"
71 #include "runtime/stackWatermarkSet.hpp"
72 #include "runtime/stubRoutines.hpp"
73 #include "runtime/synchronizer.hpp"
74 #include "runtime/vframe.inline.hpp"
75 #include "runtime/vframeArray.hpp"
76 #include "runtime/vm_version.hpp"
77 #include "utilities/copy.hpp"
78 #include "utilities/dtrace.hpp"
79 #include "utilities/events.hpp"
80 #include "utilities/hashtable.inline.hpp"
81 #include "utilities/macros.hpp"
82 #include "utilities/xmlstream.hpp"
83 #ifdef COMPILER1
84 #include "c1/c1_Runtime1.hpp"
85 #endif
86
87 // Shared stub locations
88 RuntimeStub* SharedRuntime::_wrong_method_blob;
89 RuntimeStub* SharedRuntime::_wrong_method_abstract_blob;
90 RuntimeStub* SharedRuntime::_ic_miss_blob;
91 RuntimeStub* SharedRuntime::_resolve_opt_virtual_call_blob;
92 RuntimeStub* SharedRuntime::_resolve_virtual_call_blob;
93 RuntimeStub* SharedRuntime::_resolve_static_call_blob;
94
95 DeoptimizationBlob* SharedRuntime::_deopt_blob;
96 SafepointBlob* SharedRuntime::_polling_page_vectors_safepoint_handler_blob;
97 SafepointBlob* SharedRuntime::_polling_page_safepoint_handler_blob;
98 SafepointBlob* SharedRuntime::_polling_page_return_handler_blob;
99
100 #ifdef COMPILER2
101 UncommonTrapBlob* SharedRuntime::_uncommon_trap_blob;
102 #endif // COMPILER2
103
104
105 //----------------------------generate_stubs-----------------------------------
106 void SharedRuntime::generate_stubs() {
107 _wrong_method_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method), "wrong_method_stub");
108 _wrong_method_abstract_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract), "wrong_method_abstract_stub");
109 _ic_miss_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss), "ic_miss_stub");
110 _resolve_opt_virtual_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C), "resolve_opt_virtual_call");
111 _resolve_virtual_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C), "resolve_virtual_call");
112 _resolve_static_call_blob = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C), "resolve_static_call");
113
114 AdapterHandlerLibrary::initialize();
115
116 #if COMPILER2_OR_JVMCI
117 // Vectors are generated only by C2 and JVMCI.
118 bool support_wide = is_wide_vector(MaxVectorSize);
119 if (support_wide) {
120 _polling_page_vectors_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_VECTOR_LOOP);
121 }
122 #endif // COMPILER2_OR_JVMCI
123 _polling_page_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_LOOP);
124 _polling_page_return_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_RETURN);
125
126 generate_deopt_blob();
127
128 #ifdef COMPILER2
129 generate_uncommon_trap_blob();
130 #endif // COMPILER2
131 }
132
972 // forwarded before we look at the return value.
973 THROW_(vmSymbols::java_lang_UnsatisfiedLinkError(), (void*)badAddress);
974 }
975 JNI_END
976
977 address SharedRuntime::native_method_throw_unsatisfied_link_error_entry() {
978 return CAST_FROM_FN_PTR(address, &throw_unsatisfied_link_error);
979 }
980
981 JRT_ENTRY_NO_ASYNC(void, SharedRuntime::register_finalizer(JavaThread* current, oopDesc* obj))
982 #if INCLUDE_JVMCI
983 if (!obj->klass()->has_finalizer()) {
984 return;
985 }
986 #endif // INCLUDE_JVMCI
987 assert(oopDesc::is_oop(obj), "must be a valid oop");
988 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
989 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
990 JRT_END
991
992
993 jlong SharedRuntime::get_java_tid(Thread* thread) {
994 if (thread != NULL) {
995 if (thread->is_Java_thread()) {
996 oop obj = JavaThread::cast(thread)->threadObj();
997 return (obj == NULL) ? 0 : java_lang_Thread::thread_id(obj);
998 }
999 }
1000 return 0;
1001 }
1002
1003 /**
1004 * This function ought to be a void function, but cannot be because
1005 * it gets turned into a tail-call on sparc, which runs into dtrace bug
1006 * 6254741. Once that is fixed we can remove the dummy return value.
1007 */
1008 int SharedRuntime::dtrace_object_alloc(oopDesc* o, int size) {
1009 return dtrace_object_alloc_base(Thread::current(), o, size);
1010 }
1011
1012 int SharedRuntime::dtrace_object_alloc_base(Thread* thread, oopDesc* o, int size) {
1073 return caller->attached_method_before_pc(pc);
1074 }
1075 return NULL;
1076 }
1077
1078 // Finds receiver, CallInfo (i.e. receiver method), and calling bytecode
1079 // for a call current in progress, i.e., arguments has been pushed on stack
1080 // but callee has not been invoked yet. Caller frame must be compiled.
1081 Handle SharedRuntime::find_callee_info_helper(vframeStream& vfst, Bytecodes::Code& bc,
1082 CallInfo& callinfo, TRAPS) {
1083 Handle receiver;
1084 Handle nullHandle; // create a handy null handle for exception returns
1085 JavaThread* current = THREAD;
1086
1087 assert(!vfst.at_end(), "Java frame must exist");
1088
1089 // Find caller and bci from vframe
1090 methodHandle caller(current, vfst.method());
1091 int bci = vfst.bci();
1092
1093 // Substitutability test implementation piggy backs on static call resolution
1094 Bytecodes::Code code = caller->java_code_at(bci);
1095 if (code == Bytecodes::_if_acmpeq || code == Bytecodes::_if_acmpne) {
1096 bc = Bytecodes::_invokestatic;
1097 methodHandle attached_method(THREAD, extract_attached_method(vfst));
1098 assert(attached_method.not_null(), "must have attached method");
1099 vmClasses::PrimitiveObjectMethods_klass()->initialize(CHECK_NH);
1100 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, false, CHECK_NH);
1101 #ifdef ASSERT
1102 Method* is_subst = vmClasses::PrimitiveObjectMethods_klass()->find_method(vmSymbols::isSubstitutable_name(), vmSymbols::object_object_boolean_signature());
1103 assert(callinfo.selected_method() == is_subst, "must be isSubstitutable method");
1104 #endif
1105 return receiver;
1106 }
1107
1108 Bytecode_invoke bytecode(caller, bci);
1109 int bytecode_index = bytecode.index();
1110 bc = bytecode.invoke_code();
1111
1112 methodHandle attached_method(current, extract_attached_method(vfst));
1113 if (attached_method.not_null()) {
1114 Method* callee = bytecode.static_target(CHECK_NH);
1115 vmIntrinsics::ID id = callee->intrinsic_id();
1116 // When VM replaces MH.invokeBasic/linkTo* call with a direct/virtual call,
1117 // it attaches statically resolved method to the call site.
1118 if (MethodHandles::is_signature_polymorphic(id) &&
1119 MethodHandles::is_signature_polymorphic_intrinsic(id)) {
1120 bc = MethodHandles::signature_polymorphic_intrinsic_bytecode(id);
1121
1122 // Adjust invocation mode according to the attached method.
1123 switch (bc) {
1124 case Bytecodes::_invokevirtual:
1125 if (attached_method->method_holder()->is_interface()) {
1126 bc = Bytecodes::_invokeinterface;
1127 }
1128 break;
1129 case Bytecodes::_invokeinterface:
1130 if (!attached_method->method_holder()->is_interface()) {
1131 bc = Bytecodes::_invokevirtual;
1132 }
1133 break;
1134 case Bytecodes::_invokehandle:
1135 if (!MethodHandles::is_signature_polymorphic_method(attached_method())) {
1136 bc = attached_method->is_static() ? Bytecodes::_invokestatic
1137 : Bytecodes::_invokevirtual;
1138 }
1139 break;
1140 default:
1141 break;
1142 }
1143 } else {
1144 assert(attached_method->has_scalarized_args(), "invalid use of attached method");
1145 if (!attached_method->method_holder()->is_inline_klass()) {
1146 // Ignore the attached method in this case to not confuse below code
1147 attached_method = methodHandle(current, NULL);
1148 }
1149 }
1150 }
1151
1152 assert(bc != Bytecodes::_illegal, "not initialized");
1153
1154 bool has_receiver = bc != Bytecodes::_invokestatic &&
1155 bc != Bytecodes::_invokedynamic &&
1156 bc != Bytecodes::_invokehandle;
1157 bool check_null_and_abstract = true;
1158
1159 // Find receiver for non-static call
1160 if (has_receiver) {
1161 // This register map must be update since we need to find the receiver for
1162 // compiled frames. The receiver might be in a register.
1163 RegisterMap reg_map2(current);
1164 frame stubFrame = current->last_frame();
1165 // Caller-frame is a compiled frame
1166 frame callerFrame = stubFrame.sender(®_map2);
1167 bool caller_is_c1 = false;
1168
1169 if (callerFrame.is_compiled_frame() && !callerFrame.is_deoptimized_frame()) {
1170 caller_is_c1 = callerFrame.cb()->is_compiled_by_c1();
1171 }
1172
1173 Method* callee = attached_method();
1174 if (callee == NULL) {
1175 callee = bytecode.static_target(CHECK_NH);
1176 if (callee == NULL) {
1177 THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
1178 }
1179 }
1180 if (!caller_is_c1 && callee->has_scalarized_args() && callee->method_holder()->is_inline_klass() &&
1181 InlineKlass::cast(callee->method_holder())->can_be_passed_as_fields()) {
1182 // If the receiver is an inline type that is passed as fields, no oop is available
1183 // Resolve the call without receiver null checking.
1184 assert(attached_method.not_null() && !attached_method->is_abstract(), "must have non-abstract attached method");
1185 if (bc == Bytecodes::_invokeinterface) {
1186 bc = Bytecodes::_invokevirtual; // C2 optimistically replaces interface calls by virtual calls
1187 }
1188 check_null_and_abstract = false;
1189 } else {
1190 // Retrieve from a compiled argument list
1191 receiver = Handle(current, callerFrame.retrieve_receiver(®_map2));
1192 if (receiver.is_null()) {
1193 THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
1194 }
1195 }
1196 }
1197
1198 // Resolve method
1199 if (attached_method.not_null()) {
1200 // Parameterized by attached method.
1201 LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, check_null_and_abstract, CHECK_NH);
1202 } else {
1203 // Parameterized by bytecode.
1204 constantPoolHandle constants(current, caller->constants());
1205 LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
1206 }
1207
1208 #ifdef ASSERT
1209 // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
1210 if (has_receiver && check_null_and_abstract) {
1211 assert(receiver.not_null(), "should have thrown exception");
1212 Klass* receiver_klass = receiver->klass();
1213 Klass* rk = NULL;
1214 if (attached_method.not_null()) {
1215 // In case there's resolved method attached, use its holder during the check.
1216 rk = attached_method->method_holder();
1217 } else {
1218 // Klass is already loaded.
1219 constantPoolHandle constants(current, caller->constants());
1220 rk = constants->klass_ref_at(bytecode_index, CHECK_NH);
1221 }
1222 Klass* static_receiver_klass = rk;
1223 assert(receiver_klass->is_subtype_of(static_receiver_klass),
1224 "actual receiver must be subclass of static receiver klass");
1225 if (receiver_klass->is_instance_klass()) {
1226 if (InstanceKlass::cast(receiver_klass)->is_not_initialized()) {
1227 tty->print_cr("ERROR: Klass not yet initialized!!");
1228 receiver_klass->print();
1229 }
1230 assert(!InstanceKlass::cast(receiver_klass)->is_not_initialized(), "receiver_klass must be initialized");
1248 // Hence the stack can only contain an entry_frame. We need to
1249 // find the target method from the stub frame.
1250 RegisterMap reg_map(current, false);
1251 frame fr = current->last_frame();
1252 assert(fr.is_runtime_frame(), "must be a runtimeStub");
1253 fr = fr.sender(®_map);
1254 assert(fr.is_entry_frame(), "must be");
1255 // fr is now pointing to the entry frame.
1256 callee_method = methodHandle(current, fr.entry_frame_call_wrapper()->callee_method());
1257 } else {
1258 Bytecodes::Code bc;
1259 CallInfo callinfo;
1260 find_callee_info_helper(vfst, bc, callinfo, CHECK_(methodHandle()));
1261 callee_method = methodHandle(current, callinfo.selected_method());
1262 }
1263 assert(callee_method()->is_method(), "must be");
1264 return callee_method;
1265 }
1266
1267 // Resolves a call.
1268 methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, bool* caller_is_c1, TRAPS) {
1269 methodHandle callee_method;
1270 callee_method = resolve_sub_helper(is_virtual, is_optimized, caller_is_c1, THREAD);
1271 if (JvmtiExport::can_hotswap_or_post_breakpoint()) {
1272 int retry_count = 0;
1273 while (!HAS_PENDING_EXCEPTION && callee_method->is_old() &&
1274 callee_method->method_holder() != vmClasses::Object_klass()) {
1275 // If has a pending exception then there is no need to re-try to
1276 // resolve this method.
1277 // If the method has been redefined, we need to try again.
1278 // Hack: we have no way to update the vtables of arrays, so don't
1279 // require that java.lang.Object has been updated.
1280
1281 // It is very unlikely that method is redefined more than 100 times
1282 // in the middle of resolve. If it is looping here more than 100 times
1283 // means then there could be a bug here.
1284 guarantee((retry_count++ < 100),
1285 "Could not resolve to latest version of redefined method");
1286 // method is redefined in the middle of resolve so re-try.
1287 callee_method = resolve_sub_helper(is_virtual, is_optimized, caller_is_c1, THREAD);
1288 }
1289 }
1290 return callee_method;
1291 }
1292
1293 // This fails if resolution required refilling of IC stubs
1294 bool SharedRuntime::resolve_sub_helper_internal(methodHandle callee_method, const frame& caller_frame,
1295 CompiledMethod* caller_nm, bool is_virtual, bool is_optimized,
1296 Handle receiver, CallInfo& call_info, Bytecodes::Code invoke_code, TRAPS) {
1297 StaticCallInfo static_call_info;
1298 CompiledICInfo virtual_call_info;
1299
1300 // Make sure the callee nmethod does not get deoptimized and removed before
1301 // we are done patching the code.
1302 CompiledMethod* callee = callee_method->code();
1303
1304 if (callee != NULL) {
1305 assert(callee->is_compiled(), "must be nmethod for patching");
1306 }
1307
1308 if (callee != NULL && !callee->is_in_use()) {
1309 // Patch call site to C2I adapter if callee nmethod is deoptimized or unloaded.
1310 callee = NULL;
1311 }
1312 nmethodLocker nl_callee(callee);
1313 #ifdef ASSERT
1314 address dest_entry_point = callee == NULL ? 0 : callee->entry_point(); // used below
1315 #endif
1316
1317 bool is_nmethod = caller_nm->is_nmethod();
1318 bool caller_is_c1 = caller_nm->is_compiled_by_c1();
1319
1320 if (is_virtual) {
1321 Klass* receiver_klass = NULL;
1322 if (!caller_is_c1 && callee_method->has_scalarized_args() && callee_method->method_holder()->is_inline_klass() &&
1323 InlineKlass::cast(callee_method->method_holder())->can_be_passed_as_fields()) {
1324 // If the receiver is an inline type that is passed as fields, no oop is available
1325 receiver_klass = callee_method->method_holder();
1326 } else {
1327 assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check");
1328 receiver_klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass();
1329 }
1330 bool static_bound = call_info.resolved_method()->can_be_statically_bound();
1331 CompiledIC::compute_monomorphic_entry(callee_method, receiver_klass,
1332 is_optimized, static_bound, is_nmethod, caller_is_c1, virtual_call_info,
1333 CHECK_false);
1334 } else {
1335 // static call
1336 CompiledStaticCall::compute_entry(callee_method, caller_nm, static_call_info);
1337 }
1338
1339 // grab lock, check for deoptimization and potentially patch caller
1340 {
1341 CompiledICLocker ml(caller_nm);
1342
1343 // Lock blocks for safepoint during which both nmethods can change state.
1344
1345 // Now that we are ready to patch if the Method* was redefined then
1346 // don't update call site and let the caller retry.
1347 // Don't update call site if callee nmethod was unloaded or deoptimized.
1348 // Don't update call site if callee nmethod was replaced by an other nmethod
1349 // which may happen when multiply alive nmethod (tiered compilation)
1350 // will be supported.
1351 if (!callee_method->is_old() &&
1352 (callee == NULL || (callee->is_in_use() && callee_method->code() == callee))) {
1353 NoSafepointVerifier nsv;
1354 #ifdef ASSERT
1355 // We must not try to patch to jump to an already unloaded method.
1356 if (dest_entry_point != 0) {
1366 return false;
1367 }
1368 }
1369 } else {
1370 if (VM_Version::supports_fast_class_init_checks() &&
1371 invoke_code == Bytecodes::_invokestatic &&
1372 callee_method->needs_clinit_barrier() &&
1373 callee != NULL && callee->is_compiled_by_jvmci()) {
1374 return true; // skip patching for JVMCI
1375 }
1376 CompiledStaticCall* ssc = caller_nm->compiledStaticCall_before(caller_frame.pc());
1377 if (ssc->is_clean()) ssc->set(static_call_info);
1378 }
1379 }
1380 } // unlock CompiledICLocker
1381 return true;
1382 }
1383
1384 // Resolves a call. The compilers generate code for calls that go here
1385 // and are patched with the real destination of the call.
1386 methodHandle SharedRuntime::resolve_sub_helper(bool is_virtual, bool is_optimized, bool* caller_is_c1, TRAPS) {
1387 JavaThread* current = THREAD;
1388 ResourceMark rm(current);
1389 RegisterMap cbl_map(current, false);
1390 frame caller_frame = current->last_frame().sender(&cbl_map);
1391
1392 CodeBlob* caller_cb = caller_frame.cb();
1393 guarantee(caller_cb != NULL && caller_cb->is_compiled(), "must be called from compiled method");
1394 CompiledMethod* caller_nm = caller_cb->as_compiled_method_or_null();
1395 *caller_is_c1 = caller_nm->is_compiled_by_c1();
1396
1397 // make sure caller is not getting deoptimized
1398 // and removed before we are done with it.
1399 // CLEANUP - with lazy deopt shouldn't need this lock
1400 nmethodLocker caller_lock(caller_nm);
1401
1402 // determine call info & receiver
1403 // note: a) receiver is NULL for static calls
1404 // b) an exception is thrown if receiver is NULL for non-static calls
1405 CallInfo call_info;
1406 Bytecodes::Code invoke_code = Bytecodes::_illegal;
1407 Handle receiver = find_callee_info(invoke_code, call_info, CHECK_(methodHandle()));
1408 methodHandle callee_method(current, call_info.selected_method());
1409
1410 assert((!is_virtual && invoke_code == Bytecodes::_invokestatic ) ||
1411 (!is_virtual && invoke_code == Bytecodes::_invokespecial) ||
1412 (!is_virtual && invoke_code == Bytecodes::_invokehandle ) ||
1413 (!is_virtual && invoke_code == Bytecodes::_invokedynamic) ||
1414 ( is_virtual && invoke_code != Bytecodes::_invokestatic ), "inconsistent bytecode");
1415
1472 bool successful = resolve_sub_helper_internal(callee_method, caller_frame, caller_nm,
1473 is_virtual, is_optimized, receiver,
1474 call_info, invoke_code, CHECK_(methodHandle()));
1475 if (successful) {
1476 return callee_method;
1477 } else {
1478 InlineCacheBuffer::refill_ic_stubs();
1479 }
1480 }
1481
1482 }
1483
1484
1485 // Inline caches exist only in compiled code
1486 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_ic_miss(JavaThread* current))
1487 #ifdef ASSERT
1488 RegisterMap reg_map(current, false);
1489 frame stub_frame = current->last_frame();
1490 assert(stub_frame.is_runtime_frame(), "sanity check");
1491 frame caller_frame = stub_frame.sender(®_map);
1492 assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_optimized_entry_frame(), "unexpected frame");
1493 #endif /* ASSERT */
1494
1495 methodHandle callee_method;
1496 bool is_optimized = false;
1497 bool caller_is_c1 = false;
1498 JRT_BLOCK
1499 callee_method = SharedRuntime::handle_ic_miss_helper(is_optimized, caller_is_c1, CHECK_NULL);
1500 // Return Method* through TLS
1501 current->set_vm_result_2(callee_method());
1502 JRT_BLOCK_END
1503 // return compiled code entry point after potential safepoints
1504 return entry_for_handle_wrong_method(callee_method, false, is_optimized, caller_is_c1);
1505 JRT_END
1506
1507
1508 // Handle call site that has been made non-entrant
1509 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))
1510 // 6243940 We might end up in here if the callee is deoptimized
1511 // as we race to call it. We don't want to take a safepoint if
1512 // the caller was interpreted because the caller frame will look
1513 // interpreted to the stack walkers and arguments are now
1514 // "compiled" so it is much better to make this transition
1515 // invisible to the stack walking code. The i2c path will
1516 // place the callee method in the callee_target. It is stashed
1517 // there because if we try and find the callee by normal means a
1518 // safepoint is possible and have trouble gc'ing the compiled args.
1519 RegisterMap reg_map(current, false);
1520 frame stub_frame = current->last_frame();
1521 assert(stub_frame.is_runtime_frame(), "sanity check");
1522 frame caller_frame = stub_frame.sender(®_map);
1523
1524 if (caller_frame.is_interpreted_frame() ||
1528 guarantee(callee != NULL && callee->is_method(), "bad handshake");
1529 current->set_vm_result_2(callee);
1530 current->set_callee_target(NULL);
1531 if (caller_frame.is_entry_frame() && VM_Version::supports_fast_class_init_checks()) {
1532 // Bypass class initialization checks in c2i when caller is in native.
1533 // JNI calls to static methods don't have class initialization checks.
1534 // Fast class initialization checks are present in c2i adapters and call into
1535 // SharedRuntime::handle_wrong_method() on the slow path.
1536 //
1537 // JVM upcalls may land here as well, but there's a proper check present in
1538 // LinkResolver::resolve_static_call (called from JavaCalls::call_static),
1539 // so bypassing it in c2i adapter is benign.
1540 return callee->get_c2i_no_clinit_check_entry();
1541 } else {
1542 return callee->get_c2i_entry();
1543 }
1544 }
1545
1546 // Must be compiled to compiled path which is safe to stackwalk
1547 methodHandle callee_method;
1548 bool is_static_call = false;
1549 bool is_optimized = false;
1550 bool caller_is_c1 = false;
1551 JRT_BLOCK
1552 // Force resolving of caller (if we called from compiled frame)
1553 callee_method = SharedRuntime::reresolve_call_site(is_static_call, is_optimized, caller_is_c1, CHECK_NULL);
1554 current->set_vm_result_2(callee_method());
1555 JRT_BLOCK_END
1556 // return compiled code entry point after potential safepoints
1557 return entry_for_handle_wrong_method(callee_method, is_static_call, is_optimized, caller_is_c1);
1558 JRT_END
1559
1560 // Handle abstract method call
1561 JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
1562 // Verbose error message for AbstractMethodError.
1563 // Get the called method from the invoke bytecode.
1564 vframeStream vfst(current, true);
1565 assert(!vfst.at_end(), "Java frame must exist");
1566 methodHandle caller(current, vfst.method());
1567 Bytecode_invoke invoke(caller, vfst.bci());
1568 DEBUG_ONLY( invoke.verify(); )
1569
1570 // Find the compiled caller frame.
1571 RegisterMap reg_map(current);
1572 frame stubFrame = current->last_frame();
1573 assert(stubFrame.is_runtime_frame(), "must be");
1574 frame callerFrame = stubFrame.sender(®_map);
1575 assert(callerFrame.is_compiled_frame(), "must be");
1576
1577 // Install exception and return forward entry.
1578 address res = StubRoutines::throw_AbstractMethodError_entry();
1579 JRT_BLOCK
1580 methodHandle callee(current, invoke.static_target(current));
1581 if (!callee.is_null()) {
1582 oop recv = callerFrame.retrieve_receiver(®_map);
1583 Klass *recv_klass = (recv != NULL) ? recv->klass() : NULL;
1584 res = StubRoutines::forward_exception_entry();
1585 LinkResolver::throw_abstract_method_error(callee, recv_klass, CHECK_(res));
1586 }
1587 JRT_BLOCK_END
1588 return res;
1589 JRT_END
1590
1591
1592 // resolve a static call and patch code
1593 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
1594 methodHandle callee_method;
1595 bool caller_is_c1;
1596 JRT_BLOCK
1597 callee_method = SharedRuntime::resolve_helper(false, false, &caller_is_c1, CHECK_NULL);
1598 current->set_vm_result_2(callee_method());
1599 JRT_BLOCK_END
1600 // return compiled code entry point after potential safepoints
1601 address entry = caller_is_c1 ?
1602 callee_method->verified_inline_code_entry() : callee_method->verified_code_entry();
1603 assert(entry != NULL, "Jump to zero!");
1604 return entry;
1605 JRT_END
1606
1607
1608 // resolve virtual call and update inline cache to monomorphic
1609 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
1610 methodHandle callee_method;
1611 bool caller_is_c1;
1612 JRT_BLOCK
1613 callee_method = SharedRuntime::resolve_helper(true, false, &caller_is_c1, CHECK_NULL);
1614 current->set_vm_result_2(callee_method());
1615 JRT_BLOCK_END
1616 // return compiled code entry point after potential safepoints
1617 address entry = caller_is_c1 ?
1618 callee_method->verified_inline_code_entry() : callee_method->verified_inline_ro_code_entry();
1619 assert(entry != NULL, "Jump to zero!");
1620 return entry;
1621 JRT_END
1622
1623
1624 // Resolve a virtual call that can be statically bound (e.g., always
1625 // monomorphic, so it has no inline cache). Patch code to resolved target.
1626 JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
1627 methodHandle callee_method;
1628 bool caller_is_c1;
1629 JRT_BLOCK
1630 callee_method = SharedRuntime::resolve_helper(true, true, &caller_is_c1, CHECK_NULL);
1631 current->set_vm_result_2(callee_method());
1632 JRT_BLOCK_END
1633 // return compiled code entry point after potential safepoints
1634 address entry = caller_is_c1 ?
1635 callee_method->verified_inline_code_entry() : callee_method->verified_code_entry();
1636 assert(entry != NULL, "Jump to zero!");
1637 return entry;
1638 JRT_END
1639
1640 // The handle_ic_miss_helper_internal function returns false if it failed due
1641 // to either running out of vtable stubs or ic stubs due to IC transitions
1642 // to transitional states. The needs_ic_stub_refill value will be set if
1643 // the failure was due to running out of IC stubs, in which case handle_ic_miss_helper
1644 // refills the IC stubs and tries again.
1645 bool SharedRuntime::handle_ic_miss_helper_internal(Handle receiver, CompiledMethod* caller_nm,
1646 const frame& caller_frame, methodHandle callee_method,
1647 Bytecodes::Code bc, CallInfo& call_info,
1648 bool& needs_ic_stub_refill, bool& is_optimized, bool caller_is_c1, TRAPS) {
1649 CompiledICLocker ml(caller_nm);
1650 CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
1651 bool should_be_mono = false;
1652 if (inline_cache->is_optimized()) {
1653 if (TraceCallFixup) {
1654 ResourceMark rm(THREAD);
1655 tty->print("OPTIMIZED IC miss (%s) call to", Bytecodes::name(bc));
1656 callee_method->print_short_name(tty);
1657 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1658 }
1659 is_optimized = true;
1660 should_be_mono = true;
1661 } else if (inline_cache->is_icholder_call()) {
1662 CompiledICHolder* ic_oop = inline_cache->cached_icholder();
1663 if (ic_oop != NULL) {
1664 if (!ic_oop->is_loader_alive()) {
1665 // Deferred IC cleaning due to concurrent class unloading
1666 if (!inline_cache->set_to_clean()) {
1667 needs_ic_stub_refill = true;
1668 return false;
1669 }
1670 } else if (receiver()->klass() == ic_oop->holder_klass()) {
1671 // This isn't a real miss. We must have seen that compiled code
1672 // is now available and we want the call site converted to a
1673 // monomorphic compiled call site.
1674 // We can't assert for callee_method->code() != NULL because it
1675 // could have been deoptimized in the meantime
1676 if (TraceCallFixup) {
1677 ResourceMark rm(THREAD);
1678 tty->print("FALSE IC miss (%s) converting to compiled call to", Bytecodes::name(bc));
1679 callee_method->print_short_name(tty);
1680 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1681 }
1682 should_be_mono = true;
1683 }
1684 }
1685 }
1686
1687 if (should_be_mono) {
1688 // We have a path that was monomorphic but was going interpreted
1689 // and now we have (or had) a compiled entry. We correct the IC
1690 // by using a new icBuffer.
1691 CompiledICInfo info;
1692 Klass* receiver_klass = receiver()->klass();
1693 inline_cache->compute_monomorphic_entry(callee_method,
1694 receiver_klass,
1695 inline_cache->is_optimized(),
1696 false, caller_nm->is_nmethod(),
1697 caller_nm->is_compiled_by_c1(),
1698 info, CHECK_false);
1699 if (!inline_cache->set_to_monomorphic(info)) {
1700 needs_ic_stub_refill = true;
1701 return false;
1702 }
1703 } else if (!inline_cache->is_megamorphic() && !inline_cache->is_clean()) {
1704 // Potential change to megamorphic
1705
1706 bool successful = inline_cache->set_to_megamorphic(&call_info, bc, needs_ic_stub_refill, caller_is_c1, CHECK_false);
1707 if (needs_ic_stub_refill) {
1708 return false;
1709 }
1710 if (!successful) {
1711 if (!inline_cache->set_to_clean()) {
1712 needs_ic_stub_refill = true;
1713 return false;
1714 }
1715 }
1716 } else {
1717 // Either clean or megamorphic
1718 }
1719 return true;
1720 }
1721
1722 methodHandle SharedRuntime::handle_ic_miss_helper(bool& is_optimized, bool& caller_is_c1, TRAPS) {
1723 JavaThread* current = THREAD;
1724 ResourceMark rm(current);
1725 CallInfo call_info;
1726 Bytecodes::Code bc;
1727
1728 // receiver is NULL for static calls. An exception is thrown for NULL
1729 // receivers for non-static calls
1730 Handle receiver = find_callee_info(bc, call_info, CHECK_(methodHandle()));
1731 // Compiler1 can produce virtual call sites that can actually be statically bound
1732 // If we fell thru to below we would think that the site was going megamorphic
1733 // when in fact the site can never miss. Worse because we'd think it was megamorphic
1734 // we'd try and do a vtable dispatch however methods that can be statically bound
1735 // don't have vtable entries (vtable_index < 0) and we'd blow up. So we force a
1736 // reresolution of the call site (as if we did a handle_wrong_method and not an
1737 // plain ic_miss) and the site will be converted to an optimized virtual call site
1738 // never to miss again. I don't believe C2 will produce code like this but if it
1739 // did this would still be the correct thing to do for it too, hence no ifdef.
1740 //
1741 if (call_info.resolved_method()->can_be_statically_bound()) {
1742 bool is_static_call = false;
1743 methodHandle callee_method = SharedRuntime::reresolve_call_site(is_static_call, is_optimized, caller_is_c1, CHECK_(methodHandle()));
1744 assert(!is_static_call, "IC miss at static call?");
1745 if (TraceCallFixup) {
1746 RegisterMap reg_map(current, false);
1747 frame caller_frame = current->last_frame().sender(®_map);
1748 ResourceMark rm(current);
1749 tty->print("converting IC miss to reresolve (%s) call to", Bytecodes::name(bc));
1750 callee_method->print_short_name(tty);
1751 tty->print_cr(" from pc: " INTPTR_FORMAT, p2i(caller_frame.pc()));
1752 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1753 }
1754 return callee_method;
1755 }
1756
1757 methodHandle callee_method(current, call_info.selected_method());
1758
1759 #ifndef PRODUCT
1760 Atomic::inc(&_ic_miss_ctr);
1761
1762 // Statistics & Tracing
1763 if (TraceCallFixup) {
1764 ResourceMark rm(current);
1774 // produce statistics under the lock
1775 trace_ic_miss(f.pc());
1776 }
1777 #endif
1778
1779 // install an event collector so that when a vtable stub is created the
1780 // profiler can be notified via a DYNAMIC_CODE_GENERATED event. The
1781 // event can't be posted when the stub is created as locks are held
1782 // - instead the event will be deferred until the event collector goes
1783 // out of scope.
1784 JvmtiDynamicCodeEventCollector event_collector;
1785
1786 // Update inline cache to megamorphic. Skip update if we are called from interpreted.
1787 // Transitioning IC caches may require transition stubs. If we run out
1788 // of transition stubs, we have to drop locks and perform a safepoint
1789 // that refills them.
1790 RegisterMap reg_map(current, false);
1791 frame caller_frame = current->last_frame().sender(®_map);
1792 CodeBlob* cb = caller_frame.cb();
1793 CompiledMethod* caller_nm = cb->as_compiled_method();
1794 caller_is_c1 = caller_nm->is_compiled_by_c1();
1795
1796 for (;;) {
1797 ICRefillVerifier ic_refill_verifier;
1798 bool needs_ic_stub_refill = false;
1799 bool successful = handle_ic_miss_helper_internal(receiver, caller_nm, caller_frame, callee_method,
1800 bc, call_info, needs_ic_stub_refill, is_optimized, caller_is_c1, CHECK_(methodHandle()));
1801 if (successful || !needs_ic_stub_refill) {
1802 return callee_method;
1803 } else {
1804 InlineCacheBuffer::refill_ic_stubs();
1805 }
1806 }
1807 }
1808
1809 static bool clear_ic_at_addr(CompiledMethod* caller_nm, address call_addr, bool is_static_call) {
1810 CompiledICLocker ml(caller_nm);
1811 if (is_static_call) {
1812 CompiledStaticCall* ssc = caller_nm->compiledStaticCall_at(call_addr);
1813 if (!ssc->is_clean()) {
1814 return ssc->set_to_clean();
1815 }
1816 } else {
1817 // compiled, dispatched call (which used to call an interpreted method)
1818 CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr);
1819 if (!inline_cache->is_clean()) {
1820 return inline_cache->set_to_clean();
1821 }
1822 }
1823 return true;
1824 }
1825
1826 //
1827 // Resets a call-site in compiled code so it will get resolved again.
1828 // This routines handles both virtual call sites, optimized virtual call
1829 // sites, and static call sites. Typically used to change a call sites
1830 // destination from compiled to interpreted.
1831 //
1832 methodHandle SharedRuntime::reresolve_call_site(bool& is_static_call, bool& is_optimized, bool& caller_is_c1, TRAPS) {
1833 JavaThread* current = THREAD;
1834 ResourceMark rm(current);
1835 RegisterMap reg_map(current, false);
1836 frame stub_frame = current->last_frame();
1837 assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");
1838 frame caller = stub_frame.sender(®_map);
1839
1840 // Do nothing if the frame isn't a live compiled frame.
1841 // nmethod could be deoptimized by the time we get here
1842 // so no update to the caller is needed.
1843
1844 if (caller.is_compiled_frame() && !caller.is_deoptimized_frame()) {
1845
1846 address pc = caller.pc();
1847
1848 // Check for static or virtual call
1849 CompiledMethod* caller_nm = CodeCache::find_compiled(pc);
1850 caller_is_c1 = caller_nm->is_compiled_by_c1();
1851
1852 // Default call_addr is the location of the "basic" call.
1853 // Determine the address of the call we a reresolving. With
1854 // Inline Caches we will always find a recognizable call.
1855 // With Inline Caches disabled we may or may not find a
1856 // recognizable call. We will always find a call for static
1857 // calls and for optimized virtual calls. For vanilla virtual
1858 // calls it depends on the state of the UseInlineCaches switch.
1859 //
1860 // With Inline Caches disabled we can get here for a virtual call
1861 // for two reasons:
1862 // 1 - calling an abstract method. The vtable for abstract methods
1863 // will run us thru handle_wrong_method and we will eventually
1864 // end up in the interpreter to throw the ame.
1865 // 2 - a racing deoptimization. We could be doing a vanilla vtable
1866 // call and between the time we fetch the entry address and
1867 // we jump to it the target gets deoptimized. Similar to 1
1868 // we will wind up in the interprter (thru a c2i with c2).
1869 //
1870 address call_addr = NULL;
1874 CompiledICLocker ml(caller_nm);
1875 // Location of call instruction
1876 call_addr = caller_nm->call_instruction_address(pc);
1877 }
1878 // Make sure nmethod doesn't get deoptimized and removed until
1879 // this is done with it.
1880 // CLEANUP - with lazy deopt shouldn't need this lock
1881 nmethodLocker nmlock(caller_nm);
1882
1883 if (call_addr != NULL) {
1884 RelocIterator iter(caller_nm, call_addr, call_addr+1);
1885 int ret = iter.next(); // Get item
1886 if (ret) {
1887 assert(iter.addr() == call_addr, "must find call");
1888 if (iter.type() == relocInfo::static_call_type) {
1889 is_static_call = true;
1890 } else {
1891 assert(iter.type() == relocInfo::virtual_call_type ||
1892 iter.type() == relocInfo::opt_virtual_call_type
1893 , "unexpected relocInfo. type");
1894 is_optimized = (iter.type() == relocInfo::opt_virtual_call_type);
1895 }
1896 } else {
1897 assert(!UseInlineCaches, "relocation info. must exist for this address");
1898 }
1899
1900 // Cleaning the inline cache will force a new resolve. This is more robust
1901 // than directly setting it to the new destination, since resolving of calls
1902 // is always done through the same code path. (experience shows that it
1903 // leads to very hard to track down bugs, if an inline cache gets updated
1904 // to a wrong method). It should not be performance critical, since the
1905 // resolve is only done once.
1906
1907 for (;;) {
1908 ICRefillVerifier ic_refill_verifier;
1909 if (!clear_ic_at_addr(caller_nm, call_addr, is_static_call)) {
1910 InlineCacheBuffer::refill_ic_stubs();
1911 } else {
1912 break;
1913 }
1914 }
1915 }
1916 }
1917
1918 methodHandle callee_method = find_callee_method(CHECK_(methodHandle()));
1919
1920 #ifndef PRODUCT
1921 Atomic::inc(&_wrong_method_ctr);
1922
1923 if (TraceCallFixup) {
1924 ResourceMark rm(current);
1925 tty->print("handle_wrong_method reresolving call to");
1926 callee_method->print_short_name(tty);
1927 tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
1928 }
1929 #endif
1930
1931 return callee_method;
1932 }
1933
1934 address SharedRuntime::handle_unsafe_access(JavaThread* thread, address next_pc) {
1935 // The faulting unsafe accesses should be changed to throw the error
1936 // synchronously instead. Meanwhile the faulting instruction will be
1937 // skipped over (effectively turning it into a no-op) and an
1938 // asynchronous exception will be raised which the thread will
1939 // handle at a later point. If the instruction is a load it will
1993 }
1994 } else {
1995 if (TraceCallFixup) {
1996 tty->print("already patched callsite at " INTPTR_FORMAT " to compiled code for", p2i(caller_pc));
1997 moop->print_short_name(tty);
1998 tty->print_cr(" to " INTPTR_FORMAT, p2i(entry_point));
1999 }
2000 }
2001 return false;
2002 }
2003
2004 // ---------------------------------------------------------------------------
2005 // We are calling the interpreter via a c2i. Normally this would mean that
2006 // we were called by a compiled method. However we could have lost a race
2007 // where we went int -> i2c -> c2i and so the caller could in fact be
2008 // interpreted. If the caller is compiled we attempt to patch the caller
2009 // so he no longer calls into the interpreter.
2010 JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address caller_pc))
2011 Method* moop(method);
2012
2013 // It's possible that deoptimization can occur at a call site which hasn't
2014 // been resolved yet, in which case this function will be called from
2015 // an nmethod that has been patched for deopt and we can ignore the
2016 // request for a fixup.
2017 // Also it is possible that we lost a race in that from_compiled_entry
2018 // is now back to the i2c in that case we don't need to patch and if
2019 // we did we'd leap into space because the callsite needs to use
2020 // "to interpreter" stub in order to load up the Method*. Don't
2021 // ask me how I know this...
2022
2023 CodeBlob* cb = CodeCache::find_blob(caller_pc);
2024 if (cb == NULL || !cb->is_compiled()) {
2025 return;
2026 }
2027 address entry_point = moop->from_compiled_entry_no_trampoline(cb->is_compiled_by_c1());
2028 if (entry_point == moop->get_c2i_entry()) {
2029 return;
2030 }
2031
2032 // The check above makes sure this is a nmethod.
2033 CompiledMethod* nm = cb->as_compiled_method_or_null();
2034 assert(nm, "must be");
2035
2036 // Get the return PC for the passed caller PC.
2037 address return_pc = caller_pc + frame::pc_return_offset;
2038
2039 // There is a benign race here. We could be attempting to patch to a compiled
2040 // entry point at the same time the callee is being deoptimized. If that is
2041 // the case then entry_point may in fact point to a c2i and we'd patch the
2042 // call site with the same old data. clear_code will set code() to NULL
2043 // at the end of it. If we happen to see that NULL then we can skip trying
2044 // to patch. If we hit the window where the callee has a c2i in the
2045 // from_compiled_entry and the NULL isn't present yet then we lose the race
2046 // and patch the code with the same old data. Asi es la vida.
2047
2048 if (moop->code() == NULL) return;
2383 private:
2384 enum {
2385 _basic_type_bits = 4,
2386 _basic_type_mask = right_n_bits(_basic_type_bits),
2387 _basic_types_per_int = BitsPerInt / _basic_type_bits,
2388 _compact_int_count = 3
2389 };
2390 // TO DO: Consider integrating this with a more global scheme for compressing signatures.
2391 // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive.
2392
2393 union {
2394 int _compact[_compact_int_count];
2395 int* _fingerprint;
2396 } _value;
2397 int _length; // A negative length indicates the fingerprint is in the compact form,
2398 // Otherwise _value._fingerprint is the array.
2399
2400 // Remap BasicTypes that are handled equivalently by the adapters.
2401 // These are correct for the current system but someday it might be
2402 // necessary to make this mapping platform dependent.
2403 static BasicType adapter_encoding(BasicType in) {
2404 switch (in) {
2405 case T_BOOLEAN:
2406 case T_BYTE:
2407 case T_SHORT:
2408 case T_CHAR:
2409 // They are all promoted to T_INT in the calling convention
2410 return T_INT;
2411
2412 case T_OBJECT:
2413 case T_ARRAY:
2414 // In other words, we assume that any register good enough for
2415 // an int or long is good enough for a managed pointer.
2416 #ifdef _LP64
2417 return T_LONG;
2418 #else
2419 return T_INT;
2420 #endif
2421
2422 case T_INT:
2423 case T_LONG:
2424 case T_FLOAT:
2425 case T_DOUBLE:
2426 case T_VOID:
2427 return in;
2428
2429 default:
2430 ShouldNotReachHere();
2431 return T_CONFLICT;
2432 }
2433 }
2434
2435 public:
2436 AdapterFingerPrint(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
2437 // The fingerprint is based on the BasicType signature encoded
2438 // into an array of ints with eight entries per int.
2439 int total_args_passed = (sig != NULL) ? sig->length() : 0;
2440 int* ptr;
2441 int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int;
2442 if (len <= _compact_int_count) {
2443 assert(_compact_int_count == 3, "else change next line");
2444 _value._compact[0] = _value._compact[1] = _value._compact[2] = 0;
2445 // Storing the signature encoded as signed chars hits about 98%
2446 // of the time.
2447 _length = -len;
2448 ptr = _value._compact;
2449 } else {
2450 _length = len;
2451 _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode);
2452 ptr = _value._fingerprint;
2453 }
2454
2455 // Now pack the BasicTypes with 8 per int
2456 int sig_index = 0;
2457 BasicType prev_bt = T_ILLEGAL;
2458 int vt_count = 0;
2459 for (int index = 0; index < len; index++) {
2460 int value = 0;
2461 for (int byte = 0; byte < _basic_types_per_int; byte++) {
2462 BasicType bt = T_ILLEGAL;
2463 if (sig_index < total_args_passed) {
2464 bt = sig->at(sig_index++)._bt;
2465 if (bt == T_INLINE_TYPE) {
2466 // Found start of inline type in signature
2467 assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
2468 if (sig_index == 1 && has_ro_adapter) {
2469 // With a ro_adapter, replace receiver inline type delimiter by T_VOID to prevent matching
2470 // with other adapters that have the same inline type as first argument and no receiver.
2471 bt = T_VOID;
2472 }
2473 vt_count++;
2474 } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
2475 // Found end of inline type in signature
2476 assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
2477 vt_count--;
2478 assert(vt_count >= 0, "invalid vt_count");
2479 } else if (vt_count == 0) {
2480 // Widen fields that are not part of a scalarized inline type argument
2481 bt = adapter_encoding(bt);
2482 }
2483 prev_bt = bt;
2484 }
2485 int bt_val = (bt == T_ILLEGAL) ? 0 : bt;
2486 assert((bt_val & _basic_type_mask) == bt_val, "must fit in 4 bits");
2487 value = (value << _basic_type_bits) | bt_val;
2488 }
2489 ptr[index] = value;
2490 }
2491 assert(vt_count == 0, "invalid vt_count");
2492 }
2493
2494 ~AdapterFingerPrint() {
2495 if (_length > 0) {
2496 FREE_C_HEAP_ARRAY(int, _value._fingerprint);
2497 }
2498 }
2499
2500 int value(int index) {
2501 if (_length < 0) {
2502 return _value._compact[index];
2503 }
2504 return _value._fingerprint[index];
2505 }
2506 int length() {
2507 if (_length < 0) return -_length;
2508 return _length;
2509 }
2510
2511 bool is_compact() {
2598
2599 private:
2600
2601 #ifndef PRODUCT
2602 static int _lookups; // number of calls to lookup
2603 static int _buckets; // number of buckets checked
2604 static int _equals; // number of buckets checked with matching hash
2605 static int _hits; // number of successful lookups
2606 static int _compact; // number of equals calls with compact signature
2607 #endif
2608
2609 AdapterHandlerEntry* bucket(int i) {
2610 return (AdapterHandlerEntry*)BasicHashtable<mtCode>::bucket(i);
2611 }
2612
2613 public:
2614 AdapterHandlerTable()
2615 : BasicHashtable<mtCode>(293, (sizeof(AdapterHandlerEntry))) { }
2616
2617 // Create a new entry suitable for insertion in the table
2618 AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry,
2619 address c2i_inline_entry, address c2i_inline_ro_entry,
2620 address c2i_unverified_entry, address c2i_unverified_inline_entry, address c2i_no_clinit_check_entry) {
2621 AdapterHandlerEntry* entry = (AdapterHandlerEntry*)BasicHashtable<mtCode>::new_entry(fingerprint->compute_hash());
2622 entry->init(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry,
2623 c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
2624 return entry;
2625 }
2626
2627 // Insert an entry into the table
2628 void add(AdapterHandlerEntry* entry) {
2629 int index = hash_to_index(entry->hash());
2630 add_entry(index, entry);
2631 }
2632
2633 void free_entry(AdapterHandlerEntry* entry) {
2634 entry->deallocate();
2635 BasicHashtable<mtCode>::free_entry(entry);
2636 }
2637
2638 // Find a entry with the same fingerprint if it exists
2639 AdapterHandlerEntry* lookup(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
2640 NOT_PRODUCT(_lookups++);
2641 AdapterFingerPrint fp(sig, has_ro_adapter);
2642 unsigned int hash = fp.compute_hash();
2643 int index = hash_to_index(hash);
2644 for (AdapterHandlerEntry* e = bucket(index); e != NULL; e = e->next()) {
2645 NOT_PRODUCT(_buckets++);
2646 if (e->hash() == hash) {
2647 NOT_PRODUCT(_equals++);
2648 if (fp.equals(e->fingerprint())) {
2649 #ifndef PRODUCT
2650 if (fp.is_compact()) _compact++;
2651 _hits++;
2652 #endif
2653 return e;
2654 }
2655 }
2656 }
2657 return NULL;
2658 }
2659
2660 #ifndef PRODUCT
2661 void print_statistics() {
2722 AdapterHandlerEntry* result = _current;
2723 _current = _current->next();
2724 if (_current == NULL) scan();
2725 return result;
2726 } else {
2727 return NULL;
2728 }
2729 }
2730 };
2731
2732
2733 // ---------------------------------------------------------------------------
2734 // Implementation of AdapterHandlerLibrary
2735 AdapterHandlerTable* AdapterHandlerLibrary::_adapters = NULL;
2736 AdapterHandlerEntry* AdapterHandlerLibrary::_abstract_method_handler = NULL;
2737 AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = NULL;
2738 AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = NULL;
2739 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = NULL;
2740 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = NULL;
2741 AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = NULL;
2742 const int AdapterHandlerLibrary_size = 32*K;
2743 BufferBlob* AdapterHandlerLibrary::_buffer = NULL;
2744
2745 BufferBlob* AdapterHandlerLibrary::buffer_blob() {
2746 return _buffer;
2747 }
2748
2749 extern "C" void unexpected_adapter_call() {
2750 ShouldNotCallThis();
2751 }
2752
2753 static void post_adapter_creation(const AdapterBlob* new_adapter, const AdapterHandlerEntry* entry) {
2754 char blob_id[256];
2755 jio_snprintf(blob_id,
2756 sizeof(blob_id),
2757 "%s(%s)",
2758 new_adapter->name(),
2759 entry->fingerprint()->as_string());
2760 Forte::register_stub(blob_id, new_adapter->content_begin(), new_adapter->content_end());
2761
2762 if (JvmtiExport::should_post_dynamic_code_generated()) {
2766
2767 void AdapterHandlerLibrary::initialize() {
2768 ResourceMark rm;
2769 AdapterBlob* no_arg_blob = NULL;
2770 AdapterBlob* int_arg_blob = NULL;
2771 AdapterBlob* obj_arg_blob = NULL;
2772 AdapterBlob* obj_int_arg_blob = NULL;
2773 AdapterBlob* obj_obj_arg_blob = NULL;
2774 {
2775 MutexLocker mu(AdapterHandlerLibrary_lock);
2776 assert(_adapters == NULL, "Initializing more than once");
2777
2778 _adapters = new AdapterHandlerTable();
2779
2780 // Create a special handler for abstract methods. Abstract methods
2781 // are never compiled so an i2c entry is somewhat meaningless, but
2782 // throw AbstractMethodError just in case.
2783 // Pass wrong_method_abstract for the c2i transitions to return
2784 // AbstractMethodError for invalid invocations.
2785 address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
2786 _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL),
2787 StubRoutines::throw_AbstractMethodError_entry(),
2788 wrong_method_abstract, wrong_method_abstract, wrong_method_abstract,
2789 wrong_method_abstract, wrong_method_abstract);
2790 _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
2791
2792 CompiledEntrySignature no_args;
2793 no_args.compute_calling_conventions();
2794 _no_arg_handler = create_adapter(no_arg_blob, no_args, true);
2795
2796 CompiledEntrySignature obj_args;
2797 SigEntry::add_entry(&obj_args.sig(), T_OBJECT, NULL);
2798 obj_args.compute_calling_conventions();
2799 _obj_arg_handler = create_adapter(obj_arg_blob, obj_args, true);
2800
2801 CompiledEntrySignature int_args;
2802 SigEntry::add_entry(&int_args.sig(), T_INT, NULL);
2803 int_args.compute_calling_conventions();
2804 _int_arg_handler = create_adapter(int_arg_blob, int_args, true);
2805
2806 CompiledEntrySignature obj_int_args;
2807 SigEntry::add_entry(&obj_int_args.sig(), T_OBJECT, NULL);
2808 SigEntry::add_entry(&obj_int_args.sig(), T_INT, NULL);
2809 obj_int_args.compute_calling_conventions();
2810 _obj_int_arg_handler = create_adapter(obj_int_arg_blob, obj_int_args, true);
2811
2812 CompiledEntrySignature obj_obj_args;
2813 SigEntry::add_entry(&obj_obj_args.sig(), T_OBJECT, NULL);
2814 SigEntry::add_entry(&obj_obj_args.sig(), T_OBJECT, NULL);
2815 obj_obj_args.compute_calling_conventions();
2816 _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, obj_obj_args, true);
2817
2818 assert(no_arg_blob != NULL &&
2819 obj_arg_blob != NULL &&
2820 int_arg_blob != NULL &&
2821 obj_int_arg_blob != NULL &&
2822 obj_obj_arg_blob != NULL, "Initial adapters must be properly created");
2823 }
2824 return;
2825
2826 // Outside of the lock
2827 post_adapter_creation(no_arg_blob, _no_arg_handler);
2828 post_adapter_creation(obj_arg_blob, _obj_arg_handler);
2829 post_adapter_creation(int_arg_blob, _int_arg_handler);
2830 post_adapter_creation(obj_int_arg_blob, _obj_int_arg_handler);
2831 post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler);
2832 }
2833
2834 AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint,
2835 address i2c_entry,
2836 address c2i_entry,
2837 address c2i_inline_entry,
2838 address c2i_inline_ro_entry,
2839 address c2i_unverified_entry,
2840 address c2i_unverified_inline_entry,
2841 address c2i_no_clinit_check_entry) {
2842 return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry,
2843 c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
2844 }
2845
2846 AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
2847 if (method->is_abstract()) {
2848 return NULL;
2849 }
2850 int total_args_passed = method->size_of_parameters(); // All args on stack
2851 if (total_args_passed == 0) {
2852 return _no_arg_handler;
2853 } else if (total_args_passed == 1) {
2854 if (!method->is_static() && !method->method_holder()->is_inline_klass()) {
2855 return _obj_arg_handler;
2856 }
2857 switch (method->signature()->char_at(1)) {
2858 case JVM_SIGNATURE_CLASS:
2859 case JVM_SIGNATURE_ARRAY:
2860 return _obj_arg_handler;
2861 case JVM_SIGNATURE_INT:
2862 case JVM_SIGNATURE_BOOLEAN:
2863 case JVM_SIGNATURE_CHAR:
2864 case JVM_SIGNATURE_BYTE:
2865 case JVM_SIGNATURE_SHORT:
2866 return _int_arg_handler;
2867 }
2868 } else if (total_args_passed == 2 &&
2869 !method->is_static() && !method->method_holder()->is_inline_klass()) {
2870 switch (method->signature()->char_at(1)) {
2871 case JVM_SIGNATURE_CLASS:
2872 case JVM_SIGNATURE_ARRAY:
2873 return _obj_obj_arg_handler;
2874 case JVM_SIGNATURE_INT:
2875 case JVM_SIGNATURE_BOOLEAN:
2876 case JVM_SIGNATURE_CHAR:
2877 case JVM_SIGNATURE_BYTE:
2878 case JVM_SIGNATURE_SHORT:
2879 return _obj_int_arg_handler;
2880 }
2881 }
2882 return NULL;
2883 }
2884
2885 CompiledEntrySignature::CompiledEntrySignature(Method* method) :
2886 _method(method), _num_inline_args(0), _has_inline_recv(false),
2887 _regs(NULL), _regs_cc(NULL), _regs_cc_ro(NULL),
2888 _args_on_stack(0), _args_on_stack_cc(0), _args_on_stack_cc_ro(0),
2889 _c1_needs_stack_repair(false), _c2_needs_stack_repair(false) {
2890 _sig = new GrowableArray<SigEntry>((method != NULL) ? method->size_of_parameters() : 1);
2891 _sig_cc = _sig;
2892 _sig_cc_ro = _sig;
2893 }
2894
2895 int CompiledEntrySignature::compute_scalarized_cc(GrowableArray<SigEntry>*& sig_cc, VMRegPair*& regs_cc, bool scalar_receiver) {
2896 InstanceKlass* holder = _method->method_holder();
2897 sig_cc = new GrowableArray<SigEntry>(_method->size_of_parameters());
2898 if (!_method->is_static()) {
2899 if (holder->is_inline_klass() && scalar_receiver && InlineKlass::cast(holder)->can_be_passed_as_fields()) {
2900 sig_cc->appendAll(InlineKlass::cast(holder)->extended_sig());
2901 } else {
2902 SigEntry::add_entry(sig_cc, T_OBJECT, holder->name());
2903 }
2904 }
2905 for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
2906 if (ss.type() == T_INLINE_TYPE) {
2907 InlineKlass* vk = ss.as_inline_klass(holder);
2908 if (vk->can_be_passed_as_fields()) {
2909 sig_cc->appendAll(vk->extended_sig());
2910 } else {
2911 SigEntry::add_entry(sig_cc, T_OBJECT, ss.as_symbol());
2912 }
2913 } else {
2914 SigEntry::add_entry(sig_cc, ss.type(), ss.as_symbol());
2915 }
2916 }
2917 regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, sig_cc->length() + 2);
2918 return SharedRuntime::java_calling_convention(sig_cc, regs_cc);
2919 }
2920
2921 // See if we can save space by sharing the same entry for VIEP and VIEP(RO),
2922 // or the same entry for VEP and VIEP(RO).
2923 CodeOffsets::Entries CompiledEntrySignature::c1_inline_ro_entry_type() const {
2924 if (!has_scalarized_args()) {
2925 // VEP/VIEP/VIEP(RO) all share the same entry. There's no packing.
2926 return CodeOffsets::Verified_Entry;
2927 }
2928 if (_method->is_static()) {
2929 // Static methods don't need VIEP(RO)
2930 return CodeOffsets::Verified_Entry;
2931 }
2932
2933 if (has_inline_recv()) {
2934 if (num_inline_args() == 1) {
2935 // Share same entry for VIEP and VIEP(RO).
2936 // This is quite common: we have an instance method in an InlineKlass that has
2937 // no inline type args other than <this>.
2938 return CodeOffsets::Verified_Inline_Entry;
2939 } else {
2940 assert(num_inline_args() > 1, "must be");
2941 // No sharing:
2942 // VIEP(RO) -- <this> is passed as object
2943 // VEP -- <this> is passed as fields
2944 return CodeOffsets::Verified_Inline_Entry_RO;
2945 }
2946 }
2947
2948 // Either a static method, or <this> is not an inline type
2949 if (args_on_stack_cc() != args_on_stack_cc_ro()) {
2950 // No sharing:
2951 // Some arguments are passed on the stack, and we have inserted reserved entries
2952 // into the VEP, but we never insert reserved entries into the VIEP(RO).
2953 return CodeOffsets::Verified_Inline_Entry_RO;
2954 } else {
2955 // Share same entry for VEP and VIEP(RO).
2956 return CodeOffsets::Verified_Entry;
2957 }
2958 }
2959
2960 void CompiledEntrySignature::compute_calling_conventions() {
2961 // Get the (non-scalarized) signature and check for inline type arguments
2962 if (_method != NULL) {
2963 if (!_method->is_static()) {
2964 if (_method->method_holder()->is_inline_klass() && InlineKlass::cast(_method->method_holder())->can_be_passed_as_fields()) {
2965 _has_inline_recv = true;
2966 _num_inline_args++;
2967 }
2968 SigEntry::add_entry(_sig, T_OBJECT, _method->name());
2969 }
2970 for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
2971 BasicType bt = ss.type();
2972 if (bt == T_INLINE_TYPE) {
2973 if (ss.as_inline_klass(_method->method_holder())->can_be_passed_as_fields()) {
2974 _num_inline_args++;
2975 }
2976 bt = T_OBJECT;
2977 }
2978 SigEntry::add_entry(_sig, bt, ss.as_symbol());
2979 }
2980 if (_method->is_abstract() && !has_inline_arg()) {
2981 return;
2982 }
2983 }
2984
2985 // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
2986 _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
2987 _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
2988
2989 // Now compute the scalarized calling convention if there are inline types in the signature
2990 _regs_cc = _regs;
2991 _regs_cc_ro = _regs;
2992 _args_on_stack_cc = _args_on_stack;
2993 _args_on_stack_cc_ro = _args_on_stack;
2994
2995 if (has_inline_arg() && !_method->is_native()) {
2996 _args_on_stack_cc = compute_scalarized_cc(_sig_cc, _regs_cc, /* scalar_receiver = */ true);
2997
2998 _sig_cc_ro = _sig_cc;
2999 _regs_cc_ro = _regs_cc;
3000 _args_on_stack_cc_ro = _args_on_stack_cc;
3001 if (_has_inline_recv) {
3002 // For interface calls, we need another entry point / adapter to unpack the receiver
3003 _args_on_stack_cc_ro = compute_scalarized_cc(_sig_cc_ro, _regs_cc_ro, /* scalar_receiver = */ false);
3004 }
3005
3006 // Upper bound on stack arguments to avoid hitting the argument limit and
3007 // bailing out of compilation ("unsupported incoming calling sequence").
3008 // TODO we need a reasonable limit (flag?) here
3009 if (_args_on_stack_cc > 50) {
3010 // Don't scalarize inline type arguments
3011 _sig_cc = _sig;
3012 _sig_cc_ro = _sig;
3013 _regs_cc = _regs;
3014 _regs_cc_ro = _regs;
3015 _args_on_stack_cc = _args_on_stack;
3016 _args_on_stack_cc_ro = _args_on_stack;
3017 } else {
3018 _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
3019 _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
3020 }
3021 }
3022 }
3023
3024 AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
3025 // Use customized signature handler. Need to lock around updates to
3026 // the AdapterHandlerTable (it is not safe for concurrent readers
3027 // and a single writer: this could be fixed if it becomes a
3028 // problem).
3029 assert(_adapters != NULL, "Uninitialized");
3030
3031 // Fast-path for trivial adapters
3032 AdapterHandlerEntry* entry = get_simple_adapter(method);
3033 if (entry != NULL) {
3034 return entry;
3035 }
3036
3037 ResourceMark rm;
3038 AdapterBlob* new_adapter = NULL;
3039
3040 CompiledEntrySignature ces(method());
3041 ces.compute_calling_conventions();
3042 if (ces.has_scalarized_args()) {
3043 method->set_has_scalarized_args(true);
3044 method->set_c1_needs_stack_repair(ces.c1_needs_stack_repair());
3045 method->set_c2_needs_stack_repair(ces.c2_needs_stack_repair());
3046 } else if (method->is_abstract()) {
3047 return _abstract_method_handler;
3048 }
3049
3050 {
3051 MutexLocker mu(AdapterHandlerLibrary_lock);
3052
3053 if (ces.has_scalarized_args() && method->is_abstract()) {
3054 // Save a C heap allocated version of the signature for abstract methods with scalarized inline type arguments
3055 address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
3056 entry = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL),
3057 StubRoutines::throw_AbstractMethodError_entry(),
3058 wrong_method_abstract, wrong_method_abstract, wrong_method_abstract,
3059 wrong_method_abstract, wrong_method_abstract);
3060 GrowableArray<SigEntry>* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<SigEntry>(ces.sig_cc_ro().length(), mtInternal);
3061 heap_sig->appendAll(&ces.sig_cc_ro());
3062 entry->set_sig_cc(heap_sig);
3063 return entry;
3064 }
3065
3066 // Lookup method signature's fingerprint
3067 entry = _adapters->lookup(&ces.sig_cc(), ces.regs_cc() != ces.regs_cc_ro());
3068
3069 if (entry != NULL) {
3070 #ifdef ASSERT
3071 if (VerifyAdapterSharing) {
3072 AdapterBlob* comparison_blob = NULL;
3073 AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, ces, false);
3074 assert(comparison_blob == NULL, "no blob should be created when creating an adapter for comparison");
3075 assert(comparison_entry->compare_code(entry), "code must match");
3076 // Release the one just created and return the original
3077 _adapters->free_entry(comparison_entry);
3078 }
3079 #endif
3080 return entry;
3081 }
3082
3083 entry = create_adapter(new_adapter, ces, /* allocate_code_blob */ true);
3084 }
3085
3086 // Outside of the lock
3087 if (new_adapter != NULL) {
3088 post_adapter_creation(new_adapter, entry);
3089 }
3090 return entry;
3091 }
3092
3093 AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter,
3094 CompiledEntrySignature& ces,
3095 bool allocate_code_blob) {
3096
3097 // StubRoutines::code2() is initialized after this function can be called. As a result,
3098 // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated
3099 // prior to StubRoutines::code2() being set. Checks refer to checks generated in an I2C
3100 // stub that ensure that an I2C stub is called from an interpreter frame.
3101 bool contains_all_checks = StubRoutines::code2() != NULL;
3102
3103 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3104 CodeBuffer buffer(buf);
3105 short buffer_locs[20];
3106 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3107 sizeof(buffer_locs)/sizeof(relocInfo));
3108
3109 // Make a C heap allocated version of the fingerprint to store in the adapter
3110 AdapterFingerPrint* fingerprint = new AdapterFingerPrint(&ces.sig_cc(), ces.regs_cc() != ces.regs_cc_ro());
3111 MacroAssembler _masm(&buffer);
3112 AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm,
3113 ces.args_on_stack(),
3114 &ces.sig(),
3115 ces.regs(),
3116 &ces.sig_cc(),
3117 ces.regs_cc(),
3118 &ces.sig_cc_ro(),
3119 ces.regs_cc_ro(),
3120 fingerprint,
3121 new_adapter,
3122 allocate_code_blob);
3123
3124 if (ces.has_scalarized_args()) {
3125 // Save a C heap allocated version of the scalarized signature and store it in the adapter
3126 GrowableArray<SigEntry>* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<SigEntry>(ces.sig_cc().length(), mtInternal);
3127 heap_sig->appendAll(&ces.sig_cc());
3128 entry->set_sig_cc(heap_sig);
3129 }
3130
3131 #ifdef ASSERT
3132 if (VerifyAdapterSharing) {
3133 entry->save_code(buf->code_begin(), buffer.insts_size());
3134 if (!allocate_code_blob) {
3135 return entry;
3136 }
3137 }
3138 #endif
3139
3140 NOT_PRODUCT(int insts_size = buffer.insts_size());
3141 if (new_adapter == NULL) {
3142 // CodeCache is full, disable compilation
3143 // Ought to log this but compile log is only per compile thread
3144 // and we're some non descript Java thread.
3145 return NULL;
3146 }
3147 entry->relocate(new_adapter->content_begin());
3148 #ifndef PRODUCT
3149 // debugging suppport
3150 if (PrintAdapterHandlers || PrintStubCode) {
3151 ttyLocker ttyl;
3152 entry->print_adapter_on(tty);
3153 tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)",
3154 _adapters->number_of_entries(), fingerprint->as_basic_args_string(),
3155 fingerprint->as_string(), insts_size);
3156 tty->print_cr("c2i argument handler starts at %p", entry->get_c2i_entry());
3157 if (Verbose || PrintStubCode) {
3158 address first_pc = entry->base_address();
3159 if (first_pc != NULL) {
3160 Disassembler::decode(first_pc, first_pc + insts_size, tty
3161 NOT_PRODUCT(COMMA &new_adapter->asm_remarks()));
3162 tty->cr();
3163 }
3164 }
3165 }
3166 #endif
3167
3168 // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp)
3169 // The checks are inserted only if -XX:+VerifyAdapterCalls is specified.
3170 if (contains_all_checks || !VerifyAdapterCalls) {
3171 _adapters->add(entry);
3172 }
3173 return entry;
3174 }
3175
3176 address AdapterHandlerEntry::base_address() {
3177 address base = _i2c_entry;
3178 if (base == NULL) base = _c2i_entry;
3179 assert(base <= _c2i_entry || _c2i_entry == NULL, "");
3180 assert(base <= _c2i_inline_entry || _c2i_inline_entry == NULL, "");
3181 assert(base <= _c2i_inline_ro_entry || _c2i_inline_ro_entry == NULL, "");
3182 assert(base <= _c2i_unverified_entry || _c2i_unverified_entry == NULL, "");
3183 assert(base <= _c2i_unverified_inline_entry || _c2i_unverified_inline_entry == NULL, "");
3184 assert(base <= _c2i_no_clinit_check_entry || _c2i_no_clinit_check_entry == NULL, "");
3185 return base;
3186 }
3187
3188 void AdapterHandlerEntry::relocate(address new_base) {
3189 address old_base = base_address();
3190 assert(old_base != NULL, "");
3191 ptrdiff_t delta = new_base - old_base;
3192 if (_i2c_entry != NULL)
3193 _i2c_entry += delta;
3194 if (_c2i_entry != NULL)
3195 _c2i_entry += delta;
3196 if (_c2i_inline_entry != NULL)
3197 _c2i_inline_entry += delta;
3198 if (_c2i_inline_ro_entry != NULL)
3199 _c2i_inline_ro_entry += delta;
3200 if (_c2i_unverified_entry != NULL)
3201 _c2i_unverified_entry += delta;
3202 if (_c2i_unverified_inline_entry != NULL)
3203 _c2i_unverified_inline_entry += delta;
3204 if (_c2i_no_clinit_check_entry != NULL)
3205 _c2i_no_clinit_check_entry += delta;
3206 assert(base_address() == new_base, "");
3207 }
3208
3209
3210 void AdapterHandlerEntry::deallocate() {
3211 delete _fingerprint;
3212 if (_sig_cc != NULL) {
3213 delete _sig_cc;
3214 }
3215 #ifdef ASSERT
3216 FREE_C_HEAP_ARRAY(unsigned char, _saved_code);
3217 #endif
3218 }
3219
3220
3221 #ifdef ASSERT
3222 // Capture the code before relocation so that it can be compared
3223 // against other versions. If the code is captured after relocation
3224 // then relative instructions won't be equivalent.
3225 void AdapterHandlerEntry::save_code(unsigned char* buffer, int length) {
3226 _saved_code = NEW_C_HEAP_ARRAY(unsigned char, length, mtCode);
3227 _saved_code_length = length;
3228 memcpy(_saved_code, buffer, length);
3229 }
3230
3231
3232 bool AdapterHandlerEntry::compare_code(AdapterHandlerEntry* other) {
3233 assert(_saved_code != NULL && other->_saved_code != NULL, "code not saved");
3234
3273 assert(compile_id > 0, "Must generate native wrapper");
3274
3275
3276 ResourceMark rm;
3277 BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
3278 if (buf != NULL) {
3279 CodeBuffer buffer(buf);
3280 struct { double data[20]; } locs_buf;
3281 buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo));
3282 #if defined(AARCH64)
3283 // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be
3284 // in the constant pool to ensure ordering between the barrier and oops
3285 // accesses. For native_wrappers we need a constant.
3286 buffer.initialize_consts_size(8);
3287 #endif
3288 MacroAssembler _masm(&buffer);
3289
3290 // Fill in the signature array, for the calling-convention call.
3291 const int total_args_passed = method->size_of_parameters();
3292
3293 BasicType stack_sig_bt[16];
3294 VMRegPair stack_regs[16];
3295 BasicType* sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
3296 VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
3297
3298 int i = 0;
3299 if (!method->is_static()) { // Pass in receiver first
3300 sig_bt[i++] = T_OBJECT;
3301 }
3302 SignatureStream ss(method->signature());
3303 for (; !ss.at_return_type(); ss.next()) {
3304 sig_bt[i++] = ss.type(); // Collect remaining bits of signature
3305 if (ss.type() == T_LONG || ss.type() == T_DOUBLE) {
3306 sig_bt[i++] = T_VOID; // Longs & doubles take 2 Java slots
3307 }
3308 }
3309 assert(i == total_args_passed, "");
3310 BasicType ret_type = ss.type();
3311
3312 // Now get the compiled-Java arguments layout.
3313 int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
3314
3315 // Generate the compiled-to-native wrapper code
3316 nm = SharedRuntime::generate_native_wrapper(&_masm, method, compile_id, sig_bt, regs, ret_type, critical_entry);
3317
3318 if (nm != NULL) {
3319 {
3320 MutexLocker pl(CompiledMethod_lock, Mutex::_no_safepoint_check_flag);
3321 if (nm->make_in_use()) {
3322 method->set_code(method, nm);
3323 }
3324 }
3325
3326 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_simple));
3327 if (directive->PrintAssemblyOption) {
3328 nm->print_code();
3329 }
3330 DirectivesStack::release(directive);
3511 AdapterHandlerTableIterator iter(_adapters);
3512 while (iter.has_next()) {
3513 AdapterHandlerEntry* a = iter.next();
3514 if (b == CodeCache::find_blob(a->get_i2c_entry())) {
3515 st->print("Adapter for signature: ");
3516 a->print_adapter_on(tty);
3517 return;
3518 }
3519 }
3520 assert(false, "Should have found handler");
3521 }
3522
3523 void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
3524 st->print("AHE@" INTPTR_FORMAT ": %s", p2i(this), fingerprint()->as_string());
3525 if (get_i2c_entry() != NULL) {
3526 st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
3527 }
3528 if (get_c2i_entry() != NULL) {
3529 st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
3530 }
3531 if (get_c2i_entry() != NULL) {
3532 st->print(" c2iVE: " INTPTR_FORMAT, p2i(get_c2i_inline_entry()));
3533 }
3534 if (get_c2i_entry() != NULL) {
3535 st->print(" c2iVROE: " INTPTR_FORMAT, p2i(get_c2i_inline_ro_entry()));
3536 }
3537 if (get_c2i_unverified_entry() != NULL) {
3538 st->print(" c2iUE: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
3539 }
3540 if (get_c2i_unverified_entry() != NULL) {
3541 st->print(" c2iUVE: " INTPTR_FORMAT, p2i(get_c2i_unverified_inline_entry()));
3542 }
3543 if (get_c2i_no_clinit_check_entry() != NULL) {
3544 st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
3545 }
3546 st->cr();
3547 }
3548
3549 #ifndef PRODUCT
3550
3551 void AdapterHandlerLibrary::print_statistics() {
3552 _adapters->print_statistics();
3553 }
3554
3555 #endif /* PRODUCT */
3556
3557 JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current))
3558 StackOverflow* overflow_state = current->stack_overflow_state();
3559 overflow_state->enable_stack_reserved_zone(/*check_if_disabled*/true);
3560 overflow_state->set_reserved_stack_activation(current->stack_base());
3561 JRT_END
3606 break;
3607 } else {
3608 fr = fr.java_sender();
3609 }
3610 }
3611 return activation;
3612 }
3613
3614 void SharedRuntime::on_slowpath_allocation_exit(JavaThread* current) {
3615 // After any safepoint, just before going back to compiled code,
3616 // we inform the GC that we will be doing initializing writes to
3617 // this object in the future without emitting card-marks, so
3618 // GC may take any compensating steps.
3619
3620 oop new_obj = current->vm_result();
3621 if (new_obj == NULL) return;
3622
3623 BarrierSet *bs = BarrierSet::barrier_set();
3624 bs->on_slowpath_allocation_exit(current, new_obj);
3625 }
3626
3627 // We are at a compiled code to interpreter call. We need backing
3628 // buffers for all inline type arguments. Allocate an object array to
3629 // hold them (convenient because once we're done with it we don't have
3630 // to worry about freeing it).
3631 oop SharedRuntime::allocate_inline_types_impl(JavaThread* current, methodHandle callee, bool allocate_receiver, TRAPS) {
3632 assert(InlineTypePassFieldsAsArgs, "no reason to call this");
3633 ResourceMark rm;
3634
3635 int nb_slots = 0;
3636 InstanceKlass* holder = callee->method_holder();
3637 allocate_receiver &= !callee->is_static() && holder->is_inline_klass();
3638 if (allocate_receiver) {
3639 nb_slots++;
3640 }
3641 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
3642 if (ss.type() == T_INLINE_TYPE) {
3643 nb_slots++;
3644 }
3645 }
3646 objArrayOop array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
3647 objArrayHandle array(THREAD, array_oop);
3648 int i = 0;
3649 if (allocate_receiver) {
3650 InlineKlass* vk = InlineKlass::cast(holder);
3651 oop res = vk->allocate_instance(CHECK_NULL);
3652 array->obj_at_put(i, res);
3653 i++;
3654 }
3655 for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
3656 if (ss.type() == T_INLINE_TYPE) {
3657 InlineKlass* vk = ss.as_inline_klass(holder);
3658 oop res = vk->allocate_instance(CHECK_NULL);
3659 array->obj_at_put(i, res);
3660 i++;
3661 }
3662 }
3663 return array();
3664 }
3665
3666 JRT_ENTRY(void, SharedRuntime::allocate_inline_types(JavaThread* current, Method* callee_method, bool allocate_receiver))
3667 methodHandle callee(current, callee_method);
3668 oop array = SharedRuntime::allocate_inline_types_impl(current, callee, allocate_receiver, CHECK);
3669 current->set_vm_result(array);
3670 current->set_vm_result_2(callee()); // TODO: required to keep callee live?
3671 JRT_END
3672
3673 // We're returning from an interpreted method: load each field into a
3674 // register following the calling convention
3675 JRT_LEAF(void, SharedRuntime::load_inline_type_fields_in_regs(JavaThread* current, oopDesc* res))
3676 {
3677 assert(res->klass()->is_inline_klass(), "only inline types here");
3678 ResourceMark rm;
3679 RegisterMap reg_map(current);
3680 frame stubFrame = current->last_frame();
3681 frame callerFrame = stubFrame.sender(®_map);
3682 assert(callerFrame.is_interpreted_frame(), "should be coming from interpreter");
3683
3684 InlineKlass* vk = InlineKlass::cast(res->klass());
3685
3686 const Array<SigEntry>* sig_vk = vk->extended_sig();
3687 const Array<VMRegPair>* regs = vk->return_regs();
3688
3689 if (regs == NULL) {
3690 // The fields of the inline klass don't fit in registers, bail out
3691 return;
3692 }
3693
3694 int j = 1;
3695 for (int i = 0; i < sig_vk->length(); i++) {
3696 BasicType bt = sig_vk->at(i)._bt;
3697 if (bt == T_INLINE_TYPE) {
3698 continue;
3699 }
3700 if (bt == T_VOID) {
3701 if (sig_vk->at(i-1)._bt == T_LONG ||
3702 sig_vk->at(i-1)._bt == T_DOUBLE) {
3703 j++;
3704 }
3705 continue;
3706 }
3707 int off = sig_vk->at(i)._offset;
3708 assert(off > 0, "offset in object should be positive");
3709 VMRegPair pair = regs->at(j);
3710 address loc = reg_map.location(pair.first());
3711 switch(bt) {
3712 case T_BOOLEAN:
3713 *(jboolean*)loc = res->bool_field(off);
3714 break;
3715 case T_CHAR:
3716 *(jchar*)loc = res->char_field(off);
3717 break;
3718 case T_BYTE:
3719 *(jbyte*)loc = res->byte_field(off);
3720 break;
3721 case T_SHORT:
3722 *(jshort*)loc = res->short_field(off);
3723 break;
3724 case T_INT: {
3725 *(jint*)loc = res->int_field(off);
3726 break;
3727 }
3728 case T_LONG:
3729 #ifdef _LP64
3730 *(intptr_t*)loc = res->long_field(off);
3731 #else
3732 Unimplemented();
3733 #endif
3734 break;
3735 case T_OBJECT:
3736 case T_ARRAY: {
3737 *(oop*)loc = res->obj_field(off);
3738 break;
3739 }
3740 case T_FLOAT:
3741 *(jfloat*)loc = res->float_field(off);
3742 break;
3743 case T_DOUBLE:
3744 *(jdouble*)loc = res->double_field(off);
3745 break;
3746 default:
3747 ShouldNotReachHere();
3748 }
3749 j++;
3750 }
3751 assert(j == regs->length(), "missed a field?");
3752
3753 #ifdef ASSERT
3754 VMRegPair pair = regs->at(0);
3755 address loc = reg_map.location(pair.first());
3756 assert(*(oopDesc**)loc == res, "overwritten object");
3757 #endif
3758
3759 current->set_vm_result(res);
3760 }
3761 JRT_END
3762
3763 // We've returned to an interpreted method, the interpreter needs a
3764 // reference to an inline type instance. Allocate it and initialize it
3765 // from field's values in registers.
3766 JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread* current, intptr_t res))
3767 {
3768 ResourceMark rm;
3769 RegisterMap reg_map(current);
3770 frame stubFrame = current->last_frame();
3771 frame callerFrame = stubFrame.sender(®_map);
3772
3773 #ifdef ASSERT
3774 InlineKlass* verif_vk = InlineKlass::returned_inline_klass(reg_map);
3775 #endif
3776
3777 if (!is_set_nth_bit(res, 0)) {
3778 // We're not returning with inline type fields in registers (the
3779 // calling convention didn't allow it for this inline klass)
3780 assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area");
3781 current->set_vm_result((oopDesc*)res);
3782 assert(verif_vk == NULL, "broken calling convention");
3783 return;
3784 }
3785
3786 clear_nth_bit(res, 0);
3787 InlineKlass* vk = (InlineKlass*)res;
3788 assert(verif_vk == vk, "broken calling convention");
3789 assert(Metaspace::contains((void*)res), "should be klass");
3790
3791 // Allocate handles for every oop field so they are safe in case of
3792 // a safepoint when allocating
3793 GrowableArray<Handle> handles;
3794 vk->save_oop_fields(reg_map, handles);
3795
3796 // It's unsafe to safepoint until we are here
3797 JRT_BLOCK;
3798 {
3799 JavaThread* THREAD = current;
3800 oop vt = vk->realloc_result(reg_map, handles, CHECK);
3801 current->set_vm_result(vt);
3802 }
3803 JRT_BLOCK_END;
3804 }
3805 JRT_END
3806
|