31 #include "code/nmethod.hpp"
32 #include "code/pcDesc.hpp"
33 #include "code/scopeDesc.hpp"
34 #include "compiler/compilationPolicy.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "interpreter/bytecode.inline.hpp"
39 #include "interpreter/bytecodeStream.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "interpreter/oopMapCache.hpp"
42 #include "jvm.h"
43 #include "logging/log.hpp"
44 #include "logging/logLevel.hpp"
45 #include "logging/logMessage.hpp"
46 #include "logging/logStream.hpp"
47 #include "memory/allocation.inline.hpp"
48 #include "memory/oopFactory.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "memory/universe.hpp"
51 #include "oops/constantPool.hpp"
52 #include "oops/fieldStreams.inline.hpp"
53 #include "oops/method.hpp"
54 #include "oops/objArrayKlass.hpp"
55 #include "oops/objArrayOop.inline.hpp"
56 #include "oops/oop.inline.hpp"
57 #include "oops/typeArrayOop.inline.hpp"
58 #include "oops/verifyOopClosure.hpp"
59 #include "prims/jvmtiDeferredUpdates.hpp"
60 #include "prims/jvmtiExport.hpp"
61 #include "prims/jvmtiThreadState.hpp"
62 #include "prims/methodHandles.hpp"
63 #include "prims/vectorSupport.hpp"
64 #include "runtime/atomicAccess.hpp"
65 #include "runtime/basicLock.inline.hpp"
66 #include "runtime/continuation.hpp"
67 #include "runtime/continuationEntry.inline.hpp"
68 #include "runtime/deoptimization.hpp"
69 #include "runtime/escapeBarrier.hpp"
70 #include "runtime/fieldDescriptor.inline.hpp"
71 #include "runtime/frame.inline.hpp"
72 #include "runtime/handles.inline.hpp"
73 #include "runtime/interfaceSupport.inline.hpp"
74 #include "runtime/javaThread.hpp"
75 #include "runtime/jniHandles.inline.hpp"
76 #include "runtime/keepStackGCProcessed.hpp"
77 #include "runtime/lockStack.inline.hpp"
78 #include "runtime/objectMonitor.inline.hpp"
79 #include "runtime/osThread.hpp"
80 #include "runtime/safepointVerifiers.hpp"
81 #include "runtime/sharedRuntime.hpp"
82 #include "runtime/signature.hpp"
83 #include "runtime/stackFrameStream.inline.hpp"
281 // The actual reallocation of previously eliminated objects occurs in realloc_objects,
282 // which is called from the method fetch_unroll_info_helper below.
283 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* current, int exec_mode))
284 // fetch_unroll_info() is called at the beginning of the deoptimization
285 // handler. Note this fact before we start generating temporary frames
286 // that can confuse an asynchronous stack walker. This counter is
287 // decremented at the end of unpack_frames().
288 current->inc_in_deopt_handler();
289
290 if (exec_mode == Unpack_exception) {
291 // When we get here, a callee has thrown an exception into a deoptimized
292 // frame. That throw might have deferred stack watermark checking until
293 // after unwinding. So we deal with such deferred requests here.
294 StackWatermarkSet::after_unwind(current);
295 }
296
297 return fetch_unroll_info_helper(current, exec_mode);
298 JRT_END
299
300 #ifdef COMPILER2
301 // print information about reallocated objects
302 static void print_objects(JavaThread* deoptee_thread,
303 GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
304 ResourceMark rm;
305 stringStream st; // change to logStream with logging
306 st.print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(deoptee_thread));
307 fieldDescriptor fd;
308
309 for (int i = 0; i < objects->length(); i++) {
310 ObjectValue* sv = (ObjectValue*) objects->at(i);
311 Handle obj = sv->value();
312
313 if (obj.is_null()) {
314 st.print_cr(" nullptr");
315 continue;
316 }
317
318 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
319
320 st.print(" object <" INTPTR_FORMAT "> of type ", p2i(sv->value()()));
321 k->print_value_on(&st);
322 st.print_cr(" allocated (%zu bytes)", obj->size() * HeapWordSize);
323
324 if (Verbose && k != nullptr) {
325 k->oop_print_on(obj(), &st);
326 }
327 }
328 tty->print_raw(st.freeze());
329 }
330
331 static bool rematerialize_objects(JavaThread* thread, int exec_mode, nmethod* compiled_method,
332 frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk,
333 bool& deoptimized_objects) {
334 bool realloc_failures = false;
335 assert (chunk->at(0)->scope() != nullptr,"expect only compiled java frames");
336
337 JavaThread* deoptee_thread = chunk->at(0)->thread();
338 assert(exec_mode == Deoptimization::Unpack_none || (deoptee_thread == thread),
339 "a frame can only be deoptimized by the owner thread");
340
341 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects_to_rematerialize(deoptee, map);
342
343 // The flag return_oop() indicates call sites which return oop
344 // in compiled code. Such sites include java method calls,
345 // runtime calls (for example, used to allocate new objects/arrays
346 // on slow code path) and any other calls generated in compiled code.
347 // It is not guaranteed that we can get such information here only
348 // by analyzing bytecode in deoptimized frames. This is why this flag
349 // is set during method compilation (see Compile::Process_OopMap_Node()).
350 // If the previous frame was popped or if we are dispatching an exception,
351 // we don't have an oop result.
352 bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
353 Handle return_value;
354 if (save_oop_result) {
355 // Reallocation may trigger GC. If deoptimization happened on return from
356 // call which returns oop we need to save it since it is not in oopmap.
357 oop result = deoptee.saved_oop_result(&map);
358 assert(oopDesc::is_oop_or_null(result), "must be oop");
359 return_value = Handle(thread, result);
360 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
361 if (TraceDeoptimization) {
362 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
363 tty->cr();
364 }
365 }
366 if (objects != nullptr) {
367 if (exec_mode == Deoptimization::Unpack_none) {
368 assert(thread->thread_state() == _thread_in_vm, "assumption");
369 JavaThread* THREAD = thread; // For exception macros.
370 // Clear pending OOM if reallocation fails and return true indicating allocation failure
371 realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, CHECK_AND_CLEAR_(true));
372 deoptimized_objects = true;
373 } else {
374 JavaThread* current = thread; // For JRT_BLOCK
375 JRT_BLOCK
376 realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
377 JRT_END
378 }
379 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
380 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures);
381 if (TraceDeoptimization) {
382 print_objects(deoptee_thread, objects, realloc_failures);
383 }
384 }
385 if (save_oop_result) {
386 // Restore result.
387 deoptee.set_saved_oop_result(&map, return_value());
388 }
389 return realloc_failures;
390 }
391
392 static void restore_eliminated_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures,
393 frame& deoptee, int exec_mode, bool& deoptimized_objects) {
394 JavaThread* deoptee_thread = chunk->at(0)->thread();
395 assert(!EscapeBarrier::objs_are_deoptimized(deoptee_thread, deoptee.id()), "must relock just once");
396 assert(thread == Thread::current(), "should be");
397 HandleMark hm(thread);
398 #ifndef PRODUCT
399 bool first = true;
400 #endif // !PRODUCT
401 // Start locking from outermost/oldest frame
402 for (int i = (chunk->length() - 1); i >= 0; i--) {
403 compiledVFrame* cvf = chunk->at(i);
404 assert (cvf->scope() != nullptr,"expect only compiled java frames");
405 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
406 if (monitors->is_nonempty()) {
407 bool relocked = Deoptimization::relock_objects(thread, monitors, deoptee_thread, deoptee,
434 }
435 }
436 tty->print_raw(st.freeze());
437 }
438 #endif // !PRODUCT
439 }
440 }
441 }
442
443 // Deoptimize objects, that is reallocate and relock them, just before they escape through JVMTI.
444 // The given vframes cover one physical frame.
445 bool Deoptimization::deoptimize_objects_internal(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk,
446 bool& realloc_failures) {
447 frame deoptee = chunk->at(0)->fr();
448 JavaThread* deoptee_thread = chunk->at(0)->thread();
449 nmethod* nm = deoptee.cb()->as_nmethod_or_null();
450 RegisterMap map(chunk->at(0)->register_map());
451 bool deoptimized_objects = false;
452
453 // Reallocate the non-escaping objects and restore their fields.
454 if ((DoEscapeAnalysis && EliminateAllocations) || EliminateAutoBox || EnableVectorAggressiveReboxing) {
455 realloc_failures = rematerialize_objects(thread, Unpack_none, nm, deoptee, map, chunk, deoptimized_objects);
456 }
457
458 // MonitorInfo structures used in eliminate_locks are not GC safe.
459 NoSafepointVerifier no_safepoint;
460
461 // Now relock objects if synchronization on them was eliminated.
462 if ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) {
463 restore_eliminated_locks(thread, chunk, realloc_failures, deoptee, Unpack_none, deoptimized_objects);
464 }
465 return deoptimized_objects;
466 }
467 #endif // COMPILER2
468
469 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
470 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* current, int exec_mode) {
471 JFR_ONLY(Jfr::check_and_process_sample_request(current);)
472 // When we get here we are about to unwind the deoptee frame. In order to
473 // catch not yet safe to use frames, the following stack watermark barrier
474 // poll will make such frames safe to use.
508 }
509
510 // Create a growable array of VFrames where each VFrame represents an inlined
511 // Java frame. This storage is allocated with the usual system arena.
512 assert(deoptee.is_compiled_frame(), "Wrong frame type");
513 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
514 vframe* vf = vframe::new_vframe(&deoptee, &map, current);
515 while (!vf->is_top()) {
516 assert(vf->is_compiled_frame(), "Wrong frame type");
517 chunk->push(compiledVFrame::cast(vf));
518 vf = vf->sender();
519 }
520 assert(vf->is_compiled_frame(), "Wrong frame type");
521 chunk->push(compiledVFrame::cast(vf));
522
523 bool realloc_failures = false;
524
525 #ifdef COMPILER2
526 // Reallocate the non-escaping objects and restore their fields. Then
527 // relock objects if synchronization on them was eliminated.
528 if ((DoEscapeAnalysis && EliminateAllocations) || EliminateAutoBox || EnableVectorAggressiveReboxing) {
529 bool unused;
530 realloc_failures = rematerialize_objects(current, exec_mode, nm, deoptee, map, chunk, unused);
531 }
532 #endif // COMPILER2
533
534 // Ensure that no safepoint is taken after pointers have been stored
535 // in fields of rematerialized objects. If a safepoint occurs from here on
536 // out the java state residing in the vframeArray will be missed.
537 // Locks may be rebaised in a safepoint.
538 NoSafepointVerifier no_safepoint;
539
540 #ifdef COMPILER2
541 if (((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) &&
542 !EscapeBarrier::objs_are_deoptimized(current, deoptee.id())) {
543 bool unused = false;
544 restore_eliminated_locks(current, chunk, realloc_failures, deoptee, exec_mode, unused);
545 }
546 #endif // COMPILER2
547
548 ScopeDesc* trap_scope = chunk->at(0)->scope();
695 // its caller's stack by. If the caller is a compiled frame then
696 // we pretend that the callee has no parameters so that the
697 // extension counts for the full amount of locals and not just
698 // locals-parms. This is because without a c2i adapter the parm
699 // area as created by the compiled frame will not be usable by
700 // the interpreter. (Depending on the calling convention there
701 // may not even be enough space).
702
703 // QQQ I'd rather see this pushed down into last_frame_adjust
704 // and have it take the sender (aka caller).
705
706 if (!deopt_sender.is_interpreted_frame() || caller_was_method_handle) {
707 caller_adjustment = last_frame_adjust(0, callee_locals);
708 } else if (callee_locals > callee_parameters) {
709 // The caller frame may need extending to accommodate
710 // non-parameter locals of the first unpacked interpreted frame.
711 // Compute that adjustment.
712 caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);
713 }
714
715 // If the sender is deoptimized the we must retrieve the address of the handler
716 // since the frame will "magically" show the original pc before the deopt
717 // and we'd undo the deopt.
718
719 frame_pcs[0] = Continuation::is_cont_barrier_frame(deoptee) ? StubRoutines::cont_returnBarrier() : deopt_sender.raw_pc();
720 if (Continuation::is_continuation_enterSpecial(deopt_sender)) {
721 ContinuationEntry::from_frame(deopt_sender)->set_argsize(0);
722 }
723
724 assert(CodeCache::find_blob(frame_pcs[0]) != nullptr, "bad pc");
725
726 if (current->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
727 assert(current->has_pending_exception(), "should have thrown OOME");
728 current->set_exception_oop(current->pending_exception());
729 current->clear_pending_exception();
730 exec_mode = Unpack_exception;
731 }
732
733 int caller_actual_parameters = -1; // value not used except for interpreted frames, see below
734 if (deopt_sender.is_interpreted_frame()) {
735 caller_actual_parameters = callee_parameters + (caller_was_method_handle ? 1 : 0);
1041 }
1042 }
1043
1044 Deoptimization::DeoptAction Deoptimization::_unloaded_action
1045 = Deoptimization::Action_reinterpret;
1046
1047 #ifdef COMPILER2
1048 bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, TRAPS) {
1049 Handle pending_exception(THREAD, thread->pending_exception());
1050 const char* exception_file = thread->exception_file();
1051 int exception_line = thread->exception_line();
1052 thread->clear_pending_exception();
1053
1054 bool failures = false;
1055
1056 for (int i = 0; i < objects->length(); i++) {
1057 assert(objects->at(i)->is_object(), "invalid debug information");
1058 ObjectValue* sv = (ObjectValue*) objects->at(i);
1059
1060 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1061 oop obj = nullptr;
1062
1063 bool cache_init_error = false;
1064 if (k->is_instance_klass()) {
1065 InstanceKlass* ik = InstanceKlass::cast(k);
1066 if (obj == nullptr && !cache_init_error) {
1067 InternalOOMEMark iom(THREAD);
1068 if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
1069 obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
1070 } else {
1071 obj = ik->allocate_instance(THREAD);
1072 }
1073 }
1074 } else if (k->is_typeArray_klass()) {
1075 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1076 assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
1077 int len = sv->field_size() / type2size[ak->element_type()];
1078 InternalOOMEMark iom(THREAD);
1079 obj = ak->allocate_instance(len, THREAD);
1080 } else if (k->is_objArray_klass()) {
1081 ObjArrayKlass* ak = ObjArrayKlass::cast(k);
1082 InternalOOMEMark iom(THREAD);
1083 obj = ak->allocate_instance(sv->field_size(), THREAD);
1084 }
1085
1086 if (obj == nullptr) {
1087 failures = true;
1088 }
1089
1090 assert(sv->value().is_null(), "redundant reallocation");
1091 assert(obj != nullptr || HAS_PENDING_EXCEPTION || cache_init_error, "allocation should succeed or we should get an exception");
1092 CLEAR_PENDING_EXCEPTION;
1093 sv->set_value(obj);
1094 }
1095
1096 if (failures) {
1097 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
1098 } else if (pending_exception.not_null()) {
1099 thread->set_pending_exception(pending_exception(), exception_file, exception_line);
1100 }
1101
1102 return failures;
1103 }
1104
1105 // restore elements of an eliminated type array
1106 void Deoptimization::reassign_type_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, typeArrayOop obj, BasicType type) {
1107 int index = 0;
1108
1109 for (int i = 0; i < sv->field_size(); i++) {
1110 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1111 switch(type) {
1112 case T_LONG: case T_DOUBLE: {
1113 assert(value->type() == T_INT, "Agreement.");
1114 StackValue* low =
1115 StackValue::create_stack_value(fr, reg_map, sv->field_at(++i));
1116 #ifdef _LP64
1117 jlong res = (jlong)low->get_intptr();
1118 #else
1119 jlong res = jlong_from(value->get_jint(), low->get_jint());
1120 #endif
1121 obj->long_at_put(index, res);
1122 break;
1123 }
1124
1153 default:
1154 ShouldNotReachHere();
1155 }
1156 index++;
1157 }
1158 }
1159
1160 // restore fields of an eliminated object array
1161 void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj) {
1162 for (int i = 0; i < sv->field_size(); i++) {
1163 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1164 assert(value->type() == T_OBJECT, "object element expected");
1165 obj->obj_at_put(i, value->get_obj()());
1166 }
1167 }
1168
1169 class ReassignedField {
1170 public:
1171 int _offset;
1172 BasicType _type;
1173 public:
1174 ReassignedField() {
1175 _offset = 0;
1176 _type = T_ILLEGAL;
1177 }
1178 };
1179
1180 // Gets the fields of `klass` that are eliminated by escape analysis and need to be reassigned
1181 static GrowableArray<ReassignedField>* get_reassigned_fields(InstanceKlass* klass, GrowableArray<ReassignedField>* fields) {
1182 InstanceKlass* super = klass->super();
1183 if (super != nullptr) {
1184 get_reassigned_fields(super, fields);
1185 }
1186 for (AllFieldStream fs(klass); !fs.done(); fs.next()) {
1187 if (!fs.access_flags().is_static() && !fs.field_flags().is_injected()) {
1188 ReassignedField field;
1189 field._offset = fs.offset();
1190 field._type = Signature::basic_type(fs.signature());
1191 fields->append(field);
1192 }
1193 }
1194 return fields;
1195 }
1196
1197 // Restore fields of an eliminated instance object employing the same field order used by the compiler.
1198 static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap* reg_map, ObjectValue* sv, int svIndex, oop obj) {
1199 GrowableArray<ReassignedField>* fields = get_reassigned_fields(klass, new GrowableArray<ReassignedField>());
1200 for (int i = 0; i < fields->length(); i++) {
1201 ScopeValue* scope_field = sv->field_at(svIndex);
1202 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1203 int offset = fields->at(i)._offset;
1204 BasicType type = fields->at(i)._type;
1205 switch (type) {
1206 case T_OBJECT: case T_ARRAY:
1207 assert(value->type() == T_OBJECT, "Agreement.");
1208 obj->obj_field_put(offset, value->get_obj()());
1209 break;
1210
1211 case T_INT: case T_FLOAT: { // 4 bytes.
1212 assert(value->type() == T_INT, "Agreement.");
1213 bool big_value = false;
1214 if (i+1 < fields->length() && fields->at(i+1)._type == T_INT) {
1215 if (scope_field->is_location()) {
1216 Location::Type type = ((LocationValue*) scope_field)->location().type();
1217 if (type == Location::dbl || type == Location::lng) {
1218 big_value = true;
1219 }
1220 }
1221 if (scope_field->is_constant_int()) {
1222 ScopeValue* next_scope_field = sv->field_at(svIndex + 1);
1223 if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1224 big_value = true;
1260 break;
1261
1262 case T_BYTE:
1263 assert(value->type() == T_INT, "Agreement.");
1264 obj->byte_field_put(offset, (jbyte)value->get_jint());
1265 break;
1266
1267 case T_BOOLEAN:
1268 assert(value->type() == T_INT, "Agreement.");
1269 obj->bool_field_put(offset, (jboolean)value->get_jint());
1270 break;
1271
1272 default:
1273 ShouldNotReachHere();
1274 }
1275 svIndex++;
1276 }
1277 return svIndex;
1278 }
1279
1280 // restore fields of all eliminated objects and arrays
1281 void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
1282 for (int i = 0; i < objects->length(); i++) {
1283 assert(objects->at(i)->is_object(), "invalid debug information");
1284 ObjectValue* sv = (ObjectValue*) objects->at(i);
1285 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1286 Handle obj = sv->value();
1287 assert(obj.not_null() || realloc_failures, "reallocation was missed");
1288 #ifndef PRODUCT
1289 if (PrintDeoptimizationDetails) {
1290 tty->print_cr("reassign fields for object of type %s!", k->name()->as_C_string());
1291 }
1292 #endif // !PRODUCT
1293
1294 if (obj.is_null()) {
1295 continue;
1296 }
1297
1298 if (EnableVectorSupport && VectorSupport::is_vector(k)) {
1299 assert(sv->field_size() == 1, "%s not a vector", k->name()->as_C_string());
1300 ScopeValue* payload = sv->field_at(0);
1301 if (payload->is_location() &&
1302 payload->as_LocationValue()->location().type() == Location::vector) {
1303 #ifndef PRODUCT
1304 if (PrintDeoptimizationDetails) {
1305 tty->print_cr("skip field reassignment for this vector - it should be assigned already");
1306 if (Verbose) {
1307 Handle obj = sv->value();
1308 k->oop_print_on(obj(), tty);
1309 }
1310 }
1311 #endif // !PRODUCT
1312 continue; // Such vector's value was already restored in VectorSupport::allocate_vector().
1313 }
1314 // Else fall-through to do assignment for scalar-replaced boxed vector representation
1315 // which could be restored after vector object allocation.
1316 }
1317 if (k->is_instance_klass()) {
1318 InstanceKlass* ik = InstanceKlass::cast(k);
1319 reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj());
1320 } else if (k->is_typeArray_klass()) {
1321 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1322 reassign_type_array_elements(fr, reg_map, sv, (typeArrayOop) obj(), ak->element_type());
1323 } else if (k->is_objArray_klass()) {
1324 reassign_object_array_elements(fr, reg_map, sv, (objArrayOop) obj());
1325 }
1326 }
1327 // These objects may escape when we return to Interpreter after deoptimization.
1328 // We need barrier so that stores that initialize these objects can't be reordered
1329 // with subsequent stores that make these objects accessible by other threads.
1330 OrderAccess::storestore();
1331 }
1332
1333
1334 // relock objects for which synchronization was eliminated
1335 bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInfo*>* monitors,
1336 JavaThread* deoptee_thread, frame& fr, int exec_mode, bool realloc_failures) {
1337 bool relocked_objects = false;
1338 for (int i = 0; i < monitors->length(); i++) {
1339 MonitorInfo* mon_info = monitors->at(i);
1340 if (mon_info->eliminated()) {
1341 assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
1342 relocked_objects = true;
1343 if (!mon_info->owner_is_scalar_replaced()) {
1481 xtty->begin_head("deoptimized thread='%zu' reason='%s' pc='" INTPTR_FORMAT "'",(uintx)thread->osthread()->thread_id(), trap_reason_name(reason), p2i(fr.pc()));
1482 nm->log_identity(xtty);
1483 xtty->end_head();
1484 for (ScopeDesc* sd = nm->scope_desc_at(fr.pc()); ; sd = sd->sender()) {
1485 xtty->begin_elem("jvms bci='%d'", sd->bci());
1486 xtty->method(sd->method());
1487 xtty->end_elem();
1488 if (sd->is_top()) break;
1489 }
1490 xtty->tail("deoptimized");
1491 }
1492
1493 Continuation::notify_deopt(thread, fr.sp());
1494
1495 // Patch the compiled method so that when execution returns to it we will
1496 // deopt the execution state and return to the interpreter.
1497 fr.deoptimize(thread);
1498 }
1499
1500 void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason) {
1501 // Deoptimize only if the frame comes from compile code.
1502 // Do not deoptimize the frame which is already patched
1503 // during the execution of the loops below.
1504 if (!fr.is_compiled_frame() || fr.is_deoptimized_frame()) {
1505 return;
1506 }
1507 ResourceMark rm;
1508 deoptimize_single_frame(thread, fr, reason);
1509 }
1510
1511 address Deoptimization::deoptimize_for_missing_exception_handler(nmethod* nm, bool make_not_entrant) {
1512 // there is no exception handler for this pc => deoptimize
1513 if (make_not_entrant) {
1514 nm->make_not_entrant(nmethod::InvalidationReason::MISSING_EXCEPTION_HANDLER);
1515 }
1516
1517 // Use Deoptimization::deoptimize for all of its side-effects:
1518 // gathering traps statistics, logging...
1519 // it also patches the return pc but we do not care about that
1520 // since we return a continuation to the deopt_blob below.
1521 JavaThread* thread = JavaThread::current();
|
31 #include "code/nmethod.hpp"
32 #include "code/pcDesc.hpp"
33 #include "code/scopeDesc.hpp"
34 #include "compiler/compilationPolicy.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "interpreter/bytecode.inline.hpp"
39 #include "interpreter/bytecodeStream.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "interpreter/oopMapCache.hpp"
42 #include "jvm.h"
43 #include "logging/log.hpp"
44 #include "logging/logLevel.hpp"
45 #include "logging/logMessage.hpp"
46 #include "logging/logStream.hpp"
47 #include "memory/allocation.inline.hpp"
48 #include "memory/oopFactory.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "memory/universe.hpp"
51 #include "oops/arrayOop.inline.hpp"
52 #include "oops/constantPool.hpp"
53 #include "oops/fieldStreams.inline.hpp"
54 #include "oops/flatArrayKlass.hpp"
55 #include "oops/flatArrayOop.hpp"
56 #include "oops/inlineKlass.inline.hpp"
57 #include "oops/method.hpp"
58 #include "oops/objArrayKlass.hpp"
59 #include "oops/objArrayOop.inline.hpp"
60 #include "oops/oop.inline.hpp"
61 #include "oops/typeArrayOop.inline.hpp"
62 #include "oops/verifyOopClosure.hpp"
63 #include "prims/jvmtiDeferredUpdates.hpp"
64 #include "prims/jvmtiExport.hpp"
65 #include "prims/jvmtiThreadState.hpp"
66 #include "prims/methodHandles.hpp"
67 #include "prims/vectorSupport.hpp"
68 #include "runtime/arguments.hpp"
69 #include "runtime/atomicAccess.hpp"
70 #include "runtime/basicLock.inline.hpp"
71 #include "runtime/continuation.hpp"
72 #include "runtime/continuationEntry.inline.hpp"
73 #include "runtime/deoptimization.hpp"
74 #include "runtime/escapeBarrier.hpp"
75 #include "runtime/fieldDescriptor.inline.hpp"
76 #include "runtime/frame.inline.hpp"
77 #include "runtime/handles.inline.hpp"
78 #include "runtime/interfaceSupport.inline.hpp"
79 #include "runtime/javaThread.hpp"
80 #include "runtime/jniHandles.inline.hpp"
81 #include "runtime/keepStackGCProcessed.hpp"
82 #include "runtime/lockStack.inline.hpp"
83 #include "runtime/objectMonitor.inline.hpp"
84 #include "runtime/osThread.hpp"
85 #include "runtime/safepointVerifiers.hpp"
86 #include "runtime/sharedRuntime.hpp"
87 #include "runtime/signature.hpp"
88 #include "runtime/stackFrameStream.inline.hpp"
286 // The actual reallocation of previously eliminated objects occurs in realloc_objects,
287 // which is called from the method fetch_unroll_info_helper below.
288 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* current, int exec_mode))
289 // fetch_unroll_info() is called at the beginning of the deoptimization
290 // handler. Note this fact before we start generating temporary frames
291 // that can confuse an asynchronous stack walker. This counter is
292 // decremented at the end of unpack_frames().
293 current->inc_in_deopt_handler();
294
295 if (exec_mode == Unpack_exception) {
296 // When we get here, a callee has thrown an exception into a deoptimized
297 // frame. That throw might have deferred stack watermark checking until
298 // after unwinding. So we deal with such deferred requests here.
299 StackWatermarkSet::after_unwind(current);
300 }
301
302 return fetch_unroll_info_helper(current, exec_mode);
303 JRT_END
304
305 #ifdef COMPILER2
306
307 static Klass* get_refined_array_klass(Klass* k, frame* fr, RegisterMap* map, ObjectValue* sv, TRAPS) {
308 // If it's an array, get the properties
309 if (k->is_array_klass() && !k->is_typeArray_klass()) {
310 assert(k->is_unrefined_objArray_klass(), "Expected unrefined array klass");
311 nmethod* nm = fr->cb()->as_nmethod_or_null();
312 assert(sv->has_properties(), "Property information is missing");
313 ArrayProperties props(checked_cast<ArrayProperties::Type>(StackValue::create_stack_value(fr, map, sv->properties())->get_jint()));
314 k = ObjArrayKlass::cast(k)->klass_with_properties(props, THREAD);
315 }
316 return k;
317 }
318
319 // print information about reallocated objects
320 static void print_objects(JavaThread* deoptee_thread, frame* deoptee, RegisterMap* map,
321 GrowableArray<ScopeValue*>* objects, bool realloc_failures, TRAPS) {
322 ResourceMark rm;
323 stringStream st; // change to logStream with logging
324 st.print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(deoptee_thread));
325 fieldDescriptor fd;
326
327 for (int i = 0; i < objects->length(); i++) {
328 ObjectValue* sv = (ObjectValue*) objects->at(i);
329 Handle obj = sv->value();
330
331 if (obj.is_null()) {
332 st.print_cr(" nullptr");
333 continue;
334 }
335
336 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
337 k = get_refined_array_klass(k, deoptee, map, sv, THREAD);
338
339 st.print(" object <" INTPTR_FORMAT "> of type ", p2i(sv->value()()));
340 k->print_value_on(&st);
341 st.print_cr(" allocated (%zu bytes)", obj->size() * HeapWordSize);
342
343 if (Verbose && k != nullptr) {
344 k->oop_print_on(obj(), &st);
345 }
346 }
347 tty->print_raw(st.freeze());
348 }
349
350 static bool rematerialize_objects(JavaThread* thread, int exec_mode, nmethod* compiled_method,
351 frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk,
352 bool& deoptimized_objects) {
353 bool realloc_failures = false;
354 assert (chunk->at(0)->scope() != nullptr,"expect only compiled java frames");
355
356 JavaThread* deoptee_thread = chunk->at(0)->thread();
357 assert(exec_mode == Deoptimization::Unpack_none || (deoptee_thread == thread),
358 "a frame can only be deoptimized by the owner thread");
359
360 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects_to_rematerialize(deoptee, map);
361
362 // The flag return_oop() indicates call sites which return oop
363 // in compiled code. Such sites include java method calls,
364 // runtime calls (for example, used to allocate new objects/arrays
365 // on slow code path) and any other calls generated in compiled code.
366 // It is not guaranteed that we can get such information here only
367 // by analyzing bytecode in deoptimized frames. This is why this flag
368 // is set during method compilation (see Compile::Process_OopMap_Node()).
369 // If the previous frame was popped or if we are dispatching an exception,
370 // we don't have an oop result.
371 ScopeDesc* scope = chunk->at(0)->scope();
372 bool save_oop_result = scope->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
373 // In case of the return of multiple values, we must take care
374 // of all oop return values.
375 GrowableArray<Handle> return_oops;
376 InlineKlass* vk = nullptr;
377 if (save_oop_result && scope->return_scalarized()) {
378 vk = InlineKlass::returned_inline_klass(map);
379 if (vk != nullptr) {
380 vk->save_oop_fields(map, return_oops);
381 save_oop_result = false;
382 }
383 }
384 if (save_oop_result) {
385 // Reallocation may trigger GC. If deoptimization happened on return from
386 // call which returns oop we need to save it since it is not in oopmap.
387 oop result = deoptee.saved_oop_result(&map);
388 assert(oopDesc::is_oop_or_null(result), "must be oop");
389 return_oops.push(Handle(thread, result));
390 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
391 if (TraceDeoptimization) {
392 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
393 tty->cr();
394 }
395 }
396 if (objects != nullptr || vk != nullptr) {
397 if (exec_mode == Deoptimization::Unpack_none) {
398 assert(thread->thread_state() == _thread_in_vm, "assumption");
399 JavaThread* THREAD = thread; // For exception macros.
400 // Clear pending OOM if reallocation fails and return true indicating allocation failure
401 if (vk != nullptr) {
402 realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, CHECK_AND_CLEAR_(true));
403 }
404 if (objects != nullptr) {
405 realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, CHECK_AND_CLEAR_(true));
406 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
407 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, CHECK_AND_CLEAR_(true));
408 }
409 deoptimized_objects = true;
410 } else {
411 JavaThread* current = thread; // For JRT_BLOCK
412 JRT_BLOCK
413 if (vk != nullptr) {
414 realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, THREAD);
415 }
416 if (objects != nullptr) {
417 realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
418 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
419 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, THREAD);
420 }
421 JRT_END
422 }
423 if (TraceDeoptimization && objects != nullptr) {
424 print_objects(deoptee_thread, &deoptee, &map, objects, realloc_failures, thread);
425 }
426 }
427 if (save_oop_result || vk != nullptr) {
428 // Restore result.
429 assert(return_oops.length() == 1, "no inline type");
430 deoptee.set_saved_oop_result(&map, return_oops.pop()());
431 }
432 return realloc_failures;
433 }
434
435 static void restore_eliminated_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures,
436 frame& deoptee, int exec_mode, bool& deoptimized_objects) {
437 JavaThread* deoptee_thread = chunk->at(0)->thread();
438 assert(!EscapeBarrier::objs_are_deoptimized(deoptee_thread, deoptee.id()), "must relock just once");
439 assert(thread == Thread::current(), "should be");
440 HandleMark hm(thread);
441 #ifndef PRODUCT
442 bool first = true;
443 #endif // !PRODUCT
444 // Start locking from outermost/oldest frame
445 for (int i = (chunk->length() - 1); i >= 0; i--) {
446 compiledVFrame* cvf = chunk->at(i);
447 assert (cvf->scope() != nullptr,"expect only compiled java frames");
448 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
449 if (monitors->is_nonempty()) {
450 bool relocked = Deoptimization::relock_objects(thread, monitors, deoptee_thread, deoptee,
477 }
478 }
479 tty->print_raw(st.freeze());
480 }
481 #endif // !PRODUCT
482 }
483 }
484 }
485
486 // Deoptimize objects, that is reallocate and relock them, just before they escape through JVMTI.
487 // The given vframes cover one physical frame.
488 bool Deoptimization::deoptimize_objects_internal(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk,
489 bool& realloc_failures) {
490 frame deoptee = chunk->at(0)->fr();
491 JavaThread* deoptee_thread = chunk->at(0)->thread();
492 nmethod* nm = deoptee.cb()->as_nmethod_or_null();
493 RegisterMap map(chunk->at(0)->register_map());
494 bool deoptimized_objects = false;
495
496 // Reallocate the non-escaping objects and restore their fields.
497 if (((DoEscapeAnalysis || Arguments::is_valhalla_enabled()) && EliminateAllocations) ||
498 EliminateAutoBox || EnableVectorAggressiveReboxing) {
499 realloc_failures = rematerialize_objects(thread, Unpack_none, nm, deoptee, map, chunk, deoptimized_objects);
500 }
501
502 // MonitorInfo structures used in eliminate_locks are not GC safe.
503 NoSafepointVerifier no_safepoint;
504
505 // Now relock objects if synchronization on them was eliminated.
506 if ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) {
507 restore_eliminated_locks(thread, chunk, realloc_failures, deoptee, Unpack_none, deoptimized_objects);
508 }
509 return deoptimized_objects;
510 }
511 #endif // COMPILER2
512
513 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
514 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* current, int exec_mode) {
515 JFR_ONLY(Jfr::check_and_process_sample_request(current);)
516 // When we get here we are about to unwind the deoptee frame. In order to
517 // catch not yet safe to use frames, the following stack watermark barrier
518 // poll will make such frames safe to use.
552 }
553
554 // Create a growable array of VFrames where each VFrame represents an inlined
555 // Java frame. This storage is allocated with the usual system arena.
556 assert(deoptee.is_compiled_frame(), "Wrong frame type");
557 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
558 vframe* vf = vframe::new_vframe(&deoptee, &map, current);
559 while (!vf->is_top()) {
560 assert(vf->is_compiled_frame(), "Wrong frame type");
561 chunk->push(compiledVFrame::cast(vf));
562 vf = vf->sender();
563 }
564 assert(vf->is_compiled_frame(), "Wrong frame type");
565 chunk->push(compiledVFrame::cast(vf));
566
567 bool realloc_failures = false;
568
569 #ifdef COMPILER2
570 // Reallocate the non-escaping objects and restore their fields. Then
571 // relock objects if synchronization on them was eliminated.
572 if (((DoEscapeAnalysis || Arguments::is_valhalla_enabled()) && EliminateAllocations) ||
573 EliminateAutoBox || EnableVectorAggressiveReboxing) {
574 bool unused;
575 realloc_failures = rematerialize_objects(current, exec_mode, nm, deoptee, map, chunk, unused);
576 }
577 #endif // COMPILER2
578
579 // Ensure that no safepoint is taken after pointers have been stored
580 // in fields of rematerialized objects. If a safepoint occurs from here on
581 // out the java state residing in the vframeArray will be missed.
582 // Locks may be rebaised in a safepoint.
583 NoSafepointVerifier no_safepoint;
584
585 #ifdef COMPILER2
586 if (((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) &&
587 !EscapeBarrier::objs_are_deoptimized(current, deoptee.id())) {
588 bool unused = false;
589 restore_eliminated_locks(current, chunk, realloc_failures, deoptee, exec_mode, unused);
590 }
591 #endif // COMPILER2
592
593 ScopeDesc* trap_scope = chunk->at(0)->scope();
740 // its caller's stack by. If the caller is a compiled frame then
741 // we pretend that the callee has no parameters so that the
742 // extension counts for the full amount of locals and not just
743 // locals-parms. This is because without a c2i adapter the parm
744 // area as created by the compiled frame will not be usable by
745 // the interpreter. (Depending on the calling convention there
746 // may not even be enough space).
747
748 // QQQ I'd rather see this pushed down into last_frame_adjust
749 // and have it take the sender (aka caller).
750
751 if (!deopt_sender.is_interpreted_frame() || caller_was_method_handle) {
752 caller_adjustment = last_frame_adjust(0, callee_locals);
753 } else if (callee_locals > callee_parameters) {
754 // The caller frame may need extending to accommodate
755 // non-parameter locals of the first unpacked interpreted frame.
756 // Compute that adjustment.
757 caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);
758 }
759
760 // If the sender is deoptimized we must retrieve the address of the handler
761 // since the frame will "magically" show the original pc before the deopt
762 // and we'd undo the deopt.
763
764 frame_pcs[0] = Continuation::is_cont_barrier_frame(deoptee) ? StubRoutines::cont_returnBarrier() : deopt_sender.raw_pc();
765 if (Continuation::is_continuation_enterSpecial(deopt_sender)) {
766 ContinuationEntry::from_frame(deopt_sender)->set_argsize(0);
767 }
768
769 assert(CodeCache::find_blob(frame_pcs[0]) != nullptr, "bad pc");
770
771 if (current->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
772 assert(current->has_pending_exception(), "should have thrown OOME");
773 current->set_exception_oop(current->pending_exception());
774 current->clear_pending_exception();
775 exec_mode = Unpack_exception;
776 }
777
778 int caller_actual_parameters = -1; // value not used except for interpreted frames, see below
779 if (deopt_sender.is_interpreted_frame()) {
780 caller_actual_parameters = callee_parameters + (caller_was_method_handle ? 1 : 0);
1086 }
1087 }
1088
1089 Deoptimization::DeoptAction Deoptimization::_unloaded_action
1090 = Deoptimization::Action_reinterpret;
1091
1092 #ifdef COMPILER2
1093 bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, TRAPS) {
1094 Handle pending_exception(THREAD, thread->pending_exception());
1095 const char* exception_file = thread->exception_file();
1096 int exception_line = thread->exception_line();
1097 thread->clear_pending_exception();
1098
1099 bool failures = false;
1100
1101 for (int i = 0; i < objects->length(); i++) {
1102 assert(objects->at(i)->is_object(), "invalid debug information");
1103 ObjectValue* sv = (ObjectValue*) objects->at(i);
1104
1105 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1106
1107 k = get_refined_array_klass(k, fr, reg_map, sv, THREAD);
1108
1109 // Check if the object may be null and has an additional null_marker input that needs
1110 // to be checked before using the field values. Skip re-allocation if it is null.
1111 if (k->is_inline_klass() && sv->has_properties()) {
1112 jint null_marker = StackValue::create_stack_value(fr, reg_map, sv->properties())->get_jint();
1113 if (null_marker == 0) {
1114 continue;
1115 }
1116 }
1117
1118 oop obj = nullptr;
1119 bool cache_init_error = false;
1120 if (k->is_instance_klass()) {
1121 InstanceKlass* ik = InstanceKlass::cast(k);
1122 if (obj == nullptr && !cache_init_error) {
1123 InternalOOMEMark iom(THREAD);
1124 if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
1125 obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
1126 } else {
1127 obj = ik->allocate_instance(THREAD);
1128 }
1129 }
1130 } else if (k->is_flatArray_klass()) {
1131 FlatArrayKlass* ak = FlatArrayKlass::cast(k);
1132 // Inline type array must be zeroed because not all memory is reassigned
1133 InternalOOMEMark iom(THREAD);
1134 obj = ak->allocate_instance(sv->field_size(), THREAD);
1135 } else if (k->is_typeArray_klass()) {
1136 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1137 assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
1138 int len = sv->field_size() / type2size[ak->element_type()];
1139 InternalOOMEMark iom(THREAD);
1140 obj = ak->allocate_instance(len, THREAD);
1141 } else if (k->is_refArray_klass()) {
1142 RefArrayKlass* ak = RefArrayKlass::cast(k);
1143 InternalOOMEMark iom(THREAD);
1144 obj = ak->allocate_instance(sv->field_size(), THREAD);
1145 }
1146
1147 if (obj == nullptr) {
1148 failures = true;
1149 }
1150
1151 assert(sv->value().is_null(), "redundant reallocation");
1152 assert(obj != nullptr || HAS_PENDING_EXCEPTION || cache_init_error, "allocation should succeed or we should get an exception");
1153 CLEAR_PENDING_EXCEPTION;
1154 sv->set_value(obj);
1155 }
1156
1157 if (failures) {
1158 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
1159 } else if (pending_exception.not_null()) {
1160 thread->set_pending_exception(pending_exception(), exception_file, exception_line);
1161 }
1162
1163 return failures;
1164 }
1165
1166 // We're deoptimizing at the return of a call, inline type fields are
1167 // in registers. When we go back to the interpreter, it will expect a
1168 // reference to an inline type instance. Allocate and initialize it from
1169 // the register values here.
1170 bool Deoptimization::realloc_inline_type_result(InlineKlass* vk, const RegisterMap& map, GrowableArray<Handle>& return_oops, TRAPS) {
1171 oop new_vt = vk->realloc_result(map, return_oops, THREAD);
1172 if (new_vt == nullptr) {
1173 CLEAR_PENDING_EXCEPTION;
1174 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), true);
1175 }
1176 return_oops.clear();
1177 return_oops.push(Handle(THREAD, new_vt));
1178 return false;
1179 }
1180
1181 // restore elements of an eliminated type array
1182 void Deoptimization::reassign_type_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, typeArrayOop obj, BasicType type) {
1183 int index = 0;
1184
1185 for (int i = 0; i < sv->field_size(); i++) {
1186 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1187 switch(type) {
1188 case T_LONG: case T_DOUBLE: {
1189 assert(value->type() == T_INT, "Agreement.");
1190 StackValue* low =
1191 StackValue::create_stack_value(fr, reg_map, sv->field_at(++i));
1192 #ifdef _LP64
1193 jlong res = (jlong)low->get_intptr();
1194 #else
1195 jlong res = jlong_from(value->get_jint(), low->get_jint());
1196 #endif
1197 obj->long_at_put(index, res);
1198 break;
1199 }
1200
1229 default:
1230 ShouldNotReachHere();
1231 }
1232 index++;
1233 }
1234 }
1235
1236 // restore fields of an eliminated object array
1237 void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj) {
1238 for (int i = 0; i < sv->field_size(); i++) {
1239 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1240 assert(value->type() == T_OBJECT, "object element expected");
1241 obj->obj_at_put(i, value->get_obj()());
1242 }
1243 }
1244
1245 class ReassignedField {
1246 public:
1247 int _offset;
1248 BasicType _type;
1249 InstanceKlass* _klass;
1250 bool _is_flat;
1251 bool _is_null_free;
1252 public:
1253 ReassignedField() : _offset(0), _type(T_ILLEGAL), _klass(nullptr), _is_flat(false), _is_null_free(false) { }
1254 };
1255
1256 // Gets the fields of `klass` that are eliminated by escape analysis and need to be reassigned
1257 static GrowableArray<ReassignedField>* get_reassigned_fields(InstanceKlass* klass, GrowableArray<ReassignedField>* fields) {
1258 InstanceKlass* super = klass->super();
1259 if (super != nullptr) {
1260 get_reassigned_fields(super, fields);
1261 }
1262 for (AllFieldStream fs(klass); !fs.done(); fs.next()) {
1263 if (!fs.access_flags().is_static() && !fs.field_flags().is_injected()) {
1264 ReassignedField field;
1265 field._offset = fs.offset();
1266 field._type = Signature::basic_type(fs.signature());
1267 if (fs.is_flat()) {
1268 field._is_flat = true;
1269 field._is_null_free = fs.is_null_free_inline_type();
1270 // Resolve klass of flat inline type field
1271 field._klass = InlineKlass::cast(klass->get_inline_type_field_klass(fs.index()));
1272 }
1273 fields->append(field);
1274 }
1275 }
1276 return fields;
1277 }
1278
1279 // Restore fields of an eliminated instance object employing the same field order used by the
1280 // compiler when it scalarizes an object at safepoints.
1281 static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap* reg_map, ObjectValue* sv, int svIndex, oop obj, int base_offset, TRAPS) {
1282 GrowableArray<ReassignedField>* fields = get_reassigned_fields(klass, new GrowableArray<ReassignedField>());
1283 for (int i = 0; i < fields->length(); i++) {
1284 BasicType type = fields->at(i)._type;
1285 int offset = base_offset + fields->at(i)._offset;
1286 // Check for flat inline type field before accessing the ScopeValue because it might not have any fields
1287 if (fields->at(i)._is_flat) {
1288 // Recursively re-assign flat inline type fields
1289 InstanceKlass* vk = fields->at(i)._klass;
1290 assert(vk != nullptr, "must be resolved");
1291 offset -= InlineKlass::cast(vk)->payload_offset(); // Adjust offset to omit oop header
1292 svIndex = reassign_fields_by_klass(vk, fr, reg_map, sv, svIndex, obj, offset, CHECK_0);
1293 if (!fields->at(i)._is_null_free) {
1294 ScopeValue* scope_field = sv->field_at(svIndex);
1295 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1296 int nm_offset = offset + InlineKlass::cast(vk)->null_marker_offset();
1297 obj->bool_field_put(nm_offset, value->get_jint() & 1);
1298 svIndex++;
1299 }
1300 continue; // Continue because we don't need to increment svIndex
1301 }
1302
1303 ScopeValue* scope_field = sv->field_at(svIndex);
1304 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1305 switch (type) {
1306 case T_OBJECT: case T_ARRAY:
1307 assert(value->type() == T_OBJECT, "Agreement.");
1308 obj->obj_field_put(offset, value->get_obj()());
1309 break;
1310
1311 case T_INT: case T_FLOAT: { // 4 bytes.
1312 assert(value->type() == T_INT, "Agreement.");
1313 bool big_value = false;
1314 if (i+1 < fields->length() && fields->at(i+1)._type == T_INT) {
1315 if (scope_field->is_location()) {
1316 Location::Type type = ((LocationValue*) scope_field)->location().type();
1317 if (type == Location::dbl || type == Location::lng) {
1318 big_value = true;
1319 }
1320 }
1321 if (scope_field->is_constant_int()) {
1322 ScopeValue* next_scope_field = sv->field_at(svIndex + 1);
1323 if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1324 big_value = true;
1360 break;
1361
1362 case T_BYTE:
1363 assert(value->type() == T_INT, "Agreement.");
1364 obj->byte_field_put(offset, (jbyte)value->get_jint());
1365 break;
1366
1367 case T_BOOLEAN:
1368 assert(value->type() == T_INT, "Agreement.");
1369 obj->bool_field_put(offset, (jboolean)value->get_jint());
1370 break;
1371
1372 default:
1373 ShouldNotReachHere();
1374 }
1375 svIndex++;
1376 }
1377 return svIndex;
1378 }
1379
1380 // restore fields of an eliminated inline type array
1381 void Deoptimization::reassign_flat_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, flatArrayOop obj, FlatArrayKlass* vak, TRAPS) {
1382 InlineKlass* vk = vak->element_klass();
1383 assert(vk->maybe_flat_in_array(), "should only be used for flat inline type arrays");
1384 // Adjust offset to omit oop header
1385 int base_offset = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT) - vk->payload_offset();
1386 // Initialize all elements of the flat inline type array
1387 for (int i = 0; i < sv->field_size(); i++) {
1388 ObjectValue* val = sv->field_at(i)->as_ObjectValue();
1389 int offset = base_offset + (i << Klass::layout_helper_log2_element_size(vak->layout_helper()));
1390 reassign_fields_by_klass(vk, fr, reg_map, val, 0, (oop)obj, offset, CHECK);
1391 if (!obj->is_null_free_array()) {
1392 jboolean null_marker_value;
1393 if (val->has_properties()) {
1394 null_marker_value = StackValue::create_stack_value(fr, reg_map, val->properties())->get_jint() & 1;
1395 } else {
1396 null_marker_value = 1;
1397 }
1398 obj->bool_field_put(offset + vk->null_marker_offset(), null_marker_value);
1399 }
1400 }
1401 }
1402
1403 // restore fields of all eliminated objects and arrays
1404 void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures, TRAPS) {
1405 for (int i = 0; i < objects->length(); i++) {
1406 assert(objects->at(i)->is_object(), "invalid debug information");
1407 ObjectValue* sv = (ObjectValue*) objects->at(i);
1408 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1409 k = get_refined_array_klass(k, fr, reg_map, sv, THREAD);
1410
1411 Handle obj = sv->value();
1412 assert(obj.not_null() || realloc_failures || sv->has_properties(), "reallocation was missed");
1413 #ifndef PRODUCT
1414 if (PrintDeoptimizationDetails) {
1415 tty->print_cr("reassign fields for object of type %s!", k->name()->as_C_string());
1416 }
1417 #endif // !PRODUCT
1418
1419 if (obj.is_null()) {
1420 continue;
1421 }
1422
1423 if (EnableVectorSupport && VectorSupport::is_vector(k)) {
1424 assert(sv->field_size() == 1, "%s not a vector", k->name()->as_C_string());
1425 ScopeValue* payload = sv->field_at(0);
1426 if (payload->is_location() &&
1427 payload->as_LocationValue()->location().type() == Location::vector) {
1428 #ifndef PRODUCT
1429 if (PrintDeoptimizationDetails) {
1430 tty->print_cr("skip field reassignment for this vector - it should be assigned already");
1431 if (Verbose) {
1432 Handle obj = sv->value();
1433 k->oop_print_on(obj(), tty);
1434 }
1435 }
1436 #endif // !PRODUCT
1437 continue; // Such vector's value was already restored in VectorSupport::allocate_vector().
1438 }
1439 // Else fall-through to do assignment for scalar-replaced boxed vector representation
1440 // which could be restored after vector object allocation.
1441 }
1442 if (k->is_instance_klass()) {
1443 InstanceKlass* ik = InstanceKlass::cast(k);
1444 reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj(), 0, CHECK);
1445 } else if (k->is_flatArray_klass()) {
1446 FlatArrayKlass* vak = FlatArrayKlass::cast(k);
1447 reassign_flat_array_elements(fr, reg_map, sv, (flatArrayOop) obj(), vak, CHECK);
1448 } else if (k->is_typeArray_klass()) {
1449 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1450 reassign_type_array_elements(fr, reg_map, sv, (typeArrayOop) obj(), ak->element_type());
1451 } else if (k->is_refArray_klass()) {
1452 reassign_object_array_elements(fr, reg_map, sv, (objArrayOop) obj());
1453 }
1454 }
1455 // These objects may escape when we return to Interpreter after deoptimization.
1456 // We need barrier so that stores that initialize these objects can't be reordered
1457 // with subsequent stores that make these objects accessible by other threads.
1458 OrderAccess::storestore();
1459 }
1460
1461
1462 // relock objects for which synchronization was eliminated
1463 bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInfo*>* monitors,
1464 JavaThread* deoptee_thread, frame& fr, int exec_mode, bool realloc_failures) {
1465 bool relocked_objects = false;
1466 for (int i = 0; i < monitors->length(); i++) {
1467 MonitorInfo* mon_info = monitors->at(i);
1468 if (mon_info->eliminated()) {
1469 assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
1470 relocked_objects = true;
1471 if (!mon_info->owner_is_scalar_replaced()) {
1609 xtty->begin_head("deoptimized thread='%zu' reason='%s' pc='" INTPTR_FORMAT "'",(uintx)thread->osthread()->thread_id(), trap_reason_name(reason), p2i(fr.pc()));
1610 nm->log_identity(xtty);
1611 xtty->end_head();
1612 for (ScopeDesc* sd = nm->scope_desc_at(fr.pc()); ; sd = sd->sender()) {
1613 xtty->begin_elem("jvms bci='%d'", sd->bci());
1614 xtty->method(sd->method());
1615 xtty->end_elem();
1616 if (sd->is_top()) break;
1617 }
1618 xtty->tail("deoptimized");
1619 }
1620
1621 Continuation::notify_deopt(thread, fr.sp());
1622
1623 // Patch the compiled method so that when execution returns to it we will
1624 // deopt the execution state and return to the interpreter.
1625 fr.deoptimize(thread);
1626 }
1627
1628 void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason) {
1629 // Deoptimize only if the frame comes from compiled code.
1630 // Do not deoptimize the frame which is already patched
1631 // during the execution of the loops below.
1632 if (!fr.is_compiled_frame() || fr.is_deoptimized_frame()) {
1633 return;
1634 }
1635 ResourceMark rm;
1636 deoptimize_single_frame(thread, fr, reason);
1637 }
1638
1639 address Deoptimization::deoptimize_for_missing_exception_handler(nmethod* nm, bool make_not_entrant) {
1640 // there is no exception handler for this pc => deoptimize
1641 if (make_not_entrant) {
1642 nm->make_not_entrant(nmethod::InvalidationReason::MISSING_EXCEPTION_HANDLER);
1643 }
1644
1645 // Use Deoptimization::deoptimize for all of its side-effects:
1646 // gathering traps statistics, logging...
1647 // it also patches the return pc but we do not care about that
1648 // since we return a continuation to the deopt_blob below.
1649 JavaThread* thread = JavaThread::current();
|