31 #include "code/nmethod.hpp"
32 #include "code/pcDesc.hpp"
33 #include "code/scopeDesc.hpp"
34 #include "compiler/compilationPolicy.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "interpreter/bytecode.inline.hpp"
39 #include "interpreter/bytecodeStream.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "interpreter/oopMapCache.hpp"
42 #include "jvm.h"
43 #include "logging/log.hpp"
44 #include "logging/logLevel.hpp"
45 #include "logging/logMessage.hpp"
46 #include "logging/logStream.hpp"
47 #include "memory/allocation.inline.hpp"
48 #include "memory/oopFactory.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "memory/universe.hpp"
51 #include "oops/constantPool.hpp"
52 #include "oops/fieldStreams.inline.hpp"
53 #include "oops/method.hpp"
54 #include "oops/objArrayKlass.hpp"
55 #include "oops/objArrayOop.inline.hpp"
56 #include "oops/oop.inline.hpp"
57 #include "oops/typeArrayOop.inline.hpp"
58 #include "oops/verifyOopClosure.hpp"
59 #include "prims/jvmtiDeferredUpdates.hpp"
60 #include "prims/jvmtiExport.hpp"
61 #include "prims/jvmtiThreadState.hpp"
62 #include "prims/methodHandles.hpp"
63 #include "prims/vectorSupport.hpp"
64 #include "runtime/atomicAccess.hpp"
65 #include "runtime/basicLock.inline.hpp"
66 #include "runtime/continuation.hpp"
67 #include "runtime/continuationEntry.inline.hpp"
68 #include "runtime/deoptimization.hpp"
69 #include "runtime/escapeBarrier.hpp"
70 #include "runtime/fieldDescriptor.inline.hpp"
71 #include "runtime/frame.inline.hpp"
72 #include "runtime/handles.inline.hpp"
73 #include "runtime/interfaceSupport.inline.hpp"
74 #include "runtime/javaThread.hpp"
75 #include "runtime/jniHandles.inline.hpp"
76 #include "runtime/keepStackGCProcessed.hpp"
77 #include "runtime/lockStack.inline.hpp"
78 #include "runtime/objectMonitor.inline.hpp"
79 #include "runtime/osThread.hpp"
80 #include "runtime/safepointVerifiers.hpp"
81 #include "runtime/sharedRuntime.hpp"
82 #include "runtime/signature.hpp"
83 #include "runtime/stackFrameStream.inline.hpp"
281 // The actual reallocation of previously eliminated objects occurs in realloc_objects,
282 // which is called from the method fetch_unroll_info_helper below.
283 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* current, int exec_mode))
284 // fetch_unroll_info() is called at the beginning of the deoptimization
285 // handler. Note this fact before we start generating temporary frames
286 // that can confuse an asynchronous stack walker. This counter is
287 // decremented at the end of unpack_frames().
288 current->inc_in_deopt_handler();
289
290 if (exec_mode == Unpack_exception) {
291 // When we get here, a callee has thrown an exception into a deoptimized
292 // frame. That throw might have deferred stack watermark checking until
293 // after unwinding. So we deal with such deferred requests here.
294 StackWatermarkSet::after_unwind(current);
295 }
296
297 return fetch_unroll_info_helper(current, exec_mode);
298 JRT_END
299
300 #if COMPILER2_OR_JVMCI
301 // print information about reallocated objects
302 static void print_objects(JavaThread* deoptee_thread,
303 GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
304 ResourceMark rm;
305 stringStream st; // change to logStream with logging
306 st.print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(deoptee_thread));
307 fieldDescriptor fd;
308
309 for (int i = 0; i < objects->length(); i++) {
310 ObjectValue* sv = (ObjectValue*) objects->at(i);
311 Handle obj = sv->value();
312
313 if (obj.is_null()) {
314 st.print_cr(" nullptr");
315 continue;
316 }
317
318 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
319
320 st.print(" object <" INTPTR_FORMAT "> of type ", p2i(sv->value()()));
321 k->print_value_on(&st);
322 st.print_cr(" allocated (%zu bytes)", obj->size() * HeapWordSize);
323
324 if (Verbose && k != nullptr) {
325 k->oop_print_on(obj(), &st);
326 }
327 }
328 tty->print_raw(st.freeze());
329 }
330
331 static bool rematerialize_objects(JavaThread* thread, int exec_mode, nmethod* compiled_method,
332 frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk,
333 bool& deoptimized_objects) {
334 bool realloc_failures = false;
335 assert (chunk->at(0)->scope() != nullptr,"expect only compiled java frames");
336
337 JavaThread* deoptee_thread = chunk->at(0)->thread();
338 assert(exec_mode == Deoptimization::Unpack_none || (deoptee_thread == thread),
339 "a frame can only be deoptimized by the owner thread");
340
341 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects_to_rematerialize(deoptee, map);
342
343 // The flag return_oop() indicates call sites which return oop
344 // in compiled code. Such sites include java method calls,
345 // runtime calls (for example, used to allocate new objects/arrays
346 // on slow code path) and any other calls generated in compiled code.
347 // It is not guaranteed that we can get such information here only
348 // by analyzing bytecode in deoptimized frames. This is why this flag
349 // is set during method compilation (see Compile::Process_OopMap_Node()).
350 // If the previous frame was popped or if we are dispatching an exception,
351 // we don't have an oop result.
352 bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
353 Handle return_value;
354 if (save_oop_result) {
355 // Reallocation may trigger GC. If deoptimization happened on return from
356 // call which returns oop we need to save it since it is not in oopmap.
357 oop result = deoptee.saved_oop_result(&map);
358 assert(oopDesc::is_oop_or_null(result), "must be oop");
359 return_value = Handle(thread, result);
360 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
361 if (TraceDeoptimization) {
362 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
363 tty->cr();
364 }
365 }
366 if (objects != nullptr) {
367 if (exec_mode == Deoptimization::Unpack_none) {
368 assert(thread->thread_state() == _thread_in_vm, "assumption");
369 JavaThread* THREAD = thread; // For exception macros.
370 // Clear pending OOM if reallocation fails and return true indicating allocation failure
371 realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, CHECK_AND_CLEAR_(true));
372 deoptimized_objects = true;
373 } else {
374 JavaThread* current = thread; // For JRT_BLOCK
375 JRT_BLOCK
376 realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
377 JRT_END
378 }
379 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
380 bool is_jvmci = compiled_method->is_compiled_by_jvmci();
381 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, is_jvmci);
382 if (TraceDeoptimization) {
383 print_objects(deoptee_thread, objects, realloc_failures);
384 }
385 }
386 if (save_oop_result) {
387 // Restore result.
388 deoptee.set_saved_oop_result(&map, return_value());
389 }
390 return realloc_failures;
391 }
392
393 static void restore_eliminated_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures,
394 frame& deoptee, int exec_mode, bool& deoptimized_objects) {
395 JavaThread* deoptee_thread = chunk->at(0)->thread();
396 assert(!EscapeBarrier::objs_are_deoptimized(deoptee_thread, deoptee.id()), "must relock just once");
397 assert(thread == Thread::current(), "should be");
398 HandleMark hm(thread);
399 #ifndef PRODUCT
400 bool first = true;
401 #endif // !PRODUCT
402 // Start locking from outermost/oldest frame
403 for (int i = (chunk->length() - 1); i >= 0; i--) {
404 compiledVFrame* cvf = chunk->at(i);
405 assert (cvf->scope() != nullptr,"expect only compiled java frames");
406 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
407 if (monitors->is_nonempty()) {
408 bool relocked = Deoptimization::relock_objects(thread, monitors, deoptee_thread, deoptee,
437 tty->print_raw(st.freeze());
438 }
439 #endif // !PRODUCT
440 }
441 }
442 }
443
444 // Deoptimize objects, that is reallocate and relock them, just before they escape through JVMTI.
445 // The given vframes cover one physical frame.
446 bool Deoptimization::deoptimize_objects_internal(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk,
447 bool& realloc_failures) {
448 frame deoptee = chunk->at(0)->fr();
449 JavaThread* deoptee_thread = chunk->at(0)->thread();
450 nmethod* nm = deoptee.cb()->as_nmethod_or_null();
451 RegisterMap map(chunk->at(0)->register_map());
452 bool deoptimized_objects = false;
453
454 bool const jvmci_enabled = JVMCI_ONLY(EnableJVMCI) NOT_JVMCI(false);
455
456 // Reallocate the non-escaping objects and restore their fields.
457 if (jvmci_enabled COMPILER2_PRESENT(|| (DoEscapeAnalysis && EliminateAllocations)
458 || EliminateAutoBox || EnableVectorAggressiveReboxing)) {
459 realloc_failures = rematerialize_objects(thread, Unpack_none, nm, deoptee, map, chunk, deoptimized_objects);
460 }
461
462 // MonitorInfo structures used in eliminate_locks are not GC safe.
463 NoSafepointVerifier no_safepoint;
464
465 // Now relock objects if synchronization on them was eliminated.
466 if (jvmci_enabled COMPILER2_PRESENT(|| ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks))) {
467 restore_eliminated_locks(thread, chunk, realloc_failures, deoptee, Unpack_none, deoptimized_objects);
468 }
469 return deoptimized_objects;
470 }
471 #endif // COMPILER2_OR_JVMCI
472
473 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
474 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* current, int exec_mode) {
475 JFR_ONLY(Jfr::check_and_process_sample_request(current);)
476 // When we get here we are about to unwind the deoptee frame. In order to
477 // catch not yet safe to use frames, the following stack watermark barrier
514 // Create a growable array of VFrames where each VFrame represents an inlined
515 // Java frame. This storage is allocated with the usual system arena.
516 assert(deoptee.is_compiled_frame(), "Wrong frame type");
517 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
518 vframe* vf = vframe::new_vframe(&deoptee, &map, current);
519 while (!vf->is_top()) {
520 assert(vf->is_compiled_frame(), "Wrong frame type");
521 chunk->push(compiledVFrame::cast(vf));
522 vf = vf->sender();
523 }
524 assert(vf->is_compiled_frame(), "Wrong frame type");
525 chunk->push(compiledVFrame::cast(vf));
526
527 bool realloc_failures = false;
528
529 #if COMPILER2_OR_JVMCI
530 bool const jvmci_enabled = JVMCI_ONLY(EnableJVMCI) NOT_JVMCI(false);
531
532 // Reallocate the non-escaping objects and restore their fields. Then
533 // relock objects if synchronization on them was eliminated.
534 if (jvmci_enabled COMPILER2_PRESENT( || (DoEscapeAnalysis && EliminateAllocations)
535 || EliminateAutoBox || EnableVectorAggressiveReboxing )) {
536 bool unused;
537 realloc_failures = rematerialize_objects(current, exec_mode, nm, deoptee, map, chunk, unused);
538 }
539 #endif // COMPILER2_OR_JVMCI
540
541 // Ensure that no safepoint is taken after pointers have been stored
542 // in fields of rematerialized objects. If a safepoint occurs from here on
543 // out the java state residing in the vframeArray will be missed.
544 // Locks may be rebaised in a safepoint.
545 NoSafepointVerifier no_safepoint;
546
547 #if COMPILER2_OR_JVMCI
548 if ((jvmci_enabled COMPILER2_PRESENT( || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) ))
549 && !EscapeBarrier::objs_are_deoptimized(current, deoptee.id())) {
550 bool unused = false;
551 restore_eliminated_locks(current, chunk, realloc_failures, deoptee, exec_mode, unused);
552 }
553 #endif // COMPILER2_OR_JVMCI
554
555 ScopeDesc* trap_scope = chunk->at(0)->scope();
702 // its caller's stack by. If the caller is a compiled frame then
703 // we pretend that the callee has no parameters so that the
704 // extension counts for the full amount of locals and not just
705 // locals-parms. This is because without a c2i adapter the parm
706 // area as created by the compiled frame will not be usable by
707 // the interpreter. (Depending on the calling convention there
708 // may not even be enough space).
709
710 // QQQ I'd rather see this pushed down into last_frame_adjust
711 // and have it take the sender (aka caller).
712
713 if (!deopt_sender.is_interpreted_frame() || caller_was_method_handle) {
714 caller_adjustment = last_frame_adjust(0, callee_locals);
715 } else if (callee_locals > callee_parameters) {
716 // The caller frame may need extending to accommodate
717 // non-parameter locals of the first unpacked interpreted frame.
718 // Compute that adjustment.
719 caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);
720 }
721
722 // If the sender is deoptimized the we must retrieve the address of the handler
723 // since the frame will "magically" show the original pc before the deopt
724 // and we'd undo the deopt.
725
726 frame_pcs[0] = Continuation::is_cont_barrier_frame(deoptee) ? StubRoutines::cont_returnBarrier() : deopt_sender.raw_pc();
727 if (Continuation::is_continuation_enterSpecial(deopt_sender)) {
728 ContinuationEntry::from_frame(deopt_sender)->set_argsize(0);
729 }
730
731 assert(CodeCache::find_blob(frame_pcs[0]) != nullptr, "bad pc");
732
733 #if INCLUDE_JVMCI
734 if (exceptionObject() != nullptr) {
735 current->set_exception_oop(exceptionObject());
736 exec_mode = Unpack_exception;
737 assert(array->element(0)->rethrow_exception(), "must be");
738 }
739 #endif
740
741 if (current->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
742 assert(current->has_pending_exception(), "should have thrown OOME");
1086 guarantee(ik->is_initialized(), "%s must be initialized", klass_name_str);
1087 CacheType::compute_offsets(ik);
1088 }
1089 return ik;
1090 }
1091 };
1092
1093 template<typename PrimitiveType, typename CacheType, typename BoxType> class BoxCache : public BoxCacheBase<CacheType> {
1094 PrimitiveType _low;
1095 PrimitiveType _high;
1096 jobject _cache;
1097 protected:
1098 static BoxCache<PrimitiveType, CacheType, BoxType> *_singleton;
1099 BoxCache(Thread* thread) {
1100 InstanceKlass* ik = BoxCacheBase<CacheType>::find_cache_klass(thread, CacheType::symbol());
1101 if (ik->is_in_error_state()) {
1102 _low = 1;
1103 _high = 0;
1104 _cache = nullptr;
1105 } else {
1106 objArrayOop cache = CacheType::cache(ik);
1107 assert(cache->length() > 0, "Empty cache");
1108 _low = BoxType::value(cache->obj_at(0));
1109 _high = checked_cast<PrimitiveType>(_low + cache->length() - 1);
1110 _cache = JNIHandles::make_global(Handle(thread, cache));
1111 }
1112 }
1113 ~BoxCache() {
1114 JNIHandles::destroy_global(_cache);
1115 }
1116 public:
1117 static BoxCache<PrimitiveType, CacheType, BoxType>* singleton(Thread* thread) {
1118 if (_singleton == nullptr) {
1119 BoxCache<PrimitiveType, CacheType, BoxType>* s = new BoxCache<PrimitiveType, CacheType, BoxType>(thread);
1120 if (!AtomicAccess::replace_if_null(&_singleton, s)) {
1121 delete s;
1122 }
1123 }
1124 return _singleton;
1125 }
1126 oop lookup(PrimitiveType value) {
1127 if (_low <= value && value <= _high) {
1128 int offset = checked_cast<int>(value - _low);
1129 return objArrayOop(JNIHandles::resolve_non_null(_cache))->obj_at(offset);
1130 }
1131 return nullptr;
1132 }
1133 oop lookup_raw(intptr_t raw_value, bool& cache_init_error) {
1134 if (_cache == nullptr) {
1135 cache_init_error = true;
1136 return nullptr;
1137 }
1138 // Have to cast to avoid little/big-endian problems.
1139 if (sizeof(PrimitiveType) > sizeof(jint)) {
1140 jlong value = (jlong)raw_value;
1141 return lookup(value);
1142 }
1143 PrimitiveType value = (PrimitiveType)*((jint*)&raw_value);
1144 return lookup(value);
1145 }
1146 };
1147
1148 typedef BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer> IntegerBoxCache;
1149 typedef BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long> LongBoxCache;
1151 typedef BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short> ShortBoxCache;
1152 typedef BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte> ByteBoxCache;
1153
1154 template<> BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>* BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>::_singleton = nullptr;
1155 template<> BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>* BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>::_singleton = nullptr;
1156 template<> BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>* BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>::_singleton = nullptr;
1157 template<> BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>* BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>::_singleton = nullptr;
1158 template<> BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>* BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>::_singleton = nullptr;
1159
1160 class BooleanBoxCache : public BoxCacheBase<java_lang_Boolean> {
1161 jobject _true_cache;
1162 jobject _false_cache;
1163 protected:
1164 static BooleanBoxCache *_singleton;
1165 BooleanBoxCache(Thread *thread) {
1166 InstanceKlass* ik = find_cache_klass(thread, java_lang_Boolean::symbol());
1167 if (ik->is_in_error_state()) {
1168 _true_cache = nullptr;
1169 _false_cache = nullptr;
1170 } else {
1171 _true_cache = JNIHandles::make_global(Handle(thread, java_lang_Boolean::get_TRUE(ik)));
1172 _false_cache = JNIHandles::make_global(Handle(thread, java_lang_Boolean::get_FALSE(ik)));
1173 }
1174 }
1175 ~BooleanBoxCache() {
1176 JNIHandles::destroy_global(_true_cache);
1177 JNIHandles::destroy_global(_false_cache);
1178 }
1179 public:
1180 static BooleanBoxCache* singleton(Thread* thread) {
1181 if (_singleton == nullptr) {
1182 BooleanBoxCache* s = new BooleanBoxCache(thread);
1183 if (!AtomicAccess::replace_if_null(&_singleton, s)) {
1184 delete s;
1185 }
1186 }
1187 return _singleton;
1188 }
1189 oop lookup_raw(intptr_t raw_value, bool& cache_in_error) {
1190 if (_true_cache == nullptr) {
1191 cache_in_error = true;
1192 return nullptr;
1221 }
1222 }
1223 return nullptr;
1224 }
1225 #endif // INCLUDE_JVMCI
1226
1227 #if COMPILER2_OR_JVMCI
1228 bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, TRAPS) {
1229 Handle pending_exception(THREAD, thread->pending_exception());
1230 const char* exception_file = thread->exception_file();
1231 int exception_line = thread->exception_line();
1232 thread->clear_pending_exception();
1233
1234 bool failures = false;
1235
1236 for (int i = 0; i < objects->length(); i++) {
1237 assert(objects->at(i)->is_object(), "invalid debug information");
1238 ObjectValue* sv = (ObjectValue*) objects->at(i);
1239
1240 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1241 oop obj = nullptr;
1242
1243 bool cache_init_error = false;
1244 if (k->is_instance_klass()) {
1245 #if INCLUDE_JVMCI
1246 nmethod* nm = fr->cb()->as_nmethod_or_null();
1247 if (nm->is_compiled_by_jvmci() && sv->is_auto_box()) {
1248 AutoBoxObjectValue* abv = (AutoBoxObjectValue*) sv;
1249 obj = get_cached_box(abv, fr, reg_map, cache_init_error, THREAD);
1250 if (obj != nullptr) {
1251 // Set the flag to indicate the box came from a cache, so that we can skip the field reassignment for it.
1252 abv->set_cached(true);
1253 } else if (cache_init_error) {
1254 // Results in an OOME which is valid (as opposed to a class initialization error)
1255 // and is fine for the rare case a cache initialization failing.
1256 failures = true;
1257 }
1258 }
1259 #endif // INCLUDE_JVMCI
1260
1261 InstanceKlass* ik = InstanceKlass::cast(k);
1262 if (obj == nullptr && !cache_init_error) {
1263 InternalOOMEMark iom(THREAD);
1264 if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
1265 obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
1266 } else {
1267 obj = ik->allocate_instance(THREAD);
1268 }
1269 }
1270 } else if (k->is_typeArray_klass()) {
1271 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1272 assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
1273 int len = sv->field_size() / type2size[ak->element_type()];
1274 InternalOOMEMark iom(THREAD);
1275 obj = ak->allocate_instance(len, THREAD);
1276 } else if (k->is_objArray_klass()) {
1277 ObjArrayKlass* ak = ObjArrayKlass::cast(k);
1278 InternalOOMEMark iom(THREAD);
1279 obj = ak->allocate_instance(sv->field_size(), THREAD);
1280 }
1281
1282 if (obj == nullptr) {
1283 failures = true;
1284 }
1285
1286 assert(sv->value().is_null(), "redundant reallocation");
1287 assert(obj != nullptr || HAS_PENDING_EXCEPTION || cache_init_error, "allocation should succeed or we should get an exception");
1288 CLEAR_PENDING_EXCEPTION;
1289 sv->set_value(obj);
1290 }
1291
1292 if (failures) {
1293 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
1294 } else if (pending_exception.not_null()) {
1295 thread->set_pending_exception(pending_exception(), exception_file, exception_line);
1296 }
1297
1298 return failures;
1299 }
1300
1301 #if INCLUDE_JVMCI
1302 /**
1303 * For primitive types whose kind gets "erased" at runtime (shorts become stack ints),
1304 * we need to somehow be able to recover the actual kind to be able to write the correct
1305 * amount of bytes.
1306 * For that purpose, this method assumes that, for an entry spanning n bytes at index i,
1307 * the entries at index n + 1 to n + i are 'markers'.
1308 * For example, if we were writing a short at index 4 of a byte array of size 8, the
1309 * expected form of the array would be:
1310 *
1311 * {b0, b1, b2, b3, INT, marker, b6, b7}
1312 *
1313 * Thus, in order to get back the size of the entry, we simply need to count the number
1314 * of marked entries
1315 *
1316 * @param virtualArray the virtualized byte array
1317 * @param i index of the virtual entry we are recovering
1318 * @return The number of bytes the entry spans
1319 */
1320 static int count_number_of_bytes_for_entry(ObjectValue *virtualArray, int i) {
1452 default:
1453 ShouldNotReachHere();
1454 }
1455 index++;
1456 }
1457 }
1458
1459 // restore fields of an eliminated object array
1460 void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj) {
1461 for (int i = 0; i < sv->field_size(); i++) {
1462 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1463 assert(value->type() == T_OBJECT, "object element expected");
1464 obj->obj_at_put(i, value->get_obj()());
1465 }
1466 }
1467
1468 class ReassignedField {
1469 public:
1470 int _offset;
1471 BasicType _type;
1472 public:
1473 ReassignedField() {
1474 _offset = 0;
1475 _type = T_ILLEGAL;
1476 }
1477 };
1478
1479 // Gets the fields of `klass` that are eliminated by escape analysis and need to be reassigned
1480 static GrowableArray<ReassignedField>* get_reassigned_fields(InstanceKlass* klass, GrowableArray<ReassignedField>* fields, bool is_jvmci) {
1481 InstanceKlass* super = klass->super();
1482 if (super != nullptr) {
1483 get_reassigned_fields(super, fields, is_jvmci);
1484 }
1485 for (AllFieldStream fs(klass); !fs.done(); fs.next()) {
1486 if (!fs.access_flags().is_static() && (is_jvmci || !fs.field_flags().is_injected())) {
1487 ReassignedField field;
1488 field._offset = fs.offset();
1489 field._type = Signature::basic_type(fs.signature());
1490 fields->append(field);
1491 }
1492 }
1493 return fields;
1494 }
1495
1496 // Restore fields of an eliminated instance object employing the same field order used by the compiler.
1497 static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap* reg_map, ObjectValue* sv, int svIndex, oop obj, bool is_jvmci) {
1498 GrowableArray<ReassignedField>* fields = get_reassigned_fields(klass, new GrowableArray<ReassignedField>(), is_jvmci);
1499 for (int i = 0; i < fields->length(); i++) {
1500 ScopeValue* scope_field = sv->field_at(svIndex);
1501 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1502 int offset = fields->at(i)._offset;
1503 BasicType type = fields->at(i)._type;
1504 switch (type) {
1505 case T_OBJECT: case T_ARRAY:
1506 assert(value->type() == T_OBJECT, "Agreement.");
1507 obj->obj_field_put(offset, value->get_obj()());
1508 break;
1509
1510 case T_INT: case T_FLOAT: { // 4 bytes.
1511 assert(value->type() == T_INT, "Agreement.");
1512 bool big_value = false;
1513 if (i+1 < fields->length() && fields->at(i+1)._type == T_INT) {
1514 if (scope_field->is_location()) {
1515 Location::Type type = ((LocationValue*) scope_field)->location().type();
1516 if (type == Location::dbl || type == Location::lng) {
1517 big_value = true;
1518 }
1519 }
1520 if (scope_field->is_constant_int()) {
1521 ScopeValue* next_scope_field = sv->field_at(svIndex + 1);
1522 if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1523 big_value = true;
1559 break;
1560
1561 case T_BYTE:
1562 assert(value->type() == T_INT, "Agreement.");
1563 obj->byte_field_put(offset, (jbyte)value->get_jint());
1564 break;
1565
1566 case T_BOOLEAN:
1567 assert(value->type() == T_INT, "Agreement.");
1568 obj->bool_field_put(offset, (jboolean)value->get_jint());
1569 break;
1570
1571 default:
1572 ShouldNotReachHere();
1573 }
1574 svIndex++;
1575 }
1576 return svIndex;
1577 }
1578
1579 // restore fields of all eliminated objects and arrays
1580 void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures, bool is_jvmci) {
1581 for (int i = 0; i < objects->length(); i++) {
1582 assert(objects->at(i)->is_object(), "invalid debug information");
1583 ObjectValue* sv = (ObjectValue*) objects->at(i);
1584 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1585 Handle obj = sv->value();
1586 assert(obj.not_null() || realloc_failures, "reallocation was missed");
1587 #ifndef PRODUCT
1588 if (PrintDeoptimizationDetails) {
1589 tty->print_cr("reassign fields for object of type %s!", k->name()->as_C_string());
1590 }
1591 #endif // !PRODUCT
1592
1593 if (obj.is_null()) {
1594 continue;
1595 }
1596
1597 #if INCLUDE_JVMCI
1598 // Don't reassign fields of boxes that came from a cache. Caches may be in CDS.
1599 if (sv->is_auto_box() && ((AutoBoxObjectValue*) sv)->is_cached()) {
1600 continue;
1601 }
1602 #endif // INCLUDE_JVMCI
1603 if (EnableVectorSupport && VectorSupport::is_vector(k)) {
1604 assert(sv->field_size() == 1, "%s not a vector", k->name()->as_C_string());
1605 ScopeValue* payload = sv->field_at(0);
1606 if (payload->is_location() &&
1607 payload->as_LocationValue()->location().type() == Location::vector) {
1608 #ifndef PRODUCT
1609 if (PrintDeoptimizationDetails) {
1610 tty->print_cr("skip field reassignment for this vector - it should be assigned already");
1611 if (Verbose) {
1612 Handle obj = sv->value();
1613 k->oop_print_on(obj(), tty);
1614 }
1615 }
1616 #endif // !PRODUCT
1617 continue; // Such vector's value was already restored in VectorSupport::allocate_vector().
1618 }
1619 // Else fall-through to do assignment for scalar-replaced boxed vector representation
1620 // which could be restored after vector object allocation.
1621 }
1622 if (k->is_instance_klass()) {
1623 InstanceKlass* ik = InstanceKlass::cast(k);
1624 reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj(), is_jvmci);
1625 } else if (k->is_typeArray_klass()) {
1626 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1627 reassign_type_array_elements(fr, reg_map, sv, (typeArrayOop) obj(), ak->element_type());
1628 } else if (k->is_objArray_klass()) {
1629 reassign_object_array_elements(fr, reg_map, sv, (objArrayOop) obj());
1630 }
1631 }
1632 // These objects may escape when we return to Interpreter after deoptimization.
1633 // We need barrier so that stores that initialize these objects can't be reordered
1634 // with subsequent stores that make these objects accessible by other threads.
1635 OrderAccess::storestore();
1636 }
1637
1638
1639 // relock objects for which synchronization was eliminated
1640 bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInfo*>* monitors,
1641 JavaThread* deoptee_thread, frame& fr, int exec_mode, bool realloc_failures) {
1642 bool relocked_objects = false;
1643 for (int i = 0; i < monitors->length(); i++) {
1644 MonitorInfo* mon_info = monitors->at(i);
1645 if (mon_info->eliminated()) {
1646 assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
1647 relocked_objects = true;
1648 if (!mon_info->owner_is_scalar_replaced()) {
1786 xtty->begin_head("deoptimized thread='%zu' reason='%s' pc='" INTPTR_FORMAT "'",(uintx)thread->osthread()->thread_id(), trap_reason_name(reason), p2i(fr.pc()));
1787 nm->log_identity(xtty);
1788 xtty->end_head();
1789 for (ScopeDesc* sd = nm->scope_desc_at(fr.pc()); ; sd = sd->sender()) {
1790 xtty->begin_elem("jvms bci='%d'", sd->bci());
1791 xtty->method(sd->method());
1792 xtty->end_elem();
1793 if (sd->is_top()) break;
1794 }
1795 xtty->tail("deoptimized");
1796 }
1797
1798 Continuation::notify_deopt(thread, fr.sp());
1799
1800 // Patch the compiled method so that when execution returns to it we will
1801 // deopt the execution state and return to the interpreter.
1802 fr.deoptimize(thread);
1803 }
1804
1805 void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason) {
1806 // Deoptimize only if the frame comes from compile code.
1807 // Do not deoptimize the frame which is already patched
1808 // during the execution of the loops below.
1809 if (!fr.is_compiled_frame() || fr.is_deoptimized_frame()) {
1810 return;
1811 }
1812 ResourceMark rm;
1813 deoptimize_single_frame(thread, fr, reason);
1814 }
1815
1816 address Deoptimization::deoptimize_for_missing_exception_handler(nmethod* nm, bool make_not_entrant) {
1817 // there is no exception handler for this pc => deoptimize
1818 if (make_not_entrant) {
1819 nm->make_not_entrant(nmethod::InvalidationReason::MISSING_EXCEPTION_HANDLER);
1820 }
1821
1822 // Use Deoptimization::deoptimize for all of its side-effects:
1823 // gathering traps statistics, logging...
1824 // it also patches the return pc but we do not care about that
1825 // since we return a continuation to the deopt_blob below.
1826 JavaThread* thread = JavaThread::current();
|
31 #include "code/nmethod.hpp"
32 #include "code/pcDesc.hpp"
33 #include "code/scopeDesc.hpp"
34 #include "compiler/compilationPolicy.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "interpreter/bytecode.inline.hpp"
39 #include "interpreter/bytecodeStream.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "interpreter/oopMapCache.hpp"
42 #include "jvm.h"
43 #include "logging/log.hpp"
44 #include "logging/logLevel.hpp"
45 #include "logging/logMessage.hpp"
46 #include "logging/logStream.hpp"
47 #include "memory/allocation.inline.hpp"
48 #include "memory/oopFactory.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "memory/universe.hpp"
51 #include "oops/arrayOop.inline.hpp"
52 #include "oops/constantPool.hpp"
53 #include "oops/fieldStreams.inline.hpp"
54 #include "oops/flatArrayKlass.hpp"
55 #include "oops/flatArrayOop.hpp"
56 #include "oops/inlineKlass.inline.hpp"
57 #include "oops/method.hpp"
58 #include "oops/objArrayKlass.hpp"
59 #include "oops/objArrayOop.inline.hpp"
60 #include "oops/oop.inline.hpp"
61 #include "oops/typeArrayOop.inline.hpp"
62 #include "oops/verifyOopClosure.hpp"
63 #include "prims/jvmtiDeferredUpdates.hpp"
64 #include "prims/jvmtiExport.hpp"
65 #include "prims/jvmtiThreadState.hpp"
66 #include "prims/methodHandles.hpp"
67 #include "prims/vectorSupport.hpp"
68 #include "runtime/arguments.hpp"
69 #include "runtime/atomicAccess.hpp"
70 #include "runtime/basicLock.inline.hpp"
71 #include "runtime/continuation.hpp"
72 #include "runtime/continuationEntry.inline.hpp"
73 #include "runtime/deoptimization.hpp"
74 #include "runtime/escapeBarrier.hpp"
75 #include "runtime/fieldDescriptor.inline.hpp"
76 #include "runtime/frame.inline.hpp"
77 #include "runtime/handles.inline.hpp"
78 #include "runtime/interfaceSupport.inline.hpp"
79 #include "runtime/javaThread.hpp"
80 #include "runtime/jniHandles.inline.hpp"
81 #include "runtime/keepStackGCProcessed.hpp"
82 #include "runtime/lockStack.inline.hpp"
83 #include "runtime/objectMonitor.inline.hpp"
84 #include "runtime/osThread.hpp"
85 #include "runtime/safepointVerifiers.hpp"
86 #include "runtime/sharedRuntime.hpp"
87 #include "runtime/signature.hpp"
88 #include "runtime/stackFrameStream.inline.hpp"
286 // The actual reallocation of previously eliminated objects occurs in realloc_objects,
287 // which is called from the method fetch_unroll_info_helper below.
288 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* current, int exec_mode))
289 // fetch_unroll_info() is called at the beginning of the deoptimization
290 // handler. Note this fact before we start generating temporary frames
291 // that can confuse an asynchronous stack walker. This counter is
292 // decremented at the end of unpack_frames().
293 current->inc_in_deopt_handler();
294
295 if (exec_mode == Unpack_exception) {
296 // When we get here, a callee has thrown an exception into a deoptimized
297 // frame. That throw might have deferred stack watermark checking until
298 // after unwinding. So we deal with such deferred requests here.
299 StackWatermarkSet::after_unwind(current);
300 }
301
302 return fetch_unroll_info_helper(current, exec_mode);
303 JRT_END
304
305 #if COMPILER2_OR_JVMCI
306
307 static Klass* get_refined_array_klass(Klass* k, frame* fr, RegisterMap* map, ObjectValue* sv, TRAPS) {
308 // If it's an array, get the properties
309 if (k->is_array_klass() && !k->is_typeArray_klass()) {
310 assert(k->is_unrefined_objArray_klass(), "Expected unrefined array klass");
311 nmethod* nm = fr->cb()->as_nmethod_or_null();
312 if (nm->is_compiled_by_c2()) {
313 assert(sv->has_properties(), "Property information is missing");
314 ArrayProperties props(checked_cast<ArrayProperties::Type>(StackValue::create_stack_value(fr, map, sv->properties())->get_jint()));
315 k = ObjArrayKlass::cast(k)->klass_with_properties(props, THREAD);
316 } else {
317 // TODO 8382708 JVMCI does not yet pass properties. Just go with the default for now.
318 k = ObjArrayKlass::cast(k)->klass_with_properties(ArrayProperties::Default(), THREAD);
319 }
320 }
321 return k;
322 }
323
324 // print information about reallocated objects
325 static void print_objects(JavaThread* deoptee_thread, frame* deoptee, RegisterMap* map,
326 GrowableArray<ScopeValue*>* objects, bool realloc_failures, TRAPS) {
327 ResourceMark rm;
328 stringStream st; // change to logStream with logging
329 st.print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(deoptee_thread));
330 fieldDescriptor fd;
331
332 for (int i = 0; i < objects->length(); i++) {
333 ObjectValue* sv = (ObjectValue*) objects->at(i);
334 Handle obj = sv->value();
335
336 if (obj.is_null()) {
337 st.print_cr(" nullptr");
338 continue;
339 }
340
341 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
342 k = get_refined_array_klass(k, deoptee, map, sv, THREAD);
343
344 st.print(" object <" INTPTR_FORMAT "> of type ", p2i(sv->value()()));
345 k->print_value_on(&st);
346 st.print_cr(" allocated (%zu bytes)", obj->size() * HeapWordSize);
347
348 if (Verbose && k != nullptr) {
349 k->oop_print_on(obj(), &st);
350 }
351 }
352 tty->print_raw(st.freeze());
353 }
354
355 static bool rematerialize_objects(JavaThread* thread, int exec_mode, nmethod* compiled_method,
356 frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk,
357 bool& deoptimized_objects) {
358 bool realloc_failures = false;
359 assert (chunk->at(0)->scope() != nullptr,"expect only compiled java frames");
360
361 JavaThread* deoptee_thread = chunk->at(0)->thread();
362 assert(exec_mode == Deoptimization::Unpack_none || (deoptee_thread == thread),
363 "a frame can only be deoptimized by the owner thread");
364
365 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects_to_rematerialize(deoptee, map);
366
367 // The flag return_oop() indicates call sites which return oop
368 // in compiled code. Such sites include java method calls,
369 // runtime calls (for example, used to allocate new objects/arrays
370 // on slow code path) and any other calls generated in compiled code.
371 // It is not guaranteed that we can get such information here only
372 // by analyzing bytecode in deoptimized frames. This is why this flag
373 // is set during method compilation (see Compile::Process_OopMap_Node()).
374 // If the previous frame was popped or if we are dispatching an exception,
375 // we don't have an oop result.
376 ScopeDesc* scope = chunk->at(0)->scope();
377 bool save_oop_result = scope->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
378 // In case of the return of multiple values, we must take care
379 // of all oop return values.
380 GrowableArray<Handle> return_oops;
381 InlineKlass* vk = nullptr;
382 if (save_oop_result && scope->return_scalarized()) {
383 vk = InlineKlass::returned_inline_klass(map);
384 if (vk != nullptr) {
385 vk->save_oop_fields(map, return_oops);
386 save_oop_result = false;
387 }
388 }
389 if (save_oop_result) {
390 // Reallocation may trigger GC. If deoptimization happened on return from
391 // call which returns oop we need to save it since it is not in oopmap.
392 oop result = deoptee.saved_oop_result(&map);
393 assert(oopDesc::is_oop_or_null(result), "must be oop");
394 return_oops.push(Handle(thread, result));
395 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
396 if (TraceDeoptimization) {
397 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
398 tty->cr();
399 }
400 }
401 if (objects != nullptr || vk != nullptr) {
402 if (exec_mode == Deoptimization::Unpack_none) {
403 assert(thread->thread_state() == _thread_in_vm, "assumption");
404 JavaThread* THREAD = thread; // For exception macros.
405 // Clear pending OOM if reallocation fails and return true indicating allocation failure
406 if (vk != nullptr) {
407 realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, CHECK_AND_CLEAR_(true));
408 }
409 if (objects != nullptr) {
410 realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, CHECK_AND_CLEAR_(true));
411 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
412 bool is_jvmci = compiled_method->is_compiled_by_jvmci();
413 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, is_jvmci, CHECK_AND_CLEAR_(true));
414 }
415 deoptimized_objects = true;
416 } else {
417 JavaThread* current = thread; // For JRT_BLOCK
418 JRT_BLOCK
419 if (vk != nullptr) {
420 realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, THREAD);
421 }
422 if (objects != nullptr) {
423 realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
424 guarantee(compiled_method != nullptr, "deopt must be associated with an nmethod");
425 bool is_jvmci = compiled_method->is_compiled_by_jvmci();
426 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, is_jvmci, THREAD);
427 }
428 JRT_END
429 }
430 if (TraceDeoptimization && objects != nullptr) {
431 print_objects(deoptee_thread, &deoptee, &map, objects, realloc_failures, thread);
432 }
433 }
434 if (save_oop_result || vk != nullptr) {
435 // Restore result.
436 assert(return_oops.length() == 1, "no inline type");
437 deoptee.set_saved_oop_result(&map, return_oops.pop()());
438 }
439 return realloc_failures;
440 }
441
442 static void restore_eliminated_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures,
443 frame& deoptee, int exec_mode, bool& deoptimized_objects) {
444 JavaThread* deoptee_thread = chunk->at(0)->thread();
445 assert(!EscapeBarrier::objs_are_deoptimized(deoptee_thread, deoptee.id()), "must relock just once");
446 assert(thread == Thread::current(), "should be");
447 HandleMark hm(thread);
448 #ifndef PRODUCT
449 bool first = true;
450 #endif // !PRODUCT
451 // Start locking from outermost/oldest frame
452 for (int i = (chunk->length() - 1); i >= 0; i--) {
453 compiledVFrame* cvf = chunk->at(i);
454 assert (cvf->scope() != nullptr,"expect only compiled java frames");
455 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
456 if (monitors->is_nonempty()) {
457 bool relocked = Deoptimization::relock_objects(thread, monitors, deoptee_thread, deoptee,
486 tty->print_raw(st.freeze());
487 }
488 #endif // !PRODUCT
489 }
490 }
491 }
492
493 // Deoptimize objects, that is reallocate and relock them, just before they escape through JVMTI.
494 // The given vframes cover one physical frame.
495 bool Deoptimization::deoptimize_objects_internal(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk,
496 bool& realloc_failures) {
497 frame deoptee = chunk->at(0)->fr();
498 JavaThread* deoptee_thread = chunk->at(0)->thread();
499 nmethod* nm = deoptee.cb()->as_nmethod_or_null();
500 RegisterMap map(chunk->at(0)->register_map());
501 bool deoptimized_objects = false;
502
503 bool const jvmci_enabled = JVMCI_ONLY(EnableJVMCI) NOT_JVMCI(false);
504
505 // Reallocate the non-escaping objects and restore their fields.
506 if (jvmci_enabled COMPILER2_PRESENT(|| ((DoEscapeAnalysis || Arguments::is_valhalla_enabled()) && EliminateAllocations)
507 || EliminateAutoBox || EnableVectorAggressiveReboxing)) {
508 realloc_failures = rematerialize_objects(thread, Unpack_none, nm, deoptee, map, chunk, deoptimized_objects);
509 }
510
511 // MonitorInfo structures used in eliminate_locks are not GC safe.
512 NoSafepointVerifier no_safepoint;
513
514 // Now relock objects if synchronization on them was eliminated.
515 if (jvmci_enabled COMPILER2_PRESENT(|| ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks))) {
516 restore_eliminated_locks(thread, chunk, realloc_failures, deoptee, Unpack_none, deoptimized_objects);
517 }
518 return deoptimized_objects;
519 }
520 #endif // COMPILER2_OR_JVMCI
521
522 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
523 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* current, int exec_mode) {
524 JFR_ONLY(Jfr::check_and_process_sample_request(current);)
525 // When we get here we are about to unwind the deoptee frame. In order to
526 // catch not yet safe to use frames, the following stack watermark barrier
563 // Create a growable array of VFrames where each VFrame represents an inlined
564 // Java frame. This storage is allocated with the usual system arena.
565 assert(deoptee.is_compiled_frame(), "Wrong frame type");
566 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
567 vframe* vf = vframe::new_vframe(&deoptee, &map, current);
568 while (!vf->is_top()) {
569 assert(vf->is_compiled_frame(), "Wrong frame type");
570 chunk->push(compiledVFrame::cast(vf));
571 vf = vf->sender();
572 }
573 assert(vf->is_compiled_frame(), "Wrong frame type");
574 chunk->push(compiledVFrame::cast(vf));
575
576 bool realloc_failures = false;
577
578 #if COMPILER2_OR_JVMCI
579 bool const jvmci_enabled = JVMCI_ONLY(EnableJVMCI) NOT_JVMCI(false);
580
581 // Reallocate the non-escaping objects and restore their fields. Then
582 // relock objects if synchronization on them was eliminated.
583 if (jvmci_enabled COMPILER2_PRESENT(|| ((DoEscapeAnalysis || Arguments::is_valhalla_enabled()) && EliminateAllocations)
584 || EliminateAutoBox || EnableVectorAggressiveReboxing)) {
585 bool unused;
586 realloc_failures = rematerialize_objects(current, exec_mode, nm, deoptee, map, chunk, unused);
587 }
588 #endif // COMPILER2_OR_JVMCI
589
590 // Ensure that no safepoint is taken after pointers have been stored
591 // in fields of rematerialized objects. If a safepoint occurs from here on
592 // out the java state residing in the vframeArray will be missed.
593 // Locks may be rebaised in a safepoint.
594 NoSafepointVerifier no_safepoint;
595
596 #if COMPILER2_OR_JVMCI
597 if ((jvmci_enabled COMPILER2_PRESENT( || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) ))
598 && !EscapeBarrier::objs_are_deoptimized(current, deoptee.id())) {
599 bool unused = false;
600 restore_eliminated_locks(current, chunk, realloc_failures, deoptee, exec_mode, unused);
601 }
602 #endif // COMPILER2_OR_JVMCI
603
604 ScopeDesc* trap_scope = chunk->at(0)->scope();
751 // its caller's stack by. If the caller is a compiled frame then
752 // we pretend that the callee has no parameters so that the
753 // extension counts for the full amount of locals and not just
754 // locals-parms. This is because without a c2i adapter the parm
755 // area as created by the compiled frame will not be usable by
756 // the interpreter. (Depending on the calling convention there
757 // may not even be enough space).
758
759 // QQQ I'd rather see this pushed down into last_frame_adjust
760 // and have it take the sender (aka caller).
761
762 if (!deopt_sender.is_interpreted_frame() || caller_was_method_handle) {
763 caller_adjustment = last_frame_adjust(0, callee_locals);
764 } else if (callee_locals > callee_parameters) {
765 // The caller frame may need extending to accommodate
766 // non-parameter locals of the first unpacked interpreted frame.
767 // Compute that adjustment.
768 caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);
769 }
770
771 // If the sender is deoptimized we must retrieve the address of the handler
772 // since the frame will "magically" show the original pc before the deopt
773 // and we'd undo the deopt.
774
775 frame_pcs[0] = Continuation::is_cont_barrier_frame(deoptee) ? StubRoutines::cont_returnBarrier() : deopt_sender.raw_pc();
776 if (Continuation::is_continuation_enterSpecial(deopt_sender)) {
777 ContinuationEntry::from_frame(deopt_sender)->set_argsize(0);
778 }
779
780 assert(CodeCache::find_blob(frame_pcs[0]) != nullptr, "bad pc");
781
782 #if INCLUDE_JVMCI
783 if (exceptionObject() != nullptr) {
784 current->set_exception_oop(exceptionObject());
785 exec_mode = Unpack_exception;
786 assert(array->element(0)->rethrow_exception(), "must be");
787 }
788 #endif
789
790 if (current->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
791 assert(current->has_pending_exception(), "should have thrown OOME");
1135 guarantee(ik->is_initialized(), "%s must be initialized", klass_name_str);
1136 CacheType::compute_offsets(ik);
1137 }
1138 return ik;
1139 }
1140 };
1141
1142 template<typename PrimitiveType, typename CacheType, typename BoxType> class BoxCache : public BoxCacheBase<CacheType> {
1143 PrimitiveType _low;
1144 PrimitiveType _high;
1145 jobject _cache;
1146 protected:
1147 static BoxCache<PrimitiveType, CacheType, BoxType> *_singleton;
1148 BoxCache(Thread* thread) {
1149 InstanceKlass* ik = BoxCacheBase<CacheType>::find_cache_klass(thread, CacheType::symbol());
1150 if (ik->is_in_error_state()) {
1151 _low = 1;
1152 _high = 0;
1153 _cache = nullptr;
1154 } else {
1155 refArrayOop cache = CacheType::cache(ik);
1156 assert(!cache->is_flatArray(), "box caches must be reference arrays");
1157 assert(cache->length() > 0, "Empty cache");
1158 _low = BoxType::value(cache->obj_at(0));
1159 _high = checked_cast<PrimitiveType>(_low + cache->length() - 1);
1160 _cache = JNIHandles::make_global(Handle(thread, cache));
1161 }
1162 }
1163 ~BoxCache() {
1164 JNIHandles::destroy_global(_cache);
1165 }
1166 public:
1167 static BoxCache<PrimitiveType, CacheType, BoxType>* singleton(Thread* thread) {
1168 if (_singleton == nullptr) {
1169 BoxCache<PrimitiveType, CacheType, BoxType>* s = new BoxCache<PrimitiveType, CacheType, BoxType>(thread);
1170 if (!AtomicAccess::replace_if_null(&_singleton, s)) {
1171 delete s;
1172 }
1173 }
1174 return _singleton;
1175 }
1176 oop lookup(PrimitiveType value) {
1177 if (_low <= value && value <= _high) {
1178 int offset = checked_cast<int>(value - _low);
1179 return refArrayOop(JNIHandles::resolve_non_null(_cache))->obj_at(offset);
1180 }
1181 return nullptr;
1182 }
1183 oop lookup_raw(intptr_t raw_value, bool& cache_init_error) {
1184 if (_cache == nullptr) {
1185 cache_init_error = true;
1186 return nullptr;
1187 }
1188 // Have to cast to avoid little/big-endian problems.
1189 if (sizeof(PrimitiveType) > sizeof(jint)) {
1190 jlong value = (jlong)raw_value;
1191 return lookup(value);
1192 }
1193 PrimitiveType value = (PrimitiveType)*((jint*)&raw_value);
1194 return lookup(value);
1195 }
1196 };
1197
1198 typedef BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer> IntegerBoxCache;
1199 typedef BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long> LongBoxCache;
1201 typedef BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short> ShortBoxCache;
1202 typedef BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte> ByteBoxCache;
1203
1204 template<> BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>* BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>::_singleton = nullptr;
1205 template<> BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>* BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>::_singleton = nullptr;
1206 template<> BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>* BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>::_singleton = nullptr;
1207 template<> BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>* BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>::_singleton = nullptr;
1208 template<> BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>* BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>::_singleton = nullptr;
1209
1210 class BooleanBoxCache : public BoxCacheBase<java_lang_Boolean> {
1211 jobject _true_cache;
1212 jobject _false_cache;
1213 protected:
1214 static BooleanBoxCache *_singleton;
1215 BooleanBoxCache(Thread *thread) {
1216 InstanceKlass* ik = find_cache_klass(thread, java_lang_Boolean::symbol());
1217 if (ik->is_in_error_state()) {
1218 _true_cache = nullptr;
1219 _false_cache = nullptr;
1220 } else {
1221 oop true_cache = java_lang_Boolean::get_TRUE(ik);
1222 oop false_cache = java_lang_Boolean::get_FALSE(ik);
1223 assert(true_cache != nullptr && false_cache != nullptr, "Boolean cache fields must be initialized");
1224 _true_cache = JNIHandles::make_global(Handle(thread, true_cache));
1225 _false_cache = JNIHandles::make_global(Handle(thread, false_cache));
1226 }
1227 }
1228 ~BooleanBoxCache() {
1229 JNIHandles::destroy_global(_true_cache);
1230 JNIHandles::destroy_global(_false_cache);
1231 }
1232 public:
1233 static BooleanBoxCache* singleton(Thread* thread) {
1234 if (_singleton == nullptr) {
1235 BooleanBoxCache* s = new BooleanBoxCache(thread);
1236 if (!AtomicAccess::replace_if_null(&_singleton, s)) {
1237 delete s;
1238 }
1239 }
1240 return _singleton;
1241 }
1242 oop lookup_raw(intptr_t raw_value, bool& cache_in_error) {
1243 if (_true_cache == nullptr) {
1244 cache_in_error = true;
1245 return nullptr;
1274 }
1275 }
1276 return nullptr;
1277 }
1278 #endif // INCLUDE_JVMCI
1279
1280 #if COMPILER2_OR_JVMCI
1281 bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, TRAPS) {
1282 Handle pending_exception(THREAD, thread->pending_exception());
1283 const char* exception_file = thread->exception_file();
1284 int exception_line = thread->exception_line();
1285 thread->clear_pending_exception();
1286
1287 bool failures = false;
1288
1289 for (int i = 0; i < objects->length(); i++) {
1290 assert(objects->at(i)->is_object(), "invalid debug information");
1291 ObjectValue* sv = (ObjectValue*) objects->at(i);
1292
1293 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1294
1295 k = get_refined_array_klass(k, fr, reg_map, sv, THREAD);
1296
1297 // Check if the object may be null and has an additional null_marker input that needs
1298 // to be checked before using the field values. Skip re-allocation if it is null.
1299 if (k->is_inline_klass() && sv->has_properties()) {
1300 jint null_marker = StackValue::create_stack_value(fr, reg_map, sv->properties())->get_jint();
1301 if (null_marker == 0) {
1302 continue;
1303 }
1304 }
1305
1306 oop obj = nullptr;
1307 bool cache_init_error = false;
1308 if (k->is_instance_klass()) {
1309 #if INCLUDE_JVMCI
1310 nmethod* nm = fr->cb()->as_nmethod_or_null();
1311 if (nm->is_compiled_by_jvmci() && sv->is_auto_box()) {
1312 AutoBoxObjectValue* abv = (AutoBoxObjectValue*) sv;
1313 obj = get_cached_box(abv, fr, reg_map, cache_init_error, THREAD);
1314 if (obj != nullptr) {
1315 // Set the flag to indicate the box came from a cache, so that we can skip the field reassignment for it.
1316 abv->set_cached(true);
1317 } else if (cache_init_error) {
1318 // Results in an OOME which is valid (as opposed to a class initialization error)
1319 // and is fine for the rare case a cache initialization failing.
1320 failures = true;
1321 }
1322 }
1323 #endif // INCLUDE_JVMCI
1324
1325 InstanceKlass* ik = InstanceKlass::cast(k);
1326 if (obj == nullptr && !cache_init_error) {
1327 InternalOOMEMark iom(THREAD);
1328 if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
1329 obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
1330 } else {
1331 obj = ik->allocate_instance(THREAD);
1332 }
1333 }
1334 } else if (k->is_flatArray_klass()) {
1335 FlatArrayKlass* ak = FlatArrayKlass::cast(k);
1336 // Inline type array must be zeroed because not all memory is reassigned
1337 InternalOOMEMark iom(THREAD);
1338 obj = ak->allocate_instance(sv->field_size(), THREAD);
1339 } else if (k->is_typeArray_klass()) {
1340 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1341 assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
1342 int len = sv->field_size() / type2size[ak->element_type()];
1343 InternalOOMEMark iom(THREAD);
1344 obj = ak->allocate_instance(len, THREAD);
1345 } else if (k->is_refArray_klass()) {
1346 RefArrayKlass* ak = RefArrayKlass::cast(k);
1347 InternalOOMEMark iom(THREAD);
1348 obj = ak->allocate_instance(sv->field_size(), THREAD);
1349 }
1350
1351 if (obj == nullptr) {
1352 failures = true;
1353 }
1354
1355 assert(sv->value().is_null(), "redundant reallocation");
1356 assert(obj != nullptr || HAS_PENDING_EXCEPTION || cache_init_error, "allocation should succeed or we should get an exception");
1357 CLEAR_PENDING_EXCEPTION;
1358 sv->set_value(obj);
1359 }
1360
1361 if (failures) {
1362 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
1363 } else if (pending_exception.not_null()) {
1364 thread->set_pending_exception(pending_exception(), exception_file, exception_line);
1365 }
1366
1367 return failures;
1368 }
1369
1370 // We're deoptimizing at the return of a call, inline type fields are
1371 // in registers. When we go back to the interpreter, it will expect a
1372 // reference to an inline type instance. Allocate and initialize it from
1373 // the register values here.
1374 bool Deoptimization::realloc_inline_type_result(InlineKlass* vk, const RegisterMap& map, GrowableArray<Handle>& return_oops, TRAPS) {
1375 oop new_vt = vk->realloc_result(map, return_oops, THREAD);
1376 if (new_vt == nullptr) {
1377 CLEAR_PENDING_EXCEPTION;
1378 THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), true);
1379 }
1380 return_oops.clear();
1381 return_oops.push(Handle(THREAD, new_vt));
1382 return false;
1383 }
1384
1385 #if INCLUDE_JVMCI
1386 /**
1387 * For primitive types whose kind gets "erased" at runtime (shorts become stack ints),
1388 * we need to somehow be able to recover the actual kind to be able to write the correct
1389 * amount of bytes.
1390 * For that purpose, this method assumes that, for an entry spanning n bytes at index i,
1391 * the entries at index n + 1 to n + i are 'markers'.
1392 * For example, if we were writing a short at index 4 of a byte array of size 8, the
1393 * expected form of the array would be:
1394 *
1395 * {b0, b1, b2, b3, INT, marker, b6, b7}
1396 *
1397 * Thus, in order to get back the size of the entry, we simply need to count the number
1398 * of marked entries
1399 *
1400 * @param virtualArray the virtualized byte array
1401 * @param i index of the virtual entry we are recovering
1402 * @return The number of bytes the entry spans
1403 */
1404 static int count_number_of_bytes_for_entry(ObjectValue *virtualArray, int i) {
1536 default:
1537 ShouldNotReachHere();
1538 }
1539 index++;
1540 }
1541 }
1542
1543 // restore fields of an eliminated object array
1544 void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj) {
1545 for (int i = 0; i < sv->field_size(); i++) {
1546 StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1547 assert(value->type() == T_OBJECT, "object element expected");
1548 obj->obj_at_put(i, value->get_obj()());
1549 }
1550 }
1551
1552 class ReassignedField {
1553 public:
1554 int _offset;
1555 BasicType _type;
1556 InstanceKlass* _klass;
1557 bool _is_flat;
1558 bool _is_null_free;
1559 public:
1560 ReassignedField() : _offset(0), _type(T_ILLEGAL), _klass(nullptr), _is_flat(false), _is_null_free(false) { }
1561 };
1562
1563 // Gets the fields of `klass` that are eliminated by escape analysis and need to be reassigned
1564 static GrowableArray<ReassignedField>* get_reassigned_fields(InstanceKlass* klass, GrowableArray<ReassignedField>* fields, bool is_jvmci) {
1565 InstanceKlass* super = klass->super();
1566 if (super != nullptr) {
1567 get_reassigned_fields(super, fields, is_jvmci);
1568 }
1569 for (AllFieldStream fs(klass); !fs.done(); fs.next()) {
1570 if (!fs.access_flags().is_static() && (is_jvmci || !fs.field_flags().is_injected())) {
1571 ReassignedField field;
1572 field._offset = fs.offset();
1573 field._type = Signature::basic_type(fs.signature());
1574 if (fs.is_flat()) {
1575 field._is_flat = true;
1576 field._is_null_free = fs.is_null_free_inline_type();
1577 // Resolve klass of flat inline type field
1578 field._klass = InlineKlass::cast(klass->get_inline_type_field_klass(fs.index()));
1579 }
1580 fields->append(field);
1581 }
1582 }
1583 return fields;
1584 }
1585
1586 // Restore fields of an eliminated instance object employing the same field order used by the
1587 // compiler when it scalarizes an object at safepoints.
1588 static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap* reg_map, ObjectValue* sv, int svIndex, oop obj, bool is_jvmci, int base_offset, TRAPS) {
1589 GrowableArray<ReassignedField>* fields = get_reassigned_fields(klass, new GrowableArray<ReassignedField>(), is_jvmci);
1590 for (int i = 0; i < fields->length(); i++) {
1591 BasicType type = fields->at(i)._type;
1592 int offset = base_offset + fields->at(i)._offset;
1593 // Check for flat inline type field before accessing the ScopeValue because it might not have any fields
1594 if (fields->at(i)._is_flat) {
1595 // Recursively re-assign flat inline type fields
1596 InstanceKlass* vk = fields->at(i)._klass;
1597 assert(vk != nullptr, "must be resolved");
1598 offset -= InlineKlass::cast(vk)->payload_offset(); // Adjust offset to omit oop header
1599 svIndex = reassign_fields_by_klass(vk, fr, reg_map, sv, svIndex, obj, is_jvmci, offset, CHECK_0);
1600 if (!fields->at(i)._is_null_free) {
1601 ScopeValue* scope_field = sv->field_at(svIndex);
1602 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1603 int nm_offset = offset + InlineKlass::cast(vk)->null_marker_offset();
1604 obj->bool_field_put(nm_offset, value->get_jint() & 1);
1605 svIndex++;
1606 }
1607 continue; // Continue because we don't need to increment svIndex
1608 }
1609
1610 ScopeValue* scope_field = sv->field_at(svIndex);
1611 StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);
1612 switch (type) {
1613 case T_OBJECT: case T_ARRAY:
1614 assert(value->type() == T_OBJECT, "Agreement.");
1615 obj->obj_field_put(offset, value->get_obj()());
1616 break;
1617
1618 case T_INT: case T_FLOAT: { // 4 bytes.
1619 assert(value->type() == T_INT, "Agreement.");
1620 bool big_value = false;
1621 if (i+1 < fields->length() && fields->at(i+1)._type == T_INT) {
1622 if (scope_field->is_location()) {
1623 Location::Type type = ((LocationValue*) scope_field)->location().type();
1624 if (type == Location::dbl || type == Location::lng) {
1625 big_value = true;
1626 }
1627 }
1628 if (scope_field->is_constant_int()) {
1629 ScopeValue* next_scope_field = sv->field_at(svIndex + 1);
1630 if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1631 big_value = true;
1667 break;
1668
1669 case T_BYTE:
1670 assert(value->type() == T_INT, "Agreement.");
1671 obj->byte_field_put(offset, (jbyte)value->get_jint());
1672 break;
1673
1674 case T_BOOLEAN:
1675 assert(value->type() == T_INT, "Agreement.");
1676 obj->bool_field_put(offset, (jboolean)value->get_jint());
1677 break;
1678
1679 default:
1680 ShouldNotReachHere();
1681 }
1682 svIndex++;
1683 }
1684 return svIndex;
1685 }
1686
1687 // restore fields of an eliminated inline type array
1688 void Deoptimization::reassign_flat_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, flatArrayOop obj, FlatArrayKlass* vak, bool is_jvmci, TRAPS) {
1689 InlineKlass* vk = vak->element_klass();
1690 assert(vk->maybe_flat_in_array(), "should only be used for flat inline type arrays");
1691 // Adjust offset to omit oop header
1692 int base_offset = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT) - vk->payload_offset();
1693 // Initialize all elements of the flat inline type array
1694 for (int i = 0; i < sv->field_size(); i++) {
1695 ObjectValue* val = sv->field_at(i)->as_ObjectValue();
1696 int offset = base_offset + (i << Klass::layout_helper_log2_element_size(vak->layout_helper()));
1697 reassign_fields_by_klass(vk, fr, reg_map, val, 0, (oop)obj, is_jvmci, offset, CHECK);
1698 if (!obj->is_null_free_array()) {
1699 jboolean null_marker_value;
1700 if (val->has_properties()) {
1701 null_marker_value = StackValue::create_stack_value(fr, reg_map, val->properties())->get_jint() & 1;
1702 } else {
1703 null_marker_value = 1;
1704 }
1705 obj->bool_field_put(offset + vk->null_marker_offset(), null_marker_value);
1706 }
1707 }
1708 }
1709
1710 // restore fields of all eliminated objects and arrays
1711 void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures, bool is_jvmci, TRAPS) {
1712 for (int i = 0; i < objects->length(); i++) {
1713 assert(objects->at(i)->is_object(), "invalid debug information");
1714 ObjectValue* sv = (ObjectValue*) objects->at(i);
1715 Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1716 k = get_refined_array_klass(k, fr, reg_map, sv, THREAD);
1717
1718 Handle obj = sv->value();
1719 assert(obj.not_null() || realloc_failures || sv->has_properties(), "reallocation was missed");
1720 #ifndef PRODUCT
1721 if (PrintDeoptimizationDetails) {
1722 tty->print_cr("reassign fields for object of type %s!", k->name()->as_C_string());
1723 }
1724 #endif // !PRODUCT
1725
1726 if (obj.is_null()) {
1727 continue;
1728 }
1729
1730 #if INCLUDE_JVMCI
1731 // Don't reassign fields of boxes that came from a cache. Caches may be in CDS.
1732 if (sv->is_auto_box() && ((AutoBoxObjectValue*) sv)->is_cached()) {
1733 continue;
1734 }
1735 #endif // INCLUDE_JVMCI
1736 if (EnableVectorSupport && VectorSupport::is_vector(k)) {
1737 assert(sv->field_size() == 1, "%s not a vector", k->name()->as_C_string());
1738 ScopeValue* payload = sv->field_at(0);
1739 if (payload->is_location() &&
1740 payload->as_LocationValue()->location().type() == Location::vector) {
1741 #ifndef PRODUCT
1742 if (PrintDeoptimizationDetails) {
1743 tty->print_cr("skip field reassignment for this vector - it should be assigned already");
1744 if (Verbose) {
1745 Handle obj = sv->value();
1746 k->oop_print_on(obj(), tty);
1747 }
1748 }
1749 #endif // !PRODUCT
1750 continue; // Such vector's value was already restored in VectorSupport::allocate_vector().
1751 }
1752 // Else fall-through to do assignment for scalar-replaced boxed vector representation
1753 // which could be restored after vector object allocation.
1754 }
1755 if (k->is_instance_klass()) {
1756 InstanceKlass* ik = InstanceKlass::cast(k);
1757 reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj(), is_jvmci, 0, CHECK);
1758 } else if (k->is_flatArray_klass()) {
1759 FlatArrayKlass* vak = FlatArrayKlass::cast(k);
1760 reassign_flat_array_elements(fr, reg_map, sv, (flatArrayOop) obj(), vak, is_jvmci, CHECK);
1761 } else if (k->is_typeArray_klass()) {
1762 TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1763 reassign_type_array_elements(fr, reg_map, sv, (typeArrayOop) obj(), ak->element_type());
1764 } else if (k->is_refArray_klass()) {
1765 reassign_object_array_elements(fr, reg_map, sv, (objArrayOop) obj());
1766 }
1767 }
1768 // These objects may escape when we return to Interpreter after deoptimization.
1769 // We need barrier so that stores that initialize these objects can't be reordered
1770 // with subsequent stores that make these objects accessible by other threads.
1771 OrderAccess::storestore();
1772 }
1773
1774
1775 // relock objects for which synchronization was eliminated
1776 bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInfo*>* monitors,
1777 JavaThread* deoptee_thread, frame& fr, int exec_mode, bool realloc_failures) {
1778 bool relocked_objects = false;
1779 for (int i = 0; i < monitors->length(); i++) {
1780 MonitorInfo* mon_info = monitors->at(i);
1781 if (mon_info->eliminated()) {
1782 assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
1783 relocked_objects = true;
1784 if (!mon_info->owner_is_scalar_replaced()) {
1922 xtty->begin_head("deoptimized thread='%zu' reason='%s' pc='" INTPTR_FORMAT "'",(uintx)thread->osthread()->thread_id(), trap_reason_name(reason), p2i(fr.pc()));
1923 nm->log_identity(xtty);
1924 xtty->end_head();
1925 for (ScopeDesc* sd = nm->scope_desc_at(fr.pc()); ; sd = sd->sender()) {
1926 xtty->begin_elem("jvms bci='%d'", sd->bci());
1927 xtty->method(sd->method());
1928 xtty->end_elem();
1929 if (sd->is_top()) break;
1930 }
1931 xtty->tail("deoptimized");
1932 }
1933
1934 Continuation::notify_deopt(thread, fr.sp());
1935
1936 // Patch the compiled method so that when execution returns to it we will
1937 // deopt the execution state and return to the interpreter.
1938 fr.deoptimize(thread);
1939 }
1940
1941 void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason) {
1942 // Deoptimize only if the frame comes from compiled code.
1943 // Do not deoptimize the frame which is already patched
1944 // during the execution of the loops below.
1945 if (!fr.is_compiled_frame() || fr.is_deoptimized_frame()) {
1946 return;
1947 }
1948 ResourceMark rm;
1949 deoptimize_single_frame(thread, fr, reason);
1950 }
1951
1952 address Deoptimization::deoptimize_for_missing_exception_handler(nmethod* nm, bool make_not_entrant) {
1953 // there is no exception handler for this pc => deoptimize
1954 if (make_not_entrant) {
1955 nm->make_not_entrant(nmethod::InvalidationReason::MISSING_EXCEPTION_HANDLER);
1956 }
1957
1958 // Use Deoptimization::deoptimize for all of its side-effects:
1959 // gathering traps statistics, logging...
1960 // it also patches the return pc but we do not care about that
1961 // since we return a continuation to the deopt_blob below.
1962 JavaThread* thread = JavaThread::current();
|