450 inline frame freeze_start_frame_yield_stub();
451 template<typename FKind>
452 inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
453 inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
454 inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
455 freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
456 void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame);
457 NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
458 freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
459 NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
460 NOINLINE freeze_result recurse_freeze_native_frame(frame& f, frame& caller);
461 NOINLINE void finish_freeze(const frame& f, const frame& top);
462
463 void freeze_lockstack(stackChunkOop chunk);
464
465 inline bool stack_overflow();
466
467 static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
468 : sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
469 template<typename FKind> static inline frame sender(const frame& f);
470 template<typename FKind> frame new_heap_frame(frame& f, frame& caller);
471 inline void set_top_frame_metadata_pd(const frame& hf);
472 inline void patch_pd(frame& callee, const frame& caller);
473 inline void patch_pd_unused(intptr_t* sp);
474 void adjust_interpreted_frame_unextended_sp(frame& f);
475 inline void prepare_freeze_interpreted_top_frame(frame& f);
476 static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
477
478 protected:
479 void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
480 bool freeze_fast_new_chunk(stackChunkOop chunk);
481 };
482
483 template <typename ConfigT>
484 class Freeze : public FreezeBase {
485 private:
486 stackChunkOop allocate_chunk(size_t stack_size, int argsize_md);
487
488 public:
489 inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt)
490 : FreezeBase(thread, cont, frame_sp, preempt) {}
491
492 freeze_result try_freeze_fast();
1160
1161 assert((!empty && Continuation::is_return_barrier_entry(entry.pc())) || (empty && Continuation::is_continuation_enterSpecial(entry)), "");
1162 assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
1163 #endif
1164
1165 return freeze_ok_bottom;
1166 }
1167
1168 // After freezing a frame we need to possibly adjust some values related to the caller frame.
1169 void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) {
1170 if (is_bottom_frame) {
1171 // If we're the bottom frame, we need to replace the return barrier with the real
1172 // caller's pc.
1173 address last_pc = caller.pc();
1174 assert((last_pc == nullptr) == _cont.tail()->is_empty(), "");
1175 ContinuationHelper::Frame::patch_pc(caller, last_pc);
1176 } else {
1177 assert(!caller.is_empty(), "");
1178 }
1179
1180 patch_pd(hf, caller);
1181
1182 if (f.is_interpreted_frame()) {
1183 assert(hf.is_heap_frame(), "should be");
1184 ContinuationHelper::InterpretedFrame::patch_sender_sp(hf, caller);
1185 }
1186
1187 #ifdef ASSERT
1188 if (hf.is_compiled_frame()) {
1189 if (f.is_deoptimized_frame()) { // TODO DEOPT: long term solution: unroll on freeze and patch pc
1190 log_develop_trace(continuations)("Freezing deoptimized frame");
1191 assert(f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), "");
1192 assert(f.cb()->as_nmethod()->is_deopt_pc(ContinuationHelper::Frame::real_pc(f)), "");
1193 }
1194 }
1195 #endif
1196 }
1197
1198 #ifdef ASSERT
1199 static void verify_frame_top(const frame& f, intptr_t* top) {
1200 ResourceMark rm;
1257
1258 CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
1259 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1260 caller = hf;
1261
1262 // Mark frame_method's GC epoch for class redefinition on_stack calculation.
1263 frame_method->record_gc_epoch();
1264
1265 return freeze_ok;
1266 }
1267
1268 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
1269 // See also StackChunkFrameStream<frame_kind>::frame_size()
1270 freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller,
1271 int callee_argsize /* incl. metadata */,
1272 bool callee_interpreted) {
1273 // The frame's top never includes the stack arguments to the callee
1274 intptr_t* const stack_frame_top = ContinuationHelper::CompiledFrame::frame_top(f, callee_argsize, callee_interpreted);
1275 intptr_t* const stack_frame_bottom = ContinuationHelper::CompiledFrame::frame_bottom(f);
1276 // including metadata between f and its stackargs
1277 const int argsize = ContinuationHelper::CompiledFrame::stack_argsize(f) + frame::metadata_words_at_top;
1278 const int fsize = pointer_delta_as_int(stack_frame_bottom + argsize, stack_frame_top);
1279
1280 log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d",
1281 ContinuationHelper::Frame::frame_method(f) != nullptr ?
1282 ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "",
1283 _freeze_size, fsize, argsize);
1284 // we'd rather not yield inside methods annotated with @JvmtiMountTransition
1285 assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
1286
1287 freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize);
1288 if (UNLIKELY(result > freeze_ok_bottom)) {
1289 return result;
1290 }
1291
1292 bool is_bottom_frame = result == freeze_ok_bottom;
1293 assert(!caller.is_empty() || is_bottom_frame, "");
1294
1295 DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, is_bottom_frame);)
1296
1297 frame hf = new_heap_frame<ContinuationHelper::CompiledFrame>(f, caller);
1298
1299 intptr_t* heap_frame_top = ContinuationHelper::CompiledFrame::frame_top(hf, callee_argsize, callee_interpreted);
1300
1301 copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
1302 assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
1303
1304 if (caller.is_interpreted_frame()) {
1305 // When thawing the frame we might need to add alignment (see Thaw::align)
1306 _total_align_size += frame::align_wiggle;
1307 }
1308
1309 patch(f, hf, caller, is_bottom_frame);
1310
1311 assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
1312
1313 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1314 caller = hf;
1315 return freeze_ok;
1316 }
1317
2059
2060 // Only used for preemption on ObjectLocker
2061 ObjectMonitor* _init_lock;
2062
2063 StackChunkFrameStream<ChunkFrames::Mixed> _stream;
2064
2065 NOT_PRODUCT(int _frames;)
2066
2067 protected:
2068 ThawBase(JavaThread* thread, ContinuationWrapper& cont) :
2069 _thread(thread), _cont(cont),
2070 _fastpath(nullptr) {
2071 DEBUG_ONLY(_top_unextended_sp_before_thaw = nullptr;)
2072 assert (cont.tail() != nullptr, "no last chunk");
2073 DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());)
2074 }
2075
2076 void clear_chunk(stackChunkOop chunk);
2077 template<bool check_stub>
2078 int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize);
2079 void copy_from_chunk(intptr_t* from, intptr_t* to, int size);
2080
2081 void thaw_lockstack(stackChunkOop chunk);
2082
2083 // fast path
2084 inline void prefetch_chunk_pd(void* start, int size_words);
2085 void patch_return(intptr_t* sp, bool is_last);
2086
2087 intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case);
2088 inline intptr_t* push_cleanup_continuation();
2089 inline intptr_t* push_preempt_adapter();
2090 intptr_t* redo_vmcall(JavaThread* current, frame& top);
2091 void throw_interrupted_exception(JavaThread* current, frame& top);
2092
2093 void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case);
2094 void finish_thaw(frame& f);
2095
2096 private:
2097 template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
2098 void finalize_thaw(frame& entry, int argsize);
2099
2100 inline bool seen_by_gc();
2101
2102 inline void before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame);
2103 inline void after_thaw_java_frame(const frame& f, bool bottom);
2104 inline void patch(frame& f, const frame& caller, bool bottom);
2105 void clear_bitmap_bits(address start, address end);
2106
2107 NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames, bool is_top);
2108 void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
2109 void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
2110 void recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames);
2111
2112 void push_return_frame(const frame& f);
2113 inline frame new_entry_frame();
2114 template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
2115 inline void patch_pd(frame& f, const frame& sender);
2116 inline void patch_pd(frame& f, intptr_t* caller_sp);
2117 inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
2118
2119 void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
2120
2121 static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
2122
2123 public:
2124 CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
2125 };
2126
2127 template <typename ConfigT>
2128 class Thaw : public ThawBase {
2129 public:
2130 Thaw(JavaThread* thread, ContinuationWrapper& cont) : ThawBase(thread, cont) {}
2131
2132 inline bool can_thaw_fast(stackChunkOop chunk) {
2133 return !_barriers
2134 && _thread->cont_fastpath_thread_state()
2171 assert(_base - 1 <= top() + total_size() + frame::metadata_words_at_bottom, "missed entry frame");
2172 }
2173
2174 int entry_frame_extension() const { return _argsize + (_argsize > 0 ? frame::metadata_words_at_top : 0); }
2175
2176 // top and bottom stack pointers
2177 intptr_t* sp() const { return ContinuationHelper::frame_align_pointer(_base - _thaw_size); }
2178 intptr_t* bottom_sp() const { return ContinuationHelper::frame_align_pointer(_base - entry_frame_extension()); }
2179
2180 // several operations operate on the totality of the stack being reconstructed,
2181 // including the metadata words
2182 intptr_t* top() const { return sp() - frame::metadata_words_at_bottom; }
2183 int total_size() const { return _thaw_size + frame::metadata_words_at_bottom; }
2184 };
2185
2186 inline void ThawBase::clear_chunk(stackChunkOop chunk) {
2187 chunk->set_sp(chunk->bottom());
2188 chunk->set_max_thawing_size(0);
2189 }
2190
2191 template<bool check_stub>
2192 int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) {
2193 bool empty = false;
2194 StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
2195 DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
2196 assert(chunk_sp == f.sp(), "");
2197 assert(chunk_sp == f.unextended_sp(), "");
2198
2199 int frame_size = f.cb()->frame_size();
2200 argsize = f.stack_argsize();
2201
2202 assert(!f.is_stub() || check_stub, "");
2203 if (check_stub && f.is_stub()) {
2204 // If we don't thaw the top compiled frame too, after restoring the saved
2205 // registers back in Java, we would hit the return barrier to thaw one more
2206 // frame effectively overwriting the restored registers during that call.
2207 f.next(SmallRegisterMap::instance_no_args(), true /* stop */);
2208 assert(!f.is_done(), "");
2209
2210 f.get_cb();
2211 assert(f.is_compiled(), "");
2212 frame_size += f.cb()->frame_size();
2213 argsize = f.stack_argsize();
2214
2215 if (f.cb()->as_nmethod()->is_marked_for_deoptimization()) {
2216 // The caller of the runtime stub when the continuation is preempted is not at a
2217 // Java call instruction, and so cannot rely on nmethod patching for deopt.
2218 log_develop_trace(continuations)("Deoptimizing runtime stub caller");
2219 f.to_frame().deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
2220 }
2221 }
2222
2223 f.next(SmallRegisterMap::instance_no_args(), true /* stop */);
2224 empty = f.is_done();
2225 assert(!empty || argsize == chunk->argsize(), "");
2226
2227 if (empty) {
2228 clear_chunk(chunk);
2229 } else {
2230 chunk->set_sp(chunk->sp() + frame_size);
2231 chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
2232 // We set chunk->pc to the return pc into the next frame
2233 chunk->set_pc(f.pc());
2234 #ifdef ASSERT
2235 {
2236 intptr_t* retaddr_slot = (chunk_sp
2237 + frame_size
2238 - frame::sender_sp_ret_address_offset());
2239 assert(f.pc() == ContinuationHelper::return_address_at(retaddr_slot),
2240 "unexpected pc");
2501 assert(!_cont.is_empty(), "no more frames");
2502 assert(num_frames > 0, "");
2503 assert(!heap_frame.is_empty(), "");
2504
2505 if (top_on_preempt_case && (heap_frame.is_native_frame() || heap_frame.is_runtime_frame())) {
2506 heap_frame.is_native_frame() ? recurse_thaw_native_frame(heap_frame, caller, 2) : recurse_thaw_stub_frame(heap_frame, caller, 2);
2507 } else if (!heap_frame.is_interpreted_frame()) {
2508 recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
2509 } else {
2510 recurse_thaw_interpreted_frame(heap_frame, caller, num_frames, top_on_preempt_case);
2511 }
2512 }
2513
2514 template<typename FKind>
2515 bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) {
2516 assert(num_frames > 0, "");
2517
2518 DEBUG_ONLY(_frames++;)
2519
2520 int argsize = _stream.stack_argsize();
2521
2522 _stream.next(SmallRegisterMap::instance_no_args());
2523 assert(_stream.to_frame().is_empty() == _stream.is_done(), "");
2524
2525 // we never leave a compiled caller of an interpreted frame as the top frame in the chunk
2526 // as it makes detecting that situation and adjusting unextended_sp tricky
2527 if (num_frames == 1 && !_stream.is_done() && FKind::interpreted && _stream.is_compiled()) {
2528 log_develop_trace(continuations)("thawing extra compiled frame to not leave a compiled interpreted-caller at top");
2529 num_frames++;
2530 }
2531
2532 if (num_frames == 1 || _stream.is_done()) { // end recursion
2533 finalize_thaw(caller, FKind::interpreted ? 0 : argsize);
2534 return true; // bottom
2535 } else { // recurse
2536 recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false /* top_on_preempt_case */);
2537 return false;
2538 }
2539 }
2540
2541 void ThawBase::finalize_thaw(frame& entry, int argsize) {
2542 stackChunkOop chunk = _cont.tail();
2543
2544 if (!_stream.is_done()) {
2545 assert(_stream.sp() >= chunk->sp_address(), "");
2546 chunk->set_sp(chunk->to_offset(_stream.sp()));
2547 chunk->set_pc(_stream.pc());
2574 }
2575 assert(bottom == _cont.is_entry_frame(caller), "bottom: %d is_entry_frame: %d", bottom, _cont.is_entry_frame(hf));
2576 }
2577
2578 inline void ThawBase::after_thaw_java_frame(const frame& f, bool bottom) {
2579 #ifdef ASSERT
2580 LogTarget(Trace, continuations) lt;
2581 if (lt.develop_is_enabled()) {
2582 LogStream ls(lt);
2583 ls.print_cr("thawed frame:");
2584 print_frame_layout(f, false, &ls); // f.print_on(&ls);
2585 }
2586 #endif
2587 }
2588
2589 inline void ThawBase::patch(frame& f, const frame& caller, bool bottom) {
2590 assert(!bottom || caller.fp() == _cont.entryFP(), "");
2591 if (bottom) {
2592 ContinuationHelper::Frame::patch_pc(caller, _cont.is_empty() ? caller.pc()
2593 : StubRoutines::cont_returnBarrier());
2594 } else if (_should_patch_caller_pc) {
2595 // Caller was deoptimized during thaw but we've overwritten the return address when copying f from the heap.
2596 // Also, on some platforms, if the caller is interpreted but the callee not we also need to patch.
2597
2598 #if defined(PPC64) || defined(S390)
2599 assert(caller.is_deoptimized_frame() || caller.is_interpreted_frame(), "");
2600 #else
2601 assert(caller.is_deoptimized_frame(), "");
2602 #endif
2603
2604 ContinuationHelper::Frame::patch_pc(caller, caller.raw_pc());
2605 _should_patch_caller_pc = false;
2606 }
2607
2608 patch_pd(f, caller);
2609
2610 if (f.is_interpreted_frame()) {
2611 ContinuationHelper::InterpretedFrame::patch_sender_sp(f, caller);
2612 }
2613
2614 assert(!bottom || !_cont.is_empty() || Continuation::is_continuation_entry_frame(f, nullptr), "");
2615 assert(!bottom || (_cont.is_empty() != Continuation::is_cont_barrier_frame(f)), "");
2616 assert(!caller.is_compiled_frame() || verify_deopt_state(caller), "");
2617 }
2618
2619 void ThawBase::clear_bitmap_bits(address start, address end) {
2620 assert(is_aligned(start, wordSize), "should be aligned: " PTR_FORMAT, p2i(start));
2621 assert(is_aligned(end, VMRegImpl::stack_slot_size), "should be aligned: " PTR_FORMAT, p2i(end));
2823 }
2824
2825 void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
2826 assert(hf.is_compiled_frame(), "");
2827 assert(_preempted_case || !stub_caller, "stub caller not at preemption");
2828
2829 if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
2830 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance_no_args());
2831 }
2832
2833 const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::CompiledFrame>(caller, num_frames);
2834
2835 DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
2836
2837 assert(caller.sp() == caller.unextended_sp(), "");
2838
2839 if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
2840 _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_compiled_frame
2841 }
2842
2843 // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
2844 // yet laid out in the stack, and so the original_pc is not stored in it.
2845 // As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
2846 frame f = new_stack_frame<ContinuationHelper::CompiledFrame>(hf, caller, is_bottom_frame);
2847 intptr_t* const stack_frame_top = f.sp();
2848 intptr_t* const heap_frame_top = hf.unextended_sp();
2849
2850 const int added_argsize = (is_bottom_frame || caller.is_interpreted_frame()) ? hf.compiled_frame_stack_argsize() : 0;
2851 int fsize = ContinuationHelper::CompiledFrame::size(hf) + added_argsize;
2852 assert(fsize <= (int)(caller.unextended_sp() - f.unextended_sp()), "");
2853
2854 intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom;
2855 intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom;
2856 // copy metadata, except the metadata at the top of the (unextended) entry frame
2857 int sz = fsize + frame::metadata_words_at_bottom + (is_bottom_frame && added_argsize == 0 ? 0 : frame::metadata_words_at_top);
2858
2859 // If we're the bottom-most thawed frame, we're writing to within one word from entrySP
2860 // (we might have one padding word for alignment)
2861 assert(!is_bottom_frame || (_cont.entrySP() - 1 <= to + sz && to + sz <= _cont.entrySP()), "");
2862 assert(!is_bottom_frame || hf.compiled_frame_stack_argsize() != 0 || (to + sz && to + sz == _cont.entrySP()), "");
2863
2864 copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above
2865
2866 patch(f, caller, is_bottom_frame);
2867
2868 // f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
2869 assert(!f.is_deoptimized_frame(), "");
2870 if (hf.is_deoptimized_frame()) {
2871 maybe_set_fastpath(f.sp());
2872 f.set_deoptimized();
2873 } else if (_thread->is_interp_only_mode()
|
450 inline frame freeze_start_frame_yield_stub();
451 template<typename FKind>
452 inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
453 inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
454 inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
455 freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
456 void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame);
457 NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
458 freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
459 NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
460 NOINLINE freeze_result recurse_freeze_native_frame(frame& f, frame& caller);
461 NOINLINE void finish_freeze(const frame& f, const frame& top);
462
463 void freeze_lockstack(stackChunkOop chunk);
464
465 inline bool stack_overflow();
466
467 static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
468 : sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
469 template<typename FKind> static inline frame sender(const frame& f);
470 template<typename FKind> frame new_heap_frame(frame& f, frame& caller, int size_adjust = 0);
471 inline void set_top_frame_metadata_pd(const frame& hf);
472 inline void patch_pd(frame& callee, const frame& caller, bool is_bottom_frame);
473 inline void patch_pd_unused(intptr_t* sp);
474 void adjust_interpreted_frame_unextended_sp(frame& f);
475 inline void prepare_freeze_interpreted_top_frame(frame& f);
476 static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
477
478 protected:
479 void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
480 bool freeze_fast_new_chunk(stackChunkOop chunk);
481 };
482
483 template <typename ConfigT>
484 class Freeze : public FreezeBase {
485 private:
486 stackChunkOop allocate_chunk(size_t stack_size, int argsize_md);
487
488 public:
489 inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt)
490 : FreezeBase(thread, cont, frame_sp, preempt) {}
491
492 freeze_result try_freeze_fast();
1160
1161 assert((!empty && Continuation::is_return_barrier_entry(entry.pc())) || (empty && Continuation::is_continuation_enterSpecial(entry)), "");
1162 assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
1163 #endif
1164
1165 return freeze_ok_bottom;
1166 }
1167
1168 // After freezing a frame we need to possibly adjust some values related to the caller frame.
1169 void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) {
1170 if (is_bottom_frame) {
1171 // If we're the bottom frame, we need to replace the return barrier with the real
1172 // caller's pc.
1173 address last_pc = caller.pc();
1174 assert((last_pc == nullptr) == _cont.tail()->is_empty(), "");
1175 ContinuationHelper::Frame::patch_pc(caller, last_pc);
1176 } else {
1177 assert(!caller.is_empty(), "");
1178 }
1179
1180 patch_pd(hf, caller, is_bottom_frame);
1181
1182 if (f.is_interpreted_frame()) {
1183 assert(hf.is_heap_frame(), "should be");
1184 ContinuationHelper::InterpretedFrame::patch_sender_sp(hf, caller);
1185 }
1186
1187 #ifdef ASSERT
1188 if (hf.is_compiled_frame()) {
1189 if (f.is_deoptimized_frame()) { // TODO DEOPT: long term solution: unroll on freeze and patch pc
1190 log_develop_trace(continuations)("Freezing deoptimized frame");
1191 assert(f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), "");
1192 assert(f.cb()->as_nmethod()->is_deopt_pc(ContinuationHelper::Frame::real_pc(f)), "");
1193 }
1194 }
1195 #endif
1196 }
1197
1198 #ifdef ASSERT
1199 static void verify_frame_top(const frame& f, intptr_t* top) {
1200 ResourceMark rm;
1257
1258 CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
1259 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1260 caller = hf;
1261
1262 // Mark frame_method's GC epoch for class redefinition on_stack calculation.
1263 frame_method->record_gc_epoch();
1264
1265 return freeze_ok;
1266 }
1267
1268 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
1269 // See also StackChunkFrameStream<frame_kind>::frame_size()
1270 freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller,
1271 int callee_argsize /* incl. metadata */,
1272 bool callee_interpreted) {
1273 // The frame's top never includes the stack arguments to the callee
1274 intptr_t* const stack_frame_top = ContinuationHelper::CompiledFrame::frame_top(f, callee_argsize, callee_interpreted);
1275 intptr_t* const stack_frame_bottom = ContinuationHelper::CompiledFrame::frame_bottom(f);
1276 // including metadata between f and its stackargs
1277 int argsize = ContinuationHelper::CompiledFrame::stack_argsize(f) + frame::metadata_words_at_top;
1278 int fsize = pointer_delta_as_int(stack_frame_bottom + argsize, stack_frame_top);
1279
1280 int real_frame_size = 0;
1281 bool augmented = f.was_augmented_on_entry(real_frame_size);
1282 if (augmented) {
1283 // The args reside inside the frame so clear argsize. If the caller is compiled,
1284 // this will cause the stack arguments passed by the caller to be freezed when
1285 // freezing the caller frame itself. If the caller is interpreted this will have
1286 // the effect of discarding the arg area created in the i2c stub.
1287 argsize = 0;
1288 fsize = real_frame_size - (callee_interpreted ? 0 : callee_argsize);
1289 #ifdef ASSERT
1290 nmethod* nm = f.cb()->as_nmethod();
1291 Method* method = nm->method();
1292 address return_pc = ContinuationHelper::CompiledFrame::return_pc(f);
1293 CodeBlob* caller_cb = CodeCache::find_blob_fast(return_pc);
1294 assert(nm->is_compiled_by_c2() || (caller_cb->is_nmethod() && caller_cb->as_nmethod()->is_compiled_by_c2()), "caller or callee should be c2 compiled");
1295 assert((!caller_cb->is_nmethod() && nm->is_compiled_by_c2()) ||
1296 (nm->compiler_type() != caller_cb->as_nmethod()->compiler_type()) ||
1297 (nm->is_compiled_by_c2() && !method->is_static() && method->method_holder()->is_inline_klass()),
1298 "frame should not be extended");
1299 #endif
1300 }
1301
1302 log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d augmented: %d",
1303 ContinuationHelper::Frame::frame_method(f) != nullptr ?
1304 ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "",
1305 _freeze_size, fsize, argsize, augmented);
1306 // we'd rather not yield inside methods annotated with @JvmtiMountTransition
1307 assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
1308
1309 freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize);
1310 if (UNLIKELY(result > freeze_ok_bottom)) {
1311 return result;
1312 }
1313
1314 bool is_bottom_frame = result == freeze_ok_bottom;
1315 assert(!caller.is_empty() || is_bottom_frame, "");
1316 assert(!is_bottom_frame || !augmented, "thaw extended frame without caller?");
1317
1318 DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, is_bottom_frame);)
1319
1320 frame hf = new_heap_frame<ContinuationHelper::CompiledFrame>(f, caller, augmented ? real_frame_size - f.cb()->as_nmethod()->frame_size() : 0);
1321
1322 intptr_t* heap_frame_top = ContinuationHelper::CompiledFrame::frame_top(hf, callee_argsize, callee_interpreted);
1323
1324 copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
1325 assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
1326
1327 if (caller.is_interpreted_frame()) {
1328 // When thawing the frame we might need to add alignment (see Thaw::align)
1329 _total_align_size += frame::align_wiggle;
1330 }
1331
1332 patch(f, hf, caller, is_bottom_frame);
1333
1334 assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
1335
1336 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1337 caller = hf;
1338 return freeze_ok;
1339 }
1340
2082
2083 // Only used for preemption on ObjectLocker
2084 ObjectMonitor* _init_lock;
2085
2086 StackChunkFrameStream<ChunkFrames::Mixed> _stream;
2087
2088 NOT_PRODUCT(int _frames;)
2089
2090 protected:
2091 ThawBase(JavaThread* thread, ContinuationWrapper& cont) :
2092 _thread(thread), _cont(cont),
2093 _fastpath(nullptr) {
2094 DEBUG_ONLY(_top_unextended_sp_before_thaw = nullptr;)
2095 assert (cont.tail() != nullptr, "no last chunk");
2096 DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());)
2097 }
2098
2099 void clear_chunk(stackChunkOop chunk);
2100 template<bool check_stub>
2101 int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize);
2102 int remove_scalarized_frames(StackChunkFrameStream<ChunkFrames::CompiledOnly>& scfs, int &argsize);
2103 void copy_from_chunk(intptr_t* from, intptr_t* to, int size);
2104
2105 void thaw_lockstack(stackChunkOop chunk);
2106
2107 // fast path
2108 inline void prefetch_chunk_pd(void* start, int size_words);
2109 void patch_return(intptr_t* sp, bool is_last);
2110
2111 intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case);
2112 inline intptr_t* push_cleanup_continuation();
2113 inline intptr_t* push_preempt_adapter();
2114 intptr_t* redo_vmcall(JavaThread* current, frame& top);
2115 void throw_interrupted_exception(JavaThread* current, frame& top);
2116
2117 void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case);
2118 void finish_thaw(frame& f);
2119
2120 private:
2121 template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
2122 void finalize_thaw(frame& entry, int argsize);
2123
2124 inline bool seen_by_gc();
2125
2126 inline void before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame);
2127 inline void after_thaw_java_frame(const frame& f, bool bottom);
2128 inline void patch(frame& f, const frame& caller, bool bottom);
2129 void clear_bitmap_bits(address start, address end);
2130
2131 NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames, bool is_top);
2132 void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
2133 void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
2134 void recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames);
2135
2136 void push_return_frame(const frame& f);
2137 inline frame new_entry_frame();
2138 template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom, int size_adjust = 0);
2139 inline void patch_pd(frame& f, const frame& sender);
2140 inline void patch_pd(frame& f, intptr_t* caller_sp);
2141 inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
2142
2143 void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
2144
2145 static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
2146
2147 public:
2148 CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
2149 };
2150
2151 template <typename ConfigT>
2152 class Thaw : public ThawBase {
2153 public:
2154 Thaw(JavaThread* thread, ContinuationWrapper& cont) : ThawBase(thread, cont) {}
2155
2156 inline bool can_thaw_fast(stackChunkOop chunk) {
2157 return !_barriers
2158 && _thread->cont_fastpath_thread_state()
2195 assert(_base - 1 <= top() + total_size() + frame::metadata_words_at_bottom, "missed entry frame");
2196 }
2197
2198 int entry_frame_extension() const { return _argsize + (_argsize > 0 ? frame::metadata_words_at_top : 0); }
2199
2200 // top and bottom stack pointers
2201 intptr_t* sp() const { return ContinuationHelper::frame_align_pointer(_base - _thaw_size); }
2202 intptr_t* bottom_sp() const { return ContinuationHelper::frame_align_pointer(_base - entry_frame_extension()); }
2203
2204 // several operations operate on the totality of the stack being reconstructed,
2205 // including the metadata words
2206 intptr_t* top() const { return sp() - frame::metadata_words_at_bottom; }
2207 int total_size() const { return _thaw_size + frame::metadata_words_at_bottom; }
2208 };
2209
2210 inline void ThawBase::clear_chunk(stackChunkOop chunk) {
2211 chunk->set_sp(chunk->bottom());
2212 chunk->set_max_thawing_size(0);
2213 }
2214
2215 int ThawBase::remove_scalarized_frames(StackChunkFrameStream<ChunkFrames::CompiledOnly>& f, int &argsize) {
2216 intptr_t* top = f.sp();
2217
2218 while (f.cb()->as_nmethod()->needs_stack_repair()) {
2219 f.next(SmallRegisterMap::instance_no_args(), false /* stop */);
2220 }
2221 assert(!f.is_done(), "");
2222 assert(f.is_compiled(), "");
2223
2224 intptr_t* bottom = f.sp() + f.cb()->frame_size();
2225 argsize = f.stack_argsize();
2226 return bottom - top;
2227 }
2228
2229 template<bool check_stub>
2230 int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) {
2231 bool empty = false;
2232 StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
2233 DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
2234 assert(chunk_sp == f.sp(), "");
2235 assert(chunk_sp == f.unextended_sp(), "");
2236
2237 int frame_size = f.cb()->frame_size();
2238 argsize = f.stack_argsize();
2239
2240 assert(!f.is_stub() || check_stub, "");
2241 if (check_stub && f.is_stub()) {
2242 // If we don't thaw the top compiled frame too, after restoring the saved
2243 // registers back in Java, we would hit the return barrier to thaw one more
2244 // frame effectively overwriting the restored registers during that call.
2245 f.next(SmallRegisterMap::instance_no_args(), true /* stop */);
2246 assert(!f.is_done(), "");
2247
2248 f.get_cb();
2249 assert(f.is_compiled(), "");
2250
2251 if (f.cb()->as_nmethod()->is_marked_for_deoptimization()) {
2252 // The caller of the runtime stub when the continuation is preempted is not at a
2253 // Java call instruction, and so cannot rely on nmethod patching for deopt.
2254 log_develop_trace(continuations)("Deoptimizing runtime stub caller");
2255 f.to_frame().deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
2256 }
2257
2258 if (f.cb()->as_nmethod()->needs_stack_repair()) {
2259 frame_size += remove_scalarized_frames(f, argsize);
2260 } else {
2261 frame_size += f.cb()->frame_size();
2262 argsize = f.stack_argsize();
2263 }
2264 } else if (f.cb()->as_nmethod()->needs_stack_repair()) {
2265 frame_size = remove_scalarized_frames(f, argsize);
2266 }
2267
2268 f.next(SmallRegisterMap::instance_no_args(), true /* stop */);
2269 empty = f.is_done();
2270 assert(!empty || argsize == chunk->argsize(), "");
2271
2272 if (empty) {
2273 clear_chunk(chunk);
2274 } else {
2275 chunk->set_sp(chunk->sp() + frame_size);
2276 chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
2277 // We set chunk->pc to the return pc into the next frame
2278 chunk->set_pc(f.pc());
2279 #ifdef ASSERT
2280 {
2281 intptr_t* retaddr_slot = (chunk_sp
2282 + frame_size
2283 - frame::sender_sp_ret_address_offset());
2284 assert(f.pc() == ContinuationHelper::return_address_at(retaddr_slot),
2285 "unexpected pc");
2546 assert(!_cont.is_empty(), "no more frames");
2547 assert(num_frames > 0, "");
2548 assert(!heap_frame.is_empty(), "");
2549
2550 if (top_on_preempt_case && (heap_frame.is_native_frame() || heap_frame.is_runtime_frame())) {
2551 heap_frame.is_native_frame() ? recurse_thaw_native_frame(heap_frame, caller, 2) : recurse_thaw_stub_frame(heap_frame, caller, 2);
2552 } else if (!heap_frame.is_interpreted_frame()) {
2553 recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
2554 } else {
2555 recurse_thaw_interpreted_frame(heap_frame, caller, num_frames, top_on_preempt_case);
2556 }
2557 }
2558
2559 template<typename FKind>
2560 bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) {
2561 assert(num_frames > 0, "");
2562
2563 DEBUG_ONLY(_frames++;)
2564
2565 int argsize = _stream.stack_argsize();
2566 CodeBlob* cb = _stream.cb();
2567
2568 _stream.next(SmallRegisterMap::instance_no_args());
2569 assert(_stream.to_frame().is_empty() == _stream.is_done(), "");
2570
2571 // We never leave a compiled caller of an interpreted frame as the top frame in the chunk
2572 // as it makes detecting that situation and adjusting unextended_sp tricky. We also always
2573 // thaw the caller of a frame that needs_stack_repair, as it would otherwise complicate things:
2574 // - Regardless of whether the frame was extended or not, we would need to copy the right arg
2575 // size if its greater than the one given by the normal method signature (non-scalarized).
2576 // - If the frame was indeed extended, leaving its caller as the top frame would complicate walking
2577 // the chunk (we need unextended_sp, but we only have sp).
2578 if (num_frames == 1 && !_stream.is_done() && ((FKind::interpreted && _stream.is_compiled()) || (FKind::compiled && cb->as_nmethod_or_null()->needs_stack_repair()))) {
2579 log_develop_trace(continuations)("thawing extra compiled frame to not leave a compiled interpreted-caller at top");
2580 num_frames++;
2581 }
2582
2583 if (num_frames == 1 || _stream.is_done()) { // end recursion
2584 finalize_thaw(caller, FKind::interpreted ? 0 : argsize);
2585 return true; // bottom
2586 } else { // recurse
2587 recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false /* top_on_preempt_case */);
2588 return false;
2589 }
2590 }
2591
2592 void ThawBase::finalize_thaw(frame& entry, int argsize) {
2593 stackChunkOop chunk = _cont.tail();
2594
2595 if (!_stream.is_done()) {
2596 assert(_stream.sp() >= chunk->sp_address(), "");
2597 chunk->set_sp(chunk->to_offset(_stream.sp()));
2598 chunk->set_pc(_stream.pc());
2625 }
2626 assert(bottom == _cont.is_entry_frame(caller), "bottom: %d is_entry_frame: %d", bottom, _cont.is_entry_frame(hf));
2627 }
2628
2629 inline void ThawBase::after_thaw_java_frame(const frame& f, bool bottom) {
2630 #ifdef ASSERT
2631 LogTarget(Trace, continuations) lt;
2632 if (lt.develop_is_enabled()) {
2633 LogStream ls(lt);
2634 ls.print_cr("thawed frame:");
2635 print_frame_layout(f, false, &ls); // f.print_on(&ls);
2636 }
2637 #endif
2638 }
2639
2640 inline void ThawBase::patch(frame& f, const frame& caller, bool bottom) {
2641 assert(!bottom || caller.fp() == _cont.entryFP(), "");
2642 if (bottom) {
2643 ContinuationHelper::Frame::patch_pc(caller, _cont.is_empty() ? caller.pc()
2644 : StubRoutines::cont_returnBarrier());
2645 } else if (_should_patch_caller_pc || caller.is_compiled_frame()) {
2646 // Caller was deoptimized during thaw but we've overwritten the return address when copying f from the heap.
2647 // Also, on some platforms, if the caller is interpreted but the callee not we also need to patch.
2648
2649 #if defined(PPC64) || defined(S390)
2650 assert(!_should_patch_caller_pc || caller.is_deoptimized_frame() || caller.is_interpreted_frame(), "");
2651 #else
2652 assert(!_should_patch_caller_pc || caller.is_deoptimized_frame(), "");
2653 #endif
2654
2655 ContinuationHelper::Frame::patch_pc(caller, caller.raw_pc());
2656 _should_patch_caller_pc = false;
2657 }
2658
2659 patch_pd(f, caller);
2660
2661 if (f.is_interpreted_frame()) {
2662 ContinuationHelper::InterpretedFrame::patch_sender_sp(f, caller);
2663 }
2664
2665 assert(!bottom || !_cont.is_empty() || Continuation::is_continuation_entry_frame(f, nullptr), "");
2666 assert(!bottom || (_cont.is_empty() != Continuation::is_cont_barrier_frame(f)), "");
2667 assert(!caller.is_compiled_frame() || verify_deopt_state(caller), "");
2668 }
2669
2670 void ThawBase::clear_bitmap_bits(address start, address end) {
2671 assert(is_aligned(start, wordSize), "should be aligned: " PTR_FORMAT, p2i(start));
2672 assert(is_aligned(end, VMRegImpl::stack_slot_size), "should be aligned: " PTR_FORMAT, p2i(end));
2874 }
2875
2876 void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
2877 assert(hf.is_compiled_frame(), "");
2878 assert(_preempted_case || !stub_caller, "stub caller not at preemption");
2879
2880 if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
2881 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance_no_args());
2882 }
2883
2884 const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::CompiledFrame>(caller, num_frames);
2885
2886 DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
2887
2888 assert(caller.sp() == caller.unextended_sp(), "");
2889
2890 if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
2891 _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_compiled_frame
2892 }
2893
2894 int fsize = 0;
2895 int added_argsize = 0;
2896 bool augmented = hf.was_augmented_on_entry(fsize);
2897 if (!augmented) {
2898 added_argsize = (is_bottom_frame || caller.is_interpreted_frame()) ? hf.compiled_frame_stack_argsize() : 0;
2899 fsize += added_argsize;
2900 }
2901 assert(!is_bottom_frame || !augmented, "");
2902
2903 // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
2904 // yet laid out in the stack, and so the original_pc is not stored in it.
2905 // As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
2906 frame f = new_stack_frame<ContinuationHelper::CompiledFrame>(hf, caller, is_bottom_frame, augmented ? fsize - hf.cb()->frame_size() : 0);
2907 assert((int)(caller.sp() - f.sp()) == (augmented ? fsize : f.cb()->frame_size()), "");
2908
2909 intptr_t* const stack_frame_top = f.sp();
2910 intptr_t* const heap_frame_top = hf.unextended_sp();
2911 intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom;
2912 intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom;
2913 // copy metadata, except the metadata at the top of the (unextended) entry frame
2914 int sz = fsize + frame::metadata_words_at_bottom + (is_bottom_frame && added_argsize == 0 ? 0 : frame::metadata_words_at_top);
2915
2916 // If we're the bottom-most thawed frame, we're writing to within one word from entrySP
2917 // (we might have one padding word for alignment)
2918 assert(!is_bottom_frame || (_cont.entrySP() - 1 <= to + sz && to + sz <= _cont.entrySP()), "");
2919 assert(!is_bottom_frame || hf.compiled_frame_stack_argsize() != 0 || (to + sz && to + sz == _cont.entrySP()), "");
2920
2921 copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above
2922
2923 patch(f, caller, is_bottom_frame);
2924
2925 // f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
2926 assert(!f.is_deoptimized_frame(), "");
2927 if (hf.is_deoptimized_frame()) {
2928 maybe_set_fastpath(f.sp());
2929 f.set_deoptimized();
2930 } else if (_thread->is_interp_only_mode()
|