1 /*
  2  * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef CPU_AARCH64_CONTINUATION_AARCH64_INLINE_HPP
 26 #define CPU_AARCH64_CONTINUATION_AARCH64_INLINE_HPP
 27 
 28 #include "oops/instanceStackChunkKlass.inline.hpp"
 29 #include "runtime/frame.hpp"
 30 #include "runtime/frame.inline.hpp"
 31 
 32 const int ContinuationHelper::frame_metadata = frame::sender_sp_offset;
 33 const int ContinuationHelper::align_wiggle = 1;
 34 
 35 #ifdef ASSERT
 36 bool Frame::assert_frame_laid_out(frame f) {
 37   intptr_t* sp = f.sp();
 38   address pc = *(address*)(sp - frame::sender_sp_ret_address_offset());
 39   intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset);
 40   assert (f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc));
 41   assert (f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp));
 42   return f.raw_pc() == pc && f.fp() == fp;
 43 }
 44 #endif
 45 
 46 inline intptr_t** Frame::callee_link_address(const frame& f) {
 47   return (intptr_t**)(f.sp() - frame::sender_sp_offset);
 48 }
 49 
 50 template<typename FKind>
 51 static inline intptr_t* real_fp(const frame& f) {
 52   assert (FKind::is_instance(f), "");
 53   assert (FKind::interpreted || f.cb() != nullptr, "");
 54 
 55   return FKind::interpreted ? f.fp() : f.unextended_sp() + f.cb()->frame_size();
 56 }
 57 
 58 template<typename FKind> // TODO: maybe do the same CRTP trick with Interpreted and Compiled as with hframe
 59 static inline intptr_t** link_address(const frame& f) {
 60   assert (FKind::is_instance(f), "");
 61   return FKind::interpreted
 62             ? (intptr_t**)(f.fp() + frame::link_offset)
 63             : (intptr_t**)(real_fp<FKind>(f) - frame::sender_sp_offset);
 64 }
 65 
 66 static void patch_callee_link(const frame& f, intptr_t* fp) {
 67   DEBUG_ONLY(intptr_t* orig = *Frame::callee_link_address(f));
 68   *Frame::callee_link_address(f) = fp;
 69   log_develop_trace(jvmcont)("patched link at " INTPTR_FORMAT ": " INTPTR_FORMAT " orig: " INTPTR_FORMAT, p2i(Frame::callee_link_address(f)), p2i(fp), p2i(orig));
 70 }
 71 
 72 static void patch_callee_link_relative(const frame& f, intptr_t* fp) {
 73   intptr_t* la = (intptr_t*)Frame::callee_link_address(f);
 74   intptr_t new_value = fp - la;
 75   *la = new_value;
 76   log_trace(jvmcont)("patched link at " INTPTR_FORMAT ": to relative %ld", p2i(Frame::callee_link_address(f)), new_value);
 77 }
 78 
 79 inline address* Interpreted::return_pc_address(const frame& f) {
 80   return (address*)(f.fp() + frame::return_addr_offset);
 81 }
 82 
 83 template <bool relative>
 84 void Interpreted::patch_sender_sp(frame& f, intptr_t* sp) {
 85   assert (f.is_interpreted_frame(), "");
 86   intptr_t* la = f.addr_at(frame::interpreter_frame_sender_sp_offset);
 87   *la = relative ? (intptr_t)(sp - f.fp()) : (intptr_t)sp;
 88   log_trace(jvmcont)("patched sender_sp: " INTPTR_FORMAT, *la);
 89 }
 90 
 91 inline address* Frame::return_pc_address(const frame& f) {
 92   return (address*)(f.real_fp() - 1);
 93 }
 94 
 95 // inline address* Frame::pc_address(const frame& f) {
 96 //   return (address*)(f.sp() - frame::return_addr_offset);
 97 // }
 98 
 99 inline address Frame::real_pc(const frame& f) {
100   address* pc_addr = &(((address*) f.sp())[-1]);
101   return *pc_addr;
102 }
103 
104 inline void Frame::patch_pc(const frame& f, address pc) {
105   address* pc_addr = &(((address*) f.sp())[-1]);
106   *pc_addr = pc;
107   log_develop_trace(jvmcont)("patch_pc at " INTPTR_FORMAT ": " INTPTR_FORMAT, p2i(pc_addr), p2i(pc));
108 }
109 
110 inline intptr_t* Interpreted::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame
111   // interpreter_frame_last_sp_offset, points to unextended_sp includes arguments in the frame
112   // interpreter_frame_initial_sp_offset excludes expression stack slots
113   int expression_stack_sz = expression_stack_size(f, mask);
114   intptr_t* res = *(intptr_t**)f.addr_at(frame::interpreter_frame_initial_sp_offset) - expression_stack_sz;
115   assert (res == (intptr_t*)f.interpreter_frame_monitor_end() - expression_stack_sz, "");
116   assert (res >= f.unextended_sp(),
117     "res: " INTPTR_FORMAT " initial_sp: " INTPTR_FORMAT " last_sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " expression_stack_size: %d", 
118     p2i(res), p2i(f.addr_at(frame::interpreter_frame_initial_sp_offset)), f.at(frame::interpreter_frame_last_sp_offset), p2i(f.unextended_sp()), expression_stack_sz);
119   return res;
120   // Not true, but using unextended_sp might work
121   // assert (res == f.unextended_sp(), "res: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT, p2i(res), p2i(f.unextended_sp() + 1));
122 }
123 
124 template <bool relative>
125 inline intptr_t* Interpreted::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame
126   return (intptr_t*)f.at<relative>(frame::interpreter_frame_locals_offset) + 1; // exclusive, so we add 1 word
127 }
128 
129 inline intptr_t* Interpreted::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) {
130   // tty->print_cr(">>> f.unextended_sp(): %p callee_argsize: %d callee_interpreted: %d", f.unextended_sp(), callee_argsize, callee_interpreted);
131   return f.unextended_sp() + (callee_interpreted ? callee_argsize : 0);
132 }
133 
134 template<typename FKind, typename RegisterMapT>
135 inline void ContinuationHelper::update_register_map(RegisterMapT* map, const frame& f) {
136   frame::update_map_with_saved_link(map, link_address<FKind>(f));
137 }
138 
139 template<typename RegisterMapT>
140 inline void ContinuationHelper::update_register_map_with_callee(RegisterMapT* map, const frame& f) {
141   frame::update_map_with_saved_link(map, Frame::callee_link_address(f));
142 }
143 
144 inline void ContinuationHelper::push_pd(const frame& f) {
145   log_develop_trace(jvmcont)("ContinuationHelper::push_pd: " INTPTR_FORMAT, p2i(f.fp()));
146   // os::print_location(tty, (intptr_t)f.fp());
147   *(intptr_t**)(f.sp() - frame::sender_sp_offset) = f.fp();
148 }
149 
150 // creates the yield stub frame faster than JavaThread::last_frame
151 inline frame ContinuationHelper::last_frame(JavaThread* thread) {
152   JavaFrameAnchor* anchor = thread->frame_anchor();
153   assert (anchor->last_Java_sp() != nullptr, "");
154   assert (anchor->last_Java_pc() != nullptr, "");
155 
156   assert (StubRoutines::cont_doYield_stub()->contains(anchor->last_Java_pc()), "must be");
157   assert (StubRoutines::cont_doYield_stub()->oop_maps()->count() == 1, "must be");
158 
159   return frame(anchor->last_Java_sp(), anchor->last_Java_sp(), anchor->last_Java_fp(), anchor->last_Java_pc(), nullptr, nullptr, true);
160   // return frame(anchor->last_Java_sp(), anchor->last_Java_sp(), anchor->last_Java_fp(), anchor->last_Java_pc(), 
161   //   StubRoutines::cont_doYield_stub(), StubRoutines::cont_doYield_stub()->oop_map_for_slot(0, anchor->last_Java_pc()), true);
162 }
163 
164 frame ContinuationEntry::to_frame() {
165   static CodeBlob* cb = CodeCache::find_blob(entry_pc());
166   return frame(entry_sp(), entry_sp(), entry_fp(), entry_pc(), cb);
167 }
168 
169 void ContinuationEntry::update_register_map(RegisterMap* map) {
170   intptr_t** fp = (intptr_t**)(bottom_sender_sp() - frame::sender_sp_offset);
171   frame::update_map_with_saved_link(map, fp);
172 }
173 
174 void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) {
175   anchor->set_last_Java_fp(cont->entry_fp());
176 }
177 
178 void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) {
179   intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset);
180   anchor->set_last_Java_fp(fp);
181 }
182 
183 /////
184 
185 template <typename ConfigT>
186 inline void Freeze<ConfigT>::set_top_frame_metadata_pd(const frame& hf) {
187   stackChunkOop chunk = _cont.tail();
188   assert (chunk->is_in_chunk(hf.sp() - 1), "");
189   assert (chunk->is_in_chunk(hf.sp() - frame::sender_sp_offset), "");
190 
191   *(hf.sp() - 1) = (intptr_t)hf.pc();
192 
193   intptr_t* fp_addr = hf.sp() - frame::sender_sp_offset;
194   *fp_addr = hf.is_interpreted_frame() ? (intptr_t)(hf.fp() - fp_addr) 
195                                        : (intptr_t)hf.fp();
196 
197   log_develop_trace(jvmcont)("set_top_frame_metadata_pd pc: " INTPTR_FORMAT " fp: %ld", p2i(hf.pc()), *fp_addr);
198 }
199 
200 template <typename ConfigT>
201 inline intptr_t* Freeze<ConfigT>::align_bottom(intptr_t* bottom, int argsize) {
202 #ifdef _LP64
203   bottom -= (argsize & 1);
204 #endif
205   return bottom;
206 }
207 
208 template <typename ConfigT>
209 template<typename FKind>
210 inline frame Freeze<ConfigT>::sender(const frame& f) {
211   assert (FKind::is_instance(f), "");
212   if (FKind::interpreted) {
213     return frame(f.sender_sp(), f.interpreter_frame_sender_sp(), f.link(), f.sender_pc());
214   }
215   intptr_t** link_addr = link_address<FKind>(f);
216 
217   intptr_t* sender_sp = (intptr_t*)(link_addr + frame::sender_sp_offset); //  f.unextended_sp() + (fsize/wordSize); // 
218   address sender_pc = (address) *(sender_sp-1);
219   assert(sender_sp != f.sp(), "must have changed");
220 
221   int slot = 0;
222   CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot);
223   return sender_cb != nullptr
224     ? frame(sender_sp, sender_sp, *link_addr, sender_pc, sender_cb, slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc))
225     : frame(sender_sp, sender_sp, *link_addr, sender_pc);
226 }
227 
228 template <typename ConfigT>
229 template<typename FKind> frame Freeze<ConfigT>::new_hframe(frame& f, frame& caller) {
230   assert (FKind::is_instance(f), "");
231   assert (!caller.is_interpreted_frame() || caller.unextended_sp() == (intptr_t*)caller.at<true>(frame::interpreter_frame_last_sp_offset), "");
232 
233   intptr_t *sp, *fp; // sp is really our unextended_sp
234   if (FKind::interpreted) {
235     assert ((intptr_t*)f.at<false>(frame::interpreter_frame_last_sp_offset) == nullptr || f.unextended_sp() == (intptr_t*)f.at<false>(frame::interpreter_frame_last_sp_offset), "");
236     int locals = f.interpreter_frame_method()->max_locals();
237     bool overlap_caller = caller.is_interpreted_frame() || caller.is_empty();
238     fp = caller.unextended_sp() - (locals + frame::sender_sp_offset) + (overlap_caller ? Interpreted::stack_argsize(f) : 0);
239     sp = fp - (f.fp() - f.unextended_sp());
240     assert (sp <= fp && fp <= caller.unextended_sp(), "");
241     caller.set_sp(fp + frame::sender_sp_offset);
242 
243     assert (_cont.tail()->is_in_chunk(sp), "sp: " INTPTR_FORMAT " caller.sp(): " INTPTR_FORMAT " start_address: " INTPTR_FORMAT, p2i(sp), p2i(caller.sp()), p2i(_cont.tail()->start_address()));
244     
245     frame hf(sp, sp, fp, f.pc(), nullptr, nullptr, false);
246     *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + locals - 1;
247     return hf;
248   } else {
249     fp = *(intptr_t**)(f.sp() - frame::sender_sp_offset); // we need to re-read fp because it may be an oop and we might have had a safepoint in finalize_freeze, after constructing f.
250     int fsize = FKind::size(f);
251     sp = caller.unextended_sp() - fsize;
252     if (caller.is_interpreted_frame()) {
253       int argsize = FKind::stack_argsize(f);
254       sp -= argsize;
255     }
256     caller.set_sp(sp + fsize);
257 
258     assert (_cont.tail()->is_in_chunk(sp), "sp: " INTPTR_FORMAT " caller.sp(): " INTPTR_FORMAT " start_address: " INTPTR_FORMAT, p2i(sp), p2i(caller.sp()), p2i(_cont.tail()->start_address()));
259     
260     return frame(sp, sp, fp, f.pc(), nullptr, nullptr, false);
261   }
262 }
263 
264 template <typename ConfigT>
265 inline void Freeze<ConfigT>::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) {
266   intptr_t* vfp = f.fp();
267   intptr_t* hfp = hf.fp();
268   assert (hfp == hf.unextended_sp() + (f.fp() - f.unextended_sp()), "");
269   assert ((f.at<false>(frame::interpreter_frame_last_sp_offset) != 0) || (f.unextended_sp() == f.sp()), "");
270   assert (f.fp() > (intptr_t*)f.at<false>(frame::interpreter_frame_initial_sp_offset), "");
271 
272   // at(frame::interpreter_frame_last_sp_offset) can be NULL at safepoint preempts
273   *hf.addr_at(frame::interpreter_frame_last_sp_offset) = hf.unextended_sp() - hf.fp();
274   *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + f.interpreter_frame_method()->max_locals() - 1;
275   
276   relativize(vfp, hfp, frame::interpreter_frame_initial_sp_offset); // == block_top == block_bottom
277 
278   assert ((hf.fp() - hf.unextended_sp()) == (f.fp() - f.unextended_sp()), "");
279   assert (hf.unextended_sp() == (intptr_t*)hf.at<true>(frame::interpreter_frame_last_sp_offset), "");
280   assert (hf.unextended_sp() <= (intptr_t*)hf.at<true>(frame::interpreter_frame_initial_sp_offset), "");
281   assert (hf.fp()            >  (intptr_t*)hf.at<true>(frame::interpreter_frame_initial_sp_offset), "");
282   assert (hf.fp()            <= (intptr_t*)hf.at<true>(frame::interpreter_frame_locals_offset), "");
283 }
284 
285 template <typename ConfigT>
286 template <typename FKind, bool bottom>
287 inline void Freeze<ConfigT>::patch_pd(frame& hf, const frame& caller) {
288   if (caller.is_interpreted_frame()) {
289     assert (!caller.is_empty(), "");
290     patch_callee_link_relative(caller, caller.fp());
291   } else {
292     patch_callee_link(caller, caller.fp());
293   }
294 }
295 
296 template <typename ConfigT>
297 inline void Freeze<ConfigT>::patch_chunk_pd(intptr_t* vsp, intptr_t* hsp) {
298   *(vsp - frame::sender_sp_offset) = *(hsp - frame::sender_sp_offset);
299 }
300 
301 template <typename ConfigT>
302 inline frame Thaw<ConfigT>::new_entry_frame() {
303   // if (Interpreter::contains(_cont.entryPC())) _cont.set_entrySP(_cont.entrySP() - 1);
304   intptr_t* sp = _cont.entrySP();
305   return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state
306 }
307 
308 template <typename ConfigT>
309 template<typename FKind> frame Thaw<ConfigT>::new_frame(const frame& hf, frame& caller, bool bottom) {
310   assert (FKind::is_instance(hf), "");
311 
312   if (FKind::interpreted) {
313     intptr_t* hsp = hf.unextended_sp();
314     const int fsize = Interpreted::frame_bottom<true>(hf) - hf.unextended_sp();
315     const int locals = hf.interpreter_frame_method()->max_locals();
316     intptr_t* vsp = caller.unextended_sp() - fsize;
317     intptr_t* fp = vsp + (hf.fp() - hsp);
318     if ((intptr_t)fp % 16 != 0) {
319       fp--;
320       vsp--;
321     }
322     DEBUG_ONLY(intptr_t* unextended_sp = fp + *hf.addr_at(frame::interpreter_frame_last_sp_offset);)
323     assert (vsp == unextended_sp, "vsp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT, p2i(vsp), p2i(unextended_sp));
324     caller.set_sp(fp + frame::sender_sp_offset);
325     frame f(vsp, vsp, fp, hf.pc());
326     // it's set again later in derelativize_interpreted_frame_metadata, but we need to set the locals now so that we'll have the frame's bottom
327     intptr_t offset = *hf.addr_at(frame::interpreter_frame_locals_offset);
328     assert ((int)offset == locals + frame::sender_sp_offset - 1, "");
329     *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = fp + offset;
330     assert ((intptr_t)f.fp() % 16 == 0, "");
331     return f;
332   } else {
333     int fsize = FKind::size(hf);
334     intptr_t* vsp = caller.unextended_sp() - fsize;
335     if (bottom || caller.is_interpreted_frame()) {
336       int argsize = hf.compiled_frame_stack_argsize();
337       log_develop_trace(jvmcont)("thaw_compiled_frame add argsize: fsize: %d argsize: %d fsize: %d", fsize, argsize, fsize + argsize);
338       
339       fsize += argsize;
340       vsp   -= argsize;
341       caller.set_sp(caller.sp() - argsize);
342       assert (caller.sp() == vsp + (fsize-argsize), "");
343 
344       vsp = align(hf, vsp, caller, bottom);
345     }
346 
347     assert (hf.cb() != nullptr && hf.oop_map() != nullptr, "");
348     intptr_t* fp = FKind::stub 
349       ? vsp + fsize - frame::sender_sp_offset // on AArch64, this value is used for the safepoint stub
350       : *(intptr_t**)(hf.sp() - frame::sender_sp_offset); // we need to re-read fp because it may be an oop and we might have fixed the frame.
351     return frame(vsp, vsp, fp, hf.pc(), hf.cb(), hf.oop_map()); // TODO PERF : this computes deopt state; is it necessary?
352   }
353 }
354 
355 template <typename ConfigT>
356 inline void Thaw<ConfigT>::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) {
357   *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = bottom - 1;
358 }
359 
360 template <typename ConfigT>
361 inline void Thaw<ConfigT>::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) {
362   intptr_t* vfp = f.fp();
363 
364   derelativize(vfp, frame::interpreter_frame_last_sp_offset);
365   derelativize(vfp, frame::interpreter_frame_initial_sp_offset);
366 }
367 
368 template <typename ConfigT>
369 inline intptr_t* Thaw<ConfigT>::align(const frame& hf, intptr_t* vsp, frame& caller, bool bottom) {
370   // if (caller.is_interpreted_frame()) {
371   //   // Deoptimization likes ample room between interpreted frames and compiled frames. 
372   //   // This is due to caller_adjustment calculation in Deoptimization::fetch_unroll_info_helper.
373   //   // An attempt to simplify that calculation and make more room during deopt has failed some tests.
374 
375   //   int addedWords = 0;
376 
377   //   // SharedRuntime::gen_i2c_adapter makes room that's twice as big as required for the stack-passed arguments by counting slots but subtracting words from rsp 
378   //   assert (VMRegImpl::stack_slot_size == 4, "");
379   //   int argsize = hf.compiled_frame_stack_argsize();
380   //   assert (argsize >= 0, "");
381   //   addedWords += (argsize /* / 2*/) >> LogBytesPerWord; // Not sure why dividing by 2 is not big enough.
382 
383   //   log_develop_trace(jvmcont)("Aligning compiled frame 0: " INTPTR_FORMAT " -> " INTPTR_FORMAT, p2i(vsp), p2i(vsp - addedWords));
384   //   vsp -= addedWords;
385   //   log_develop_trace(jvmcont)("Aligning sender sp: " INTPTR_FORMAT " -> " INTPTR_FORMAT, p2i(caller.sp()), p2i(caller.sp() - addedWords));
386   //   caller.set_sp(caller.sp() - addedWords);
387   // }
388 #ifdef _LP64
389   if (((intptr_t)vsp & 0xf) != 0) {
390     log_develop_trace(jvmcont)("Aligning compiled frame 1: " INTPTR_FORMAT " -> " INTPTR_FORMAT, p2i(vsp), p2i(vsp - 1));
391     assert(caller.is_interpreted_frame() || (bottom && hf.compiled_frame_stack_argsize() % 2 != 0), "");
392     vsp--;
393 
394     log_develop_trace(jvmcont)("Aligning sender sp: " INTPTR_FORMAT " -> " INTPTR_FORMAT, p2i(caller.sp()), p2i(caller.sp() - 1));
395     caller.set_sp(caller.sp() - 1);
396   }
397   assert((intptr_t)vsp % 16 == 0, "");
398 #endif
399 
400   return vsp;
401 }
402 
403 template <typename ConfigT>
404 template<typename FKind, bool bottom>
405 inline void Thaw<ConfigT>::patch_pd(frame& f, const frame& caller) {
406   assert (!bottom || caller.fp() == _cont.entryFP(), "caller.fp: " INTPTR_FORMAT " entryFP: " INTPTR_FORMAT, p2i(caller.fp()), p2i(_cont.entryFP()));
407   patch_callee_link(caller, caller.fp());
408 }
409 
410 template <typename ConfigT>
411 intptr_t* Thaw<ConfigT>::push_interpreter_return_frame(intptr_t* sp) {
412   address pc = StubRoutines::cont_interpreter_forced_preempt_return();
413   intptr_t* fp = sp - frame::sender_sp_offset;
414 
415   log_develop_trace(jvmcont)("push_interpreter_return_frame initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(sp), p2i(sp - ContinuationHelper::frame_metadata), p2i(fp));
416 
417   sp = align_down(sp, 16);
418   assert((intptr_t)sp % 16 == 0, "");
419 
420   sp -= ContinuationHelper::frame_metadata;
421   *(address*)(sp - frame::sender_sp_ret_address_offset()) = pc;
422   *(intptr_t**)(sp - frame::sender_sp_offset) = fp;
423   return sp;
424 }
425 
426 template <typename ConfigT>
427 void Thaw<ConfigT>::patch_chunk_pd(intptr_t* sp) {
428   intptr_t* fp = _cont.entryFP();
429   *(intptr_t**)(sp - frame::sender_sp_offset) = fp;
430   log_develop_trace(jvmcont)("thaw_chunk patching fp at " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(sp - frame::sender_sp_offset), p2i(fp));
431 }
432 
433 template <typename ConfigT>
434 inline void Thaw<ConfigT>::prefetch_chunk_pd(void* start, int size) {
435   size <<= LogBytesPerWord;
436   Prefetch::read(start, size);
437   Prefetch::read(start, size - 64);
438 }
439 
440 template <typename ConfigT>
441 inline intptr_t* Thaw<ConfigT>::align_chunk(intptr_t* vsp) {
442 #ifdef _LP64
443   vsp = align_down(vsp, 16);
444   assert((intptr_t)vsp % 16 == 0, "");
445 #endif
446   return vsp;
447 }
448 ////////
449 
450 #endif // CPU_AARCH64_CONTINUATION_AARCH64_INLINE_HPP