1 /*
2 * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2026 IBM Corporation. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef CPU_S390_CONTINUATION_S390_INLINE_HPP
27 #define CPU_S390_CONTINUATION_S390_INLINE_HPP
28
29 #include "oops/stackChunkOop.inline.hpp"
30 #include "runtime/frame.hpp"
31 #include "runtime/frame.inline.hpp"
32
33 inline void patch_callee_link(const frame& f, intptr_t* fp) {
34 *ContinuationHelper::Frame::callee_link_address(f) = fp;
35 }
36
37 inline void patch_callee_link_relative(const frame& f, intptr_t* fp) {
38 intptr_t* la = (intptr_t*)ContinuationHelper::Frame::callee_link_address(f);
39 intptr_t new_value = fp - la;
40 *la = new_value;
41 }
42
43 inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) {
44 stackChunkOop chunk = _cont.tail();
45 assert(chunk->is_in_chunk(hf.sp()), "hf.sp()=" PTR_FORMAT, p2i(hf.sp()));
46
47 hf.own_abi()->return_pc = (uint64_t)hf.pc();
48 if (hf.is_interpreted_frame()) {
49 patch_callee_link_relative(hf, hf.fp());
50 } else {
51 #ifdef ASSERT
52 // See also FreezeBase::patch_pd()
53 patch_callee_link(hf, (intptr_t*)badAddress);
54 #endif
55 }
56 }
57
58 template<typename FKind>
59 inline frame FreezeBase::sender(const frame& f) {
60 assert(FKind::is_instance(f), "");
61
62 if (FKind::interpreted) {
63 return frame(f.sender_sp(), f.sender_pc(), f.interpreter_frame_sender_sp());
64 }
65
66 intptr_t* sender_sp = f.sender_sp();
67 address sender_pc = f.sender_pc();
68 assert(sender_sp != f.sp(), "must have changed");
69 int slot = 0;
70 CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot);
71 return sender_cb != nullptr
72 ? frame(sender_sp, sender_sp, nullptr, sender_pc, sender_cb, slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc))
73 : frame(sender_sp, sender_pc, sender_sp);
74 }
75
76 template<typename FKind> frame FreezeBase::new_heap_frame(frame& f, frame& caller, int size_adjust) {
77 assert(FKind::is_instance(f), "");
78 intptr_t *sp, *fp;
79 if (FKind::interpreted) {
80 intptr_t locals_offset = *f.addr_at(_z_ijava_idx(locals));
81
82 // If the caller.is_empty(), i.e. we're freezing into an empty chunk, then we set
83 // the chunk's argsize in finalize_freeze and make room for it above the unextended_sp
84 // See also comment on StackChunkFrameStream<frame_kind>::interpreter_frame_size()
85
86 int overlap =
87 (caller.is_interpreted_frame() || caller.is_empty())
88 ? ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top
89 : 0;
90
91 // Calculate the new frame's FP in the heap chunk.
92 // Starting from caller's unextended_sp, we:
93 // - subtract 1 for the z_parent_ijava_frame_abi (which sits just below the locals)
94 // - subtract locals_offset (distance from FP to locals in the original frame)
95 // - add overlap (to account for shared stack args when caller is interpreted or empty)
96 // This positions FP such that locals are correctly placed relative to the caller's frame.
97 fp = caller.unextended_sp() - 1 - locals_offset + overlap;
98
99 // esp points one slot below the last argument
100 intptr_t* x86_64_like_unextended_sp = f.interpreter_frame_esp() + 1 - frame::metadata_words_at_top;
101
102 sp = fp - (f.fp() - x86_64_like_unextended_sp);
103 assert (sp <= fp && (fp <= caller.unextended_sp() || caller.is_interpreted_frame()),
104 "sp=" PTR_FORMAT " fp=" PTR_FORMAT " caller.unextended_sp()=" PTR_FORMAT " caller.is_interpreted_frame()=%d",
105 p2i(sp), p2i(fp), p2i(caller.unextended_sp()), caller.is_interpreted_frame());
106 caller.set_sp(fp);
107
108 assert(_cont.tail()->is_in_chunk(sp), "");
109
110 frame hf(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */);
111 // frame_top() and frame_bottom() read these before relativize_interpreted_frame_metadata() is called
112 *hf.addr_at(_z_ijava_idx(locals)) = locals_offset;
113 *hf.addr_at(_z_ijava_idx(esp)) = f.interpreter_frame_esp() - f.fp();
114 return hf;
115 } else {
116 int fsize = FKind::size(f);
117 sp = caller.unextended_sp() - fsize;
118 if (caller.is_interpreted_frame()) {
119 // If the caller is interpreted, our stackargs are not supposed to overlap with it
120 // so we make more room by moving sp down by argsize
121 int argsize = FKind::stack_argsize(f);
122 sp -= argsize + frame::metadata_words_at_top;
123 }
124 fp = sp + fsize;
125 caller.set_sp(fp);
126
127 assert(_cont.tail()->is_in_chunk(sp), "");
128
129 return frame(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */);
130 }
131 }
132
133 void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) {
134 // Nothing to do on s390 and ppc. On x86/aarch64/riscv, the unextended_sp is stored
135 // in interpreter_frame_last_sp and needs to be restored from there. On s390/ppc,
136 // the frame structure doesn't have interpreter_frame_last_sp; instead, the unextended_sp
137 // is directly maintained in the frame and doesn't need adjustment.
138 }
139
140 inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) {
141 // Nothing to do. We don't save a last sp because we cannot use sp as esp.
142 // Instead the top frame is trimmed when making an i2i call. The original
143 // top_frame_sp is set when the frame is pushed (see generate_fixed_frame()).
144 // An interpreter top frame that was just thawed is resized to top_frame_sp by the
145 // resume adapter (see generate_cont_resume_interpreter_adapter()). So the assertion is
146 // false, if we freeze again right after thawing as we do when redoing a vm call wasn't
147 // successful.
148 assert(_thread->interp_redoing_vm_call() ||
149 ((intptr_t*)f.at_relative(_z_ijava_idx(top_frame_sp)) == f.unextended_sp()),
150 "top_frame_sp:" PTR_FORMAT " usp:" PTR_FORMAT, f.at_relative(_z_ijava_idx(top_frame_sp)), p2i(f.unextended_sp()));
151 }
152
153 inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) {
154 intptr_t* vfp = f.fp();
155 intptr_t* hfp = hf.fp();
156 assert(f.fp() > (intptr_t*)f.interpreter_frame_esp(), "");
157
158 // There is alignment padding between vfp and f's locals array in the original
159 // frame, because we freeze the padding (see recurse_freeze_interpreted_frame)
160 // in order to keep the same relativized locals pointer, we don't need to change it here.
161
162 // Make sure that monitors is already relativized.
163 assert(hf.at_absolute(_z_ijava_idx(monitors)) <= -(frame::z_ijava_state_size / wordSize), "");
164 // Make sure that esp is already relativized.
165 assert(hf.at_absolute(_z_ijava_idx(esp)) <= hf.at_absolute(_z_ijava_idx(monitors)), "");
166 // top_frame_sp is already relativized
167
168 // hfp == hf.sp() + (f.fp() - f.sp()) is not true on ppc because the stack frame has room for
169 // the maximal expression stack and the expression stack in the heap frame is trimmed.
170 assert(hf.fp() == hf.interpreter_frame_esp() + (f.fp() - f.interpreter_frame_esp()), "");
171 assert(hf.fp() <= (intptr_t*)hf.at(_z_ijava_idx(locals)), "");
172 }
173
174 inline void FreezeBase::patch_pd(frame& hf, const frame& caller, bool is_bottom_frame) {
175 if (caller.is_interpreted_frame()) {
176 assert(!caller.is_empty(), "");
177 patch_callee_link_relative(caller, caller.fp());
178 }
179 #ifdef ASSERT
180 else {
181 // For compiled frames the back link is actually redundant. It gets computed
182 // as unextended_sp + frame_size.
183
184 // Note a difference from x86_64: the link is not made relative if the caller
185 // is a compiled frame because there rbp is used as a non-volatile register by
186 // c1/c2 so it could be a computed value local to the caller.
187
188 // See also:
189 // - FreezeBase::set_top_frame_metadata_pd
190 // - StackChunkFrameStream<frame_kind>::fp()
191 // - UseContinuationFastPath: compiled frames are copied in a batch w/o patching the back link.
192 // The backlinks are restored when thawing (see Thaw<ConfigT>::patch_caller_links())
193 patch_callee_link(hf, (intptr_t*)badAddress);
194 }
195 #endif
196 }
197
198 inline void FreezeBase::patch_pd_unused(intptr_t* sp) {
199 }
200
201 inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) {
202 // Nothing to do. The backchain is reconstructed when thawing (see Thaw<ConfigT>::patch_caller_links())
203 }
204
205 inline intptr_t* AnchorMark::anchor_mark_set_pd() {
206 // Nothing to do on s390 because the interpreter does not use SP as expression stack pointer.
207 // Instead there is a dedicated register Z_esp which is not affected by VM calls.
208 return _top_frame.sp();
209 }
210
211 inline void AnchorMark::anchor_mark_clear_pd() {
212 // Nothing to do. See anchor_mark_set_pd().
213 }
214
215 inline frame ThawBase::new_entry_frame() {
216 intptr_t* sp = _cont.entrySP();
217 return frame(sp, _cont.entryPC(), sp, _cont.entryFP());
218 }
219
220 template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom, int size_adjust) {
221 assert(FKind::is_instance(hf), "");
222
223 assert(is_aligned(caller.fp(), frame::frame_alignment), PTR_FORMAT, p2i(caller.fp()));
224 // caller.sp() can be unaligned. This is fixed below.
225 if (FKind::interpreted) {
226 // Note: we have to overlap with the caller, at least if it is interpreted, to match the
227 // max_thawing_size calculation during freeze. See also comment above.
228 intptr_t* heap_sp = hf.unextended_sp();
229 const int fsize = ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp();
230 const int overlap = !caller.is_interpreted_frame() ? 0
231 : ContinuationHelper::InterpretedFrame::stack_argsize(hf) + frame::metadata_words_at_top;
232 intptr_t* frame_sp = caller.unextended_sp() + overlap - fsize;
233 intptr_t* fp = frame_sp + (hf.fp() - heap_sp);
234 // align fp
235 int padding = fp - align_down(fp, frame::frame_alignment);
236 fp -= padding;
237 // alignment of sp is done by callee or in finish_thaw()
238 frame_sp -= padding;
239
240 // On s390 esp points to the first free slot on the expression stack (see frame_s390.hpp).
241 // The assertion verifies that frame_sp + metadata_words_at_top points to the slot above esp,
242 // which corresponds to the last parameter position.
243 DEBUG_ONLY(intptr_t* esp = fp + *hf.addr_at(_z_ijava_idx(esp));)
244 assert(frame_sp + frame::metadata_words_at_top == esp+1, " frame_sp=" PTR_FORMAT " esp=" PTR_FORMAT, p2i(frame_sp), p2i(esp));
245 caller.set_sp(fp);
246 frame f(frame_sp, hf.pc(), frame_sp, fp);
247 // we need to set the locals so that the caller of new_stack_frame() can call
248 // ContinuationHelper::InterpretedFrame::frame_bottom
249 // copy relativized locals from the heap frame
250 *f.addr_at(_z_ijava_idx(locals)) = *hf.addr_at(_z_ijava_idx(locals));
251
252 return f;
253 } else {
254 int fsize = FKind::size(hf);
255 int argsize = FKind::stack_argsize(hf);
256 intptr_t* frame_sp = caller.sp() - fsize;
257
258 if ((bottom && argsize > 0) || caller.is_interpreted_frame()) {
259 assert(!_should_patch_caller_pc, "what??");
260 _should_patch_caller_pc = caller.is_interpreted_frame();
261 frame_sp -= argsize + frame::metadata_words_at_top;
262 frame_sp = align_down(frame_sp, frame::alignment_in_bytes);
263 caller.set_sp(frame_sp + fsize);
264 }
265
266 assert(hf.cb() != nullptr, "");
267 assert(hf.oop_map() != nullptr, "");
268 intptr_t* fp = frame_sp + fsize;
269 return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false);
270 }
271 }
272
273 inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) {
274 // Make sure that monitors is still relativized.
275 assert(f.at_absolute(_z_ijava_idx(monitors)) <= -(frame::z_ijava_state_size / wordSize), "");
276 // Make sure that esp is still relativized.
277 assert(f.at_absolute(_z_ijava_idx(esp)) <= f.at_absolute(_z_ijava_idx(monitors)), "");
278 // Keep top_frame_sp relativized.
279 }
280
281 inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) {
282 // Unused. Alignment is done directly in new_stack_frame() / finish_thaw().
283 return nullptr;
284 }
285
286 inline void ThawBase::patch_pd(frame& f, const frame& caller) {
287 patch_callee_link(caller, caller.fp());
288 // Prevent assertion if f gets deoptimized right away before it's fully initialized
289 f.mark_not_fully_initialized();
290 }
291
292 inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
293 assert(f.own_abi()->callers_sp == (uint64_t)caller_sp, "should have been fixed by patch_caller_links");
294 }
295
296 inline intptr_t* ThawBase::push_cleanup_continuation() {
297 frame enterSpecial = new_entry_frame();
298 frame::z_common_abi* enterSpecial_abi = (frame::z_common_abi*)enterSpecial.sp();
299
300 enterSpecial_abi->return_pc = (intptr_t)ContinuationEntry::cleanup_pc();
301
302 log_develop_trace(continuations, preempt)("push_cleanup_continuation enterSpecial sp: " INTPTR_FORMAT " cleanup pc: " INTPTR_FORMAT,
303 p2i(enterSpecial_abi),
304 p2i(ContinuationEntry::cleanup_pc()));
305
306 return enterSpecial.sp();
307 }
308
309 inline intptr_t* ThawBase::push_preempt_adapter() {
310 frame enterSpecial = new_entry_frame();
311 frame::z_common_abi* enterSpecial_abi = (frame::z_common_abi*)enterSpecial.sp();
312
313 enterSpecial_abi->return_pc = (intptr_t)StubRoutines::cont_preempt_stub();
314
315 log_develop_trace(continuations, preempt)("push_preempt_adapter enterSpecial sp: " INTPTR_FORMAT " adapter pc: " INTPTR_FORMAT,
316 p2i(enterSpecial_abi),
317 p2i(StubRoutines::cont_preempt_stub()));
318
319 return enterSpecial.sp();
320 }
321
322 template <typename ConfigT>
323 inline void Thaw<ConfigT>::patch_caller_links(intptr_t* sp, intptr_t* bottom) {
324 for (intptr_t* callers_sp; sp < bottom; sp = callers_sp) {
325 address pc = (address)((frame::z_java_abi*) sp)->return_pc;
326 assert(pc != nullptr, "");
327 // see ThawBase::patch_return() which gets called just before
328 bool is_entry_frame = pc == StubRoutines::cont_returnBarrier() || pc == _cont.entryPC();
329 if (is_entry_frame) {
330 callers_sp = _cont.entryFP();
331 } else {
332 assert(!Interpreter::contains(pc), "sp:" PTR_FORMAT " pc:" PTR_FORMAT, p2i(sp), p2i(pc));
333 CodeBlob* cb = CodeCache::find_blob(pc);
334 callers_sp = sp + cb->frame_size();
335 }
336 // set the back link
337 ((frame::z_java_abi*) sp)->callers_sp = (intptr_t) callers_sp;
338 }
339 }
340
341 inline void ThawBase::prefetch_chunk_pd(void* start, int size) {
342 // TODO: implement in future;
343 }
344
345 #endif // CPU_S390_CONTINUATION_S390_INLINE_HPP