1 /*
2 * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP
26 #define CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP
27
28 #include "runtime/continuationHelper.hpp"
29
30 #include "runtime/continuationEntry.inline.hpp"
31 #include "runtime/frame.inline.hpp"
32 #include "runtime/registerMap.hpp"
33 #include "utilities/macros.hpp"
34
35 template<typename FKind>
36 static inline intptr_t** link_address(const frame& f) {
37 assert(FKind::is_instance(f), "");
38 return FKind::interpreted
39 ? (intptr_t**)(f.fp() + frame::link_offset)
40 : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - frame::sender_sp_offset);
41 }
42
43 static inline void patch_return_pc_with_preempt_stub(frame& f) {
44 if (f.is_runtime_frame()) {
45 // Unlike x86 we don't know where in the callee frame the return pc is
46 // saved so we can't patch the return from the VM call back to Java.
47 // Instead, we will patch the return from the runtime stub back to the
48 // compiled method so that the target returns to the preempt cleanup stub.
49 intptr_t* caller_sp = f.sp() + f.cb()->frame_size();
50 ContinuationHelper::patch_return_address_at(&caller_sp[-1],
51 StubRoutines::cont_preempt_stub());
52 } else {
53 // The target will check for preemption once it returns to the interpreter
54 // or the native wrapper code and will manually jump to the preempt stub.
55 JavaThread *thread = JavaThread::current();
56 DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();)
57 assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0,
58 "preemptable VM call not using call_VM_preemptable");
59 thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub());
60 }
61 }
62
63 inline int ContinuationHelper::frame_align_words(int size) {
64 #ifdef _LP64
65 return size & 1;
66 #else
67 return 0;
68 #endif
69 }
70
71 inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) {
72 #ifdef _LP64
73 sp = align_down(sp, frame::frame_alignment);
74 #endif
75 return sp;
76 }
77
78 template<typename FKind>
79 inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) {
80 frame::update_map_with_saved_link(map, link_address<FKind>(f));
81 }
82
83 inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) {
84 frame::update_map_with_saved_link(map, ContinuationHelper::Frame::callee_link_address(f));
85 }
86
87 inline void ContinuationHelper::push_pd(const frame& f) {
88 *(intptr_t**)(f.sp() - frame::sender_sp_offset) = f.fp();
89 }
90
91 #define CPU_OVERRIDES_RETURN_ADDRESS_ACCESSORS
92
93 inline address ContinuationHelper::return_address_at(intptr_t* sp) {
94 return pauth_strip_verifiable(*(address*)sp);
95 }
96
97 inline void ContinuationHelper::patch_return_address_at(intptr_t* sp,
98 address pc) {
99 *(address*)sp = pauth_sign_return_address(pc);
100 }
101
102 inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* entry) {
103 anchor->set_last_Java_fp(entry->entry_fp());
104 }
105
106 inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) {
107 intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset);
108 anchor->set_last_Java_fp(fp);
109 }
110
111 #ifdef ASSERT
112 inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) {
113 intptr_t* sp = f.sp();
114 address pc = ContinuationHelper::return_address_at(
115 sp - frame::sender_sp_ret_address_offset());
116 intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset);
117 assert(f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc));
118 assert(f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp));
119 return f.raw_pc() == pc && f.fp() == fp;
120 }
121 #endif
122
123 inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) {
124 return (intptr_t**)(f.sp() - frame::sender_sp_offset);
125 }
126
127 inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) {
128 frame::CompiledFramePointers cfp = f.compiled_frame_details();
129 return cfp.sender_pc_addr;
130 }
131
132 inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) {
133 return (address*)(f.fp() + frame::return_addr_offset);
134 }
135
136 inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, const frame& caller) {
137 intptr_t* sp = caller.unextended_sp();
138 assert(f.is_interpreted_frame(), "");
139 intptr_t* la = f.addr_at(frame::interpreter_frame_sender_sp_offset);
140 *la = f.is_heap_frame() ? (intptr_t)(sp - f.fp()) : (intptr_t)sp;
141 }
142
143 inline address ContinuationHelper::Frame::real_pc(const frame& f) {
144 // Always used in assertions. Just strip it.
145 address* pc_addr = &(((address*) f.sp())[-1]);
146 return pauth_strip_pointer(*pc_addr);
147 }
148
149 inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
150 address* pc_addr = &(((address*) f.sp())[-1]);
151 *pc_addr = pauth_sign_return_address(pc);
152 }
153
154 inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame
155 // interpreter_frame_last_sp_offset, points to unextended_sp includes arguments in the frame
156 // interpreter_frame_initial_sp_offset excludes expression stack slots
157 int expression_stack_sz = expression_stack_size(f, mask);
158 intptr_t* res = (intptr_t*)f.at_relative(frame::interpreter_frame_initial_sp_offset) - expression_stack_sz;
159 assert(res == (intptr_t*)f.interpreter_frame_monitor_end() - expression_stack_sz, "");
160 assert(res >= f.unextended_sp(),
161 "res: " INTPTR_FORMAT " initial_sp: " INTPTR_FORMAT " last_sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " expression_stack_size: %d",
162 p2i(res), p2i(f.addr_at(frame::interpreter_frame_initial_sp_offset)), f.at_relative_or_null(frame::interpreter_frame_last_sp_offset),
163 p2i(f.unextended_sp()), expression_stack_sz);
164 return res;
165 }
166
167 inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame
168 return (intptr_t*)f.at_relative(frame::interpreter_frame_locals_offset) + 1; // exclusive, so we add 1 word
169 }
170
171 inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) {
172 return f.unextended_sp() + (callee_interpreted ? callee_argsize : 0);
173 }
174
175 inline intptr_t* ContinuationHelper::InterpretedFrame::callers_sp(const frame& f) {
176 return f.fp() + frame::metadata_words;
177 }
178
179 #endif // CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP