1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/nativeInst.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "logging/log.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "runtime/frame.inline.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/registerMap.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/formatBuffer.hpp"
40
41 static int slow_path_size(nmethod* nm) {
42 // The slow path code is out of line with C2
43 return nm->is_compiled_by_c2() ? 0 : 6;
44 }
45
46 // This is the offset of the entry barrier relative to where the frame is completed.
47 // If any code changes between the end of the verified entry where the entry
48 // barrier resides, and the completion of the frame, then
49 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
50 // not find the expected native instruction at this offset, which needs updating.
51 // Note that this offset is invariant of PreserveFramePointer.
52 static int entry_barrier_offset(nmethod* nm) {
53 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
54 switch (bs_asm->nmethod_patching_type()) {
55 case NMethodPatchingType::stw_instruction_and_data_patch:
56 return -4 * (4 + slow_path_size(nm));
57 case NMethodPatchingType::conc_instruction_and_data_patch:
58 return -4 * (10 + slow_path_size(nm));
59 }
60 ShouldNotReachHere();
61 return 0;
62 }
63
64 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
65 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
66 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
67 nm->stub_contains(reinterpret_cast<address>(result)),
68 "guard must be in nmethod code");
69 return result;
70 }
71
72 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
73 // arming/verification.
74 // An entrypoint is defined as a tuple of <instr. address, guard address>:
75 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
76 // * The guard address is the address where the guard value of that entrypoint resides.
77 //
78 // Each nmethod has at least one entrypoint. The default must always be well-defined
79 // (neither instruction nor guard are nullptr).
80 //
81 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
82 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
83 //
84 // The mapping of C1-compiled methods (scalarization used) looks as follows:
85 // * alt1: verified entry point
86 // * alt2 (optional): verified inline ro entry point
87 //
88 // The mapping of C2-compiled methods (scalarization used) looks as follows:
89 // * alt1: verified inline entry point
90 // * alt2 (optional): verified inline ro entry point
91 //
92 // In other scenarios, neither alt1 nor alt2 are defined.
93 class NativeNMethodBarrier {
94 private:
95 // The addresses of the instructions that act as the guards.
96 address _default_entry_instruction;
97 address _verified_alt1_instruction;
98 address _verified_alt2_instruction;
99 // Pointers representing the actual guard values themselves.
100 int* _default_entry_guard;
101 int* _verified_alt1_guard;
102 int* _verified_alt2_guard;
103
104 public:
105 NativeNMethodBarrier(nmethod* nm) :
106 _default_entry_instruction(nullptr),
107 _verified_alt1_instruction(nullptr),
108 _verified_alt2_instruction(nullptr),
109 _default_entry_guard(nullptr),
110 _verified_alt1_guard(nullptr),
111 _verified_alt2_guard(nullptr)
112 {
113 // The default entry point has a known address. The guard address can be
114 // decoded from the literal in the instruction. Verification will confirm
115 // that this instruction corresponds to a load.
116 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
117 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
118
119 // If the nmethod has scalarized arguments, then there are more entry
120 // points, each with their own nmethod entry barrier.
121 if (!nm->is_osr_method() && nm->method()->has_scalarized_args()) {
122 assert(nm->verified_entry_point() != nm->verified_inline_entry_point(), "scalarized entry point not found");
123 address method_body = nm->is_compiled_by_c1() ? nm->verified_inline_entry_point() : nm->verified_entry_point();
124 int barrier_offset = _default_entry_instruction - method_body;
125
126 // Set the first alternative entry point.
127 address entry_point2 = nm->is_compiled_by_c1() ? nm->verified_entry_point() : nm->verified_inline_entry_point();
128 _verified_alt1_instruction = entry_point2 + barrier_offset;
129 assert(_default_entry_instruction != _verified_alt1_instruction, "sanity");
130 _verified_alt1_guard = decode_guard_from_instruction(nm, _verified_alt1_instruction);
131
132 // If there is a second alternative entry point, set it too.
133 if (method_body != nm->verified_inline_ro_entry_point() && entry_point2 != nm->verified_inline_ro_entry_point()) {
134 _verified_alt2_instruction = nm->verified_inline_ro_entry_point() + barrier_offset;
135 _verified_alt2_guard = decode_guard_from_instruction(nm, _verified_alt2_instruction);
136 assert(_default_entry_instruction != _verified_alt2_instruction &&
137 _verified_alt1_instruction != _verified_alt2_instruction,
138 "sanity");
139 }
140 }
141 // Perform the checking as verification.
142 err_msg msg("%s", "");
143 assert(check_barriers(msg), "%s", msg.buffer());
144 }
145
146 // Gets the value of the default entry guard.
147 // This does not consider the alternative entrypoints, as these should
148 // all be consistent. It is up to the caller to enforce this.
149 int get_default_guard_value() {
150 return AtomicAccess::load_acquire(_default_entry_guard);
151 }
152
153 // Sets the value for all barriers.
154 void set_values(int value, int bit_mask) {
155 set_value_impl(_default_entry_guard, value, bit_mask);
156 if (_verified_alt1_guard != nullptr) {
157 set_value_impl(_verified_alt1_guard, value, bit_mask);
158 }
159 if (_verified_alt2_guard != nullptr) {
160 set_value_impl(_verified_alt2_guard, value, bit_mask);
161 }
162 }
163
164 // Verifies that all potential barriers are correct.
165 bool check_barriers(err_msg& msg) {
166 // The default entry barrier should always be checked.
167 if (!check_barrier_impl(_default_entry_instruction, msg)) {
168 return false;
169 }
170 // Check the alternative entry barriers only if they are specified.
171 // Note that the guard values are already validated at construction time,
172 // if they fall out of the nmethod range, this will be caught earlier.
173 if (_verified_alt1_instruction != nullptr &&
174 !check_barrier_impl(_verified_alt1_instruction, msg)) {
175 return false;
176 }
177 if (_verified_alt2_instruction != nullptr &&
178 !check_barrier_impl(_verified_alt2_instruction, msg)) {
179 return false;
180 }
181 return true;
182 }
183
184 private:
185 // Sets the value for a single barrier.
186 void set_value_impl(int* guard, int value, int bit_mask) {
187 if (bit_mask == ~0) {
188 AtomicAccess::release_store(guard, value);
189 return;
190 }
191 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
192 value &= bit_mask;
193 int old_value = AtomicAccess::load(guard);
194 while (true) {
195 // Only bits in the mask are changed
196 int new_value = value | (old_value & ~bit_mask);
197 if (new_value == old_value) break;
198 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
199 if (v == old_value) break;
200 old_value = v;
201 }
202 }
203
204 // Checks the validity of a single barrier.
205 bool check_barrier_impl(address& instruction, err_msg& msg) {
206 uint32_t* addr = (uint32_t*) instruction;
207 uint32_t inst = *addr;
208 if ((inst & 0xff000000) != 0x18000000) {
209 msg.print("Nmethod entry barrier did not start with ldr (literal) as expected. "
210 "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(addr), inst);
211 return false;
212 }
213 return true;
214 }
215 };
216
217 /* We're called from an nmethod when we need to deoptimize it. We do
218 this by throwing away the nmethod's frame and jumping to the
219 ic_miss stub. This looks like there has been an IC miss at the
220 entry of the nmethod, so we resolve the call, which will fall back
221 to the interpreter if the nmethod has been unloaded. */
222 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
223
224 typedef struct {
225 intptr_t *sp; intptr_t *fp; address lr; address pc;
226 } frame_pointers_t;
227
228 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
229
230 JavaThread *thread = JavaThread::current();
231 RegisterMap reg_map(thread,
232 RegisterMap::UpdateMap::skip,
233 RegisterMap::ProcessFrames::include,
234 RegisterMap::WalkContinuation::skip);
235 frame frame = thread->last_frame();
236
237 assert(frame.is_compiled_frame() || frame.is_native_frame(), "must be");
238 assert(frame.cb() == nm, "must be");
239 frame = frame.sender(®_map);
240
241 LogTarget(Trace, nmethod, barrier) out;
242 if (out.is_enabled()) {
243 ResourceMark mark;
244 log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
245 nm->method()->name_and_sig_as_C_string(),
246 nm, *(address *) return_address_ptr, nm->is_osr_method(), thread,
247 thread->name(), frame.sp(), nm->verified_entry_point());
248 }
249
250 new_frame->sp = frame.sp();
251 new_frame->fp = frame.fp();
252 new_frame->lr = frame.pc();
253 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
254 }
255
256 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
257 if (!supports_entry_barrier(nm)) {
258 return;
259 }
260
261 if (value == disarmed_guard_value()) {
262 // The patching epoch is incremented before the nmethod is disarmed. Disarming
263 // is performed with a release store. In the nmethod entry barrier, the values
264 // are read in the opposite order, such that the load of the nmethod guard
265 // acquires the patching epoch. This way, the guard is guaranteed to block
266 // entries to the nmethod, until it has safely published the requirement for
267 // further fencing by mutators, before they are allowed to enter.
268 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
269 bs_asm->increment_patching_epoch();
270 }
271
272 // Enable WXWrite: the function is called directly from nmethod_entry_barrier
273 // stub.
274 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
275
276 NativeNMethodBarrier barrier(nm);
277 barrier.set_values(value, bit_mask);
278 }
279
280 int BarrierSetNMethod::guard_value(nmethod* nm) {
281 if (!supports_entry_barrier(nm)) {
282 return disarmed_guard_value();
283 }
284
285 NativeNMethodBarrier barrier(nm);
286 return barrier.get_default_guard_value();
287 }