1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/nativeInst.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "logging/log.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "runtime/frame.inline.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/registerMap.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/formatBuffer.hpp"
40 #if INCLUDE_JVMCI
41 #include "jvmci/jvmciRuntime.hpp"
42 #endif
43
44 static int slow_path_size(nmethod* nm) {
45 // The slow path code is out of line with C2
46 return nm->is_compiled_by_c2() ? 0 : 6;
47 }
48
49 // This is the offset of the entry barrier relative to where the frame is completed.
50 // If any code changes between the end of the verified entry where the entry
51 // barrier resides, and the completion of the frame, then
52 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
53 // not find the expected native instruction at this offset, which needs updating.
54 // Note that this offset is invariant of PreserveFramePointer.
55 static int entry_barrier_offset(nmethod* nm) {
56 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
57 switch (bs_asm->nmethod_patching_type()) {
58 case NMethodPatchingType::stw_instruction_and_data_patch:
59 return -4 * (4 + slow_path_size(nm));
60 case NMethodPatchingType::conc_instruction_and_data_patch:
61 return -4 * (10 + slow_path_size(nm));
62 }
63 ShouldNotReachHere();
64 return 0;
65 }
66
67 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
68 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
69 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
70 nm->stub_contains(reinterpret_cast<address>(result)),
71 "guard must be in nmethod code");
72 return result;
73 }
74
75 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
76 // arming/verification.
77 // An entrypoint is defined as a tuple of <instr. address, guard address>:
78 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
79 // * The guard address is the address where the guard value of that entrypoint resides.
80 //
81 // Each nmethod has at least one entrypoint. The default must always be well-defined
82 // (neither instruction nor guard are nullptr).
83 //
84 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
85 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
86 //
87 // The mapping of C1-compiled methods (scalarization used) looks as follows:
88 // * alt1: verified entry point
89 // * alt2 (optional): verified inline ro entry point
90 //
91 // The mapping of C2-compiled methods (scalarization used) looks as follows:
92 // * alt1: verified inline entry point
93 // * alt2 (optional): verified inline ro entry point
94 //
95 // In other scenarios, neither alt1 nor alt2 are defined.
96 class NativeNMethodBarrier {
97 private:
98 // The addresses of the instructions that act as the guards.
99 address _default_entry_instruction;
100 address _verified_alt1_instruction;
101 address _verified_alt2_instruction;
102 // Pointers representing the actual guard values themselves.
103 int* _default_entry_guard;
104 int* _verified_alt1_guard;
105 int* _verified_alt2_guard;
106
107 public:
108 NativeNMethodBarrier(nmethod* nm) :
109 _default_entry_instruction(nullptr),
110 _verified_alt1_instruction(nullptr),
111 _verified_alt2_instruction(nullptr),
112 _default_entry_guard(nullptr),
113 _verified_alt1_guard(nullptr),
114 _verified_alt2_guard(nullptr) {
115 #if INCLUDE_JVMCI
116 if (nm->is_compiled_by_jvmci()) {
117 address pc = nm->code_begin() + nm->jvmci_nmethod_data()->nmethod_entry_patch_offset();
118 RelocIterator iter(nm, pc, pc + 4);
119 guarantee(iter.next(), "missing relocs");
120 guarantee(iter.type() == relocInfo::section_word_type, "unexpected reloc");
121
122 // Only set the verified entry, the others are not supported.
123 _default_entry_guard = (int*) iter.section_word_reloc()->target();
124 _default_entry_instruction = pc;
125 } else
126 #endif
127 {
128 // The default entry point has a known address. The guard address can be
129 // decoded from the literal in the instruction. Verification will confirm
130 // that this instruction corresponds to a load.
131 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
132 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
133
134 // If the nmethod has scalarized arguments, then there are more entry
135 // points, each with their own nmethod entry barrier.
136 if (!nm->is_osr_method() && nm->method()->has_scalarized_args()) {
137 assert(nm->verified_entry_point() != nm->verified_inline_entry_point(), "scalarized entry point not found");
138 address method_body = nm->is_compiled_by_c1() ? nm->verified_inline_entry_point() : nm->verified_entry_point();
139 int barrier_offset = _default_entry_instruction - method_body;
140
141 // Set the first alternative entry point.
142 address entry_point2 = nm->is_compiled_by_c1() ? nm->verified_entry_point() : nm->verified_inline_entry_point();
143 _verified_alt1_instruction = entry_point2 + barrier_offset;
144 assert(_default_entry_instruction != _verified_alt1_instruction, "sanity");
145 _verified_alt1_guard = decode_guard_from_instruction(nm, _verified_alt1_instruction);
146
147 // If there is a second alternative entry point, set it too.
148 if (method_body != nm->verified_inline_ro_entry_point() && entry_point2 != nm->verified_inline_ro_entry_point()) {
149 _verified_alt2_instruction = nm->verified_inline_ro_entry_point() + barrier_offset;
150 _verified_alt2_guard = decode_guard_from_instruction(nm, _verified_alt2_instruction);
151 assert(_default_entry_instruction != _verified_alt2_instruction &&
152 _verified_alt1_instruction != _verified_alt2_instruction,
153 "sanity");
154 }
155 }
156 // Perform the checking as verification.
157 err_msg msg("%s", "");
158 assert(check_barriers(msg), "%s", msg.buffer());
159 }
160 }
161
162 // Gets the value of the default entry guard.
163 // This does not consider the alternative entrypoints, as these should
164 // all be consistent. It is up to the caller to enforce this.
165 int get_default_guard_value() {
166 return AtomicAccess::load_acquire(_default_entry_guard);
167 }
168
169 // Sets the value for all barriers.
170 void set_values(int value, int bit_mask) {
171 set_value_impl(_default_entry_guard, value, bit_mask);
172 if (_verified_alt1_guard != nullptr) {
173 set_value_impl(_verified_alt1_guard, value, bit_mask);
174 }
175 if (_verified_alt2_guard != nullptr) {
176 set_value_impl(_verified_alt2_guard, value, bit_mask);
177 }
178 }
179
180 // Verifies that all potential barriers are correct.
181 bool check_barriers(err_msg& msg) {
182 // The default entry barrier should always be checked.
183 if (!check_barrier_impl(_default_entry_instruction, msg)) {
184 return false;
185 }
186 // Check the alternative entry barriers only if they are specified.
187 // Note that the guard values are already validated at construction time,
188 // if they fall out of the nmethod range, this will be caught earlier.
189 if (_verified_alt1_instruction != nullptr &&
190 !check_barrier_impl(_verified_alt1_instruction, msg)) {
191 return false;
192 }
193 if (_verified_alt2_instruction != nullptr &&
194 !check_barrier_impl(_verified_alt2_instruction, msg)) {
195 return false;
196 }
197 return true;
198 }
199
200 private:
201 // Sets the value for a single barrier.
202 void set_value_impl(int* guard, int value, int bit_mask) {
203 if (bit_mask == ~0) {
204 AtomicAccess::release_store(guard, value);
205 return;
206 }
207 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
208 value &= bit_mask;
209 int old_value = AtomicAccess::load(guard);
210 while (true) {
211 // Only bits in the mask are changed
212 int new_value = value | (old_value & ~bit_mask);
213 if (new_value == old_value) break;
214 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
215 if (v == old_value) break;
216 old_value = v;
217 }
218 }
219
220 // Checks the validity of a single barrier.
221 bool check_barrier_impl(address& instruction, err_msg& msg) {
222 uint32_t* addr = (uint32_t*) instruction;
223 uint32_t inst = *addr;
224 if ((inst & 0xff000000) != 0x18000000) {
225 msg.print("Nmethod entry barrier did not start with ldr (literal) as expected. "
226 "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(addr), inst);
227 return false;
228 }
229 return true;
230 }
231 };
232
233 /* We're called from an nmethod when we need to deoptimize it. We do
234 this by throwing away the nmethod's frame and jumping to the
235 ic_miss stub. This looks like there has been an IC miss at the
236 entry of the nmethod, so we resolve the call, which will fall back
237 to the interpreter if the nmethod has been unloaded. */
238 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
239
240 typedef struct {
241 intptr_t *sp; intptr_t *fp; address lr; address pc;
242 } frame_pointers_t;
243
244 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
245
246 JavaThread *thread = JavaThread::current();
247 RegisterMap reg_map(thread,
248 RegisterMap::UpdateMap::skip,
249 RegisterMap::ProcessFrames::include,
250 RegisterMap::WalkContinuation::skip);
251 frame frame = thread->last_frame();
252
253 assert(frame.is_compiled_frame() || frame.is_native_frame(), "must be");
254 assert(frame.cb() == nm, "must be");
255 frame = frame.sender(®_map);
256
257 LogTarget(Trace, nmethod, barrier) out;
258 if (out.is_enabled()) {
259 ResourceMark mark;
260 log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
261 nm->method()->name_and_sig_as_C_string(),
262 nm, *(address *) return_address_ptr, nm->is_osr_method(), thread,
263 thread->name(), frame.sp(), nm->verified_entry_point());
264 }
265
266 new_frame->sp = frame.sp();
267 new_frame->fp = frame.fp();
268 new_frame->lr = frame.pc();
269 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
270 }
271
272 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
273 if (!supports_entry_barrier(nm)) {
274 return;
275 }
276
277 if (value == disarmed_guard_value()) {
278 // The patching epoch is incremented before the nmethod is disarmed. Disarming
279 // is performed with a release store. In the nmethod entry barrier, the values
280 // are read in the opposite order, such that the load of the nmethod guard
281 // acquires the patching epoch. This way, the guard is guaranteed to block
282 // entries to the nmethod, until it has safely published the requirement for
283 // further fencing by mutators, before they are allowed to enter.
284 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
285 bs_asm->increment_patching_epoch();
286 }
287
288 // Enable WXWrite: the function is called directly from nmethod_entry_barrier
289 // stub.
290 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
291
292 NativeNMethodBarrier barrier(nm);
293 barrier.set_values(value, bit_mask);
294 }
295
296 int BarrierSetNMethod::guard_value(nmethod* nm) {
297 if (!supports_entry_barrier(nm)) {
298 return disarmed_guard_value();
299 }
300
301 NativeNMethodBarrier barrier(nm);
302 return barrier.get_default_guard_value();
303 }
304
305 #if INCLUDE_JVMCI
306 bool BarrierSetNMethod::verify_barrier(nmethod* nm, err_msg& msg) {
307 NativeNMethodBarrier barrier(nm);
308 return barrier.check_barriers(msg);
309 }
310 #endif