1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/nativeInst.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "logging/log.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "runtime/frame.inline.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/registerMap.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/formatBuffer.hpp"
40
41 static int slow_path_size(nmethod* nm) {
42 // The slow path code is out of line with C2
43 return nm->is_compiled_by_c2() ? 0 : 6;
44 }
45
46 // This is the offset of the entry barrier relative to where the frame is completed.
47 // If any code changes between the end of the verified entry where the entry
48 // barrier resides, and the completion of the frame, then
49 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
50 // not find the expected native instruction at this offset, which needs updating.
51 // Note that this offset is invariant of PreserveFramePointer.
52 static int entry_barrier_offset(nmethod* nm) {
53 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
54 switch (bs_asm->nmethod_patching_type()) {
55 case NMethodPatchingType::stw_instruction_and_data_patch:
56 return -4 * (4 + slow_path_size(nm));
57 case NMethodPatchingType::conc_instruction_and_data_patch:
58 return -4 * (10 + slow_path_size(nm));
59 }
60 ShouldNotReachHere();
61 return 0;
62 }
63
64 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
65 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
66 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
67 nm->stub_contains(reinterpret_cast<address>(result)),
68 "guard must be in nmethod code");
69 return result;
70 }
71
72 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
73 // arming/verification.
74 // An entrypoint is defined as a tuple of <instr. address, guard address>:
75 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
76 // * The guard address is the address where the guard value of that entrypoint resides.
77 //
78 // Each nmethod has at least one entrypoint. The default must always be well-defined
79 // (neither instruction nor guard are nullptr).
80 //
81 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
82 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
83 //
84 // The mapping of C1-compiled methods (scalarization used) looks as follows:
85 // * alt1: verified entry point
86 // * alt2 (optional): verified inline ro entry point
87 //
88 // The mapping of C2-compiled methods (scalarization used) looks as follows:
89 // * alt1: verified inline entry point
90 // * alt2 (optional): verified inline ro entry point
91 //
92 // In other scenarios, neither alt1 nor alt2 are defined.
93 class NativeNMethodBarrier {
94 private:
95 // The addresses of the instructions that act as the guards.
96 address _default_entry_instruction;
97 address _verified_alt1_instruction;
98 address _verified_alt2_instruction;
99 // Pointers representing the actual guard values themselves.
100 int* _default_entry_guard;
101 int* _verified_alt1_guard;
102 int* _verified_alt2_guard;
103
104 public:
105 NativeNMethodBarrier(nmethod* nm) :
106 _default_entry_instruction(nullptr),
107 _verified_alt1_instruction(nullptr),
108 _verified_alt2_instruction(nullptr),
109 _default_entry_guard(nullptr),
110 _verified_alt1_guard(nullptr),
111 _verified_alt2_guard(nullptr)
112 {
113 // The default entry point has a known address. The guard address can be
114 // decoded from the literal in the instruction. Verification will confirm
115 // that this instruction corresponds to a load.
116 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
117 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
118
119 // If the nmethod has scalarized arguments, then there are more entry
120 // points, each with their own nmethod entry barrier.
121 if (!nm->is_osr_method() && nm->method()->has_scalarized_args()) {
122 assert(nm->verified_entry_point() != nm->verified_inline_entry_point(), "scalarized entry point not found");
123 address method_body = nm->is_compiled_by_c1() ? nm->verified_inline_entry_point() : nm->verified_entry_point();
124 int barrier_offset = _default_entry_instruction - method_body;
125
126 // Set the first alternative entry point.
127 address entry_point2 = nm->is_compiled_by_c1() ? nm->verified_entry_point() : nm->verified_inline_entry_point();
128 _verified_alt1_instruction = entry_point2 + barrier_offset;
129 assert(_default_entry_instruction != _verified_alt1_instruction, "sanity");
130 _verified_alt1_guard = decode_guard_from_instruction(nm, _verified_alt1_instruction);
131
132 // If there is a second alternative entry point, set it too.
133 if (method_body != nm->verified_inline_ro_entry_point() && entry_point2 != nm->verified_inline_ro_entry_point()) {
134 _verified_alt2_instruction = nm->verified_inline_ro_entry_point() + barrier_offset;
135 _verified_alt2_guard = decode_guard_from_instruction(nm, _verified_alt2_instruction);
136 assert(_default_entry_instruction != _verified_alt2_instruction &&
137 _verified_alt1_instruction != _verified_alt2_instruction,
138 "sanity");
139 }
140 }
141 // Perform the checking as verification.
142 err_msg msg("%s", "");
143 assert(check_barriers(msg), "%s", msg.buffer());
144 }
145
146 // Gets the value of the default entry guard.
147 // This does not consider the alternative entrypoints, as these should
148 // all be consistent. It is up to the caller to enforce this.
149 int get_default_guard_value() {
150 return AtomicAccess::load_acquire(_default_entry_guard);
151 }
152
153 // Sets the value for all barriers.
154 void set_values(int value, int bit_mask) {
155 set_value_impl(_default_entry_guard, value, bit_mask);
156 if (_verified_alt1_guard != nullptr) {
157 set_value_impl(_verified_alt1_guard, value, bit_mask);
158 }
159 if (_verified_alt2_guard != nullptr) {
160 set_value_impl(_verified_alt2_guard, value, bit_mask);
161 }
162 }
163
164 // Verifies that all potential barriers are correct.
165 bool check_barriers(err_msg& msg) {
166 // The default entry barrier should always be checked.
167 if (!check_barrier_impl(_default_entry_instruction, msg)) {
168 return false;
169 }
170 // Check the alternative entry barriers only if they are specified.
171 // Note that the guard values are already validated at construction time,
172 // if they fall out of the nmethod range, this will be caught earlier.
173 if (_verified_alt1_instruction != nullptr &&
174 !check_barrier_impl(_verified_alt1_instruction, msg)) {
175 return false;
176 }
177 if (_verified_alt2_instruction != nullptr &&
178 !check_barrier_impl(_verified_alt2_instruction, msg)) {
179 return false;
180 }
181 return true;
182 }
183
184 private:
185 // Sets the value for a single barrier.
186 void set_value_impl(int* guard, int value, int bit_mask) {
187 if (bit_mask == ~0) {
188 AtomicAccess::release_store(guard, value);
189 return;
190 }
191 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
192 value &= bit_mask;
193 int old_value = AtomicAccess::load(guard);
194 while (true) {
195 // Only bits in the mask are changed
196 int new_value = value | (old_value & ~bit_mask);
197 if (new_value == old_value) break;
198 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
199 if (v == old_value) break;
200 old_value = v;
201 }
202 }
203
204 // Checks the validity of a single barrier.
205 // The first instruction of the nmethod entry barrier is an ldrw (literal)
206 // instruction. Verify that it's really there, so the offsets are not skewed.
207 bool check_barrier_impl(address& instruction, err_msg& msg) {
208 NativeInstruction* ni = nativeInstruction_at(instruction);
209 if (!ni->is_ldrw_gpr_literal()) {
210 msg.print("Nmethod entry barrier did not start with ldrw (literal) as expected. "
211 "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(instruction), ni->encoding());
212 return false;
213 }
214 return true;
215 }
216 };
217
218 /* We're called from an nmethod when we need to deoptimize it. We do
219 this by throwing away the nmethod's frame and jumping to the
220 ic_miss stub. This looks like there has been an IC miss at the
221 entry of the nmethod, so we resolve the call, which will fall back
222 to the interpreter if the nmethod has been unloaded. */
223 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
224
225 typedef struct {
226 intptr_t *sp; intptr_t *fp; address lr; address pc;
227 } frame_pointers_t;
228
229 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
230
231 JavaThread *thread = JavaThread::current();
232 RegisterMap reg_map(thread,
233 RegisterMap::UpdateMap::skip,
234 RegisterMap::ProcessFrames::include,
235 RegisterMap::WalkContinuation::skip);
236 frame frame = thread->last_frame();
237
238 assert(frame.is_compiled_frame() || frame.is_native_frame(), "must be");
239 assert(frame.cb() == nm, "must be");
240 frame = frame.sender(®_map);
241
242 LogTarget(Trace, nmethod, barrier) out;
243 if (out.is_enabled()) {
244 ResourceMark mark;
245 log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
246 nm->method()->name_and_sig_as_C_string(),
247 nm, *(address *) return_address_ptr, nm->is_osr_method(), thread,
248 thread->name(), frame.sp(), nm->verified_entry_point());
249 }
250
251 new_frame->sp = frame.sp();
252 new_frame->fp = frame.fp();
253 new_frame->lr = frame.pc();
254 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
255 }
256
257 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
258 if (!supports_entry_barrier(nm)) {
259 return;
260 }
261
262 if (value == disarmed_guard_value()) {
263 // The patching epoch is incremented before the nmethod is disarmed. Disarming
264 // is performed with a release store. In the nmethod entry barrier, the values
265 // are read in the opposite order, such that the load of the nmethod guard
266 // acquires the patching epoch. This way, the guard is guaranteed to block
267 // entries to the nmethod, until it has safely published the requirement for
268 // further fencing by mutators, before they are allowed to enter.
269 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
270 bs_asm->increment_patching_epoch();
271 }
272
273 // Enable WXWrite: the function is called directly from nmethod_entry_barrier
274 // stub.
275 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
276
277 NativeNMethodBarrier barrier(nm);
278 barrier.set_values(value, bit_mask);
279 }
280
281 int BarrierSetNMethod::guard_value(nmethod* nm) {
282 if (!supports_entry_barrier(nm)) {
283 return disarmed_guard_value();
284 }
285
286 NativeNMethodBarrier barrier(nm);
287 return barrier.get_default_guard_value();
288 }