1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/nativeInst.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/barrierSetNMethod.hpp"
32 #include "logging/log.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "runtime/frame.inline.hpp"
35 #include "runtime/javaThread.hpp"
36 #include "runtime/registerMap.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 static int slow_path_size(nmethod* nm) {
42 // The slow path code is out of line with C2.
43 return nm->is_compiled_by_c2() ? 0 : 4;
44 }
45
46 static int entry_barrier_offset(nmethod* nm) {
47 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
48 switch (bs_asm->nmethod_patching_type()) {
49 case NMethodPatchingType::stw_instruction_and_data_patch:
50 return -4 * (5 + slow_path_size(nm));
51 case NMethodPatchingType::conc_instruction_and_data_patch:
52 return -4 * ((UseZtso ? 14 : 16) + slow_path_size(nm));
53 }
54 ShouldNotReachHere();
55 return 0;
56 }
57
58 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
59 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
60 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
61 nm->stub_contains(reinterpret_cast<address>(result)),
62 "guard must be in nmethod code");
63 return result;
64 }
65
66 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
67 // arming/verification.
68 // An entrypoint is defined as a tuple of <instr. address, guard address>:
69 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
70 // * The guard address is the address where the guard value of that entrypoint resides.
71 //
72 // Each nmethod has at least one entrypoint. The default must always be well-defined
73 // (neither instruction nor guard are nullptr).
74 //
75 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
76 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
77 //
78 // The mapping of C1-compiled methods (scalarization used) looks as follows:
79 // * alt1: verified entry point
80 // * alt2 (optional): verified inline ro entry point
81 //
82 // The mapping of C2-compiled methods (scalarization used) looks as follows:
83 // * alt1: verified inline entry point
84 // * alt2 (optional): verified inline ro entry point
85 //
86 // In other scenarios, neither alt1 nor alt2 are defined.
87 class NativeNMethodBarrier {
88 private:
89 // The addresses of the instructions that act as the guards.
90 address _default_entry_instruction;
91 address _verified_alt1_instruction;
92 address _verified_alt2_instruction;
93 // Pointers representing the actual guard values themselves.
94 int* _default_entry_guard;
95 int* _verified_alt1_guard;
96 int* _verified_alt2_guard;
97
98 public:
99 NativeNMethodBarrier(nmethod* nm) :
100 _default_entry_instruction(nullptr),
101 _verified_alt1_instruction(nullptr),
102 _verified_alt2_instruction(nullptr),
103 _default_entry_guard(nullptr),
104 _verified_alt1_guard(nullptr),
105 _verified_alt2_guard(nullptr)
106 {
107 // The default entry point has a known address. The guard address can be
108 // decoded from the literal in the instruction. Verification will confirm
109 // that this instruction corresponds to a load.
110 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
111
112 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
113
114 // If the nmethod has scalarized arguments, then there are more entry
115 // points, each with their own nmethod entry barrier.
116 assert(nm->is_osr_method() || !nm->method()->has_scalarized_args(), "unimplemented");
117
118 // Perform the checking as verification.
119 err_msg msg("%s", "");
120 assert(check_barriers(msg), "%s", msg.buffer());
121 }
122
123 // Gets the value of the default entry guard.
124 // This does not consider the alternative entrypoints, as these should
125 // all be consistent. It is up to the caller to enforce this.
126 int get_default_guard_value() {
127 return AtomicAccess::load_acquire(_default_entry_guard);
128 }
129
130 // Sets the value for all barriers.
131 void set_values(int value, int bit_mask) {
132 set_value_impl(_default_entry_guard, value, bit_mask);
133 if (_verified_alt1_guard != nullptr) {
134 set_value_impl(_verified_alt1_guard, value, bit_mask);
135 }
136 if (_verified_alt2_guard != nullptr) {
137 set_value_impl(_verified_alt2_guard, value, bit_mask);
138 }
139 }
140
141 // Verifies that all potential barriers are correct.
142 bool check_barriers(err_msg& msg) {
143 // The default entry barrier should always be checked.
144 if (!check_barrier_impl(_default_entry_instruction, msg)) {
145 return false;
146 }
147 // Check the alternative entry barriers only if they are specified.
148 // Note that the guard values are already validated at construction time,
149 // if they fall out of the nmethod range, this will be caught earlier.
150 if (_verified_alt1_instruction != nullptr &&
151 !check_barrier_impl(_verified_alt1_instruction, msg)) {
152 return false;
153 }
154 if (_verified_alt2_instruction != nullptr &&
155 !check_barrier_impl(_verified_alt2_instruction, msg)) {
156 return false;
157 }
158 return true;
159 }
160
161 private:
162 // Sets the value for a single barrier.
163 void set_value_impl(int* guard, int value, int bit_mask) {
164 if (bit_mask == ~0) {
165 AtomicAccess::release_store(guard, value);
166 return;
167 }
168 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
169 value &= bit_mask;
170 int old_value = AtomicAccess::load(guard);
171 while (true) {
172 // Only bits in the mask are changed
173 int new_value = value | (old_value & ~bit_mask);
174 if (new_value == old_value) break;
175 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
176 if (v == old_value) break;
177 old_value = v;
178 }
179 }
180
181 // Checks the validity of a single barrier.
182 bool check_barrier_impl(address& instruction, err_msg& msg) const;
183 };
184
185 // Store the instruction bitmask, bits and name for checking the barrier.
186 struct CheckInsn {
187 uint32_t mask;
188 uint32_t bits;
189 const char *name;
190 };
191
192 static const struct CheckInsn barrierInsn[] = {
193 { 0x00000fff, 0x00000297, "auipc t0, 0 " },
194 { 0x000fffff, 0x0002e283, "lwu t0, guard_offset(t0)" },
195 /* ...... */
196 /* ...... */
197 /* guard: */
198 /* 32bit nmethod guard value */
199 };
200
201 // The encodings must match the instructions emitted by
202 // BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific
203 // register numbers and immediate values in the encoding.
204 bool NativeNMethodBarrier::check_barrier_impl(address& instruction, err_msg& msg) const {
205 address addr = instruction;
206 for (unsigned int i = 0; i < sizeof(barrierInsn) / sizeof(struct CheckInsn); i++) {
207 uint32_t inst = Assembler::ld_instr(addr);
208 if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) {
209 msg.print("Nmethod entry barrier did not start with auipc & lwu as expected. "
210 "Addr: " INTPTR_FORMAT " Code: 0x%x not an %s instruction.", p2i(addr), inst, barrierInsn[i].name);
211 return false;
212 }
213 addr += 4;
214 }
215 return true;
216 }
217
218
219 /* We're called from an nmethod when we need to deoptimize it. We do
220 this by throwing away the nmethod's frame and jumping to the
221 ic_miss stub. This looks like there has been an IC miss at the
222 entry of the nmethod, so we resolve the call, which will fall back
223 to the interpreter if the nmethod has been unloaded. */
224 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
225
226 typedef struct {
227 intptr_t *sp; intptr_t *fp; address ra; address pc;
228 } frame_pointers_t;
229
230 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
231
232 JavaThread *thread = JavaThread::current();
233 RegisterMap reg_map(thread,
234 RegisterMap::UpdateMap::skip,
235 RegisterMap::ProcessFrames::include,
236 RegisterMap::WalkContinuation::skip);
237 frame frame = thread->last_frame();
238
239 assert(frame.is_compiled_frame() || frame.is_native_frame(), "must be");
240 assert(frame.cb() == nm, "must be");
241 frame = frame.sender(®_map);
242
243 LogTarget(Trace, nmethod, barrier) out;
244 if (out.is_enabled()) {
245 ResourceMark mark;
246 log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
247 nm->method()->name_and_sig_as_C_string(),
248 nm, *(address *) return_address_ptr, nm->is_osr_method(), thread,
249 thread->name(), frame.sp(), nm->verified_entry_point());
250 }
251
252 new_frame->sp = frame.sp();
253 new_frame->fp = frame.fp();
254 new_frame->ra = frame.pc();
255 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
256 }
257
258 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
259 if (!supports_entry_barrier(nm)) {
260 return;
261 }
262
263 if (value == disarmed_guard_value()) {
264 // The patching epoch is incremented before the nmethod is disarmed. Disarming
265 // is performed with a release store. In the nmethod entry barrier, the values
266 // are read in the opposite order, such that the load of the nmethod guard
267 // acquires the patching epoch. This way, the guard is guaranteed to block
268 // entries to the nmethod, until it has safely published the requirement for
269 // further fencing by mutators, before they are allowed to enter.
270 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
271 bs_asm->increment_patching_epoch();
272 }
273
274 NativeNMethodBarrier barrier(nm);
275 barrier.set_values(value, bit_mask);
276 }
277
278 int BarrierSetNMethod::guard_value(nmethod* nm) {
279 if (!supports_entry_barrier(nm)) {
280 return disarmed_guard_value();
281 }
282
283 NativeNMethodBarrier barrier(nm);
284 return barrier.get_default_guard_value();
285 }