5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "code/codeCache.hpp"
26 #include "code/nativeInst.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "logging/log.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "runtime/frame.inline.hpp"
33 #include "runtime/javaThread.hpp"
34 #include "runtime/registerMap.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "utilities/align.hpp"
37 #include "utilities/debug.hpp"
38 #include "utilities/formatBuffer.hpp"
39
40 static int slow_path_size(nmethod* nm) {
41 // The slow path code is out of line with C2
42 return nm->is_compiled_by_c2() ? 0 : 6;
43 }
44
45 // This is the offset of the entry barrier relative to where the frame is completed.
46 // If any code changes between the end of the verified entry where the entry
47 // barrier resides, and the completion of the frame, then
48 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
49 // not find the expected native instruction at this offset, which needs updating.
50 // Note that this offset is invariant of PreserveFramePointer.
51 static int entry_barrier_offset(nmethod* nm) {
52 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
53 switch (bs_asm->nmethod_patching_type()) {
54 case NMethodPatchingType::stw_instruction_and_data_patch:
55 return -4 * (4 + slow_path_size(nm));
56 case NMethodPatchingType::conc_instruction_and_data_patch:
57 return -4 * (10 + slow_path_size(nm));
58 }
59 ShouldNotReachHere();
60 return 0;
61 }
62
63 class NativeNMethodBarrier {
64 address _instruction_address;
65 int* _guard_addr;
66 nmethod* _nm;
67
68 address instruction_address() const { return _instruction_address; }
69
70 int *guard_addr() {
71 return _guard_addr;
72 }
73
74 int local_guard_offset(nmethod* nm) {
75 // It's the last instruction
76 return (-entry_barrier_offset(nm)) - 4;
77 }
78
79 public:
80 NativeNMethodBarrier(nmethod* nm): _nm(nm) {
81 _instruction_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
82 if (nm->is_compiled_by_c2()) {
83 // With c2 compiled code, the guard is out-of-line in a stub
84 // We find it using the RelocIterator.
85 RelocIterator iter(nm);
86 while (iter.next()) {
87 if (iter.type() == relocInfo::entry_guard_type) {
88 entry_guard_Relocation* const reloc = iter.entry_guard_reloc();
89 _guard_addr = reinterpret_cast<int*>(reloc->addr());
90 return;
91 }
92 }
93 ShouldNotReachHere();
94 }
95 _guard_addr = reinterpret_cast<int*>(instruction_address() + local_guard_offset(nm));
96 }
97
98 int get_value() {
99 return AtomicAccess::load_acquire(guard_addr());
100 }
101
102 void set_value(int value, int bit_mask) {
103 if (bit_mask == ~0) {
104 AtomicAccess::release_store(guard_addr(), value);
105 return;
106 }
107 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
108 value &= bit_mask;
109 int old_value = AtomicAccess::load(guard_addr());
110 while (true) {
111 // Only bits in the mask are changed
112 int new_value = value | (old_value & ~bit_mask);
113 if (new_value == old_value) break;
114 int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
115 if (v == old_value) break;
116 old_value = v;
117 }
118 }
119
120 bool check_barrier(err_msg& msg) const;
121 void verify() const {
122 err_msg msg("%s", "");
123 assert(check_barrier(msg), "%s", msg.buffer());
124 }
125 };
126
127 // The first instruction of the nmethod entry barrier is an ldrw (literal)
128 // instruction. Verify that it's really there, so the offsets are not skewed.
129 bool NativeNMethodBarrier::check_barrier(err_msg& msg) const {
130 NativeInstruction* ni = nativeInstruction_at(instruction_address());
131 if (!ni->is_ldrw_gpr_literal()) {
132 msg.print("Nmethod entry barrier did not start with ldrw (literal) as expected. "
133 "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(instruction_address()), ni->encoding());
134 return false;
135 }
136 return true;
137 }
138
139
140 /* We're called from an nmethod when we need to deoptimize it. We do
141 this by throwing away the nmethod's frame and jumping to the
142 ic_miss stub. This looks like there has been an IC miss at the
143 entry of the nmethod, so we resolve the call, which will fall back
144 to the interpreter if the nmethod has been unloaded. */
145 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
146
147 typedef struct {
148 intptr_t *sp; intptr_t *fp; address lr; address pc;
149 } frame_pointers_t;
150
151 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
152
153 JavaThread *thread = JavaThread::current();
154 RegisterMap reg_map(thread,
155 RegisterMap::UpdateMap::skip,
156 RegisterMap::ProcessFrames::include,
157 RegisterMap::WalkContinuation::skip);
158 frame frame = thread->last_frame();
159
180 if (!supports_entry_barrier(nm)) {
181 return;
182 }
183
184 if (value == disarmed_guard_value()) {
185 // The patching epoch is incremented before the nmethod is disarmed. Disarming
186 // is performed with a release store. In the nmethod entry barrier, the values
187 // are read in the opposite order, such that the load of the nmethod guard
188 // acquires the patching epoch. This way, the guard is guaranteed to block
189 // entries to the nmethod, until it has safely published the requirement for
190 // further fencing by mutators, before they are allowed to enter.
191 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
192 bs_asm->increment_patching_epoch();
193 }
194
195 // Enable WXWrite: the function is called directly from nmethod_entry_barrier
196 // stub.
197 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
198
199 NativeNMethodBarrier barrier(nm);
200 barrier.set_value(value, bit_mask);
201 }
202
203 int BarrierSetNMethod::guard_value(nmethod* nm) {
204 if (!supports_entry_barrier(nm)) {
205 return disarmed_guard_value();
206 }
207
208 NativeNMethodBarrier barrier(nm);
209 return barrier.get_value();
210 }
|
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/nativeInst.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "logging/log.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "runtime/frame.inline.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/registerMap.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/formatBuffer.hpp"
40
41 static int slow_path_size(nmethod* nm) {
42 // The slow path code is out of line with C2
43 return nm->is_compiled_by_c2() ? 0 : 6;
44 }
45
46 // This is the offset of the entry barrier relative to where the frame is completed.
47 // If any code changes between the end of the verified entry where the entry
48 // barrier resides, and the completion of the frame, then
49 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
50 // not find the expected native instruction at this offset, which needs updating.
51 // Note that this offset is invariant of PreserveFramePointer.
52 static int entry_barrier_offset(nmethod* nm) {
53 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
54 switch (bs_asm->nmethod_patching_type()) {
55 case NMethodPatchingType::stw_instruction_and_data_patch:
56 return -4 * (4 + slow_path_size(nm));
57 case NMethodPatchingType::conc_instruction_and_data_patch:
58 return -4 * (10 + slow_path_size(nm));
59 }
60 ShouldNotReachHere();
61 return 0;
62 }
63
64 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
65 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
66 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
67 nm->stub_contains(reinterpret_cast<address>(result)),
68 "guard must be in nmethod code");
69 return result;
70 }
71
72 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
73 // arming/verification.
74 // An entrypoint is defined as a tuple of <instr. address, guard address>:
75 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
76 // * The guard address is the address where the guard value of that entrypoint resides.
77 //
78 // Each nmethod has at least one entrypoint. The default must always be well-defined
79 // (neither instruction nor guard are nullptr).
80 //
81 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
82 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
83 //
84 // The mapping of C1-compiled methods (scalarization used) looks as follows:
85 // * alt1: verified entry point
86 // * alt2 (optional): verified inline ro entry point
87 //
88 // The mapping of C2-compiled methods (scalarization used) looks as follows:
89 // * alt1: verified inline entry point
90 // * alt2 (optional): verified inline ro entry point
91 //
92 // In other scenarios, neither alt1 nor alt2 are defined.
93 class NativeNMethodBarrier {
94 private:
95 // The addresses of the instructions that act as the guards.
96 address _default_entry_instruction;
97 address _verified_alt1_instruction;
98 address _verified_alt2_instruction;
99 // Pointers representing the actual guard values themselves.
100 int* _default_entry_guard;
101 int* _verified_alt1_guard;
102 int* _verified_alt2_guard;
103
104 public:
105 NativeNMethodBarrier(nmethod* nm) :
106 _default_entry_instruction(nullptr),
107 _verified_alt1_instruction(nullptr),
108 _verified_alt2_instruction(nullptr),
109 _default_entry_guard(nullptr),
110 _verified_alt1_guard(nullptr),
111 _verified_alt2_guard(nullptr)
112 {
113 // The default entry point has a known address. The guard address can be
114 // decoded from the literal in the instruction. Verification will confirm
115 // that this instruction corresponds to a load.
116 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
117 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
118
119 // If the nmethod has scalarized arguments, then there are more entry
120 // points, each with their own nmethod entry barrier.
121 if (!nm->is_osr_method() && nm->method()->has_scalarized_args()) {
122 assert(nm->verified_entry_point() != nm->verified_inline_entry_point(), "scalarized entry point not found");
123 address method_body = nm->is_compiled_by_c1() ? nm->verified_inline_entry_point() : nm->verified_entry_point();
124 int barrier_offset = _default_entry_instruction - method_body;
125
126 // Set the first alternative entry point.
127 address entry_point2 = nm->is_compiled_by_c1() ? nm->verified_entry_point() : nm->verified_inline_entry_point();
128 _verified_alt1_instruction = entry_point2 + barrier_offset;
129 assert(_default_entry_instruction != _verified_alt1_instruction, "sanity");
130 _verified_alt1_guard = decode_guard_from_instruction(nm, _verified_alt1_instruction);
131
132 // If there is a second alternative entry point, set it too.
133 if (method_body != nm->verified_inline_ro_entry_point() && entry_point2 != nm->verified_inline_ro_entry_point()) {
134 _verified_alt2_instruction = nm->verified_inline_ro_entry_point() + barrier_offset;
135 _verified_alt2_guard = decode_guard_from_instruction(nm, _verified_alt2_instruction);
136 assert(_default_entry_instruction != _verified_alt2_instruction &&
137 _verified_alt1_instruction != _verified_alt2_instruction,
138 "sanity");
139 }
140 }
141 // Perform the checking as verification.
142 err_msg msg("%s", "");
143 assert(check_barriers(msg), "%s", msg.buffer());
144 }
145
146 // Gets the value of the default entry guard.
147 // This does not consider the alternative entrypoints, as these should
148 // all be consistent. It is up to the caller to enforce this.
149 int get_default_guard_value() {
150 return AtomicAccess::load_acquire(_default_entry_guard);
151 }
152
153 // Sets the value for all barriers.
154 void set_values(int value, int bit_mask) {
155 set_value_impl(_default_entry_guard, value, bit_mask);
156 if (_verified_alt1_guard != nullptr) {
157 set_value_impl(_verified_alt1_guard, value, bit_mask);
158 }
159 if (_verified_alt2_guard != nullptr) {
160 set_value_impl(_verified_alt2_guard, value, bit_mask);
161 }
162 }
163
164 // Verifies that all potential barriers are correct.
165 bool check_barriers(err_msg& msg) {
166 // The default entry barrier should always be checked.
167 if (!check_barrier_impl(_default_entry_instruction, msg)) {
168 return false;
169 }
170 // Check the alternative entry barriers only if they are specified.
171 // Note that the guard values are already validated at construction time,
172 // if they fall out of the nmethod range, this will be caught earlier.
173 if (_verified_alt1_instruction != nullptr &&
174 !check_barrier_impl(_verified_alt1_instruction, msg)) {
175 return false;
176 }
177 if (_verified_alt2_instruction != nullptr &&
178 !check_barrier_impl(_verified_alt2_instruction, msg)) {
179 return false;
180 }
181 return true;
182 }
183
184 private:
185 // Sets the value for a single barrier.
186 void set_value_impl(int* guard, int value, int bit_mask) {
187 if (bit_mask == ~0) {
188 AtomicAccess::release_store(guard, value);
189 return;
190 }
191 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
192 value &= bit_mask;
193 int old_value = AtomicAccess::load(guard);
194 while (true) {
195 // Only bits in the mask are changed
196 int new_value = value | (old_value & ~bit_mask);
197 if (new_value == old_value) break;
198 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
199 if (v == old_value) break;
200 old_value = v;
201 }
202 }
203
204 // Checks the validity of a single barrier.
205 // The first instruction of the nmethod entry barrier is an ldrw (literal)
206 // instruction. Verify that it's really there, so the offsets are not skewed.
207 bool check_barrier_impl(address& instruction, err_msg& msg) {
208 NativeInstruction* ni = nativeInstruction_at(instruction);
209 if (!ni->is_ldrw_gpr_literal()) {
210 msg.print("Nmethod entry barrier did not start with ldrw (literal) as expected. "
211 "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(instruction), ni->encoding());
212 return false;
213 }
214 return true;
215 }
216 };
217
218 /* We're called from an nmethod when we need to deoptimize it. We do
219 this by throwing away the nmethod's frame and jumping to the
220 ic_miss stub. This looks like there has been an IC miss at the
221 entry of the nmethod, so we resolve the call, which will fall back
222 to the interpreter if the nmethod has been unloaded. */
223 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
224
225 typedef struct {
226 intptr_t *sp; intptr_t *fp; address lr; address pc;
227 } frame_pointers_t;
228
229 frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5);
230
231 JavaThread *thread = JavaThread::current();
232 RegisterMap reg_map(thread,
233 RegisterMap::UpdateMap::skip,
234 RegisterMap::ProcessFrames::include,
235 RegisterMap::WalkContinuation::skip);
236 frame frame = thread->last_frame();
237
258 if (!supports_entry_barrier(nm)) {
259 return;
260 }
261
262 if (value == disarmed_guard_value()) {
263 // The patching epoch is incremented before the nmethod is disarmed. Disarming
264 // is performed with a release store. In the nmethod entry barrier, the values
265 // are read in the opposite order, such that the load of the nmethod guard
266 // acquires the patching epoch. This way, the guard is guaranteed to block
267 // entries to the nmethod, until it has safely published the requirement for
268 // further fencing by mutators, before they are allowed to enter.
269 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
270 bs_asm->increment_patching_epoch();
271 }
272
273 // Enable WXWrite: the function is called directly from nmethod_entry_barrier
274 // stub.
275 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
276
277 NativeNMethodBarrier barrier(nm);
278 barrier.set_values(value, bit_mask);
279 }
280
281 int BarrierSetNMethod::guard_value(nmethod* nm) {
282 if (!supports_entry_barrier(nm)) {
283 return disarmed_guard_value();
284 }
285
286 NativeNMethodBarrier barrier(nm);
287 return barrier.get_default_guard_value();
288 }
|