6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "code/codeCache.hpp"
27 #include "code/nativeInst.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "logging/log.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "runtime/frame.inline.hpp"
33 #include "runtime/javaThread.hpp"
34 #include "runtime/registerMap.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "utilities/align.hpp"
37 #include "utilities/debug.hpp"
38
39 static int slow_path_size(nmethod* nm) {
40 // The slow path code is out of line with C2.
41 return nm->is_compiled_by_c2() ? 0 : 4;
42 }
43
44 static int entry_barrier_offset(nmethod* nm) {
45 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
46 switch (bs_asm->nmethod_patching_type()) {
47 case NMethodPatchingType::stw_instruction_and_data_patch:
48 return -4 * (5 + slow_path_size(nm));
49 case NMethodPatchingType::conc_instruction_and_data_patch:
50 return -4 * ((UseZtso ? 14 : 16) + slow_path_size(nm));
51 }
52 ShouldNotReachHere();
53 return 0;
54 }
55
56 class NativeNMethodBarrier {
57 address _instruction_address;
58 int* _guard_addr;
59 nmethod* _nm;
60
61 address instruction_address() const { return _instruction_address; }
62
63 int *guard_addr() {
64 return _guard_addr;
65 }
66
67 int local_guard_offset(nmethod* nm) {
68 // It's the last instruction
69 return (-entry_barrier_offset(nm)) - 4;
70 }
71
72 public:
73 NativeNMethodBarrier(nmethod* nm): _nm(nm) {
74 _instruction_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
75 if (nm->is_compiled_by_c2()) {
76 // With c2 compiled code, the guard is out-of-line in a stub
77 // We find it using the RelocIterator.
78 RelocIterator iter(nm);
79 while (iter.next()) {
80 if (iter.type() == relocInfo::entry_guard_type) {
81 entry_guard_Relocation* const reloc = iter.entry_guard_reloc();
82 _guard_addr = reinterpret_cast<int*>(reloc->addr());
83 return;
84 }
85 }
86
87 ShouldNotReachHere();
88 }
89 _guard_addr = reinterpret_cast<int*>(instruction_address() + local_guard_offset(nm));
90
91 // Perform the checking as verification.
92 err_msg msg("%s", "");
93 assert(check_barrier(msg), "%s", msg.buffer());
94 }
95
96 int get_value() {
97 return AtomicAccess::load_acquire(guard_addr());
98 }
99
100 void set_value(int value, int bit_mask) {
101 if (bit_mask == ~0) {
102 AtomicAccess::release_store(guard_addr(), value);
103 return;
104 }
105 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
106 value &= bit_mask;
107 int old_value = AtomicAccess::load(guard_addr());
108 while (true) {
109 // Only bits in the mask are changed
110 int new_value = value | (old_value & ~bit_mask);
111 if (new_value == old_value) break;
112 int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
113 if (v == old_value) break;
114 old_value = v;
115 }
116 }
117
118 bool check_barrier(err_msg& msg) const;
119 };
120
121 // Store the instruction bitmask, bits and name for checking the barrier.
122 struct CheckInsn {
123 uint32_t mask;
124 uint32_t bits;
125 const char *name;
126 };
127
128 static const struct CheckInsn barrierInsn[] = {
129 { 0x00000fff, 0x00000297, "auipc t0, 0 " },
130 { 0x000fffff, 0x0002e283, "lwu t0, guard_offset(t0)" },
131 /* ...... */
132 /* ...... */
133 /* guard: */
134 /* 32bit nmethod guard value */
135 };
136
137 // The encodings must match the instructions emitted by
138 // BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific
139 // register numbers and immediate values in the encoding.
140 bool NativeNMethodBarrier::check_barrier(err_msg& msg) const {
141 address addr = instruction_address();
142 for (unsigned int i = 0; i < sizeof(barrierInsn) / sizeof(struct CheckInsn); i++) {
143 uint32_t inst = Assembler::ld_instr(addr);
144 if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) {
145 msg.print("Nmethod entry barrier did not start with auipc & lwu as expected. "
146 "Addr: " INTPTR_FORMAT " Code: 0x%x not an %s instruction.", p2i(addr), inst, barrierInsn[i].name);
147 return false;
148 }
149 addr += 4;
150 }
151 return true;
152 }
153
154
155 /* We're called from an nmethod when we need to deoptimize it. We do
156 this by throwing away the nmethod's frame and jumping to the
157 ic_miss stub. This looks like there has been an IC miss at the
158 entry of the nmethod, so we resolve the call, which will fall back
159 to the interpreter if the nmethod has been unloaded. */
160 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
161
191 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
192 }
193
194 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
195 if (!supports_entry_barrier(nm)) {
196 return;
197 }
198
199 if (value == disarmed_guard_value()) {
200 // The patching epoch is incremented before the nmethod is disarmed. Disarming
201 // is performed with a release store. In the nmethod entry barrier, the values
202 // are read in the opposite order, such that the load of the nmethod guard
203 // acquires the patching epoch. This way, the guard is guaranteed to block
204 // entries to the nmethod, until it has safely published the requirement for
205 // further fencing by mutators, before they are allowed to enter.
206 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
207 bs_asm->increment_patching_epoch();
208 }
209
210 NativeNMethodBarrier barrier(nm);
211 barrier.set_value(value, bit_mask);
212 }
213
214 int BarrierSetNMethod::guard_value(nmethod* nm) {
215 if (!supports_entry_barrier(nm)) {
216 return disarmed_guard_value();
217 }
218
219 NativeNMethodBarrier barrier(nm);
220 return barrier.get_value();
221 }
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/nativeInst.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/barrierSetNMethod.hpp"
32 #include "logging/log.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "runtime/frame.inline.hpp"
35 #include "runtime/javaThread.hpp"
36 #include "runtime/registerMap.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 static int slow_path_size(nmethod* nm) {
42 // The slow path code is out of line with C2.
43 return nm->is_compiled_by_c2() ? 0 : 4;
44 }
45
46 static int entry_barrier_offset(nmethod* nm) {
47 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
48 switch (bs_asm->nmethod_patching_type()) {
49 case NMethodPatchingType::stw_instruction_and_data_patch:
50 return -4 * (5 + slow_path_size(nm));
51 case NMethodPatchingType::conc_instruction_and_data_patch:
52 return -4 * ((UseZtso ? 14 : 16) + slow_path_size(nm));
53 }
54 ShouldNotReachHere();
55 return 0;
56 }
57
58 static int* decode_guard_from_instruction(nmethod* nm, address& instruction) {
59 int* result = reinterpret_cast<int*>(MacroAssembler::target_addr_for_insn(instruction));
60 assert(nm->insts_contains(reinterpret_cast<address>(result)) ||
61 nm->stub_contains(reinterpret_cast<address>(result)),
62 "guard must be in nmethod code");
63 return result;
64 }
65
66 // The NativeNMethodBarrier class encapsulates up to three entrypoints and handles their
67 // arming/verification.
68 // An entrypoint is defined as a tuple of <instr. address, guard address>:
69 // * The instr. address corresponds to the ldr of the guard value of that entrypoint.
70 // * The guard address is the address where the guard value of that entrypoint resides.
71 //
72 // Each nmethod has at least one entrypoint. The default must always be well-defined
73 // (neither instruction nor guard are nullptr).
74 //
75 // When using the scalarized calling convention, up to two additional (verified) entrypoints,
76 // alt1 and alt2 can be present. The meaning of these depends on who compiled the nmethod.
77 //
78 // The mapping of C1-compiled methods (scalarization used) looks as follows:
79 // * alt1: verified entry point
80 // * alt2 (optional): verified inline ro entry point
81 //
82 // The mapping of C2-compiled methods (scalarization used) looks as follows:
83 // * alt1: verified inline entry point
84 // * alt2 (optional): verified inline ro entry point
85 //
86 // In other scenarios, neither alt1 nor alt2 are defined.
87 class NativeNMethodBarrier {
88 private:
89 // The addresses of the instructions that act as the guards.
90 address _default_entry_instruction;
91 address _verified_alt1_instruction;
92 address _verified_alt2_instruction;
93 // Pointers representing the actual guard values themselves.
94 int* _default_entry_guard;
95 int* _verified_alt1_guard;
96 int* _verified_alt2_guard;
97
98 public:
99 NativeNMethodBarrier(nmethod* nm) :
100 _default_entry_instruction(nullptr),
101 _verified_alt1_instruction(nullptr),
102 _verified_alt2_instruction(nullptr),
103 _default_entry_guard(nullptr),
104 _verified_alt1_guard(nullptr),
105 _verified_alt2_guard(nullptr)
106 {
107 // The default entry point has a known address. The guard address can be
108 // decoded from the literal in the instruction. Verification will confirm
109 // that this instruction corresponds to a load.
110 _default_entry_instruction = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
111
112 _default_entry_guard = decode_guard_from_instruction(nm, _default_entry_instruction);
113
114 // If the nmethod has scalarized arguments, then there are more entry
115 // points, each with their own nmethod entry barrier.
116 assert(nm->is_osr_method() || !nm->method()->has_scalarized_args(), "unimplemented");
117
118 // Perform the checking as verification.
119 err_msg msg("%s", "");
120 assert(check_barriers(msg), "%s", msg.buffer());
121 }
122
123 // Gets the value of the default entry guard.
124 // This does not consider the alternative entrypoints, as these should
125 // all be consistent. It is up to the caller to enforce this.
126 int get_default_guard_value() {
127 return AtomicAccess::load_acquire(_default_entry_guard);
128 }
129
130 // Sets the value for all barriers.
131 void set_values(int value, int bit_mask) {
132 set_value_impl(_default_entry_guard, value, bit_mask);
133 if (_verified_alt1_guard != nullptr) {
134 set_value_impl(_verified_alt1_guard, value, bit_mask);
135 }
136 if (_verified_alt2_guard != nullptr) {
137 set_value_impl(_verified_alt2_guard, value, bit_mask);
138 }
139 }
140
141 // Verifies that all potential barriers are correct.
142 bool check_barriers(err_msg& msg) {
143 // The default entry barrier should always be checked.
144 if (!check_barrier_impl(_default_entry_instruction, msg)) {
145 return false;
146 }
147 // Check the alternative entry barriers only if they are specified.
148 // Note that the guard values are already validated at construction time,
149 // if they fall out of the nmethod range, this will be caught earlier.
150 if (_verified_alt1_instruction != nullptr &&
151 !check_barrier_impl(_verified_alt1_instruction, msg)) {
152 return false;
153 }
154 if (_verified_alt2_instruction != nullptr &&
155 !check_barrier_impl(_verified_alt2_instruction, msg)) {
156 return false;
157 }
158 return true;
159 }
160
161 private:
162 // Sets the value for a single barrier.
163 void set_value_impl(int* guard, int value, int bit_mask) {
164 if (bit_mask == ~0) {
165 AtomicAccess::release_store(guard, value);
166 return;
167 }
168 assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
169 value &= bit_mask;
170 int old_value = AtomicAccess::load(guard);
171 while (true) {
172 // Only bits in the mask are changed
173 int new_value = value | (old_value & ~bit_mask);
174 if (new_value == old_value) break;
175 int v = AtomicAccess::cmpxchg(guard, old_value, new_value, memory_order_release);
176 if (v == old_value) break;
177 old_value = v;
178 }
179 }
180
181 // Checks the validity of a single barrier.
182 bool check_barrier_impl(address& instruction, err_msg& msg) const;
183 };
184
185 // Store the instruction bitmask, bits and name for checking the barrier.
186 struct CheckInsn {
187 uint32_t mask;
188 uint32_t bits;
189 const char *name;
190 };
191
192 static const struct CheckInsn barrierInsn[] = {
193 { 0x00000fff, 0x00000297, "auipc t0, 0 " },
194 { 0x000fffff, 0x0002e283, "lwu t0, guard_offset(t0)" },
195 /* ...... */
196 /* ...... */
197 /* guard: */
198 /* 32bit nmethod guard value */
199 };
200
201 // The encodings must match the instructions emitted by
202 // BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific
203 // register numbers and immediate values in the encoding.
204 bool NativeNMethodBarrier::check_barrier_impl(address& instruction, err_msg& msg) const {
205 address addr = instruction;
206 for (unsigned int i = 0; i < sizeof(barrierInsn) / sizeof(struct CheckInsn); i++) {
207 uint32_t inst = Assembler::ld_instr(addr);
208 if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) {
209 msg.print("Nmethod entry barrier did not start with auipc & lwu as expected. "
210 "Addr: " INTPTR_FORMAT " Code: 0x%x not an %s instruction.", p2i(addr), inst, barrierInsn[i].name);
211 return false;
212 }
213 addr += 4;
214 }
215 return true;
216 }
217
218
219 /* We're called from an nmethod when we need to deoptimize it. We do
220 this by throwing away the nmethod's frame and jumping to the
221 ic_miss stub. This looks like there has been an IC miss at the
222 entry of the nmethod, so we resolve the call, which will fall back
223 to the interpreter if the nmethod has been unloaded. */
224 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
225
255 new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
256 }
257
258 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
259 if (!supports_entry_barrier(nm)) {
260 return;
261 }
262
263 if (value == disarmed_guard_value()) {
264 // The patching epoch is incremented before the nmethod is disarmed. Disarming
265 // is performed with a release store. In the nmethod entry barrier, the values
266 // are read in the opposite order, such that the load of the nmethod guard
267 // acquires the patching epoch. This way, the guard is guaranteed to block
268 // entries to the nmethod, until it has safely published the requirement for
269 // further fencing by mutators, before they are allowed to enter.
270 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
271 bs_asm->increment_patching_epoch();
272 }
273
274 NativeNMethodBarrier barrier(nm);
275 barrier.set_values(value, bit_mask);
276 }
277
278 int BarrierSetNMethod::guard_value(nmethod* nm) {
279 if (!supports_entry_barrier(nm)) {
280 return disarmed_guard_value();
281 }
282
283 NativeNMethodBarrier barrier(nm);
284 return barrier.get_default_guard_value();
285 }
|