166 }
167 BarrierSetNMethodArmClosure cl(_current_phase);
168 Threads::threads_do(&cl);
169
170 #if (defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
171 // We clear the patching epoch when disarming nmethods, so that
172 // the counter won't overflow.
173 BarrierSetAssembler::clear_patching_epoch();
174 #endif
175 }
176
177 int BarrierSetNMethod::nmethod_stub_entry_barrier(address* return_address_ptr) {
178 address return_address = *return_address_ptr;
179 AARCH64_PORT_ONLY(return_address = pauth_strip_pointer(return_address));
180 CodeBlob* cb = CodeCache::find_blob(return_address);
181 assert(cb != nullptr, "invariant");
182
183 nmethod* nm = cb->as_nmethod();
184 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
185
186 // Called upon first entry after being armed
187 bool may_enter = !bs_nm->is_not_entrant(nm) && bs_nm->nmethod_entry_barrier(nm);
188 assert(!nm->is_osr_method() || may_enter, "OSR nmethods should always be entrant after migration");
189
190 if (may_enter) {
191 // In case a concurrent thread disarmed the nmethod, we need to ensure the new instructions
192 // are made visible, by using a cross modify fence. Note that this is synchronous cross modifying
193 // code, where the existence of new instructions is communicated via data (the guard value).
194 // This cross modify fence is only needed when the nmethod entry barrier modifies the
195 // instructions. Not all platforms currently do that, so if this check becomes expensive,
196 // it can be made conditional on the nmethod_patching_type.
197 OrderAccess::cross_modify_fence();
198
199 // Diagnostic option to force deoptimization 1 in 10 times. It is otherwise
200 // a very rare event.
201 if (DeoptimizeNMethodBarriersALot && !nm->is_osr_method()) {
202 static Atomic<uint32_t> counter{0};
203 if (counter.add_then_fetch(1u) % 10 == 0) {
204 may_enter = false;
205 }
206 }
207 }
208
209 if (!may_enter) {
210 log_trace(nmethod, barrier)("Deoptimizing nmethod: " PTR_FORMAT, p2i(nm));
211 bs_nm->deoptimize(nm, return_address_ptr);
212 }
213 return may_enter ? 0 : 1;
214 }
215
216 bool BarrierSetNMethod::nmethod_osr_entry_barrier(nmethod* nm) {
217 assert(nm->is_osr_method(), "Should not reach here");
218 log_trace(nmethod, barrier)("Running osr nmethod entry barrier: " PTR_FORMAT, p2i(nm));
219 bool result = nmethod_entry_barrier(nm);
220 OrderAccess::cross_modify_fence();
221 return result;
222 }
223
224 oop BarrierSetNMethod::oop_load_no_keepalive(const nmethod* nm, int index) {
225 return NativeAccess<AS_NO_KEEPALIVE>::oop_load(nm->oop_addr_at(index));
226 }
227
228 oop BarrierSetNMethod::oop_load_phantom(const nmethod* nm, int index) {
229 return NativeAccess<ON_PHANTOM_OOP_REF>::oop_load(nm->oop_addr_at(index));
230 }
231
232 // Make the nmethod permanently not-entrant, so that nmethod_stub_entry_barrier() will call
233 // deoptimize() to redirect the caller to SharedRuntime::get_handle_wrong_method_stub().
234 // A sticky armed bit is set and other bits are preserved. As a result, a call to
235 // nmethod_stub_entry_barrier() may appear to be spurious, because is_armed() still returns
236 // false and nmethod_entry_barrier() is not called.
237 void BarrierSetNMethod::make_not_entrant(nmethod* nm) {
238 set_guard_value(nm, not_entrant, not_entrant);
239 }
|
166 }
167 BarrierSetNMethodArmClosure cl(_current_phase);
168 Threads::threads_do(&cl);
169
170 #if (defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
171 // We clear the patching epoch when disarming nmethods, so that
172 // the counter won't overflow.
173 BarrierSetAssembler::clear_patching_epoch();
174 #endif
175 }
176
177 int BarrierSetNMethod::nmethod_stub_entry_barrier(address* return_address_ptr) {
178 address return_address = *return_address_ptr;
179 AARCH64_PORT_ONLY(return_address = pauth_strip_pointer(return_address));
180 CodeBlob* cb = CodeCache::find_blob(return_address);
181 assert(cb != nullptr, "invariant");
182
183 nmethod* nm = cb->as_nmethod();
184 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
185
186 log_trace(nmethod, barrier)("Running nmethod entry barrier: %d " PTR_FORMAT, nm->compile_id(), p2i(nm));
187 // Called upon first entry after being armed
188 bool may_enter = !bs_nm->is_not_entrant(nm) && bs_nm->nmethod_entry_barrier(nm);
189 assert(!nm->is_osr_method() || may_enter, "OSR nmethods should always be entrant after migration");
190
191 if (may_enter) {
192 // In case a concurrent thread disarmed the nmethod, we need to ensure the new instructions
193 // are made visible, by using a cross modify fence. Note that this is synchronous cross modifying
194 // code, where the existence of new instructions is communicated via data (the guard value).
195 // This cross modify fence is only needed when the nmethod entry barrier modifies the
196 // instructions. Not all platforms currently do that, so if this check becomes expensive,
197 // it can be made conditional on the nmethod_patching_type.
198 OrderAccess::cross_modify_fence();
199
200 // Diagnostic option to force deoptimization 1 in 10 times. It is otherwise
201 // a very rare event.
202 if (DeoptimizeNMethodBarriersALot && !nm->is_osr_method()) {
203 static Atomic<uint32_t> counter{0};
204 if (counter.add_then_fetch(1u) % 10 == 0) {
205 may_enter = false;
206 }
207 }
208 }
209
210 if (may_enter) {
211 MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
212 nm->set_used();
213 } else {
214 log_trace(nmethod, barrier)("Deoptimizing nmethod: " PTR_FORMAT, p2i(nm));
215 bs_nm->deoptimize(nm, return_address_ptr);
216 }
217 return may_enter ? 0 : 1;
218 }
219
220 bool BarrierSetNMethod::nmethod_osr_entry_barrier(nmethod* nm) {
221 assert(nm->is_osr_method(), "Should not reach here");
222 log_trace(nmethod, barrier)("Running osr nmethod entry barrier: %d " PTR_FORMAT, nm->compile_id(), p2i(nm));
223 bool result = nmethod_entry_barrier(nm);
224 if (result) {
225 nm->set_used();
226 }
227 OrderAccess::cross_modify_fence();
228 return result;
229 }
230
231 oop BarrierSetNMethod::oop_load_no_keepalive(const nmethod* nm, int index) {
232 return NativeAccess<AS_NO_KEEPALIVE>::oop_load(nm->oop_addr_at(index));
233 }
234
235 oop BarrierSetNMethod::oop_load_phantom(const nmethod* nm, int index) {
236 return NativeAccess<ON_PHANTOM_OOP_REF>::oop_load(nm->oop_addr_at(index));
237 }
238
239 // Make the nmethod permanently not-entrant, so that nmethod_stub_entry_barrier() will call
240 // deoptimize() to redirect the caller to SharedRuntime::get_handle_wrong_method_stub().
241 // A sticky armed bit is set and other bits are preserved. As a result, a call to
242 // nmethod_stub_entry_barrier() may appear to be spurious, because is_armed() still returns
243 // false and nmethod_entry_barrier() is not called.
244 void BarrierSetNMethod::make_not_entrant(nmethod* nm) {
245 set_guard_value(nm, not_entrant, not_entrant);
246 }
|