1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "code/codeCache.hpp"
 27 #include "code/nmethod.hpp"
 28 #include "gc/shared/barrierSet.hpp"
 29 #include "gc/shared/barrierSetAssembler.hpp"
 30 #include "gc/shared/barrierSetNMethod.hpp"
 31 #include "gc/shared/collectedHeap.hpp"
 32 #include "logging/log.hpp"
 33 #include "memory/iterator.hpp"
 34 #include "memory/universe.hpp"
 35 #include "oops/access.inline.hpp"
 36 #include "oops/method.inline.hpp"
 37 #include "runtime/frame.inline.hpp"
 38 #include "runtime/javaThread.hpp"
 39 #include "runtime/threadWXSetters.inline.hpp"
 40 #include "runtime/threads.hpp"
 41 #include "utilities/debug.hpp"
 42 #if INCLUDE_JVMCI
 43 #include "jvmci/jvmciRuntime.hpp"
 44 #endif
 45 
 46 int BarrierSetNMethod::disarmed_guard_value() const {
 47   return *disarmed_guard_value_address();
 48 }
 49 
 50 bool BarrierSetNMethod::supports_entry_barrier(nmethod* nm) {
 51   if (nm->method()->is_method_handle_intrinsic()) {
 52     return false;
 53   }
 54 
 55   if (nm->method()->is_continuation_enter_intrinsic()) {
 56     return false;
 57   }
 58 
 59   if (nm->method()->is_continuation_yield_intrinsic()) {
 60     return false;
 61   }
 62 
 63   if (nm->method()->is_continuation_native_intrinsic()) {
 64     guarantee(false, "Unknown Continuation native intrinsic");
 65     return false;
 66   }
 67 
 68   if (nm->is_native_method() || nm->is_compiled_by_c2() || nm->is_compiled_by_c1()) {
 69     return true;
 70   }
 71 
 72 #if INCLUDE_JVMCI
 73   if (nm->is_compiled_by_jvmci() && nm->jvmci_nmethod_data()->has_entry_barrier()) {
 74     return true;
 75   }
 76 #endif
 77 
 78   return false;
 79 }
 80 
 81 void BarrierSetNMethod::disarm(nmethod* nm) {
 82   set_guard_value(nm, disarmed_guard_value());
 83 }
 84 
 85 bool BarrierSetNMethod::is_armed(nmethod* nm) {
 86   return guard_value(nm) != disarmed_guard_value();
 87 }
 88 
 89 bool BarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
 90   class OopKeepAliveClosure : public OopClosure {
 91   public:
 92     virtual void do_oop(oop* p) {
 93       // Loads on nmethod oops are phantom strength.
 94       //
 95       // Note that we could have used NativeAccess<ON_PHANTOM_OOP_REF>::oop_load(p),
 96       // but that would have *required* us to convert the returned LoadOopProxy to an oop,
 97       // or else keep alive load barrier will never be called. It's the LoadOopProxy-to-oop
 98       // conversion that performs the load barriers. This is too subtle, so we instead
 99       // perform an explicit keep alive call.
100       oop obj = NativeAccess<ON_PHANTOM_OOP_REF | AS_NO_KEEPALIVE>::oop_load(p);
101       if (obj != nullptr) {
102         Universe::heap()->keep_alive(obj);
103       }
104     }
105 
106     virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
107   };
108 
109   // If the nmethod is the only thing pointing to the oops, and we are using a
110   // SATB GC, then it is important that this code marks them live.
111   // Also, with concurrent GC, it is possible that frames in continuation stack
112   // chunks are not visited if they are allocated after concurrent GC started.
113   OopKeepAliveClosure cl;
114   nm->oops_do(&cl);
115 
116   // CodeCache unloading support
117   nm->mark_as_maybe_on_stack();
118 
119   disarm(nm);
120 
121   return true;
122 }
123 
124 int* BarrierSetNMethod::disarmed_guard_value_address() const {
125   return (int*) &_current_phase;
126 }
127 
128 ByteSize BarrierSetNMethod::thread_disarmed_guard_value_offset() const {
129   return Thread::nmethod_disarmed_guard_value_offset();
130 }
131 
132 class BarrierSetNMethodArmClosure : public ThreadClosure {
133 private:
134   int _disarmed_guard_value;
135 
136 public:
137   BarrierSetNMethodArmClosure(int disarmed_guard_value) :
138       _disarmed_guard_value(disarmed_guard_value) {}
139 
140   virtual void do_thread(Thread* thread) {
141     thread->set_nmethod_disarmed_guard_value(_disarmed_guard_value);
142   }
143 };
144 
145 void BarrierSetNMethod::arm_all_nmethods() {
146   // Change to a new global GC phase. Doing this requires changing the thread-local
147   // disarm value for all threads, to reflect the new GC phase.
148   // We wrap around at INT_MAX. That means that we assume nmethods won't have ABA
149   // problems in their nmethod disarm values after INT_MAX - 1 GCs. Every time a GC
150   // completes, ABA problems are removed, but if a concurrent GC is started and then
151   // aborted N times, that is when there could be ABA problems. If there are anything
152   // close to INT_MAX - 1 GCs starting without being able to finish, something is
153   // seriously wrong.
154   ++_current_phase;
155   if (_current_phase == INT_MAX) {
156     _current_phase = 1;
157   }
158   BarrierSetNMethodArmClosure cl(_current_phase);
159   Threads::threads_do(&cl);
160 
161 #if (defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
162   // We clear the patching epoch when disarming nmethods, so that
163   // the counter won't overflow.
164   BarrierSetAssembler::clear_patching_epoch();
165 #endif
166 }
167 
168 int BarrierSetNMethod::nmethod_stub_entry_barrier(address* return_address_ptr) {
169   // Enable WXWrite: the function is called directly from nmethod_entry_barrier
170   // stub.
171   MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
172 
173   address return_address = *return_address_ptr;
174   AARCH64_PORT_ONLY(return_address = pauth_strip_pointer(return_address));
175   CodeBlob* cb = CodeCache::find_blob(return_address);
176   assert(cb != nullptr, "invariant");
177 
178   nmethod* nm = cb->as_nmethod();
179   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
180 
181   if (!bs_nm->is_armed(nm)) {
182     return 0;
183   }
184 
185   assert(!nm->is_osr_method(), "Should not reach here");
186   // Called upon first entry after being armed
187   bool may_enter = bs_nm->nmethod_entry_barrier(nm);
188 
189   // In case a concurrent thread disarmed the nmethod, we need to ensure the new instructions
190   // are made visible, by using a cross modify fence. Note that this is synchronous cross modifying
191   // code, where the existence of new instructions is communicated via data (the guard value).
192   // This cross modify fence is only needed when the nmethod entry barrier modifies the
193   // instructions. Not all platforms currently do that, so if this check becomes expensive,
194   // it can be made conditional on the nmethod_patching_type.
195   OrderAccess::cross_modify_fence();
196 
197   // Diagnostic option to force deoptimization 1 in 3 times. It is otherwise
198   // a very rare event.
199   if (DeoptimizeNMethodBarriersALot) {
200     static volatile uint32_t counter=0;
201     if (Atomic::add(&counter, 1u) % 3 == 0) {
202       may_enter = false;
203     }
204   }
205 
206   if (!may_enter) {
207     log_trace(nmethod, barrier)("Deoptimizing nmethod: " PTR_FORMAT, p2i(nm));
208     bs_nm->deoptimize(nm, return_address_ptr);
209   }
210   return may_enter ? 0 : 1;
211 }
212 
213 bool BarrierSetNMethod::nmethod_osr_entry_barrier(nmethod* nm) {
214   // This check depends on the invariant that all nmethods that are deoptimized / made not entrant
215   // are NOT disarmed.
216   // This invariant is important because a method can be deoptimized after the method have been
217   // resolved / looked up by OSR by another thread. By not deoptimizing them we guarantee that
218   // a deoptimized method will always hit the barrier and come to the same conclusion - deoptimize
219   if (!is_armed(nm)) {
220     return true;
221   }
222 
223   assert(nm->is_osr_method(), "Should not reach here");
224   log_trace(nmethod, barrier)("Running osr nmethod entry barrier: " PTR_FORMAT, p2i(nm));
225   bool result = nmethod_entry_barrier(nm);
226   OrderAccess::cross_modify_fence();
227   return result;
228 }