1 /*
  2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "code/relocInfo.hpp"
 26 #include "code/nmethod.hpp"
 27 #include "code/icBuffer.hpp"
 28 #include "gc/shared/barrierSet.hpp"
 29 #include "gc/shared/barrierSetNMethod.hpp"
 30 #include "gc/shared/suspendibleThreadSet.hpp"
 31 #include "gc/z/zBarrier.inline.hpp"
 32 #include "gc/z/zGlobals.hpp"
 33 #include "gc/z/zLock.inline.hpp"
 34 #include "gc/z/zNMethod.hpp"
 35 #include "gc/z/zNMethodData.hpp"
 36 #include "gc/z/zNMethodTable.hpp"
 37 #include "gc/z/zTask.hpp"
 38 #include "gc/z/zWorkers.hpp"
 39 #include "logging/log.hpp"
 40 #include "memory/allocation.inline.hpp"
 41 #include "memory/iterator.hpp"
 42 #include "memory/resourceArea.hpp"
 43 #include "memory/universe.hpp"
 44 #include "oops/oop.inline.hpp"
 45 #include "runtime/atomic.hpp"
 46 #include "utilities/debug.hpp"
 47 
 48 static ZNMethodData* gc_data(const nmethod* nm) {
 49   return nm->gc_data<ZNMethodData>();
 50 }
 51 
 52 static void set_gc_data(nmethod* nm, ZNMethodData* data) {
 53   return nm->set_gc_data<ZNMethodData>(data);
 54 }
 55 
 56 void ZNMethod::attach_gc_data(nmethod* nm) {
 57   GrowableArray<oop*> immediate_oops;
 58   bool non_immediate_oops = false;
 59 
 60   // Find all oop relocations
 61   RelocIterator iter(nm);
 62   while (iter.next()) {
 63     if (iter.type() != relocInfo::oop_type) {
 64       // Not an oop
 65       continue;
 66     }
 67 
 68     oop_Relocation* r = iter.oop_reloc();
 69 
 70     if (!r->oop_is_immediate()) {
 71       // Non-immediate oop found
 72       non_immediate_oops = true;
 73       continue;
 74     }
 75 
 76     if (r->oop_value() != NULL) {
 77       // Non-NULL immediate oop found. NULL oops can safely be
 78       // ignored since the method will be re-registered if they
 79       // are later patched to be non-NULL.
 80       immediate_oops.push(r->oop_addr());
 81     }
 82   }
 83 
 84   // Attach GC data to nmethod
 85   ZNMethodData* data = gc_data(nm);
 86   if (data == NULL) {
 87     data = new ZNMethodData();
 88     set_gc_data(nm, data);
 89   }
 90 
 91   // Attach oops in GC data
 92   ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops);
 93   ZNMethodDataOops* const old_oops = data->swap_oops(new_oops);
 94   ZNMethodDataOops::destroy(old_oops);
 95 }
 96 
 97 ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) {
 98   return gc_data(nm)->lock();
 99 }
100 
101 void ZNMethod::log_register(const nmethod* nm) {
102   LogTarget(Trace, gc, nmethod) log;
103   if (!log.is_enabled()) {
104     return;
105   }
106 
107   const ZNMethodDataOops* const oops = gc_data(nm)->oops();
108 
109   log.print("Register NMethod: %s.%s (" PTR_FORMAT "), "
110             "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s",
111             nm->method()->method_holder()->external_name(),
112             nm->method()->name()->as_C_string(),
113             p2i(nm),
114             nm->compiler_name(),
115             nm->oops_count() - 1,
116             oops->immediates_count(),
117             oops->has_non_immediates() ? "Yes" : "No");
118 
119   LogTarget(Trace, gc, nmethod, oops) log_oops;
120   if (!log_oops.is_enabled()) {
121     return;
122   }
123 
124   // Print nmethod oops table
125   {
126     oop* const begin = nm->oops_begin();
127     oop* const end = nm->oops_end();
128     for (oop* p = begin; p < end; p++) {
129       const oop o = Atomic::load(p); // C1 PatchingStub may replace it concurrently.
130       const char* external_name = (o == nullptr) ? "N/A" : o->klass()->external_name();
131       log_oops.print("           Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)",
132                      (p - begin), p2i(o), external_name);
133     }
134   }
135 
136   // Print nmethod immediate oops
137   {
138     oop** const begin = oops->immediates_begin();
139     oop** const end = oops->immediates_end();
140     for (oop** p = begin; p < end; p++) {
141       log_oops.print("  ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)",
142                      (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name());
143     }
144   }
145 }
146 
147 void ZNMethod::log_unregister(const nmethod* nm) {
148   LogTarget(Debug, gc, nmethod) log;
149   if (!log.is_enabled()) {
150     return;
151   }
152 
153   log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")",
154             nm->method()->method_holder()->external_name(),
155             nm->method()->name()->as_C_string(),
156             p2i(nm));
157 }
158 
159 void ZNMethod::register_nmethod(nmethod* nm) {
160   ResourceMark rm;
161 
162   // Create and attach gc data
163   attach_gc_data(nm);
164 
165   log_register(nm);
166 
167   ZNMethodTable::register_nmethod(nm);
168 
169   // Disarm nmethod entry barrier
170   disarm(nm);
171 }
172 
173 void ZNMethod::unregister_nmethod(nmethod* nm) {
174   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
175 
176   if (Thread::current()->is_Code_cache_sweeper_thread()) {
177     // The sweeper must wait for any ongoing iteration to complete
178     // before it can unregister an nmethod.
179     ZNMethodTable::wait_until_iteration_done();
180   }
181 
182   ResourceMark rm;
183 
184   log_unregister(nm);
185 
186   ZNMethodTable::unregister_nmethod(nm);
187 }
188 
189 void ZNMethod::flush_nmethod(nmethod* nm) {
190   // Destroy GC data
191   delete gc_data(nm);
192 }
193 
194 bool ZNMethod::supports_entry_barrier(nmethod* nm) {
195   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
196   return bs->supports_entry_barrier(nm);
197 }
198 
199 bool ZNMethod::is_armed(nmethod* nm) {
200   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
201   return bs->is_armed(nm);
202 }
203 
204 void ZNMethod::disarm(nmethod* nm) {
205   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
206   bs->disarm(nm);
207 }
208 
209 void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
210   ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
211   if (!nm->is_alive()) {
212     return;
213   }
214 
215   ZNMethod::nmethod_oops_do_inner(nm, cl);
216 }
217 
218 void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
219   // Process oops table
220   {
221     oop* const begin = nm->oops_begin();
222     oop* const end = nm->oops_end();
223     for (oop* p = begin; p < end; p++) {
224       if (!Universe::contains_non_oop_word(p)) {
225         cl->do_oop(p);
226       }
227     }
228   }
229 
230   ZNMethodDataOops* const oops = gc_data(nm)->oops();
231 
232   // Process immediate oops
233   {
234     oop** const begin = oops->immediates_begin();
235     oop** const end = oops->immediates_end();
236     for (oop** p = begin; p < end; p++) {
237       if (*p != Universe::non_oop_word()) {
238         cl->do_oop(*p);
239       }
240     }
241   }
242 
243   // Process non-immediate oops
244   if (oops->has_non_immediates()) {
245     nm->fix_oop_relocations();
246   }
247 }
248 
249 class ZNMethodOopClosure : public OopClosure {
250 public:
251   virtual void do_oop(oop* p) {
252     if (ZResurrection::is_blocked()) {
253       ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(p);
254     } else {
255       ZBarrier::load_barrier_on_root_oop_field(p);
256     }
257   }
258 
259   virtual void do_oop(narrowOop* p) {
260     ShouldNotReachHere();
261   }
262 };
263 
264 void ZNMethod::nmethod_oops_barrier(nmethod* nm) {
265   ZNMethodOopClosure cl;
266   nmethod_oops_do_inner(nm, &cl);
267 }
268 
269 void ZNMethod::nmethods_do_begin() {
270   ZNMethodTable::nmethods_do_begin();
271 }
272 
273 void ZNMethod::nmethods_do_end() {
274   ZNMethodTable::nmethods_do_end();
275 }
276 
277 void ZNMethod::nmethods_do(NMethodClosure* cl) {
278   ZNMethodTable::nmethods_do(cl);
279 }
280 
281 class ZNMethodUnlinkClosure : public NMethodClosure {
282 private:
283   bool          _unloading_occurred;
284   volatile bool _failed;
285 
286   void set_failed() {
287     Atomic::store(&_failed, true);
288   }
289 
290   void unlink(nmethod* nm) {
291     // Unlinking of the dependencies must happen before the
292     // handshake separating unlink and purge.
293     nm->flush_dependencies(false /* delete_immediately */);
294 
295     // unlink_from_method will take the CompiledMethod_lock.
296     // In this case we don't strictly need it when unlinking nmethods from
297     // the Method, because it is only concurrently unlinked by
298     // the entry barrier, which acquires the per nmethod lock.
299     nm->unlink_from_method();
300 
301     if (nm->is_osr_method()) {
302       // Invalidate the osr nmethod before the handshake. The nmethod
303       // will be made unloaded after the handshake. Then invalidate_osr_method()
304       // will be called again, which will be a no-op.
305       nm->invalidate_osr_method();
306     }
307   }
308 
309 public:
310   ZNMethodUnlinkClosure(bool unloading_occurred) :
311       _unloading_occurred(unloading_occurred),
312       _failed(false) {}
313 
314   virtual void do_nmethod(nmethod* nm) {
315     if (failed()) {
316       return;
317     }
318 
319     if (!nm->is_alive()) {
320       return;
321     }
322 
323     if (nm->is_unloading()) {
324       ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
325       unlink(nm);
326       return;
327     }
328 
329     ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
330 
331     if (ZNMethod::is_armed(nm)) {
332       // Heal oops and disarm
333       ZNMethod::nmethod_oops_barrier(nm);
334       ZNMethod::disarm(nm);
335     }
336 
337     // Clear compiled ICs and exception caches
338     if (!nm->unload_nmethod_caches(_unloading_occurred)) {
339       set_failed();
340     }
341   }
342 
343   bool failed() const {
344     return Atomic::load(&_failed);
345   }
346 };
347 
348 class ZNMethodUnlinkTask : public ZTask {
349 private:
350   ZNMethodUnlinkClosure _cl;
351   ICRefillVerifier*     _verifier;
352 
353 public:
354   ZNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
355       ZTask("ZNMethodUnlinkTask"),
356       _cl(unloading_occurred),
357       _verifier(verifier) {
358     ZNMethodTable::nmethods_do_begin();
359   }
360 
361   ~ZNMethodUnlinkTask() {
362     ZNMethodTable::nmethods_do_end();
363   }
364 
365   virtual void work() {
366     ICRefillVerifierMark mark(_verifier);
367     ZNMethodTable::nmethods_do(&_cl);
368   }
369 
370   bool success() const {
371     return !_cl.failed();
372   }
373 };
374 
375 void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) {
376   for (;;) {
377     ICRefillVerifier verifier;
378 
379     {
380       ZNMethodUnlinkTask task(unloading_occurred, &verifier);
381       workers->run(&task);
382       if (task.success()) {
383         return;
384       }
385     }
386 
387     // Cleaning failed because we ran out of transitional IC stubs,
388     // so we have to refill and try again. Refilling requires taking
389     // a safepoint, so we temporarily leave the suspendible thread set.
390     SuspendibleThreadSetLeaver sts;
391     InlineCacheBuffer::refill_ic_stubs();
392   }
393 }
394 
395 class ZNMethodPurgeClosure : public NMethodClosure {
396 public:
397   virtual void do_nmethod(nmethod* nm) {
398     if (nm->is_alive() && nm->is_unloading()) {
399       nm->make_unloaded();
400     }
401   }
402 };
403 
404 class ZNMethodPurgeTask : public ZTask {
405 private:
406   ZNMethodPurgeClosure _cl;
407 
408 public:
409   ZNMethodPurgeTask() :
410       ZTask("ZNMethodPurgeTask"),
411       _cl() {
412     ZNMethodTable::nmethods_do_begin();
413   }
414 
415   ~ZNMethodPurgeTask() {
416     ZNMethodTable::nmethods_do_end();
417   }
418 
419   virtual void work() {
420     ZNMethodTable::nmethods_do(&_cl);
421   }
422 };
423 
424 void ZNMethod::purge(ZWorkers* workers) {
425   ZNMethodPurgeTask task;
426   workers->run(&task);
427 }