1 /*
  2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "code/compiledMethod.inline.hpp"
 26 #include "code/relocInfo.hpp"
 27 #include "code/nmethod.hpp"
 28 #include "code/icBuffer.hpp"
 29 #include "gc/shared/barrierSet.hpp"
 30 #include "gc/shared/barrierSetNMethod.hpp"
 31 #include "gc/shared/suspendibleThreadSet.hpp"
 32 #include "gc/z/zBarrier.inline.hpp"
 33 #include "gc/z/zGlobals.hpp"
 34 #include "gc/z/zLock.inline.hpp"
 35 #include "gc/z/zNMethod.hpp"
 36 #include "gc/z/zNMethodData.hpp"
 37 #include "gc/z/zNMethodTable.hpp"
 38 #include "gc/z/zTask.hpp"
 39 #include "gc/z/zWorkers.hpp"
 40 #include "logging/log.hpp"
 41 #include "memory/allocation.inline.hpp"
 42 #include "memory/iterator.hpp"
 43 #include "memory/resourceArea.hpp"
 44 #include "memory/universe.hpp"
 45 #include "oops/oop.inline.hpp"
 46 #include "runtime/atomic.hpp"
 47 #include "utilities/debug.hpp"
 48 
 49 static ZNMethodData* gc_data(const nmethod* nm) {
 50   return nm->gc_data<ZNMethodData>();
 51 }
 52 
 53 static void set_gc_data(nmethod* nm, ZNMethodData* data) {
 54   return nm->set_gc_data<ZNMethodData>(data);
 55 }
 56 
 57 void ZNMethod::attach_gc_data(nmethod* nm) {
 58   GrowableArray<oop*> immediate_oops;
 59   bool non_immediate_oops = false;
 60 
 61   // Find all oop relocations
 62   RelocIterator iter(nm);
 63   while (iter.next()) {
 64     if (iter.type() != relocInfo::oop_type) {
 65       // Not an oop
 66       continue;
 67     }
 68 
 69     oop_Relocation* r = iter.oop_reloc();
 70 
 71     if (!r->oop_is_immediate()) {
 72       // Non-immediate oop found
 73       non_immediate_oops = true;
 74       continue;
 75     }
 76 
 77     if (r->oop_value() != NULL) {
 78       // Non-NULL immediate oop found. NULL oops can safely be
 79       // ignored since the method will be re-registered if they
 80       // are later patched to be non-NULL.
 81       immediate_oops.push(r->oop_addr());
 82     }
 83   }
 84 
 85   // Attach GC data to nmethod
 86   ZNMethodData* data = gc_data(nm);
 87   if (data == NULL) {
 88     data = new ZNMethodData();
 89     set_gc_data(nm, data);
 90   }
 91 
 92   // Attach oops in GC data
 93   ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops);
 94   ZNMethodDataOops* const old_oops = data->swap_oops(new_oops);
 95   ZNMethodDataOops::destroy(old_oops);
 96 }
 97 
 98 ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) {
 99   return gc_data(nm)->lock();
100 }
101 
102 void ZNMethod::log_register(const nmethod* nm) {
103   LogTarget(Trace, gc, nmethod) log;
104   if (!log.is_enabled()) {
105     return;
106   }
107 
108   const ZNMethodDataOops* const oops = gc_data(nm)->oops();
109 
110   log.print("Register NMethod: %s.%s (" PTR_FORMAT "), "
111             "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s",
112             nm->method()->method_holder()->external_name(),
113             nm->method()->name()->as_C_string(),
114             p2i(nm),
115             nm->compiler_name(),
116             nm->oops_count() - 1,
117             oops->immediates_count(),
118             oops->has_non_immediates() ? "Yes" : "No");
119 
120   LogTarget(Trace, gc, nmethod, oops) log_oops;
121   if (!log_oops.is_enabled()) {
122     return;
123   }
124 
125   // Print nmethod oops table
126   {
127     oop* const begin = nm->oops_begin();
128     oop* const end = nm->oops_end();
129     for (oop* p = begin; p < end; p++) {
130       const oop o = Atomic::load(p); // C1 PatchingStub may replace it concurrently.
131       const char* external_name = (o == nullptr) ? "N/A" : o->klass()->external_name();
132       log_oops.print("           Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)",
133                      (p - begin), p2i(o), external_name);
134     }
135   }
136 
137   // Print nmethod immediate oops
138   {
139     oop** const begin = oops->immediates_begin();
140     oop** const end = oops->immediates_end();
141     for (oop** p = begin; p < end; p++) {
142       log_oops.print("  ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)",
143                      (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name());
144     }
145   }
146 }
147 
148 void ZNMethod::log_unregister(const nmethod* nm) {
149   LogTarget(Debug, gc, nmethod) log;
150   if (!log.is_enabled()) {
151     return;
152   }
153 
154   log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")",
155             nm->method()->method_holder()->external_name(),
156             nm->method()->name()->as_C_string(),
157             p2i(nm));
158 }
159 
160 void ZNMethod::register_nmethod(nmethod* nm) {
161   ResourceMark rm;
162 
163   // Create and attach gc data
164   attach_gc_data(nm);
165 
166   log_register(nm);
167 
168   ZNMethodTable::register_nmethod(nm);
169 
170   // Disarm nmethod entry barrier
171   disarm(nm);
172 }
173 
174 void ZNMethod::unregister_nmethod(nmethod* nm) {
175   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
176 
177   if (Thread::current()->is_Code_cache_sweeper_thread()) {
178     // The sweeper must wait for any ongoing iteration to complete
179     // before it can unregister an nmethod.
180     ZNMethodTable::wait_until_iteration_done();
181   }
182 
183   ResourceMark rm;
184 
185   log_unregister(nm);
186 
187   ZNMethodTable::unregister_nmethod(nm);
188 }
189 
190 void ZNMethod::flush_nmethod(nmethod* nm) {
191   // Destroy GC data
192   delete gc_data(nm);
193 }
194 
195 bool ZNMethod::supports_entry_barrier(nmethod* nm) {
196   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
197   return bs->supports_entry_barrier(nm);
198 }
199 
200 bool ZNMethod::is_armed(nmethod* nm) {
201   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
202   return bs->is_armed(nm);
203 }
204 
205 void ZNMethod::disarm(nmethod* nm) {
206   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
207   bs->disarm(nm);
208 }
209 
210 void ZNMethod::arm(nmethod* nm, int arm_value) {
211   BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
212   if (bs != NULL) {
213     bs->arm(nm, arm_value);
214   }
215 }
216 
217 void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
218   ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
219   if (!nm->is_alive()) {
220     return;
221   }
222 
223   ZNMethod::nmethod_oops_do_inner(nm, cl);
224 }
225 
226 void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
227   // Process oops table
228   {
229     oop* const begin = nm->oops_begin();
230     oop* const end = nm->oops_end();
231     for (oop* p = begin; p < end; p++) {
232       if (!Universe::contains_non_oop_word(p)) {
233         cl->do_oop(p);
234       }
235     }
236   }
237 
238   ZNMethodDataOops* const oops = gc_data(nm)->oops();
239 
240   // Process immediate oops
241   {
242     oop** const begin = oops->immediates_begin();
243     oop** const end = oops->immediates_end();
244     for (oop** p = begin; p < end; p++) {
245       if (*p != Universe::non_oop_word()) {
246         cl->do_oop(*p);
247       }
248     }
249   }
250 
251   // Process non-immediate oops
252   if (oops->has_non_immediates()) {
253     nm->fix_oop_relocations();
254   }
255 }
256 
257 class ZNMethodOopClosure : public OopClosure {
258 public:
259   virtual void do_oop(oop* p) {
260     if (ZResurrection::is_blocked()) {
261       ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(p);
262     } else {
263       ZBarrier::load_barrier_on_root_oop_field(p);
264     }
265   }
266 
267   virtual void do_oop(narrowOop* p) {
268     ShouldNotReachHere();
269   }
270 };
271 
272 void ZNMethod::nmethod_oops_barrier(nmethod* nm) {
273   ZNMethodOopClosure cl;
274   nmethod_oops_do_inner(nm, &cl);
275 }
276 
277 void ZNMethod::nmethods_do_begin() {
278   ZNMethodTable::nmethods_do_begin();
279 }
280 
281 void ZNMethod::nmethods_do_end() {
282   ZNMethodTable::nmethods_do_end();
283 }
284 
285 void ZNMethod::nmethods_do(NMethodClosure* cl) {
286   ZNMethodTable::nmethods_do(cl);
287 }
288 
289 class ZNMethodUnlinkClosure : public NMethodClosure {
290 private:
291   bool          _unloading_occurred;
292   volatile bool _failed;
293 
294   void set_failed() {
295     Atomic::store(&_failed, true);
296   }
297 
298   void unlink(nmethod* nm) {
299     // Unlinking of the dependencies must happen before the
300     // handshake separating unlink and purge.
301     nm->flush_dependencies(false /* delete_immediately */);
302 
303     // unlink_from_method will take the CompiledMethod_lock.
304     // In this case we don't strictly need it when unlinking nmethods from
305     // the Method, because it is only concurrently unlinked by
306     // the entry barrier, which acquires the per nmethod lock.
307     nm->unlink_from_method();
308 
309     if (nm->is_osr_method()) {
310       // Invalidate the osr nmethod before the handshake. The nmethod
311       // will be made unloaded after the handshake. Then invalidate_osr_method()
312       // will be called again, which will be a no-op.
313       nm->invalidate_osr_method();
314     }
315   }
316 
317 public:
318   ZNMethodUnlinkClosure(bool unloading_occurred) :
319       _unloading_occurred(unloading_occurred),
320       _failed(false) {}
321 
322   virtual void do_nmethod(nmethod* nm) {
323     if (failed()) {
324       return;
325     }
326 
327     if (!nm->is_alive()) {
328       return;
329     }
330 
331     if (nm->is_unloading()) {
332       ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
333       unlink(nm);
334       return;
335     }
336 
337     ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
338 
339     if (ZNMethod::is_armed(nm)) {
340       // Heal oops and disarm
341       ZNMethod::nmethod_oops_barrier(nm);
342       ZNMethod::arm(nm, 0);
343     }
344 
345     // Clear compiled ICs and exception caches
346     if (!nm->unload_nmethod_caches(_unloading_occurred)) {
347       set_failed();
348     }
349   }
350 
351   bool failed() const {
352     return Atomic::load(&_failed);
353   }
354 };
355 
356 class ZNMethodUnlinkTask : public ZTask {
357 private:
358   ZNMethodUnlinkClosure _cl;
359   ICRefillVerifier*     _verifier;
360 
361 public:
362   ZNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
363       ZTask("ZNMethodUnlinkTask"),
364       _cl(unloading_occurred),
365       _verifier(verifier) {
366     ZNMethodTable::nmethods_do_begin();
367   }
368 
369   ~ZNMethodUnlinkTask() {
370     ZNMethodTable::nmethods_do_end();
371   }
372 
373   virtual void work() {
374     ICRefillVerifierMark mark(_verifier);
375     ZNMethodTable::nmethods_do(&_cl);
376   }
377 
378   bool success() const {
379     return !_cl.failed();
380   }
381 };
382 
383 void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) {
384   for (;;) {
385     ICRefillVerifier verifier;
386 
387     {
388       ZNMethodUnlinkTask task(unloading_occurred, &verifier);
389       workers->run(&task);
390       if (task.success()) {
391         return;
392       }
393     }
394 
395     // Cleaning failed because we ran out of transitional IC stubs,
396     // so we have to refill and try again. Refilling requires taking
397     // a safepoint, so we temporarily leave the suspendible thread set.
398     SuspendibleThreadSetLeaver sts;
399     InlineCacheBuffer::refill_ic_stubs();
400   }
401 }
402 
403 class ZNMethodPurgeClosure : public NMethodClosure {
404 public:
405   virtual void do_nmethod(nmethod* nm) {
406     if (nm->is_alive() && nm->is_unloading()) {
407       nm->make_unloaded();
408     }
409   }
410 };
411 
412 class ZNMethodPurgeTask : public ZTask {
413 private:
414   ZNMethodPurgeClosure _cl;
415 
416 public:
417   ZNMethodPurgeTask() :
418       ZTask("ZNMethodPurgeTask"),
419       _cl() {
420     ZNMethodTable::nmethods_do_begin();
421   }
422 
423   ~ZNMethodPurgeTask() {
424     ZNMethodTable::nmethods_do_end();
425   }
426 
427   virtual void work() {
428     ZNMethodTable::nmethods_do(&_cl);
429   }
430 };
431 
432 void ZNMethod::purge(ZWorkers* workers) {
433   ZNMethodPurgeTask task;
434   workers->run(&task);
435 }