1 /*
  2  * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 
 27 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 28 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 29 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 30 #include "memory/resourceArea.hpp"
 31 
 32 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) :
 33   _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) {
 34 
 35   if (!oops.is_empty()) {
 36     _oops_count = oops.length();
 37     _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
 38     for (int c = 0; c < _oops_count; c++) {
 39       _oops[c] = oops.at(c);
 40     }
 41   }
 42   _has_non_immed_oops = non_immediate_oops;
 43 
 44   assert_same_oops();
 45 }
 46 
 47 ShenandoahNMethod::~ShenandoahNMethod() {
 48   if (_oops != NULL) {
 49     FREE_C_HEAP_ARRAY(oop*, _oops);
 50   }
 51 }
 52 
 53 class ShenandoahHasCSetOopClosure : public OopClosure {
 54 private:
 55   ShenandoahHeap* const _heap;
 56   bool                  _has_cset_oops;
 57 
 58 public:
 59   ShenandoahHasCSetOopClosure(ShenandoahHeap *heap) :
 60     _heap(heap),
 61     _has_cset_oops(false) {
 62   }
 63 
 64   bool has_cset_oops() const {
 65     return _has_cset_oops;
 66   }
 67 
 68   void do_oop(oop* p) {
 69     oop value = RawAccess<>::oop_load(p);
 70     if (!_has_cset_oops && _heap->in_collection_set(value)) {
 71       _has_cset_oops = true;
 72     }
 73   }
 74 
 75   void do_oop(narrowOop* p) {
 76     ShouldNotReachHere();
 77   }
 78 };
 79 
 80 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
 81   ShenandoahHasCSetOopClosure cl(heap);
 82   oops_do(&cl);
 83   return cl.has_cset_oops();
 84 }
 85 
 86 void ShenandoahNMethod::update() {
 87   ResourceMark rm;
 88   bool non_immediate_oops = false;
 89   GrowableArray<oop*> oops;
 90 
 91   detect_reloc_oops(nm(), oops, non_immediate_oops);
 92   if (oops.length() != _oops_count) {
 93     if (_oops != NULL) {
 94       FREE_C_HEAP_ARRAY(oop*, _oops);
 95       _oops = NULL;
 96     }
 97 
 98     _oops_count = oops.length();
 99     if (_oops_count > 0) {
100       _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
101     }
102   }
103 
104   for (int index = 0; index < _oops_count; index ++) {
105     _oops[index] = oops.at(index);
106   }
107   _has_non_immed_oops = non_immediate_oops;
108 
109   assert_same_oops();
110 }
111 
112 void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) {
113   has_non_immed_oops = false;
114   // Find all oops relocations
115   RelocIterator iter(nm);
116   while (iter.next()) {
117     if (iter.type() != relocInfo::oop_type) {
118       // Not an oop
119       continue;
120     }
121 
122     oop_Relocation* r = iter.oop_reloc();
123     if (!r->oop_is_immediate()) {
124       // Non-immediate oop found
125       has_non_immed_oops = true;
126       continue;
127     }
128 
129     oop value = r->oop_value();
130     if (value != NULL) {
131       oop* addr = r->oop_addr();
132       shenandoah_assert_correct(addr, value);
133       shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
134       shenandoah_assert_not_forwarded(addr, value);
135       // Non-NULL immediate oop found. NULL oops can safely be
136       // ignored since the method will be re-registered if they
137       // are later patched to be non-NULL.
138       oops.push(addr);
139     }
140   }
141 }
142 
143 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
144   ResourceMark rm;
145   bool non_immediate_oops = false;
146   GrowableArray<oop*> oops;
147 
148   detect_reloc_oops(nm, oops, non_immediate_oops);
149   return new ShenandoahNMethod(nm, oops, non_immediate_oops);
150 }
151 
152 void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
153   ShenandoahNMethod* data = gc_data(nm);
154   assert(data != NULL, "Sanity");
155   assert(data->lock()->owned_by_self(), "Must hold the lock");
156 
157   ShenandoahHeap* const heap = ShenandoahHeap::heap();
158   if (heap->is_concurrent_weak_root_in_progress() ||
159       heap->is_concurrent_strong_root_in_progress()) {
160     ShenandoahEvacOOMScope evac_scope;
161     heal_nmethod_metadata(data);
162   } else if (heap->is_concurrent_mark_in_progress()) {
163     ShenandoahKeepAliveClosure cl;
164     data->oops_do(&cl);
165   } else {
166     // There is possibility that GC is cancelled when it arrives final mark.
167     // In this case, concurrent root phase is skipped and degenerated GC should be
168     // followed, where nmethods are disarmed.
169     assert(heap->cancelled_gc(), "What else?");
170   }
171 }
172 
173 #ifdef ASSERT
174 void ShenandoahNMethod::assert_correct() {
175   ShenandoahHeap* heap = ShenandoahHeap::heap();
176   for (int c = 0; c < _oops_count; c++) {
177     oop *loc = _oops[c];
178     assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
179     oop o = RawAccess<>::oop_load(loc);
180     shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
181   }
182 
183   oop* const begin = _nm->oops_begin();
184   oop* const end = _nm->oops_end();
185   for (oop* p = begin; p < end; p++) {
186     if (*p != Universe::non_oop_word()) {
187       oop o = RawAccess<>::oop_load(p);
188       shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress());
189     }
190   }
191 }
192 
193 class ShenandoahNMethodOopDetector : public OopClosure {
194 private:
195   ResourceMark rm; // For growable array allocation below.
196   GrowableArray<oop*> _oops;
197 
198 public:
199   ShenandoahNMethodOopDetector() : _oops(10) {};
200 
201   void do_oop(oop* o) {
202     _oops.append(o);
203   }
204   void do_oop(narrowOop* o) {
205     fatal("NMethods should not have compressed oops embedded.");
206   }
207 
208   GrowableArray<oop*>* oops() {
209     return &_oops;
210   }
211 
212   bool has_oops() {
213     return !_oops.is_empty();
214   }
215 };
216 
217 void ShenandoahNMethod::assert_same_oops(bool allow_dead) {
218   ShenandoahNMethodOopDetector detector;
219   nm()->oops_do(&detector, allow_dead);
220 
221   GrowableArray<oop*>* oops = detector.oops();
222 
223   int count = _oops_count;
224   for (int index = 0; index < _oops_count; index ++) {
225     assert(oops->contains(_oops[index]), "Must contain this oop");
226   }
227 
228   for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) {
229     if (*p == Universe::non_oop_word()) continue;
230     count++;
231     assert(oops->contains(p), "Must contain this oop");
232   }
233 
234   if (oops->length() < count) {
235     stringStream debug_stream;
236     debug_stream.print_cr("detected locs: %d", oops->length());
237     for (int i = 0; i < oops->length(); i++) {
238       debug_stream.print_cr("-> " PTR_FORMAT, p2i(oops->at(i)));
239     }
240     debug_stream.print_cr("recorded oops: %d", _oops_count);
241     for (int i = 0; i < _oops_count; i++) {
242       debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i]));
243     }
244     GrowableArray<oop*> check;
245     bool non_immed;
246     detect_reloc_oops(nm(), check, non_immed);
247     debug_stream.print_cr("check oops: %d", check.length());
248     for (int i = 0; i < check.length(); i++) {
249       debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i)));
250     }
251     fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s",
252           oops->length(), _oops_count, count, p2i(nm()->oops_begin()), p2i(nm()->oops_end()), debug_stream.as_string());
253   }
254 }
255 #endif
256 
257 ShenandoahNMethodTable::ShenandoahNMethodTable() :
258   _heap(ShenandoahHeap::heap()),
259   _index(0),
260   _itr_cnt(0) {
261   _list = new ShenandoahNMethodList(minSize);
262 }
263 
264 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
265   assert(_list != NULL, "Sanity");
266   _list->release();
267 }
268 
269 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
270   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
271   assert(_index >= 0 && _index <= _list->size(), "Sanity");
272 
273   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
274   ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL);
275 
276   if (data != NULL) {
277     assert(contain(nm), "Must have been registered");
278     assert(nm == data->nm(), "Must be same nmethod");
279     data->update();
280   } else {
281     data = ShenandoahNMethod::for_nmethod(nm);
282     assert(data != NULL, "Sanity");
283     ShenandoahNMethod::attach_gc_data(nm, data);
284     ShenandoahLocker locker(&_lock);
285     log_register_nmethod(nm);
286     append(data);
287   }
288   // Disarm new nmethod
289   ShenandoahNMethod::disarm_nmethod(nm);
290 }
291 
292 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
293   assert_locked_or_safepoint(CodeCache_lock);
294 
295   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
296   assert(data != NULL, "Sanity");
297   if (Thread::current()->is_Code_cache_sweeper_thread()) {
298     wait_until_concurrent_iteration_done();
299   }
300   log_unregister_nmethod(nm);
301   ShenandoahLocker locker(&_lock);
302   assert(contain(nm), "Must have been registered");
303 
304   ShenandoahReentrantLocker data_locker(data->lock());
305   data->mark_unregistered();
306 }
307 
308 void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) {
309   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
310   assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread");
311   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
312   assert(data != NULL, "Sanity");
313 
314   // Can not alter the array when iteration is in progress
315   wait_until_concurrent_iteration_done();
316   log_flush_nmethod(nm);
317 
318   ShenandoahLocker locker(&_lock);
319   int idx = index_of(nm);
320   assert(idx >= 0 && idx < _index, "Invalid index");
321   ShenandoahNMethod::attach_gc_data(nm, NULL);
322   remove(idx);
323 }
324 
325 bool ShenandoahNMethodTable::contain(nmethod* nm) const {
326   return index_of(nm) != -1;
327 }
328 
329 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
330   assert(index >= 0 && index < _index, "Out of bound");
331   return _list->at(index);
332 }
333 
334 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
335   for (int index = 0; index < length(); index ++) {
336     if (at(index)->nm() == nm) {
337       return index;
338     }
339   }
340   return -1;
341 }
342 
343 void ShenandoahNMethodTable::remove(int idx) {
344   shenandoah_assert_locked_or_safepoint(CodeCache_lock);
345   assert(!iteration_in_progress(), "Can not happen");
346   assert(_index >= 0 && _index <= _list->size(), "Sanity");
347 
348   assert(idx >= 0 && idx < _index, "Out of bound");
349   ShenandoahNMethod* snm = _list->at(idx);
350   ShenandoahNMethod* tmp = _list->at(_index - 1);
351   _list->set(idx, tmp);
352   _index --;
353 
354   delete snm;
355 }
356 
357 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
358   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
359   while (iteration_in_progress()) {
360     CodeCache_lock->wait_without_safepoint_check();
361   }
362 }
363 
364 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
365   if (is_full()) {
366     int new_size = 2 * _list->size();
367     // Rebuild table and replace current one
368     rebuild(new_size);
369   }
370 
371   _list->set(_index++,  snm);
372   assert(_index >= 0 && _index <= _list->size(), "Sanity");
373 }
374 
375 void ShenandoahNMethodTable::rebuild(int size) {
376   ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size);
377   new_list->transfer(_list, _index);
378 
379   // Release old list
380   _list->release();
381   _list = new_list;
382 }
383 
384 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
385   _itr_cnt++;
386   return new ShenandoahNMethodTableSnapshot(this);
387 }
388 
389 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
390   assert(iteration_in_progress(), "Why we here?");
391   assert(snapshot != NULL, "No snapshot");
392   _itr_cnt--;
393 
394   delete snapshot;
395 }
396 
397 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) {
398   LogTarget(Debug, gc, nmethod) log;
399   if (!log.is_enabled()) {
400     return;
401   }
402 
403   ResourceMark rm;
404   log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)",
405             nm->method()->method_holder()->external_name(),
406             nm->method()->name()->as_C_string(),
407             p2i(nm),
408             nm->compiler_name());
409 }
410 
411 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) {
412   LogTarget(Debug, gc, nmethod) log;
413   if (!log.is_enabled()) {
414     return;
415   }
416 
417   ResourceMark rm;
418   log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]",
419             nm->method()->method_holder()->external_name(),
420             nm->method()->name()->as_C_string(),
421             p2i(nm));
422 }
423 
424 void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) {
425   LogTarget(Debug, gc, nmethod) log;
426   if (!log.is_enabled()) {
427     return;
428   }
429 
430   ResourceMark rm;
431   log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm));
432 }
433 
434 #ifdef ASSERT
435 void ShenandoahNMethodTable::assert_nmethods_correct() {
436   assert_locked_or_safepoint(CodeCache_lock);
437 
438   for (int index = 0; index < length(); index ++) {
439     ShenandoahNMethod* m = _list->at(index);
440     // Concurrent unloading may have dead nmethods to be cleaned by sweeper
441     if (m->is_unregistered()) continue;
442     m->assert_correct();
443   }
444 }
445 #endif
446 
447 
448 ShenandoahNMethodList::ShenandoahNMethodList(int size) :
449   _size(size), _ref_count(1) {
450   _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
451 }
452 
453 ShenandoahNMethodList::~ShenandoahNMethodList() {
454   assert(_list != NULL, "Sanity");
455   assert(_ref_count == 0, "Must be");
456   FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _list);
457 }
458 
459 void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
460   assert(limit <= size(), "Sanity");
461   ShenandoahNMethod** old_list = list->list();
462   for (int index = 0; index < limit; index++) {
463     _list[index] = old_list[index];
464   }
465 }
466 
467 ShenandoahNMethodList* ShenandoahNMethodList::acquire() {
468   assert_locked_or_safepoint(CodeCache_lock);
469   _ref_count++;
470   return this;
471 }
472 
473 void ShenandoahNMethodList::release() {
474   assert_locked_or_safepoint(CodeCache_lock);
475   _ref_count--;
476   if (_ref_count == 0) {
477     delete this;
478   }
479 }
480 
481 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
482   _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) {
483 }
484 
485 ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() {
486   _list->release();
487 }
488 
489 void ShenandoahNMethodTableSnapshot::parallel_blobs_do(CodeBlobClosure *f) {
490   size_t stride = 256; // educated guess
491 
492   ShenandoahNMethod** const list = _list->list();
493 
494   size_t max = (size_t)_limit;
495   while (_claimed < max) {
496     size_t cur = Atomic::fetch_and_add(&_claimed, stride);
497     size_t start = cur;
498     size_t end = MIN2(cur + stride, max);
499     if (start >= max) break;
500 
501     for (size_t idx = start; idx < end; idx++) {
502       ShenandoahNMethod* nmr = list[idx];
503       assert(nmr != NULL, "Sanity");
504       if (nmr->is_unregistered()) {
505         continue;
506       }
507 
508       // A nmethod can become a zombie before it is unregistered.
509       if (nmr->nm()->is_alive()) {
510         nmr->assert_correct();
511         f->do_code_blob(nmr->nm());
512       }
513     }
514   }
515 }
516 
517 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
518   size_t stride = 256; // educated guess
519 
520   ShenandoahNMethod** list = _list->list();
521   size_t max = (size_t)_limit;
522   while (_claimed < max) {
523     size_t cur = Atomic::fetch_and_add(&_claimed, stride);
524     size_t start = cur;
525     size_t end = MIN2(cur + stride, max);
526     if (start >= max) break;
527 
528     for (size_t idx = start; idx < end; idx++) {
529       ShenandoahNMethod* data = list[idx];
530       assert(data != NULL, "Should not be NULL");
531       if (!data->is_unregistered()) {
532         cl->do_nmethod(data->nm());
533       }
534     }
535   }
536 }
537 
538 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) :
539   _table(table), _table_snapshot(NULL) {
540 }
541 
542 void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() {
543   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
544   _table_snapshot = _table->snapshot_for_iteration();
545 }
546 
547 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) {
548   assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()");
549   _table_snapshot->concurrent_nmethods_do(cl);
550 }
551 
552 void ShenandoahConcurrentNMethodIterator::nmethods_do_end() {
553   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
554   _table->finish_iteration(_table_snapshot);
555   CodeCache_lock->notify_all();
556 }