1 /*
  2  * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 
 27 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 28 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 29 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 30 #include "memory/resourceArea.hpp"
 31 
 32 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) :
 33   _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) {
 34 
 35   if (!oops.is_empty()) {
 36     _oops_count = oops.length();
 37     _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
 38     for (int c = 0; c < _oops_count; c++) {
 39       _oops[c] = oops.at(c);
 40     }
 41   }
 42   _has_non_immed_oops = non_immediate_oops;
 43 
 44   assert_same_oops();
 45 }
 46 
 47 ShenandoahNMethod::~ShenandoahNMethod() {
 48   if (_oops != NULL) {
 49     FREE_C_HEAP_ARRAY(oop*, _oops);
 50   }
 51 }
 52 
 53 class ShenandoahHasCSetOopClosure : public OopClosure {
 54 private:
 55   ShenandoahHeap* const _heap;
 56   bool                  _has_cset_oops;
 57 
 58 public:
 59   ShenandoahHasCSetOopClosure(ShenandoahHeap *heap) :
 60     _heap(heap),
 61     _has_cset_oops(false) {
 62   }
 63 
 64   bool has_cset_oops() const {
 65     return _has_cset_oops;
 66   }
 67 
 68   void do_oop(oop* p) {
 69     oop value = RawAccess<>::oop_load(p);
 70     if (!_has_cset_oops && _heap->in_collection_set(value)) {
 71       _has_cset_oops = true;
 72     }
 73   }
 74 
 75   void do_oop(narrowOop* p) {
 76     ShouldNotReachHere();
 77   }
 78 };
 79 
 80 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
 81   ShenandoahHasCSetOopClosure cl(heap);
 82   oops_do(&cl);
 83   return cl.has_cset_oops();
 84 }
 85 
 86 void ShenandoahNMethod::update() {
 87   ResourceMark rm;
 88   bool non_immediate_oops = false;
 89   GrowableArray<oop*> oops;
 90 
 91   detect_reloc_oops(nm(), oops, non_immediate_oops);
 92   if (oops.length() != _oops_count) {
 93     if (_oops != NULL) {
 94       FREE_C_HEAP_ARRAY(oop*, _oops);
 95       _oops = NULL;
 96     }
 97 
 98     _oops_count = oops.length();
 99     if (_oops_count > 0) {
100       _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
101     }
102   }
103 
104   for (int index = 0; index < _oops_count; index ++) {
105     _oops[index] = oops.at(index);
106   }
107   _has_non_immed_oops = non_immediate_oops;
108 
109   assert_same_oops();
110 }
111 
112 void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) {
113   has_non_immed_oops = false;
114   // Find all oops relocations
115   RelocIterator iter(nm);
116   while (iter.next()) {
117     if (iter.type() != relocInfo::oop_type) {
118       // Not an oop
119       continue;
120     }
121 
122     oop_Relocation* r = iter.oop_reloc();
123     if (!r->oop_is_immediate()) {
124       // Non-immediate oop found
125       has_non_immed_oops = true;
126       continue;
127     }
128 
129     oop value = r->oop_value();
130     if (value != NULL) {
131       oop* addr = r->oop_addr();
132       shenandoah_assert_correct(addr, value);
133       shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
134       shenandoah_assert_not_forwarded(addr, value);
135       // Non-NULL immediate oop found. NULL oops can safely be
136       // ignored since the method will be re-registered if they
137       // are later patched to be non-NULL.
138       oops.push(addr);
139     }
140   }
141 }
142 
143 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
144   ResourceMark rm;
145   bool non_immediate_oops = false;
146   GrowableArray<oop*> oops;
147 
148   detect_reloc_oops(nm, oops, non_immediate_oops);
149   return new ShenandoahNMethod(nm, oops, non_immediate_oops);
150 }
151 
152 void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
153   ShenandoahNMethod* data = gc_data(nm);
154   assert(data != NULL, "Sanity");
155   assert(data->lock()->owned_by_self(), "Must hold the lock");
156 
157   ShenandoahHeap* const heap = ShenandoahHeap::heap();
158   if (heap->is_concurrent_mark_in_progress()) {
159     ShenandoahKeepAliveClosure cl;
160     data->oops_do(&cl);
161   } else if (heap->is_concurrent_weak_root_in_progress() ||
162              heap->is_concurrent_strong_root_in_progress()) {
163     ShenandoahEvacOOMScope evac_scope;
164     heal_nmethod_metadata(data);
165   } else {
166     // There is possibility that GC is cancelled when it arrives final mark.
167     // In this case, concurrent root phase is skipped and degenerated GC should be
168     // followed, where nmethods are disarmed.
169     assert(heap->cancelled_gc(), "What else?");
170   }
171 }
172 
173 #ifdef ASSERT
174 void ShenandoahNMethod::assert_correct() {
175   ShenandoahHeap* heap = ShenandoahHeap::heap();
176   for (int c = 0; c < _oops_count; c++) {
177     oop *loc = _oops[c];
178     assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
179     oop o = RawAccess<>::oop_load(loc);
180     shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
181   }
182 
183   oop* const begin = _nm->oops_begin();
184   oop* const end = _nm->oops_end();
185   for (oop* p = begin; p < end; p++) {
186     if (*p != Universe::non_oop_word()) {
187       oop o = RawAccess<>::oop_load(p);
188       shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress());
189     }
190   }
191 }
192 
193 class ShenandoahNMethodOopDetector : public OopClosure {
194 private:
195   ResourceMark rm; // For growable array allocation below.
196   GrowableArray<oop*> _oops;
197 
198 public:
199   ShenandoahNMethodOopDetector() : _oops(10) {};
200 
201   void do_oop(oop* o) {
202     _oops.append(o);
203   }
204   void do_oop(narrowOop* o) {
205     fatal("NMethods should not have compressed oops embedded.");
206   }
207 
208   GrowableArray<oop*>* oops() {
209     return &_oops;
210   }
211 
212   bool has_oops() {
213     return !_oops.is_empty();
214   }
215 };
216 
217 void ShenandoahNMethod::assert_same_oops(bool allow_dead) {
218   ShenandoahNMethodOopDetector detector;
219   nm()->oops_do(&detector, allow_dead);
220 
221   GrowableArray<oop*>* oops = detector.oops();
222 
223   int count = _oops_count;
224   for (int index = 0; index < _oops_count; index ++) {
225     assert(oops->contains(_oops[index]), "Must contain this oop");
226   }
227 
228   for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) {
229     if (*p == Universe::non_oop_word()) continue;
230     count++;
231     assert(oops->contains(p), "Must contain this oop");
232   }
233 
234   if (oops->length() < count) {
235     stringStream debug_stream;
236     debug_stream.print_cr("detected locs: %d", oops->length());
237     for (int i = 0; i < oops->length(); i++) {
238       debug_stream.print_cr("-> " PTR_FORMAT, p2i(oops->at(i)));
239     }
240     debug_stream.print_cr("recorded oops: %d", _oops_count);
241     for (int i = 0; i < _oops_count; i++) {
242       debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i]));
243     }
244     GrowableArray<oop*> check;
245     bool non_immed;
246     detect_reloc_oops(nm(), check, non_immed);
247     debug_stream.print_cr("check oops: %d", check.length());
248     for (int i = 0; i < check.length(); i++) {
249       debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i)));
250     }
251     fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s",
252           oops->length(), _oops_count, count, p2i(nm()->oops_begin()), p2i(nm()->oops_end()), debug_stream.as_string());
253   }
254 }
255 #endif
256 
257 ShenandoahNMethodTable::ShenandoahNMethodTable() :
258   _heap(ShenandoahHeap::heap()),
259   _index(0),
260   _itr_cnt(0) {
261   _list = new ShenandoahNMethodList(minSize);
262 }
263 
264 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
265   assert(_list != NULL, "Sanity");
266   _list->release();
267 }
268 
269 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
270   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
271   assert(_index >= 0 && _index <= _list->size(), "Sanity");
272 
273   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
274 
275   if (data != NULL) {
276     assert(contain(nm), "Must have been registered");
277     assert(nm == data->nm(), "Must be same nmethod");
278     // Prevent updating a nmethod while concurrent iteration is in progress.
279     wait_until_concurrent_iteration_done();
280     ShenandoahReentrantLocker data_locker(data->lock());
281     data->update();
282   } else {
283     // For a new nmethod, we can safely append it to the list, because
284     // concurrent iteration will not touch it.
285     data = ShenandoahNMethod::for_nmethod(nm);
286     assert(data != NULL, "Sanity");
287     ShenandoahNMethod::attach_gc_data(nm, data);
288     ShenandoahLocker locker(&_lock);
289     log_register_nmethod(nm);
290     append(data);
291   }
292   // Disarm new nmethod
293   ShenandoahNMethod::disarm_nmethod(nm);
294 }
295 
296 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
297   assert_locked_or_safepoint(CodeCache_lock);
298 
299   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
300   assert(data != NULL, "Sanity");
301   if (Thread::current()->is_Code_cache_sweeper_thread()) {
302     wait_until_concurrent_iteration_done();
303   }
304   log_unregister_nmethod(nm);
305   ShenandoahLocker locker(&_lock);
306   assert(contain(nm), "Must have been registered");
307 
308   ShenandoahReentrantLocker data_locker(data->lock());
309   data->mark_unregistered();
310 }
311 
312 void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) {
313   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
314   assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread");
315   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
316   assert(data != NULL, "Sanity");
317 
318   // Can not alter the array when iteration is in progress
319   wait_until_concurrent_iteration_done();
320   log_flush_nmethod(nm);
321 
322   ShenandoahLocker locker(&_lock);
323   int idx = index_of(nm);
324   assert(idx >= 0 && idx < _index, "Invalid index");
325   ShenandoahNMethod::attach_gc_data(nm, NULL);
326   remove(idx);
327 }
328 
329 bool ShenandoahNMethodTable::contain(nmethod* nm) const {
330   return index_of(nm) != -1;
331 }
332 
333 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
334   assert(index >= 0 && index < _index, "Out of bound");
335   return _list->at(index);
336 }
337 
338 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
339   for (int index = 0; index < length(); index ++) {
340     if (at(index)->nm() == nm) {
341       return index;
342     }
343   }
344   return -1;
345 }
346 
347 void ShenandoahNMethodTable::remove(int idx) {
348   shenandoah_assert_locked_or_safepoint(CodeCache_lock);
349   assert(!iteration_in_progress(), "Can not happen");
350   assert(_index >= 0 && _index <= _list->size(), "Sanity");
351 
352   assert(idx >= 0 && idx < _index, "Out of bound");
353   ShenandoahNMethod* snm = _list->at(idx);
354   ShenandoahNMethod* tmp = _list->at(_index - 1);
355   _list->set(idx, tmp);
356   _index --;
357 
358   delete snm;
359 }
360 
361 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
362   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
363   while (iteration_in_progress()) {
364     CodeCache_lock->wait_without_safepoint_check();
365   }
366 }
367 
368 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
369   if (is_full()) {
370     int new_size = 2 * _list->size();
371     // Rebuild table and replace current one
372     rebuild(new_size);
373   }
374 
375   _list->set(_index++,  snm);
376   assert(_index >= 0 && _index <= _list->size(), "Sanity");
377 }
378 
379 void ShenandoahNMethodTable::rebuild(int size) {
380   ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size);
381   new_list->transfer(_list, _index);
382 
383   // Release old list
384   _list->release();
385   _list = new_list;
386 }
387 
388 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
389   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
390   _itr_cnt++;
391   return new ShenandoahNMethodTableSnapshot(this);
392 }
393 
394 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
395   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
396   assert(iteration_in_progress(), "Why we here?");
397   assert(snapshot != NULL, "No snapshot");
398   _itr_cnt--;
399 
400   delete snapshot;
401 }
402 
403 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) {
404   LogTarget(Debug, gc, nmethod) log;
405   if (!log.is_enabled()) {
406     return;
407   }
408 
409   ResourceMark rm;
410   log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)",
411             nm->method()->method_holder()->external_name(),
412             nm->method()->name()->as_C_string(),
413             p2i(nm),
414             nm->compiler_name());
415 }
416 
417 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) {
418   LogTarget(Debug, gc, nmethod) log;
419   if (!log.is_enabled()) {
420     return;
421   }
422 
423   ResourceMark rm;
424   log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]",
425             nm->method()->method_holder()->external_name(),
426             nm->method()->name()->as_C_string(),
427             p2i(nm));
428 }
429 
430 void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) {
431   LogTarget(Debug, gc, nmethod) log;
432   if (!log.is_enabled()) {
433     return;
434   }
435 
436   ResourceMark rm;
437   log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm));
438 }
439 
440 #ifdef ASSERT
441 void ShenandoahNMethodTable::assert_nmethods_correct() {
442   assert_locked_or_safepoint(CodeCache_lock);
443 
444   for (int index = 0; index < length(); index ++) {
445     ShenandoahNMethod* m = _list->at(index);
446     // Concurrent unloading may have dead nmethods to be cleaned by sweeper
447     if (m->is_unregistered()) continue;
448     m->assert_correct();
449   }
450 }
451 #endif
452 
453 
454 ShenandoahNMethodList::ShenandoahNMethodList(int size) :
455   _size(size), _ref_count(1) {
456   _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
457 }
458 
459 ShenandoahNMethodList::~ShenandoahNMethodList() {
460   assert(_list != NULL, "Sanity");
461   assert(_ref_count == 0, "Must be");
462   FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _list);
463 }
464 
465 void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
466   assert(limit <= size(), "Sanity");
467   ShenandoahNMethod** old_list = list->list();
468   for (int index = 0; index < limit; index++) {
469     _list[index] = old_list[index];
470   }
471 }
472 
473 ShenandoahNMethodList* ShenandoahNMethodList::acquire() {
474   assert_locked_or_safepoint(CodeCache_lock);
475   _ref_count++;
476   return this;
477 }
478 
479 void ShenandoahNMethodList::release() {
480   assert_locked_or_safepoint(CodeCache_lock);
481   _ref_count--;
482   if (_ref_count == 0) {
483     delete this;
484   }
485 }
486 
487 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
488   _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) {
489 }
490 
491 ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() {
492   _list->release();
493 }
494 
495 void ShenandoahNMethodTableSnapshot::parallel_blobs_do(CodeBlobClosure *f) {
496   size_t stride = 256; // educated guess
497 
498   ShenandoahNMethod** const list = _list->list();
499 
500   size_t max = (size_t)_limit;
501   while (_claimed < max) {
502     size_t cur = Atomic::fetch_and_add(&_claimed, stride, memory_order_relaxed);
503     size_t start = cur;
504     size_t end = MIN2(cur + stride, max);
505     if (start >= max) break;
506 
507     for (size_t idx = start; idx < end; idx++) {
508       ShenandoahNMethod* nmr = list[idx];
509       assert(nmr != NULL, "Sanity");
510       if (nmr->is_unregistered()) {
511         continue;
512       }
513 
514       // A nmethod can become a zombie before it is unregistered.
515       if (nmr->nm()->is_alive()) {
516         nmr->assert_correct();
517         f->do_code_blob(nmr->nm());
518       }
519     }
520   }
521 }
522 
523 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
524   size_t stride = 256; // educated guess
525 
526   ShenandoahNMethod** list = _list->list();
527   size_t max = (size_t)_limit;
528   while (_claimed < max) {
529     size_t cur = Atomic::fetch_and_add(&_claimed, stride, memory_order_relaxed);
530     size_t start = cur;
531     size_t end = MIN2(cur + stride, max);
532     if (start >= max) break;
533 
534     for (size_t idx = start; idx < end; idx++) {
535       ShenandoahNMethod* data = list[idx];
536       assert(data != NULL, "Should not be NULL");
537       if (!data->is_unregistered()) {
538         cl->do_nmethod(data->nm());
539       }
540     }
541   }
542 }
543 
544 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) :
545   _table(table), _table_snapshot(NULL) {
546 }
547 
548 void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() {
549   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
550   _table_snapshot = _table->snapshot_for_iteration();
551 }
552 
553 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) {
554   assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()");
555   _table_snapshot->concurrent_nmethods_do(cl);
556 }
557 
558 void ShenandoahConcurrentNMethodIterator::nmethods_do_end() {
559   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
560   _table->finish_iteration(_table_snapshot);
561   CodeCache_lock->notify_all();
562 }