1 /*
   2  * Copyright (c) 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  27 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  28 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  29 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
  30 #include "memory/resourceArea.hpp"
  31 
  32 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) :
  33   _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) {
  34 
  35   if (!oops.is_empty()) {
  36     _oops_count = oops.length();
  37     _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
  38     for (int c = 0; c < _oops_count; c++) {
  39       _oops[c] = oops.at(c);
  40     }
  41   }
  42   _has_non_immed_oops = non_immediate_oops;
  43 
  44   assert_same_oops();
  45 }
  46 
  47 ShenandoahNMethod::~ShenandoahNMethod() {
  48   if (_oops != NULL) {
  49     FREE_C_HEAP_ARRAY(oop*, _oops);
  50   }
  51 }
  52 
  53 class ShenandoahHasCSetOopClosure : public OopClosure {
  54 private:
  55   ShenandoahHeap* const _heap;
  56   bool                  _has_cset_oops;
  57 
  58 public:
  59   ShenandoahHasCSetOopClosure() :
  60     _heap(ShenandoahHeap::heap()),
  61     _has_cset_oops(false) {
  62   }
  63 
  64   bool has_cset_oops() const {
  65     return _has_cset_oops;
  66   }
  67 
  68   void do_oop(oop* p) {
  69     oop value = RawAccess<>::oop_load(p);
  70     if (!_has_cset_oops && _heap->in_collection_set(value)) {
  71       _has_cset_oops = true;
  72     }
  73   }
  74 
  75   void do_oop(narrowOop* p) {
  76     ShouldNotReachHere();
  77   }
  78 };
  79 
  80 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
  81   ShenandoahHasCSetOopClosure cl;
  82   oops_do(&cl);
  83   return cl.has_cset_oops();
  84 }
  85 
  86 void ShenandoahNMethod::update() {
  87   ResourceMark rm;
  88   bool non_immediate_oops = false;
  89   GrowableArray<oop*> oops;
  90 
  91   detect_reloc_oops(nm(), oops, non_immediate_oops);
  92   if (oops.length() != _oops_count) {
  93     if (_oops != NULL) {
  94       FREE_C_HEAP_ARRAY(oop*, _oops);
  95       _oops = NULL;
  96     }
  97 
  98     _oops_count = oops.length();
  99     if (_oops_count > 0) {
 100       _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
 101     }
 102   }
 103 
 104   for (int index = 0; index < _oops_count; index ++) {
 105     _oops[index] = oops.at(index);
 106   }
 107   _has_non_immed_oops = non_immediate_oops;
 108 
 109   assert_same_oops();
 110 }
 111 
 112 void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) {
 113   for (int c = 0; c < _oops_count; c ++) {
 114     oops->do_oop(_oops[c]);
 115   }
 116 
 117   oop* const begin = _nm->oops_begin();
 118   oop* const end = _nm->oops_end();
 119   for (oop* p = begin; p < end; p++) {
 120     if (*p != Universe::non_oop_word()) {
 121       oops->do_oop(p);
 122     }
 123   }
 124 
 125   if (fix_relocations && _has_non_immed_oops) {
 126     _nm->fix_oop_relocations();
 127   }
 128 }
 129 
 130 void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) {
 131   has_non_immed_oops = false;
 132   // Find all oops relocations
 133   RelocIterator iter(nm);
 134   while (iter.next()) {
 135     if (iter.type() != relocInfo::oop_type) {
 136       // Not an oop
 137       continue;
 138     }
 139 
 140     oop_Relocation* r = iter.oop_reloc();
 141     if (!r->oop_is_immediate()) {
 142       // Non-immediate oop found
 143       has_non_immed_oops = true;
 144       continue;
 145     }
 146 
 147     if (r->oop_value() != NULL) {
 148       // Non-NULL immediate oop found. NULL oops can safely be
 149       // ignored since the method will be re-registered if they
 150       // are later patched to be non-NULL.
 151       oops.push(r->oop_addr());
 152     }
 153   }
 154 }
 155 
 156 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
 157   ResourceMark rm;
 158   bool non_immediate_oops = false;
 159   GrowableArray<oop*> oops;
 160 
 161   detect_reloc_oops(nm, oops, non_immediate_oops);
 162 
 163   // No embedded oops
 164   if(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() &&
 165     oops.is_empty() && nm->oops_begin() >= nm->oops_end()) {
 166     return NULL;
 167   }
 168 
 169   return new ShenandoahNMethod(nm, oops, non_immediate_oops);
 170 }
 171 
 172 void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
 173   ShenandoahNMethod* data = gc_data(nm);
 174   assert(data != NULL, "Sanity");
 175   assert(data->lock()->owned_by_self(), "Must hold the lock");
 176 
 177   ShenandoahEvacuateUpdateRootsClosure cl;
 178   data->oops_do(&cl, true /*fix relocation*/);
 179 }
 180 
 181 #ifdef ASSERT
 182 void ShenandoahNMethod::assert_alive_and_correct() {
 183   assert(_nm->is_alive(), "only alive nmethods here");
 184   ShenandoahHeap* heap = ShenandoahHeap::heap();
 185   for (int c = 0; c < _oops_count; c++) {
 186     oop *loc = _oops[c];
 187     assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
 188     oop o = RawAccess<>::oop_load(loc);
 189     shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
 190   }
 191 
 192   oop* const begin = _nm->oops_begin();
 193   oop* const end = _nm->oops_end();
 194   for (oop* p = begin; p < end; p++) {
 195     if (*p != Universe::non_oop_word()) {
 196       oop o = RawAccess<>::oop_load(p);
 197       shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress());
 198     }
 199   }
 200 }
 201 
 202 class ShenandoahNMethodOopDetector : public OopClosure {
 203 private:
 204   ResourceMark rm; // For growable array allocation below.
 205   GrowableArray<oop*> _oops;
 206 
 207 public:
 208   ShenandoahNMethodOopDetector() : _oops(10) {};
 209 
 210   void do_oop(oop* o) {
 211     _oops.append(o);
 212   }
 213   void do_oop(narrowOop* o) {
 214     fatal("NMethods should not have compressed oops embedded.");
 215   }
 216 
 217   GrowableArray<oop*>* oops() {
 218     return &_oops;
 219   }
 220 
 221   bool has_oops() {
 222     return !_oops.is_empty();
 223   }
 224 };
 225 
 226 void ShenandoahNMethod::assert_same_oops(bool allow_dead) {
 227   ShenandoahNMethodOopDetector detector;
 228   nm()->oops_do(&detector, allow_dead);
 229 
 230   GrowableArray<oop*>* oops = detector.oops();
 231 
 232   assert(oops->length() == oop_count(), "Must match");
 233 
 234   for (int index = 0; index < _oops_count; index ++) {
 235     assert(oops->contains(_oops[index]), "Must contain this oop");
 236   }
 237 
 238   for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) {
 239     assert(oops->contains(p), "Must contain this oop");
 240   }
 241 }
 242 
 243 void ShenandoahNMethod::assert_no_oops(nmethod* nm, bool allow_dead) {
 244   ShenandoahNMethodOopDetector detector;
 245   nm->oops_do(&detector, allow_dead);
 246   assert(detector.oops()->length() == 0, "Should not have oops");
 247 }
 248 #endif
 249 
 250 ShenandoahNMethodTable::ShenandoahNMethodTable() :
 251   _heap(ShenandoahHeap::heap()),
 252   _size(minSize),
 253   _index(0),
 254   _iteration_in_progress(false) {
 255   _array = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, _size, mtGC);
 256 }
 257 
 258 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
 259   assert(_array != NULL, "Sanity");
 260   FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _array);
 261 }
 262 
 263 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
 264   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
 265   assert(_index >= 0 && _index <= _size, "Sanity");
 266 
 267   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
 268   ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL);
 269 
 270   if (data != NULL) {
 271     assert(contain(nm), "Must have been registered");
 272     data->update();
 273   } else {
 274     data = ShenandoahNMethod::for_nmethod(nm);
 275     if (data == NULL) {
 276       assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(),
 277              "Only possible when concurrent class unloading is off");
 278       return;
 279     }
 280     ShenandoahNMethod::attach_gc_data(nm, data);
 281     ShenandoahLocker locker(&_lock);
 282     log_register_nmethod(nm);
 283     append(data);
 284   }
 285   // Disarm new nmethod
 286   ShenandoahNMethod::disarm_nmethod(nm);
 287 }
 288 
 289 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
 290   assert_locked_or_safepoint(CodeCache_lock);
 291 
 292   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
 293   if (data == NULL) {
 294     assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(),
 295            "Only possible when concurrent class unloading is off");
 296     ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/);
 297     return;
 298   }
 299 
 300   if (Thread::current()->is_Code_cache_sweeper_thread()) {
 301     wait_until_concurrent_iteration_done();
 302   }
 303   log_unregister_nmethod(nm);
 304   ShenandoahLocker locker(&_lock);
 305   assert(contain(nm), "Must have been registered");
 306 
 307   ShenandoahReentrantLocker data_locker(data->lock());
 308   data->mark_unregistered();
 309 }
 310 
 311 void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) {
 312   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
 313   assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread");
 314   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
 315   assert(data != NULL || !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(),
 316          "Only possible when concurrent class unloading is off");
 317   if (data == NULL) {
 318     ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/);
 319     return;
 320   }
 321 
 322   // Can not alter the array when iteration is in progress
 323   wait_until_concurrent_iteration_done();
 324   log_flush_nmethod(nm);
 325 
 326   ShenandoahLocker locker(&_lock);
 327   int idx = index_of(nm);
 328   assert(idx >= 0 && idx < _index, "Invalid index");
 329   ShenandoahNMethod::attach_gc_data(nm, NULL);
 330   remove(idx);
 331 }
 332 
 333 bool ShenandoahNMethodTable::contain(nmethod* nm) const {
 334   return index_of(nm) != -1;
 335 }
 336 
 337 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
 338   assert(index >= 0 && index < _index, "Out of bound");
 339   return _array[index];
 340 }
 341 
 342 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
 343   for (int index = 0; index < length(); index ++) {
 344     if (_array[index]->nm() == nm) {
 345       return index;
 346     }
 347   }
 348   return -1;
 349 }
 350 
 351 void ShenandoahNMethodTable::remove(int idx) {
 352   shenandoah_assert_locked_or_safepoint(CodeCache_lock);
 353   assert(!_iteration_in_progress, "Can not happen");
 354   assert(_index >= 0 && _index <= _size, "Sanity");
 355 
 356   assert(idx >= 0 && idx < _index, "Out of bound");
 357   ShenandoahNMethod* snm = _array[idx];
 358 
 359   _index --;
 360   _array[idx] = _array[_index];
 361 
 362   delete snm;
 363 }
 364 
 365 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
 366   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
 367   while (iteration_in_progress()) {
 368     CodeCache_lock->wait_without_safepoint_check();
 369   }
 370 }
 371 
 372 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
 373   if (is_full()) {
 374     int new_size = 2 * _size;
 375     ShenandoahNMethod** old_table = _array;
 376 
 377     // Rebuild table and replace current one
 378     rebuild(new_size);
 379 
 380     // An iteration is in progress over early snapshot,
 381     // can not release the array until iteration is completed
 382     if (!iteration_in_progress()) {
 383       FREE_C_HEAP_ARRAY(ShenandoahNMethod*, old_table);
 384     }
 385   }
 386 
 387   _array[_index ++] = snm;
 388   assert(_index >= 0 && _index <= _size, "Sanity");
 389 }
 390 
 391 void ShenandoahNMethodTable::rebuild(int size) {
 392   ShenandoahNMethod** arr = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
 393   for (int index = 0; index < _index; index ++) {
 394       arr[index] = _array[index];
 395   }
 396   _array = arr;
 397   _size = size;
 398 }
 399 
 400 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
 401   assert(!iteration_in_progress(), "Already in progress");
 402   _iteration_in_progress = true;
 403 
 404   return new ShenandoahNMethodTableSnapshot(this);
 405 }
 406 
 407 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
 408   assert(iteration_in_progress(), "Why we here?");
 409   assert(snapshot != NULL, "No snapshot");
 410   _iteration_in_progress = false;
 411 
 412   // Table has been rebuilt during iteration, free old table
 413   if (snapshot->_array != _array) {
 414     FREE_C_HEAP_ARRAY(ShenandoahNMethod*, snapshot->_array);
 415   }
 416   delete snapshot;
 417 }
 418 
 419 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) {
 420   LogTarget(Debug, gc, nmethod) log;
 421   if (!log.is_enabled()) {
 422     return;
 423   }
 424 
 425   ResourceMark rm;
 426   log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)",
 427             nm->method()->method_holder()->external_name(),
 428             nm->method()->name()->as_C_string(),
 429             p2i(nm),
 430             nm->compiler_name());
 431 }
 432 
 433 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) {
 434   LogTarget(Debug, gc, nmethod) log;
 435   if (!log.is_enabled()) {
 436     return;
 437   }
 438 
 439   ResourceMark rm;
 440   log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]",
 441             nm->method()->method_holder()->external_name(),
 442             nm->method()->name()->as_C_string(),
 443             p2i(nm));
 444 }
 445 
 446 void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) {
 447   LogTarget(Debug, gc, nmethod) log;
 448   if (!log.is_enabled()) {
 449     return;
 450   }
 451 
 452   ResourceMark rm;
 453   log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm));
 454 }
 455 
 456 #ifdef ASSERT
 457 void ShenandoahNMethodTable::assert_nmethods_alive_and_correct() {
 458   assert_locked_or_safepoint(CodeCache_lock);
 459 
 460   for (int index = 0; index < length(); index ++) {
 461     ShenandoahNMethod* m = _array[index];
 462     // Concurrent unloading may have dead nmethods to be cleaned by sweeper
 463     if (m->is_unregistered()) continue;
 464     m->assert_alive_and_correct();
 465   }
 466 }
 467 #endif
 468 
 469 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
 470   _heap(ShenandoahHeap::heap()), _table(table), _array(table->_array), _length(table->_index), _claimed(0) {
 471 }
 472 
 473 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
 474   size_t stride = 256; // educated guess
 475 
 476   ShenandoahNMethod** list = _array;
 477   size_t max = (size_t)_length;
 478   while (_claimed < max) {
 479     size_t cur = Atomic::add(stride, &_claimed) - stride;
 480     size_t start = cur;
 481     size_t end = MIN2(cur + stride, max);
 482     if (start >= max) break;
 483 
 484     for (size_t idx = start; idx < end; idx++) {
 485       ShenandoahNMethod* data = list[idx];
 486       assert(data != NULL, "Should not be NULL");
 487       if (!data->is_unregistered()) {
 488         cl->do_nmethod(data->nm());
 489       }
 490     }
 491   }
 492 }
 493 
 494 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) :
 495   _table(table), _table_snapshot(NULL) {
 496 }
 497 
 498 void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() {
 499   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
 500   assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(),
 501          "Only for concurrent class unloading");
 502   _table_snapshot = _table->snapshot_for_iteration();
 503 }
 504 
 505 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) {
 506   assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()");
 507   _table_snapshot->concurrent_nmethods_do(cl);
 508 }
 509 
 510 void ShenandoahConcurrentNMethodIterator::nmethods_do_end() {
 511   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
 512   assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(),
 513          "Only for concurrent class unloading");
 514   _table->finish_iteration(_table_snapshot);
 515   CodeCache_lock->notify_all();
 516 }