< prev index next >

src/hotspot/share/code/aotCodeCache.cpp

Print this page

  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 
  26 #include "asm/macroAssembler.hpp"
  27 #include "cds/aotCacheAccess.hpp"
  28 #include "cds/aotMetaspace.hpp"
  29 #include "cds/cds_globals.hpp"
  30 #include "cds/cdsConfig.hpp"
  31 #include "cds/heapShared.hpp"







  32 #include "classfile/javaAssertions.hpp"





  33 #include "code/aotCodeCache.hpp"

  34 #include "code/codeCache.hpp"






  35 #include "gc/shared/gcConfig.hpp"
  36 #include "logging/logStream.hpp"
  37 #include "memory/memoryReserver.hpp"






  38 #include "runtime/deoptimization.hpp"
  39 #include "runtime/flags/flagSetting.hpp"
  40 #include "runtime/globals_extension.hpp"

  41 #include "runtime/java.hpp"

  42 #include "runtime/mutexLocker.hpp"
  43 #include "runtime/os.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubInfo.hpp"
  46 #include "runtime/stubRoutines.hpp"


  47 #include "utilities/copy.hpp"



  48 #ifdef COMPILER1

  49 #include "c1/c1_Runtime1.hpp"
  50 #endif






  51 #ifdef COMPILER2
  52 #include "opto/runtime.hpp"
  53 #endif



  54 #if INCLUDE_G1GC
  55 #include "gc/g1/g1BarrierSetRuntime.hpp"
  56 #endif
  57 #if INCLUDE_SHENANDOAHGC
  58 #include "gc/shenandoah/shenandoahRuntime.hpp"
  59 #endif
  60 #if INCLUDE_ZGC
  61 #include "gc/z/zBarrierSetRuntime.hpp"
  62 #endif



  63 
  64 #include <errno.h>
  65 #include <sys/stat.h>
  66 
  67 const char* aot_code_entry_kind_name[] = {
  68 #define DECL_KIND_STRING(kind) XSTR(kind),
  69   DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
  70 #undef DECL_KIND_STRING
  71 };
  72 










  73 static void report_load_failure() {
  74   if (AbortVMOnAOTCodeFailure) {
  75     vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
  76   }
  77   log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
  78   AOTCodeCache::disable_caching();
  79 }
  80 
  81 static void report_store_failure() {
  82   if (AbortVMOnAOTCodeFailure) {
  83     tty->print_cr("Unable to create AOT Code Cache.");
  84     vm_abort(false);
  85   }
  86   log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
  87   AOTCodeCache::disable_caching();
  88 }
  89 
  90 // The sequence of AOT code caching flags and parametters settings.
  91 //
  92 // 1. The initial AOT code caching flags setting is done

 107 
 108 // Next methods determine which action we do with AOT code depending
 109 // on phase of AOT process: assembly or production.
 110 
 111 bool AOTCodeCache::is_dumping_adapter() {
 112   return AOTAdapterCaching && is_on_for_dump();
 113 }
 114 
 115 bool AOTCodeCache::is_using_adapter()   {
 116   return AOTAdapterCaching && is_on_for_use();
 117 }
 118 
 119 bool AOTCodeCache::is_dumping_stub() {
 120   return AOTStubCaching && is_on_for_dump();
 121 }
 122 
 123 bool AOTCodeCache::is_using_stub()   {
 124   return AOTStubCaching && is_on_for_use();
 125 }
 126 
 127 // Next methods could be called regardless AOT code cache status.
 128 // Initially they are called during flags parsing and finilized














 129 // in AOTCodeCache::initialize().
 130 void AOTCodeCache::enable_caching() {

 131   FLAG_SET_ERGO_IF_DEFAULT(AOTStubCaching, true);
 132   FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true);
 133 }
 134 
 135 void AOTCodeCache::disable_caching() {

 136   FLAG_SET_ERGO(AOTStubCaching, false);
 137   FLAG_SET_ERGO(AOTAdapterCaching, false);
 138 }
 139 
 140 bool AOTCodeCache::is_caching_enabled() {
 141   return AOTStubCaching || AOTAdapterCaching;
 142 }
 143 
 144 static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
 145   assert(AOTCodeEntry::is_valid_entry_kind(kind), "invalid AOTCodeEntry kind %d", (int)kind);
 146   // There can be a conflict of id between an Adapter and *Blob, but that should not cause any functional issue
 147   // becasue both id and kind are used to find an entry, and that combination should be unique
 148   if (kind == AOTCodeEntry::Adapter) {
 149     return id;
 150   } else if (kind == AOTCodeEntry::SharedBlob) {
 151     assert(StubInfo::is_shared(static_cast<BlobId>(id)), "not a shared blob id %d", id);
 152     return id;
 153   } else if (kind == AOTCodeEntry::C1Blob) {
 154     assert(StubInfo::is_c1(static_cast<BlobId>(id)), "not a c1 blob id %d", id);
 155     return id;
 156   } else {
 157     // kind must be AOTCodeEntry::C2Blob
 158     assert(StubInfo::is_c2(static_cast<BlobId>(id)), "not a c2 blob id %d", id);
 159     return id;
 160   }
 161 }
 162 
 163 static uint _max_aot_code_size = 0;
 164 uint AOTCodeCache::max_aot_code_size() {
 165   return _max_aot_code_size;
 166 }
 167 














 168 // It is called from AOTMetaspace::initialize_shared_spaces()
 169 // which is called from universe_init().
 170 // At this point all AOT class linking seetings are finilized
 171 // and AOT cache is open so we can map AOT code region.
 172 void AOTCodeCache::initialize() {




 173 #if defined(ZERO) || !(defined(AMD64) || defined(AARCH64))
 174   log_info(aot, codecache, init)("AOT Code Cache is not supported on this platform.");
 175   disable_caching();
 176   return;
 177 #else
 178   if (FLAG_IS_DEFAULT(AOTCache)) {
 179     log_info(aot, codecache, init)("AOT Code Cache is not used: AOTCache is not specified.");
 180     disable_caching();
 181     return; // AOTCache must be specified to dump and use AOT code
 182   }
 183 
 184   // Disable stubs caching until JDK-8357398 is fixed.
 185   FLAG_SET_ERGO(AOTStubCaching, false);
 186 
 187   if (VerifyOops) {
 188     // Disable AOT stubs caching when VerifyOops flag is on.
 189     // Verify oops code generated a lot of C strings which overflow
 190     // AOT C string table (which has fixed size).
 191     // AOT C string table will be reworked later to handle such cases.
 192     //
 193     // Note: AOT adapters are not affected - they don't have oop operations.
 194     log_info(aot, codecache, init)("AOT Stubs Caching is not supported with VerifyOops.");
 195     FLAG_SET_ERGO(AOTStubCaching, false);
 196   }
 197 
 198   bool is_dumping = false;
 199   bool is_using   = false;
 200   if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) {
 201     is_dumping = true;
 202     enable_caching();
 203     is_dumping = is_caching_enabled();
 204   } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) {
 205     enable_caching();
 206     is_using = is_caching_enabled();
 207   } else {
 208     log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used.");
 209     disable_caching();
 210     return; // nothing to do
 211   }
 212   if (!(is_dumping || is_using)) {

 213     disable_caching();
 214     return; // AOT code caching disabled on command line
 215   }

 216   _max_aot_code_size = AOTCodeMaxSize;
 217   if (!FLAG_IS_DEFAULT(AOTCodeMaxSize)) {
 218     if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) {
 219       _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity());
 220       log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K));
 221     }
 222   }
 223   size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0;
 224   if (is_using && aot_code_size == 0) {
 225     log_info(aot, codecache, init)("AOT Code Cache is empty");
 226     disable_caching();
 227     return;
 228   }
 229   if (!open_cache(is_dumping, is_using)) {
 230     if (is_using) {
 231       report_load_failure();
 232     } else {
 233       report_store_failure();
 234     }
 235     return;
 236   }
 237   if (is_dumping) {

 238     FLAG_SET_DEFAULT(ForceUnreachable, true);
 239   }
 240   FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
 241 #endif // defined(AMD64) || defined(AARCH64)
 242 }
 243 
 244 static AOTCodeCache*  opened_cache = nullptr; // Use this until we verify the cache
 245 AOTCodeCache* AOTCodeCache::_cache = nullptr;
 246 DEBUG_ONLY( bool AOTCodeCache::_passed_init2 = false; )
 247 
 248 // It is called after universe_init() when all GC settings are finalized.
 249 void AOTCodeCache::init2() {
 250   DEBUG_ONLY( _passed_init2 = true; )
 251   if (opened_cache == nullptr) {
 252     return;
 253   }
 254   if (!opened_cache->verify_config()) {

 255     delete opened_cache;
 256     opened_cache = nullptr;
 257     report_load_failure();
 258     return;
 259   }
 260 
 261   // initialize the table of external routines so we can save



 262   // generated code blobs that reference them
 263   AOTCodeAddressTable* table = opened_cache->_table;
 264   assert(table != nullptr, "should be initialized already");
 265   table->init_extrs();
 266 
 267   // Now cache and address table are ready for AOT code generation
 268   _cache = opened_cache;







 269 }
 270 
 271 bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
 272   opened_cache = new AOTCodeCache(is_dumping, is_using);
 273   if (opened_cache->failed()) {
 274     delete opened_cache;
 275     opened_cache = nullptr;
 276     return false;
 277   }
 278   return true;
 279 }
 280 

















 281 void AOTCodeCache::close() {
 282   if (is_on()) {
 283     delete _cache; // Free memory
 284     _cache = nullptr;
 285     opened_cache = nullptr;
 286   }
 287 }
 288 











































 289 #define DATA_ALIGNMENT HeapWordSize
 290 
 291 AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
 292   _load_header(nullptr),
 293   _load_buffer(nullptr),
 294   _store_buffer(nullptr),
 295   _C_store_buffer(nullptr),
 296   _write_position(0),
 297   _load_size(0),
 298   _store_size(0),
 299   _for_use(is_using),
 300   _for_dump(is_dumping),
 301   _closing(false),
 302   _failed(false),
 303   _lookup_failed(false),


 304   _table(nullptr),
 305   _load_entries(nullptr),
 306   _search_entries(nullptr),
 307   _store_entries(nullptr),
 308   _C_strings_buf(nullptr),
 309   _store_entries_cnt(0)


 310 {
 311   // Read header at the begining of cache
 312   if (_for_use) {
 313     // Read cache
 314     size_t load_size = AOTCacheAccess::get_aot_code_region_size();
 315     ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode);
 316     if (!rs.is_reserved()) {
 317       log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size);
 318       set_failed();
 319       return;
 320     }
 321     if (!AOTCacheAccess::map_aot_code_region(rs)) {
 322       log_warning(aot, codecache, init)("Failed to read/mmap cached code region into AOT Code Cache");
 323       set_failed();
 324       return;
 325     }


 326 
 327     _load_size = (uint)load_size;
 328     _load_buffer = (char*)rs.base();
 329     assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
 330     log_debug(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " at AOT Code Cache", _load_size, p2i(_load_buffer));
 331 
 332     _load_header = (Header*)addr(0);
 333     if (!_load_header->verify(_load_size)) {
 334       set_failed();
 335       return;
 336     }
 337     log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count());
 338     log_debug(aot, codecache, init)("  Adapters:  total=%u", _load_header->adapters_count());
 339     log_debug(aot, codecache, init)("  Shared Blobs: total=%u", _load_header->shared_blobs_count());
 340     log_debug(aot, codecache, init)("  C1 Blobs: total=%u", _load_header->C1_blobs_count());
 341     log_debug(aot, codecache, init)("  C2 Blobs: total=%u", _load_header->C2_blobs_count());


 342     log_debug(aot, codecache, init)("  AOT code cache size: %u bytes", _load_header->cache_size());
 343 
 344     // Read strings
 345     load_strings();
 346   }
 347   if (_for_dump) {
 348     _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
 349     _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
 350     // Entries allocated at the end of buffer in reverse (as on stack).
 351     _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
 352     log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size());
 353   }
 354   _table = new AOTCodeAddressTable();
 355 }
 356 







 357 void AOTCodeCache::init_early_stubs_table() {
 358   AOTCodeAddressTable* table = addr_table();
 359   if (table != nullptr) {
 360     table->init_early_stubs();
 361   }
 362 }
 363 
 364 void AOTCodeCache::init_shared_blobs_table() {
 365   AOTCodeAddressTable* table = addr_table();
 366   if (table != nullptr) {
 367     table->init_shared_blobs();
 368   }
 369 }
 370 







 371 void AOTCodeCache::init_early_c1_table() {
 372   AOTCodeAddressTable* table = addr_table();
 373   if (table != nullptr) {
 374     table->init_early_c1();
 375   }
 376 }
 377 














 378 AOTCodeCache::~AOTCodeCache() {
 379   if (_closing) {
 380     return; // Already closed
 381   }
 382   // Stop any further access to cache.

 383   _closing = true;
 384 





 385   MutexLocker ml(Compile_lock);
 386   if (for_dump()) { // Finalize cache
 387     finish_write();
 388   }
 389   _load_buffer = nullptr;
 390   if (_C_store_buffer != nullptr) {
 391     FREE_C_HEAP_ARRAY(char, _C_store_buffer);
 392     _C_store_buffer = nullptr;
 393     _store_buffer = nullptr;
 394   }
 395   if (_table != nullptr) {
 396     MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
 397     delete _table;
 398     _table = nullptr;
 399   }
 400 }
 401 
 402 void AOTCodeCache::Config::record() {
 403   _flags = 0;
 404 #ifdef ASSERT
 405   _flags |= debugVM;
 406 #endif
 407   if (UseCompressedOops) {
 408     _flags |= compressedOops;
 409   }
 410   if (UseCompressedClassPointers) {
 411     _flags |= compressedClassPointers;
 412   }
 413   if (UseTLAB) {
 414     _flags |= useTLAB;
 415   }
 416   if (JavaAssertions::systemClassDefault()) {
 417     _flags |= systemClassAssertions;
 418   }
 419   if (JavaAssertions::userClassDefault()) {
 420     _flags |= userClassAssertions;
 421   }
 422   if (EnableContended) {
 423     _flags |= enableContendedPadding;
 424   }
 425   if (RestrictContended) {
 426     _flags |= restrictContendedPadding;
 427   }




 428   _compressedOopShift    = CompressedOops::shift();
 429   _compressedOopBase     = CompressedOops::base();
 430   _compressedKlassShift  = CompressedKlassPointers::shift();

 431   _contendedPaddingWidth = ContendedPaddingWidth;


 432   _gc                    = (uint)Universe::heap()->kind();

 433 }
 434 
 435 bool AOTCodeCache::Config::verify() const {
 436   // First checks affect all cached AOT code
 437 #ifdef ASSERT
 438   if ((_flags & debugVM) == 0) {
 439     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
 440     return false;
 441   }
 442 #else
 443   if ((_flags & debugVM) != 0) {
 444     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
 445     return false;
 446   }
 447 #endif
 448 






 449   CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
 450   if (aot_gc != Universe::heap()->kind()) {
 451     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
 452     return false;
 453   }
 454 




























 455   if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
 456     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s", UseCompressedClassPointers ? "false" : "true");
 457     return false;
 458   }
 459   if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
 460     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
 461     return false;
 462   }
 463 
 464   // The following checks do not affect AOT adapters caching


 465 
 466   if (((_flags & compressedOops) != 0) != UseCompressedOops) {
 467     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s", UseCompressedOops ? "false" : "true");
 468     AOTStubCaching = false;
 469   }
 470   if (_compressedOopShift != (uint)CompressedOops::shift()) {
 471     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
 472     AOTStubCaching = false;
 473   }
 474 
 475   // This should be the last check as it only disables AOTStubCaching
 476   if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
 477     log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
 478     AOTStubCaching = false;






































 479   }
 480 
 481   return true;
 482 }
 483 
 484 bool AOTCodeCache::Header::verify(uint load_size) const {
 485   if (_version != AOT_CODE_VERSION) {
 486     log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
 487     return false;
 488   }
 489   if (load_size < _cache_size) {
 490     log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
 491     return false;
 492   }
 493   return true;
 494 }
 495 


 496 AOTCodeCache* AOTCodeCache::open_for_use() {
 497   if (AOTCodeCache::is_on_for_use()) {
 498     return AOTCodeCache::cache();
 499   }
 500   return nullptr;
 501 }
 502 
 503 AOTCodeCache* AOTCodeCache::open_for_dump() {
 504   if (AOTCodeCache::is_on_for_dump()) {
 505     AOTCodeCache* cache = AOTCodeCache::cache();
 506     cache->clear_lookup_failed(); // Reset bit
 507     return cache;
 508   }
 509   return nullptr;
 510 }
 511 
 512 void copy_bytes(const char* from, address to, uint size) {












 513   assert((int)size > 0, "sanity");
 514   memcpy(to, from, size);
 515   log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
 516 }
 517 
 518 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) {
 519   _cache = cache;
 520   _entry = entry;
 521   _load_buffer = cache->cache_buffer();
 522   _read_position = 0;









 523   _lookup_failed = false;
 524 }
 525 
 526 void AOTCodeReader::set_read_position(uint pos) {
 527   if (pos == _read_position) {
 528     return;
 529   }
 530   assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
 531   _read_position = pos;
 532 }
 533 
 534 bool AOTCodeCache::set_write_position(uint pos) {
 535   if (pos == _write_position) {
 536     return true;
 537   }
 538   if (_store_size < _write_position) {
 539     _store_size = _write_position; // Adjust during write
 540   }
 541   assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
 542   _write_position = pos;

 585   if (nbytes == 0) {
 586     return 0;
 587   }
 588   uint new_position = _write_position + nbytes;
 589   if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
 590     log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.",
 591                                 nbytes, _write_position);
 592     set_failed();
 593     report_store_failure();
 594     return 0;
 595   }
 596   copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
 597   log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
 598   _write_position += nbytes;
 599   if (_store_size < _write_position) {
 600     _store_size = _write_position;
 601   }
 602   return nbytes;
 603 }
 604 

































































 605 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
 606   return (void*)(cache->add_entry());
 607 }
 608 
 609 static bool check_entry(AOTCodeEntry::Kind kind, uint id, AOTCodeEntry* entry) {
 610   if (entry->kind() == kind) {
 611     assert(entry->id() == id, "sanity");
 612     return true; // Found




 613   }
 614   return false;
 615 }
 616 
 617 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id) {
 618   assert(_for_use, "sanity");
 619   uint count = _load_header->entries_count();
 620   if (_load_entries == nullptr) {
 621     // Read it
 622     _search_entries = (uint*)addr(_load_header->entries_offset()); // [id, index]
 623     _load_entries = (AOTCodeEntry*)(_search_entries + 2 * count);
 624     log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
 625   }
 626   // Binary search
 627   int l = 0;
 628   int h = count - 1;
 629   while (l <= h) {
 630     int mid = (l + h) >> 1;
 631     int ix = mid * 2;
 632     uint is = _search_entries[ix];
 633     if (is == id) {
 634       int index = _search_entries[ix + 1];
 635       AOTCodeEntry* entry = &(_load_entries[index]);
 636       if (check_entry(kind, id, entry)) {
 637         return entry; // Found
 638       }
 639       // Linear search around to handle id collission
 640       for (int i = mid - 1; i >= l; i--) { // search back
 641         ix = i * 2;
 642         is = _search_entries[ix];
 643         if (is != id) {
 644           break;
 645         }
 646         index = _search_entries[ix + 1];
 647         AOTCodeEntry* entry = &(_load_entries[index]);
 648         if (check_entry(kind, id, entry)) {
 649           return entry; // Found
 650         }
 651       }
 652       for (int i = mid + 1; i <= h; i++) { // search forward
 653         ix = i * 2;
 654         is = _search_entries[ix];
 655         if (is != id) {
 656           break;
 657         }
 658         index = _search_entries[ix + 1];
 659         AOTCodeEntry* entry = &(_load_entries[index]);
 660         if (check_entry(kind, id, entry)) {
 661           return entry; // Found
 662         }
 663       }
 664       break; // Not found match
 665     } else if (is < id) {
 666       l = mid + 1;
 667     } else {
 668       h = mid - 1;
 669     }
 670   }
 671   return nullptr;
 672 }
 673 
 674 extern "C" {
 675   static int uint_cmp(const void *i, const void *j) {
 676     uint a = *(uint *)i;
 677     uint b = *(uint *)j;
 678     return a > b ? 1 : a < b ? -1 : 0;
























































 679   }
 680 }
 681 

















 682 bool AOTCodeCache::finish_write() {
 683   if (!align_write()) {
 684     return false;
 685   }
 686   uint strings_offset = _write_position;
 687   int strings_count = store_strings();
 688   if (strings_count < 0) {
 689     return false;
 690   }
 691   if (!align_write()) {
 692     return false;
 693   }
 694   uint strings_size = _write_position - strings_offset;
 695 
 696   uint entries_count = 0; // Number of entrant (useful) code entries
 697   uint entries_offset = _write_position;


 698 
 699   uint store_count = _store_entries_cnt;
 700   if (store_count > 0) {
 701     uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header),  DATA_ALIGNMENT);
 702     uint code_count = store_count;
 703     uint search_count = code_count * 2;
 704     uint search_size = search_count * sizeof(uint);
 705     uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
 706     // _write_position includes size of code and strings
 707     uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
 708     uint total_size = header_size + _write_position + code_alignment + search_size + entries_size;




 709     assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size());
 710 
 711     // Create ordered search table for entries [id, index];
 712     uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
 713     // Allocate in AOT Cache buffer
 714     char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT);
 715     char* start = align_up(buffer, DATA_ALIGNMENT);
 716     char* current = start + header_size; // Skip header
 717 








 718     AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
 719     uint adapters_count = 0;
 720     uint shared_blobs_count = 0;
 721     uint C1_blobs_count = 0;
 722     uint C2_blobs_count = 0;
 723     uint max_size = 0;
 724     // AOTCodeEntry entries were allocated in reverse in store buffer.
 725     // Process them in reverse order to cache first code first.
 726     for (int i = store_count - 1; i >= 0; i--) {
 727       entries_address[i].set_next(nullptr); // clear pointers before storing data
 728       uint size = align_up(entries_address[i].size(), DATA_ALIGNMENT);



























 729       if (size > max_size) {
 730         max_size = size;
 731       }
 732       copy_bytes((_store_buffer + entries_address[i].offset()), (address)current, size);
 733       entries_address[i].set_offset(current - start); // New offset
 734       current += size;
 735       uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry));
 736       if (n != sizeof(AOTCodeEntry)) {
 737         FREE_C_HEAP_ARRAY(uint, search);
 738         return false;







 739       }
 740       search[entries_count*2 + 0] = entries_address[i].id();








 741       search[entries_count*2 + 1] = entries_count;
 742       entries_count++;
 743       AOTCodeEntry::Kind kind = entries_address[i].kind();
 744       if (kind == AOTCodeEntry::Adapter) {
 745         adapters_count++;
 746       } else if (kind == AOTCodeEntry::SharedBlob) {
 747         shared_blobs_count++;
 748       } else if (kind == AOTCodeEntry::C1Blob) {
 749         C1_blobs_count++;
 750       } else if (kind == AOTCodeEntry::C2Blob) {
 751         C2_blobs_count++;
 752       }
 753     }
 754     if (entries_count == 0) {
 755       log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires");



 756       FREE_C_HEAP_ARRAY(uint, search);
 757       return true; // Nothing to write
 758     }
 759     assert(entries_count <= store_count, "%d > %d", entries_count, store_count);

 760     // Write strings
 761     if (strings_count > 0) {
 762       copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
 763       strings_offset = (current - start); // New offset
 764       current += strings_size;
 765     }
 766 
 767     uint new_entries_offset = (current - start); // New offset
 768     // Sort and store search table
 769     qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
 770     search_size = 2 * entries_count * sizeof(uint);
 771     copy_bytes((const char*)search, (address)current, search_size);
 772     FREE_C_HEAP_ARRAY(uint, search);
 773     current += search_size;
 774 
 775     // Write entries
 776     entries_size = entries_count * sizeof(AOTCodeEntry); // New size
 777     copy_bytes((_store_buffer + entries_offset), (address)current, entries_size);
 778     current += entries_size;
 779     uint size = (current - start);
 780     assert(size <= total_size, "%d > %d", size , total_size);
 781 
 782     log_debug(aot, codecache, exit)("  Adapters:  total=%u", adapters_count);
 783     log_debug(aot, codecache, exit)("  Shared Blobs:  total=%d", shared_blobs_count);
 784     log_debug(aot, codecache, exit)("  C1 Blobs:      total=%d", C1_blobs_count);
 785     log_debug(aot, codecache, exit)("  C2 Blobs:      total=%d", C2_blobs_count);
 786     log_debug(aot, codecache, exit)("  AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
 787 
 788     // Finalize header
 789     AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
 790     header->init(size, (uint)strings_count, strings_offset,
 791                  entries_count, new_entries_offset,
 792                  adapters_count, shared_blobs_count,
 793                  C1_blobs_count, C2_blobs_count);




 794 
 795     log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count);
 796   }
 797   return true;
 798 }
 799 
 800 //------------------Store/Load AOT code ----------------------
 801 
 802 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
 803   AOTCodeCache* cache = open_for_dump();
 804   if (cache == nullptr) {
 805     return false;
 806   }
 807   assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
 808 
 809   if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
 810     return false;
 811   }
 812   if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
 813     return false;
 814   }
 815   log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);

 850     return false;
 851   }
 852   CodeBlob::archive_blob(&blob, archive_buffer);
 853 
 854   uint reloc_data_size = blob.relocation_size();
 855   n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size);
 856   if (n != reloc_data_size) {
 857     return false;
 858   }
 859 
 860   bool has_oop_maps = false;
 861   if (blob.oop_maps() != nullptr) {
 862     if (!cache->write_oop_map_set(blob)) {
 863       return false;
 864     }
 865     has_oop_maps = true;
 866   }
 867 
 868 #ifndef PRODUCT
 869   // Write asm remarks
 870   if (!cache->write_asm_remarks(blob)) {
 871     return false;
 872   }
 873   if (!cache->write_dbg_strings(blob)) {
 874     return false;
 875   }
 876 #endif /* PRODUCT */
 877 
 878   if (!cache->write_relocations(blob)) {
 879     if (!cache->failed()) {
 880       // We may miss an address in AOT table - skip this code blob.
 881       cache->set_write_position(entry_position);
 882     }
 883     return false;
 884   }
 885 
 886   uint entry_size = cache->_write_position - entry_position;
 887   AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
 888                                                 entry_position, entry_size, name_offset, name_size,
 889                                                 blob_offset, has_oop_maps, blob.content_begin());
 890   log_debug(aot, codecache, stubs)("Wrote code blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
 891   return true;
 892 }
 893 

 899 
 900 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
 901   AOTCodeCache* cache = open_for_use();
 902   if (cache == nullptr) {
 903     return nullptr;
 904   }
 905   assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
 906 
 907   if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
 908     return nullptr;
 909   }
 910   if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
 911     return nullptr;
 912   }
 913   log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
 914 
 915   AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
 916   if (entry == nullptr) {
 917     return nullptr;
 918   }
 919   AOTCodeReader reader(cache, entry);
 920   CodeBlob* blob = reader.compile_code_blob(name);
 921 
 922   log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
 923                                    (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
 924   return blob;
 925 }
 926 
 927 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
 928   assert(AOTCodeEntry::is_blob(entry_kind),
 929          "wrong entry kind for blob id %s", StubInfo::name(id));
 930   return load_code_blob(entry_kind, (uint)id, StubInfo::name(id));
 931 }
 932 
 933 CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
 934   uint entry_position = _entry->offset();
 935 
 936   // Read name
 937   uint name_offset = entry_position + _entry->name_offset();
 938   uint name_size = _entry->name_size(); // Includes '/0'
 939   const char* stored_name = addr(name_offset);
 940 
 941   if (strncmp(stored_name, name, (name_size - 1)) != 0) {
 942     log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'",
 943                                        stored_name, name);
 944     set_lookup_failed(); // Skip this blob
 945     return nullptr;
 946   }
 947 
 948   // Read archived code blob
 949   uint offset = entry_position + _entry->blob_offset();
 950   CodeBlob* archived_blob = (CodeBlob*)addr(offset);
 951   offset += archived_blob->size();
 952 
 953   address reloc_data = (address)addr(offset);
 954   offset += archived_blob->relocation_size();
 955   set_read_position(offset);
 956 
 957   ImmutableOopMapSet* oop_maps = nullptr;
 958   if (_entry->has_oop_maps()) {
 959     oop_maps = read_oop_map_set();
 960   }
 961 
 962   CodeBlob* code_blob = CodeBlob::create(archived_blob,
 963                                          stored_name,
 964                                          reloc_data,
 965                                          oop_maps
 966                                         );
 967   if (code_blob == nullptr) { // no space left in CodeCache
 968     return nullptr;
 969   }
 970 
 971 #ifndef PRODUCT
 972   code_blob->asm_remarks().init();
 973   read_asm_remarks(code_blob->asm_remarks());
 974   code_blob->dbg_strings().init();
 975   read_dbg_strings(code_blob->dbg_strings());
 976 #endif // PRODUCT
 977 
 978   fix_relocations(code_blob);
 979 
 980 #ifdef ASSERT
 981   LogStreamHandle(Trace, aot, codecache, stubs) log;
 982   if (log.is_enabled()) {
 983     FlagSetting fs(PrintRelocations, true);
 984     code_blob->print_on(&log);
 985   }
 986 #endif
 987   return code_blob;
 988 }
 989 
 990 // ------------ process code and data --------------
 991 
 992 // Can't use -1. It is valid value for jump to iteself destination
 993 // used by static call stub: see NativeJump::jump_destination().
 994 #define BAD_ADDRESS_ID -2
 995 
 996 bool AOTCodeCache::write_relocations(CodeBlob& code_blob) {
 997   GrowableArray<uint> reloc_data;
 998   RelocIterator iter(&code_blob);
 999   LogStreamHandle(Trace, aot, codecache, reloc) log;
1000   while (iter.next()) {
1001     int idx = reloc_data.append(0); // default value
1002     switch (iter.type()) {
1003       case relocInfo::none:
1004         break;
1005       case relocInfo::runtime_call_type: {
1006         // Record offset of runtime destination
1007         CallRelocation* r = (CallRelocation*)iter.reloc();
1008         address dest = r->destination();
1009         if (dest == r->addr()) { // possible call via trampoline on Aarch64
1010           dest = (address)-1;    // do nothing in this case when loading this relocation
1011         }
1012         int id = _table->id_for_address(dest, iter, &code_blob);
1013         if (id == BAD_ADDRESS_ID) {
1014           return false;
1015         }
1016         reloc_data.at_put(idx, id);
1017         break;
1018       }
1019       case relocInfo::runtime_call_w_cp_type:
1020         log_debug(aot, codecache, reloc)("runtime_call_w_cp_type relocation is not implemented");
1021         return false;
1022       case relocInfo::external_word_type: {
1023         // Record offset of runtime target
1024         address target = ((external_word_Relocation*)iter.reloc())->target();
1025         int id = _table->id_for_address(target, iter, &code_blob);
1026         if (id == BAD_ADDRESS_ID) {
1027           return false;
1028         }
1029         reloc_data.at_put(idx, id);
1030         break;
1031       }
1032       case relocInfo::internal_word_type:
1033         break;
1034       case relocInfo::section_word_type:
1035         break;
1036       case relocInfo::post_call_nop_type:
1037         break;
1038       default:
















































































































































































































































































































































































































































































































































































1039         log_debug(aot, codecache, reloc)("relocation %d unimplemented", (int)iter.type());
1040         return false;
1041         break;
1042     }
1043     if (log.is_enabled()) {
1044       iter.print_current_on(&log);
1045     }
1046   }
1047 
1048   // Write additional relocation data: uint per relocation
1049   // Write the count first
1050   int count = reloc_data.length();
1051   write_bytes(&count, sizeof(int));
1052   for (GrowableArrayIterator<uint> iter = reloc_data.begin();
1053        iter != reloc_data.end(); ++iter) {
1054     uint value = *iter;
1055     int n = write_bytes(&value, sizeof(uint));
1056     if (n != sizeof(uint)) {
1057       return false;
1058     }
1059   }
1060   return true;
1061 }
1062 
1063 void AOTCodeReader::fix_relocations(CodeBlob* code_blob) {
1064   LogStreamHandle(Trace, aot, reloc) log;
1065   uint offset = read_position();
1066   int count = *(int*)addr(offset);
1067   offset += sizeof(int);
1068   if (log.is_enabled()) {
1069     log.print_cr("======== extra relocations count=%d", count);
1070   }
1071   uint* reloc_data = (uint*)addr(offset);
1072   offset += (count * sizeof(uint));
1073   set_read_position(offset);
1074 
1075   RelocIterator iter(code_blob);
1076   int j = 0;
1077   while (iter.next()) {
1078     switch (iter.type()) {
1079       case relocInfo::none:
1080         break;













































1081       case relocInfo::runtime_call_type: {
1082         address dest = _cache->address_for_id(reloc_data[j]);
1083         if (dest != (address)-1) {
1084           ((CallRelocation*)iter.reloc())->set_destination(dest);
1085         }
1086         break;
1087       }
1088       case relocInfo::runtime_call_w_cp_type:
1089         // this relocation should not be in cache (see write_relocations)
1090         assert(false, "runtime_call_w_cp_type relocation is not implemented");
1091         break;
1092       case relocInfo::external_word_type: {
1093         address target = _cache->address_for_id(reloc_data[j]);
1094         // Add external address to global table
1095         int index = ExternalsRecorder::find_index(target);
1096         // Update index in relocation
1097         Relocation::add_jint(iter.data(), index);
1098         external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
1099         assert(reloc->target() == target, "sanity");
1100         reloc->set_value(target); // Patch address in the code
1101         break;
1102       }
1103       case relocInfo::internal_word_type: {
1104         internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
1105         r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
1106         break;
1107       }
1108       case relocInfo::section_word_type: {
1109         section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
1110         r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
1111         break;




















































































































































































































































































































































































































































































































































1112       }
1113       case relocInfo::post_call_nop_type:
1114         break;
1115       default:
1116         assert(false,"relocation %d unimplemented", (int)iter.type());
1117         break;
1118     }

1119     if (log.is_enabled()) {
1120       iter.print_current_on(&log);









1121     }
1122     j++;
1123   }
1124   assert(j == count, "sanity");
1125 }
1126 
1127 bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
1128   ImmutableOopMapSet* oopmaps = cb.oop_maps();
1129   int oopmaps_size = oopmaps->nr_of_bytes();
1130   if (!write_bytes(&oopmaps_size, sizeof(int))) {
1131     return false;
1132   }
1133   uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
1134   if (n != (uint)oopmaps->nr_of_bytes()) {
1135     return false;
1136   }
1137   return true;
1138 }
1139 
1140 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
1141   uint offset = read_position();
1142   int size = *(int *)addr(offset);
1143   offset += sizeof(int);
1144   ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
1145   offset += size;
1146   set_read_position(offset);
1147   return oopmaps;
1148 }
1149 













1150 #ifndef PRODUCT
1151 bool AOTCodeCache::write_asm_remarks(CodeBlob& cb) {
1152   // Write asm remarks
1153   uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
1154   if (count_ptr == nullptr) {
1155     return false;
1156   }
1157   uint count = 0;
1158   bool result = cb.asm_remarks().iterate([&] (uint offset, const char* str) -> bool {
1159     log_trace(aot, codecache, stubs)("asm remark offset=%d, str='%s'", offset, str);
1160     uint n = write_bytes(&offset, sizeof(uint));
1161     if (n != sizeof(uint)) {
1162       return false;
1163     }
1164     const char* cstr = add_C_string(str);
1165     int id = _table->id_for_C_string((address)cstr);
1166     assert(id != -1, "asm remark string '%s' not found in AOTCodeAddressTable", str);
1167     n = write_bytes(&id, sizeof(int));
1168     if (n != sizeof(int)) {
1169       return false;







1170     }
1171     count += 1;
1172     return true;
1173   });
1174   *count_ptr = count;
1175   return result;
1176 }
1177 
1178 void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks) {
1179   // Read asm remarks
1180   uint offset = read_position();
1181   uint count = *(uint *)addr(offset);
1182   offset += sizeof(uint);
1183   for (uint i = 0; i < count; i++) {
1184     uint remark_offset = *(uint *)addr(offset);
1185     offset += sizeof(uint);
1186     int remark_string_id = *(uint *)addr(offset);
1187     offset += sizeof(int);
1188     const char* remark = (const char*)_cache->address_for_C_string(remark_string_id);






1189     asm_remarks.insert(remark_offset, remark);
1190   }
1191   set_read_position(offset);
1192 }
1193 
1194 bool AOTCodeCache::write_dbg_strings(CodeBlob& cb) {
1195   // Write dbg strings
1196   uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
1197   if (count_ptr == nullptr) {
1198     return false;
1199   }
1200   uint count = 0;
1201   bool result = cb.dbg_strings().iterate([&] (const char* str) -> bool {
1202     log_trace(aot, codecache, stubs)("dbg string=%s", str);
1203     const char* cstr = add_C_string(str);
1204     int id = _table->id_for_C_string((address)cstr);
1205     assert(id != -1, "db string '%s' not found in AOTCodeAddressTable", str);
1206     uint n = write_bytes(&id, sizeof(int));
1207     if (n != sizeof(int)) {
1208       return false;







1209     }
1210     count += 1;
1211     return true;
1212   });
1213   *count_ptr = count;
1214   return result;
1215 }
1216 
1217 void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) {
1218   // Read dbg strings
1219   uint offset = read_position();
1220   uint count = *(uint *)addr(offset);
1221   offset += sizeof(uint);
1222   for (uint i = 0; i < count; i++) {
1223     int string_id = *(uint *)addr(offset);
1224     offset += sizeof(int);
1225     const char* str = (const char*)_cache->address_for_C_string(string_id);






1226     dbg_strings.insert(str);
1227   }
1228   set_read_position(offset);
1229 }
1230 #endif // PRODUCT
1231 
1232 //======================= AOTCodeAddressTable ===============
1233 
1234 // address table ids for generated routines, external addresses and C
1235 // string addresses are partitioned into positive integer ranges
1236 // defined by the following positive base and max values
1237 // i.e. [_extrs_base, _extrs_base + _extrs_max -1],
1238 //      [_blobs_base, _blobs_base + _blobs_max -1],
1239 //      ...
1240 //      [_c_str_base, _c_str_base + _c_str_max -1],
1241 
1242 #define _extrs_max 100
1243 #define _stubs_max 3
1244 
1245 #define _shared_blobs_max 20
1246 #define _C1_blobs_max 10
1247 #define _blobs_max (_shared_blobs_max+_C1_blobs_max)
1248 #define _all_max (_extrs_max+_stubs_max+_blobs_max)
1249 
1250 #define _extrs_base 0
1251 #define _stubs_base (_extrs_base + _extrs_max)
1252 #define _shared_blobs_base (_stubs_base + _stubs_max)
1253 #define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max)

1254 #define _blobs_end  (_shared_blobs_base + _blobs_max)



1255 
1256 #define SET_ADDRESS(type, addr)                           \
1257   {                                                       \
1258     type##_addr[type##_length++] = (address) (addr);      \
1259     assert(type##_length <= type##_max, "increase size"); \
1260   }
1261 
1262 static bool initializing_extrs = false;
1263 
1264 void AOTCodeAddressTable::init_extrs() {
1265   if (_extrs_complete || initializing_extrs) return; // Done already
1266 
1267   assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting");
1268 
1269   initializing_extrs = true;
1270   _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
1271 
1272   _extrs_length = 0;
1273 
1274   // Record addresses of VM runtime methods
1275   SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite);
1276   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method);
1277   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract);
1278   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss);
1279 #if defined(AARCH64) && !defined(ZERO)
1280   SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper);
1281 #endif
1282   {
1283     // Required by Shared blobs
1284     SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info);
1285     SET_ADDRESS(_extrs, Deoptimization::unpack_frames);
1286     SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception);
1287     SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C);
1288     SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C);
1289     SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C);
1290     SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
1291     SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError);
1292     SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError);
1293     SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError);
1294     SET_ADDRESS(_extrs, SharedRuntime::throw_NullPointerException_at_call);










1295   }
1296 
1297 #ifdef COMPILER1
1298   {
1299     // Required by C1 blobs
1300     SET_ADDRESS(_extrs, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc));
1301     SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address);
1302     SET_ADDRESS(_extrs, SharedRuntime::register_finalizer);
1303     SET_ADDRESS(_extrs, Runtime1::is_instance_of);
1304     SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc);
1305     SET_ADDRESS(_extrs, Runtime1::check_abort_on_vm_exception);
1306     SET_ADDRESS(_extrs, Runtime1::new_instance);
1307     SET_ADDRESS(_extrs, Runtime1::counter_overflow);
1308     SET_ADDRESS(_extrs, Runtime1::new_type_array);
1309     SET_ADDRESS(_extrs, Runtime1::new_object_array);
1310     SET_ADDRESS(_extrs, Runtime1::new_multi_array);
1311     SET_ADDRESS(_extrs, Runtime1::throw_range_check_exception);
1312     SET_ADDRESS(_extrs, Runtime1::throw_index_exception);
1313     SET_ADDRESS(_extrs, Runtime1::throw_div0_exception);
1314     SET_ADDRESS(_extrs, Runtime1::throw_null_pointer_exception);
1315     SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception);
1316     SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception);
1317     SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error);
1318     SET_ADDRESS(_extrs, Runtime1::is_instance_of);
1319     SET_ADDRESS(_extrs, Runtime1::monitorenter);
1320     SET_ADDRESS(_extrs, Runtime1::monitorexit);
1321     SET_ADDRESS(_extrs, Runtime1::deoptimize);
1322     SET_ADDRESS(_extrs, Runtime1::access_field_patching);
1323     SET_ADDRESS(_extrs, Runtime1::move_klass_patching);
1324     SET_ADDRESS(_extrs, Runtime1::move_mirror_patching);
1325     SET_ADDRESS(_extrs, Runtime1::move_appendix_patching);
1326     SET_ADDRESS(_extrs, Runtime1::predicate_failed_trap);
1327     SET_ADDRESS(_extrs, Runtime1::unimplemented_entry);
1328     SET_ADDRESS(_extrs, Thread::current);
1329     SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr());





1330 #ifndef PRODUCT
1331     SET_ADDRESS(_extrs, os::breakpoint);
1332 #endif
1333   }
1334 #endif
1335 
1336 #ifdef COMPILER2
1337   {
1338     // Required by C2 blobs
1339     SET_ADDRESS(_extrs, Deoptimization::uncommon_trap);
1340     SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C);
1341     SET_ADDRESS(_extrs, OptoRuntime::new_instance_C);
1342     SET_ADDRESS(_extrs, OptoRuntime::new_array_C);
1343     SET_ADDRESS(_extrs, OptoRuntime::new_array_nozero_C);
1344     SET_ADDRESS(_extrs, OptoRuntime::multianewarray2_C);
1345     SET_ADDRESS(_extrs, OptoRuntime::multianewarray3_C);
1346     SET_ADDRESS(_extrs, OptoRuntime::multianewarray4_C);
1347     SET_ADDRESS(_extrs, OptoRuntime::multianewarray5_C);
1348     SET_ADDRESS(_extrs, OptoRuntime::multianewarrayN_C);
1349 #if INCLUDE_JVMTI
1350     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_start);
1351     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_end);
1352     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_mount);
1353     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_unmount);
1354 #endif
1355     SET_ADDRESS(_extrs, OptoRuntime::complete_monitor_locking_C);
1356     SET_ADDRESS(_extrs, OptoRuntime::monitor_notify_C);
1357     SET_ADDRESS(_extrs, OptoRuntime::monitor_notifyAll_C);
1358     SET_ADDRESS(_extrs, OptoRuntime::rethrow_C);
1359     SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C);
1360     SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C);
1361 #if defined(AARCH64)
1362     SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure);
1363 #endif // AARCH64


1364   }
1365 #endif // COMPILER2
1366 
1367 #if INCLUDE_G1GC
1368   SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry);
1369 #endif

1370 #if INCLUDE_SHENANDOAHGC
1371   SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre);






1372   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom);
1373   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);

1374 #endif

1375 #if INCLUDE_ZGC
1376   SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
1377 #if defined(AMD64)
1378   SET_ADDRESS(_extrs, &ZPointerLoadShift);
1379 #endif






















































1380 #endif

1381 #ifndef ZERO
1382 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
1383   SET_ADDRESS(_extrs, MacroAssembler::debug64);
1384 #endif



1385 #endif // ZERO
1386 






1387   _extrs_complete = true;
1388   log_debug(aot, codecache, init)("External addresses recorded");
1389 }
1390 
1391 static bool initializing_early_stubs = false;
1392 
1393 void AOTCodeAddressTable::init_early_stubs() {
1394   if (_complete || initializing_early_stubs) return; // Done already
1395   initializing_early_stubs = true;
1396   _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
1397   _stubs_length = 0;
1398   SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry());
1399 
1400   {
1401     // Required by C1 blobs
1402 #if defined(AMD64) && !defined(ZERO)
1403     SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip());
1404     SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup());
1405 #endif // AMD64
1406   }
1407 
1408   _early_stubs_complete = true;
1409   log_info(aot, codecache, init)("Early stubs recorded");
1410 }
1411 
1412 static bool initializing_shared_blobs = false;
1413 
1414 void AOTCodeAddressTable::init_shared_blobs() {
1415   if (_complete || initializing_shared_blobs) return; // Done already
1416   initializing_shared_blobs = true;
1417   address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode);
1418 
1419   // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel
1420   _shared_blobs_addr = blobs_addr;
1421   _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max;

1422 
1423   _shared_blobs_length = 0;
1424   _C1_blobs_length = 0;

1425 
1426   // clear the address table
1427   memset(blobs_addr, 0, sizeof(address)* _blobs_max);
1428 
1429   // Record addresses of generated code blobs
1430   SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub());
1431   SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub());
1432   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack());
1433   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception());
1434   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution());
1435   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls());












1436 #if INCLUDE_JVMCI
1437   if (EnableJVMCI) {
1438     SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap());
1439     SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap());
1440   }
1441 #endif





1442 

1443   _shared_blobs_complete = true;
1444   log_debug(aot, codecache, init)("Early shared blobs recorded");





















































































































































































1445   _complete = true;

1446 }
1447 
1448 void AOTCodeAddressTable::init_early_c1() {
1449 #ifdef COMPILER1
1450   // Runtime1 Blobs
1451   StubId id = StubInfo::stub_base(StubGroup::C1);
1452   // include forward_exception in range we publish
1453   StubId limit = StubInfo::next(StubId::c1_forward_exception_id);
1454   for (; id != limit; id = StubInfo::next(id)) {
1455     if (Runtime1::blob_for(id) == nullptr) {
1456       log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
1457       continue;
1458     }
1459     if (Runtime1::entry_for(id) == nullptr) {
1460       log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
1461       continue;
1462     }
1463     address entry = Runtime1::entry_for(id);
1464     SET_ADDRESS(_C1_blobs, entry);
1465   }
1466 #endif // COMPILER1
1467   assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
1468   _early_c1_complete = true;
1469 }
1470 




















































































1471 #undef SET_ADDRESS
1472 
1473 AOTCodeAddressTable::~AOTCodeAddressTable() {
1474   if (_extrs_addr != nullptr) {
1475     FREE_C_HEAP_ARRAY(address, _extrs_addr);
1476   }
1477   if (_stubs_addr != nullptr) {
1478     FREE_C_HEAP_ARRAY(address, _stubs_addr);
1479   }
1480   if (_shared_blobs_addr != nullptr) {
1481     FREE_C_HEAP_ARRAY(address, _shared_blobs_addr);
1482   }
1483 }
1484 
1485 #ifdef PRODUCT
1486 #define MAX_STR_COUNT 200
1487 #else
1488 #define MAX_STR_COUNT 500
1489 #endif
1490 #define _c_str_max  MAX_STR_COUNT
1491 static const int _c_str_base = _all_max;
1492 
1493 static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings
1494 static const char* _C_strings[MAX_STR_COUNT]    = {nullptr}; // Our duplicates
1495 static int _C_strings_count = 0;
1496 static int _C_strings_s[MAX_STR_COUNT] = {0};
1497 static int _C_strings_id[MAX_STR_COUNT] = {0};
1498 static int _C_strings_used = 0;
1499 
1500 void AOTCodeCache::load_strings() {
1501   uint strings_count  = _load_header->strings_count();
1502   if (strings_count == 0) {
1503     return;
1504   }
1505   uint strings_offset = _load_header->strings_offset();
1506   uint* string_lengths = (uint*)addr(strings_offset);
1507   strings_offset += (strings_count * sizeof(uint));
1508   uint strings_size = _load_header->entries_offset() - strings_offset;
1509   // We have to keep cached strings longer than _cache buffer
1510   // because they are refernced from compiled code which may
1511   // still be executed on VM exit after _cache is freed.
1512   char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
1513   memcpy(p, addr(strings_offset), strings_size);
1514   _C_strings_buf = p;
1515   assert(strings_count <= MAX_STR_COUNT, "sanity");
1516   for (uint i = 0; i < strings_count; i++) {
1517     _C_strings[i] = p;
1518     uint len = string_lengths[i];
1519     _C_strings_s[i] = i;
1520     _C_strings_id[i] = i;
1521     p += len;
1522   }
1523   assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
1524   _C_strings_count = strings_count;
1525   _C_strings_used  = strings_count;
1526   log_debug(aot, codecache, init)("  Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset);
1527 }
1528 

1630   assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
1631   if (idx == -1) {
1632     return (address)-1;
1633   }
1634   uint id = (uint)idx;
1635   // special case for symbols based relative to os::init
1636   if (id > (_c_str_base + _c_str_max)) {
1637     return (address)os::init + idx;
1638   }
1639   if (idx < 0) {
1640     fatal("Incorrect id %d for AOT Code Cache addresses table", id);
1641     return nullptr;
1642   }
1643   // no need to compare unsigned id against 0
1644   if (/* id >= _extrs_base && */ id < _extrs_length) {
1645     return _extrs_addr[id - _extrs_base];
1646   }
1647   if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
1648     return _stubs_addr[id - _stubs_base];
1649   }



1650   if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) {
1651     return _shared_blobs_addr[id - _shared_blobs_base];
1652   }
1653   if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
1654     return _C1_blobs_addr[id - _C1_blobs_base];
1655   }






1656   if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
1657     return address_for_C_string(id - _c_str_base);
1658   }
1659   fatal("Incorrect id %d for AOT Code Cache addresses table", id);
1660   return nullptr;
1661 }
1662 
1663 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) {
1664   assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
1665   int id = -1;
1666   if (addr == (address)-1) { // Static call stub has jump to itself
1667     return id;
1668   }




1669   // Seach for C string
1670   id = id_for_C_string(addr);
1671   if (id >= 0) {
1672     return id + _c_str_base;
1673   }
1674   if (StubRoutines::contains(addr)) {
1675     // Search in stubs
1676     id = search_address(addr, _stubs_addr, _stubs_length);
1677     if (id < 0) {
1678       StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
1679       if (desc == nullptr) {
1680         desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset);
1681       }
1682       const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
1683       assert(false, "Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
1684     } else {
1685       return id + _stubs_base;
1686     }
1687   } else {
1688     CodeBlob* cb = CodeCache::find_blob(addr);
1689     if (cb != nullptr) {
1690       // Search in code blobs
1691       int id_base = _shared_blobs_base;
1692       id = search_address(addr, _shared_blobs_addr, _blobs_max);
1693       if (id < 0) {











1694         assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name());
1695       } else {
1696         return id_base + id;
1697       }
1698     } else {
1699       // Search in runtime functions
1700       id = search_address(addr, _extrs_addr, _extrs_length);
1701       if (id < 0) {
1702         ResourceMark rm;
1703         const int buflen = 1024;
1704         char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
1705         int offset = 0;
1706         if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
1707           if (offset > 0) {
1708             // Could be address of C string
1709             uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
1710             log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
1711                                       p2i(addr), dist, (const char*)addr);







1712             assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
1713             return dist;
1714           }
1715 #ifdef ASSERT
1716           reloc.print_current_on(tty);
1717           code_blob->print_on(tty);
1718           code_blob->print_code_on(tty);
1719           assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);
1720 #endif
1721         } else {
1722 #ifdef ASSERT
1723           reloc.print_current_on(tty);
1724           code_blob->print_on(tty);
1725           code_blob->print_code_on(tty);
1726           os::find(addr, tty);
1727           assert(false, "Address " INTPTR_FORMAT " for <unknown>/('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr);
1728 #endif
1729         }
1730       } else {
1731         return _extrs_base + id;
1732       }
1733     }
1734   }
1735   return id;
1736 }
1737 




























































































































































































































1738 // This is called after initialize() but before init2()
1739 // and _cache is not set yet.
1740 void AOTCodeCache::print_on(outputStream* st) {
1741   if (opened_cache != nullptr && opened_cache->for_use()) {
1742     st->print_cr("\nAOT Code Cache");






























1743     uint count = opened_cache->_load_header->entries_count();
1744     uint* search_entries = (uint*)opened_cache->addr(opened_cache->_load_header->entries_offset()); // [id, index]
1745     AOTCodeEntry* load_entries = (AOTCodeEntry*)(search_entries + 2 * count);
1746 
1747     for (uint i = 0; i < count; i++) {
1748       // Use search_entries[] to order ouput
1749       int index = search_entries[2*i + 1];
1750       AOTCodeEntry* entry = &(load_entries[index]);
1751 
1752       uint entry_position = entry->offset();
1753       uint name_offset = entry->name_offset() + entry_position;
1754       const char* saved_name = opened_cache->addr(name_offset);
1755 
1756       st->print_cr("%4u: %10s idx:%4u Id:%u size=%u '%s'",
1757                    i, aot_code_entry_kind_name[entry->kind()], index, entry->id(), entry->size(), saved_name);









1758     }
1759   }
1760 }























































  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 
  26 #include "asm/macroAssembler.hpp"
  27 #include "cds/aotCacheAccess.hpp"
  28 #include "cds/aotMetaspace.hpp"
  29 #include "cds/cds_globals.hpp"
  30 #include "cds/cdsConfig.hpp"
  31 #include "cds/heapShared.hpp"
  32 #include "ci/ciConstant.hpp"
  33 #include "ci/ciEnv.hpp"
  34 #include "ci/ciField.hpp"
  35 #include "ci/ciMethod.hpp"
  36 #include "ci/ciMethodData.hpp"
  37 #include "ci/ciObject.hpp"
  38 #include "ci/ciUtilities.inline.hpp"
  39 #include "classfile/javaAssertions.hpp"
  40 #include "classfile/stringTable.hpp"
  41 #include "classfile/symbolTable.hpp"
  42 #include "classfile/systemDictionary.hpp"
  43 #include "classfile/vmClasses.hpp"
  44 #include "classfile/vmIntrinsics.hpp"
  45 #include "code/aotCodeCache.hpp"
  46 #include "code/codeBlob.hpp"
  47 #include "code/codeCache.hpp"
  48 #include "code/oopRecorder.inline.hpp"
  49 #include "compiler/abstractCompiler.hpp"
  50 #include "compiler/compilationPolicy.hpp"
  51 #include "compiler/compileBroker.hpp"
  52 #include "compiler/compileTask.hpp"
  53 #include "gc/g1/g1BarrierSetRuntime.hpp"
  54 #include "gc/shared/gcConfig.hpp"
  55 #include "logging/logStream.hpp"
  56 #include "memory/memoryReserver.hpp"
  57 #include "memory/universe.hpp"
  58 #include "oops/klass.inline.hpp"
  59 #include "oops/method.inline.hpp"
  60 #include "oops/trainingData.hpp"
  61 #include "prims/jvmtiThreadState.hpp"
  62 #include "runtime/atomicAccess.hpp"
  63 #include "runtime/deoptimization.hpp"
  64 #include "runtime/flags/flagSetting.hpp"
  65 #include "runtime/globals_extension.hpp"
  66 #include "runtime/handles.inline.hpp"
  67 #include "runtime/java.hpp"
  68 #include "runtime/jniHandles.inline.hpp"
  69 #include "runtime/mutexLocker.hpp"
  70 #include "runtime/os.inline.hpp"
  71 #include "runtime/sharedRuntime.hpp"
  72 #include "runtime/stubCodeGenerator.hpp"
  73 #include "runtime/stubRoutines.hpp"
  74 #include "runtime/threadIdentifier.hpp"
  75 #include "runtime/timerTrace.hpp"
  76 #include "utilities/copy.hpp"
  77 #include "utilities/formatBuffer.hpp"
  78 #include "utilities/ostream.hpp"
  79 #include "utilities/spinYield.hpp"
  80 #ifdef COMPILER1
  81 #include "c1/c1_LIRAssembler.hpp"
  82 #include "c1/c1_Runtime1.hpp"
  83 #include "gc/g1/c1/g1BarrierSetC1.hpp"
  84 #include "gc/shared/c1/barrierSetC1.hpp"
  85 #if INCLUDE_SHENANDOAHGC
  86 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  87 #endif // INCLUDE_SHENANDOAHGC
  88 #include "gc/z/c1/zBarrierSetC1.hpp"
  89 #endif // COMPILER1
  90 #ifdef COMPILER2
  91 #include "opto/runtime.hpp"
  92 #endif
  93 #if INCLUDE_JVMCI
  94 #include "jvmci/jvmci.hpp"
  95 #endif
  96 #if INCLUDE_G1GC
  97 #include "gc/g1/g1BarrierSetRuntime.hpp"
  98 #endif
  99 #if INCLUDE_SHENANDOAHGC
 100 #include "gc/shenandoah/shenandoahRuntime.hpp"
 101 #endif
 102 #if INCLUDE_ZGC
 103 #include "gc/z/zBarrierSetRuntime.hpp"
 104 #endif
 105 #if defined(X86) && !defined(ZERO)
 106 #include "rdtsc_x86.hpp"
 107 #endif
 108 
 109 #include <errno.h>
 110 #include <sys/stat.h>
 111 
 112 const char* aot_code_entry_kind_name[] = {
 113 #define DECL_KIND_STRING(kind) XSTR(kind),
 114   DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
 115 #undef DECL_KIND_STRING
 116 };
 117 
 118 static elapsedTimer _t_totalLoad;
 119 static elapsedTimer _t_totalPreload;
 120 static elapsedTimer _t_totalRegister;
 121 static elapsedTimer _t_totalFind;
 122 static elapsedTimer _t_totalStore;
 123 
 124 static bool enable_timers() {
 125   return CITime || log_is_enabled(Info, init);
 126 }
 127 
 128 static void report_load_failure() {
 129   if (AbortVMOnAOTCodeFailure) {
 130     vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
 131   }
 132   log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
 133   AOTCodeCache::disable_caching();
 134 }
 135 
 136 static void report_store_failure() {
 137   if (AbortVMOnAOTCodeFailure) {
 138     tty->print_cr("Unable to create AOT Code Cache.");
 139     vm_abort(false);
 140   }
 141   log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
 142   AOTCodeCache::disable_caching();
 143 }
 144 
 145 // The sequence of AOT code caching flags and parametters settings.
 146 //
 147 // 1. The initial AOT code caching flags setting is done

 162 
 163 // Next methods determine which action we do with AOT code depending
 164 // on phase of AOT process: assembly or production.
 165 
 166 bool AOTCodeCache::is_dumping_adapter() {
 167   return AOTAdapterCaching && is_on_for_dump();
 168 }
 169 
 170 bool AOTCodeCache::is_using_adapter()   {
 171   return AOTAdapterCaching && is_on_for_use();
 172 }
 173 
 174 bool AOTCodeCache::is_dumping_stub() {
 175   return AOTStubCaching && is_on_for_dump();
 176 }
 177 
 178 bool AOTCodeCache::is_using_stub()   {
 179   return AOTStubCaching && is_on_for_use();
 180 }
 181 
 182 bool AOTCodeCache::is_dumping_code() {
 183   return AOTCodeCaching && is_on_for_dump();
 184 }
 185 
 186 bool AOTCodeCache::is_using_code() {
 187   return AOTCodeCaching && is_on_for_use();
 188 }
 189 
 190 // This is used before AOTCodeCahe is initialized
 191 // but after AOT (CDS) Cache flags consistency is checked.
 192 bool AOTCodeCache::maybe_dumping_code() {
 193   return AOTCodeCaching && CDSConfig::is_dumping_final_static_archive();
 194 }
 195 
 196 // Next methods could be called regardless of AOT code cache status.
 197 // Initially they are called during AOT flags parsing and finilized
 198 // in AOTCodeCache::initialize().
 199 void AOTCodeCache::enable_caching() {
 200   FLAG_SET_ERGO_IF_DEFAULT(AOTCodeCaching, true);
 201   FLAG_SET_ERGO_IF_DEFAULT(AOTStubCaching, true);
 202   FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true);
 203 }
 204 
 205 void AOTCodeCache::disable_caching() {
 206   FLAG_SET_ERGO(AOTCodeCaching, false);
 207   FLAG_SET_ERGO(AOTStubCaching, false);
 208   FLAG_SET_ERGO(AOTAdapterCaching, false);
 209 }
 210 
 211 bool AOTCodeCache::is_caching_enabled() {
 212   return AOTCodeCaching || AOTStubCaching || AOTAdapterCaching;
 213 }
 214 
 215 static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
 216   assert(AOTCodeEntry::is_valid_entry_kind(kind), "invalid AOTCodeEntry kind %d", (int)kind);
 217   // There can be a conflict of id between an Adapter and *Blob, but that should not cause any functional issue
 218   // becasue both id and kind are used to find an entry, and that combination should be unique
 219   if (kind == AOTCodeEntry::Adapter) {
 220     return id;
 221   } else if (kind == AOTCodeEntry::SharedBlob) {
 222     assert(StubInfo::is_shared(static_cast<BlobId>(id)), "not a shared blob id %d", id);
 223     return id;
 224   } else if (kind == AOTCodeEntry::C1Blob) {
 225     assert(StubInfo::is_c1(static_cast<BlobId>(id)), "not a c1 blob id %d", id);
 226     return id;
 227   } else {
 228     // kind must be AOTCodeEntry::C2Blob
 229     assert(StubInfo::is_c2(static_cast<BlobId>(id)), "not a c2 blob id %d", id);
 230     return id;
 231   }
 232 }
 233 
 234 static uint _max_aot_code_size = 0;
 235 uint AOTCodeCache::max_aot_code_size() {
 236   return _max_aot_code_size;
 237 }
 238 
 239 bool AOTCodeCache::is_code_load_thread_on() {
 240   return UseAOTCodeLoadThread && AOTCodeCaching;
 241 }
 242 
 243 bool AOTCodeCache::allow_const_field(ciConstant& value) {
 244   ciEnv* env = CURRENT_ENV;
 245   precond(env != nullptr);
 246   assert(!env->is_precompile() || is_dumping_code(), "AOT compilation should be enabled");
 247   return !env->is_precompile() // Restrict only when we generate AOT code
 248         // Can not trust primitive too   || !is_reference_type(value.basic_type())
 249         // May disable this too for now  || is_reference_type(value.basic_type()) && value.as_object()->should_be_constant()
 250         ;
 251 }
 252 
 253 // It is called from AOTMetaspace::initialize_shared_spaces()
 254 // which is called from universe_init().
 255 // At this point all AOT class linking seetings are finilized
 256 // and AOT cache is open so we can map AOT code region.
 257 void AOTCodeCache::initialize() {
 258   if (!is_caching_enabled()) {
 259     log_info(aot, codecache, init)("AOT Code Cache is not used: disabled.");
 260     return;
 261   }
 262 #if defined(ZERO) || !(defined(AMD64) || defined(AARCH64))
 263   log_info(aot, codecache, init)("AOT Code Cache is not supported on this platform.");
 264   disable_caching();
 265   return;
 266 #else
 267   assert(!FLAG_IS_DEFAULT(AOTCache), "AOTCache should be specified");




 268 
 269   // Disable stubs caching until JDK-8357398 is fixed.
 270   FLAG_SET_ERGO(AOTStubCaching, false);
 271 
 272   if (VerifyOops) {
 273     // Disable AOT stubs caching when VerifyOops flag is on.
 274     // Verify oops code generated a lot of C strings which overflow
 275     // AOT C string table (which has fixed size).
 276     // AOT C string table will be reworked later to handle such cases.
 277     //
 278     // Note: AOT adapters are not affected - they don't have oop operations.
 279     log_info(aot, codecache, init)("AOT Stubs Caching is not supported with VerifyOops.");
 280     FLAG_SET_ERGO(AOTStubCaching, false);
 281   }
 282 
 283   bool is_dumping = false;
 284   bool is_using   = false;
 285   if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) {


 286     is_dumping = is_caching_enabled();
 287   } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) {

 288     is_using = is_caching_enabled();
 289   }
 290   if (ClassInitBarrierMode > 0 && !(is_dumping && AOTCodeCaching)) {
 291     log_info(aot, codecache, init)("Set ClassInitBarrierMode to 0 because AOT Code dumping is off.");
 292     FLAG_SET_ERGO(ClassInitBarrierMode, 0);
 293   }
 294   if (!(is_dumping || is_using)) {
 295     log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used.");
 296     disable_caching();
 297     return; // AOT code caching disabled on command line
 298   }
 299   // Reserve AOT Cache region when we dumping AOT code.
 300   _max_aot_code_size = AOTCodeMaxSize;
 301   if (is_dumping && !FLAG_IS_DEFAULT(AOTCodeMaxSize)) {
 302     if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) {
 303       _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity());
 304       log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K));
 305     }
 306   }
 307   size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0;
 308   if (is_using && aot_code_size == 0) {
 309     log_info(aot, codecache, init)("AOT Code Cache is empty");
 310     disable_caching();
 311     return;
 312   }
 313   if (!open_cache(is_dumping, is_using)) {
 314     if (is_using) {
 315       report_load_failure();
 316     } else {
 317       report_store_failure();
 318     }
 319     return;
 320   }
 321   if (is_dumping) {
 322     FLAG_SET_DEFAULT(FoldStableValues, false);
 323     FLAG_SET_DEFAULT(ForceUnreachable, true);
 324   }
 325   FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
 326 #endif // defined(AMD64) || defined(AARCH64)
 327 }
 328 
 329 static AOTCodeCache*  opened_cache = nullptr; // Use this until we verify the cache
 330 AOTCodeCache* AOTCodeCache::_cache = nullptr;
 331 DEBUG_ONLY( bool AOTCodeCache::_passed_init2 = false; )
 332 
 333 // It is called after universe_init() when all GC settings are finalized.
 334 void AOTCodeCache::init2() {
 335   DEBUG_ONLY( _passed_init2 = true; )
 336   if (opened_cache == nullptr) {
 337     return;
 338   }
 339   // After Universe initialized
 340   if (!opened_cache->verify_config_on_use()) { // Check on AOT code loading
 341     delete opened_cache;
 342     opened_cache = nullptr;
 343     report_load_failure();
 344     return;
 345   }
 346 
 347   // initialize aot runtime constants as appropriate to this runtime
 348   AOTRuntimeConstants::initialize_from_runtime();
 349 
 350   // initialize the table of external routines and initial stubs so we can save
 351   // generated code blobs that reference them
 352   AOTCodeAddressTable* table = opened_cache->_table;
 353   assert(table != nullptr, "should be initialized already");
 354   table->init_extrs();
 355 
 356   // Now cache and address table are ready for AOT code generation
 357   _cache = opened_cache;
 358 
 359   // Set ClassInitBarrierMode after all checks since it affects code generation
 360   if (is_dumping_code()) {
 361     FLAG_SET_ERGO_IF_DEFAULT(ClassInitBarrierMode, 1);
 362   } else {
 363     FLAG_SET_ERGO(ClassInitBarrierMode, 0);
 364   }
 365 }
 366 
 367 bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
 368   opened_cache = new AOTCodeCache(is_dumping, is_using);
 369   if (opened_cache->failed()) {
 370     delete opened_cache;
 371     opened_cache = nullptr;
 372     return false;
 373   }
 374   return true;
 375 }
 376 
 377 static void print_helper(nmethod* nm, outputStream* st) {
 378   AOTCodeCache::iterate([&](AOTCodeEntry* e) {
 379     if (e->method() == nm->method()) {
 380       ResourceMark rm;
 381       stringStream ss;
 382       ss.print("A%s%d", (e->for_preload() ? "P" : ""), e->comp_level());
 383       ss.print("[%s%s%s]",
 384                (e->is_loaded()   ? "L" : ""),
 385                (e->load_fail()   ? "F" : ""),
 386                (e->not_entrant() ? "I" : ""));
 387       ss.print("#%d", e->comp_id());
 388 
 389       st->print(" %s", ss.freeze());
 390     }
 391   });
 392 }
 393 
 394 void AOTCodeCache::close() {
 395   if (is_on()) {
 396     delete _cache; // Free memory
 397     _cache = nullptr;
 398     opened_cache = nullptr;
 399   }
 400 }
 401 
 402 class CachedCodeDirectory : public CachedCodeDirectoryInternal {
 403 public:
 404   uint _aot_code_size;
 405   char* _aot_code_data;
 406 
 407   void set_aot_code_data(uint size, char* aot_data) {
 408     _aot_code_size = size;
 409     AOTCacheAccess::set_pointer(&_aot_code_data, aot_data);
 410   }
 411 
 412   static CachedCodeDirectory* create();
 413 };
 414 
 415 // Storing AOT code in the AOT code region (ac) of AOT Cache:
 416 //
 417 // [1] Use CachedCodeDirectory to keep track of all of data related to AOT code.
 418 //     E.g., you can build a hashtable to record what methods have been archived.
 419 //
 420 // [2] Memory for all data for AOT code, including CachedCodeDirectory, should be
 421 //     allocated using AOTCacheAccess::allocate_aot_code_region().
 422 //
 423 // [3] CachedCodeDirectory must be the very first allocation.
 424 //
 425 // [4] Two kinds of pointer can be stored:
 426 //     - A pointer p that points to metadata. AOTCacheAccess::can_generate_aot_code(p) must return true.
 427 //     - A pointer to a buffer returned by AOTCacheAccess::allocate_aot_code_region().
 428 //       (It's OK to point to an interior location within this buffer).
 429 //     Such pointers must be stored using AOTCacheAccess::set_pointer()
 430 //
 431 // The buffers allocated by AOTCacheAccess::allocate_aot_code_region() are in a contiguous region. At runtime, this
 432 // region is mapped to the process address space. All the pointers in this buffer are relocated as necessary
 433 // (e.g., to account for the runtime location of the CodeCache).
 434 //
 435 // This is always at the very beginning of the mmaped CDS "ac" (AOT code) region
 436 static CachedCodeDirectory* _aot_code_directory = nullptr;
 437 
 438 CachedCodeDirectory* CachedCodeDirectory::create() {
 439   assert(AOTCacheAccess::is_aot_code_region_empty(), "must be");
 440   CachedCodeDirectory* dir = (CachedCodeDirectory*)AOTCacheAccess::allocate_aot_code_region(sizeof(CachedCodeDirectory));
 441   dir->dumptime_init_internal();
 442   return dir;
 443 }
 444 
 445 #define DATA_ALIGNMENT HeapWordSize
 446 
 447 AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
 448   _load_header(nullptr),
 449   _load_buffer(nullptr),
 450   _store_buffer(nullptr),
 451   _C_store_buffer(nullptr),
 452   _write_position(0),
 453   _load_size(0),
 454   _store_size(0),
 455   _for_use(is_using),
 456   _for_dump(is_dumping),
 457   _closing(false),
 458   _failed(false),
 459   _lookup_failed(false),
 460   _for_preload(false),
 461   _has_clinit_barriers(false),
 462   _table(nullptr),
 463   _load_entries(nullptr),
 464   _search_entries(nullptr),
 465   _store_entries(nullptr),
 466   _C_strings_buf(nullptr),
 467   _store_entries_cnt(0),
 468   _compile_id(0),
 469   _comp_level(0)
 470 {
 471   // Read header at the begining of cache
 472   if (_for_use) {
 473     // Read cache
 474     size_t load_size = AOTCacheAccess::get_aot_code_region_size();
 475     ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode);
 476     if (!rs.is_reserved()) {
 477       log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size);
 478       set_failed();
 479       return;
 480     }
 481     if (!AOTCacheAccess::map_aot_code_region(rs)) {
 482       log_warning(aot, codecache, init)("Failed to read/mmap AOT code region (ac) into AOT Code Cache");
 483       set_failed();
 484       return;
 485     }
 486     _aot_code_directory = (CachedCodeDirectory*)rs.base();
 487     _aot_code_directory->runtime_init_internal();
 488 
 489     _load_size = _aot_code_directory->_aot_code_size;
 490     _load_buffer = _aot_code_directory->_aot_code_data;
 491     assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
 492     log_info(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " from AOT Code Cache", _load_size, p2i(_load_buffer));
 493 
 494     _load_header = (Header*)addr(0);
 495     if (!_load_header->verify(_load_size)) {
 496       set_failed();
 497       return;
 498     }
 499     log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count());
 500     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Adapter], _load_header->adapters_count());
 501     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::SharedBlob], _load_header->shared_blobs_count());
 502     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::C1Blob], _load_header->C1_blobs_count());
 503     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::C2Blob], _load_header->C2_blobs_count());
 504     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Stub], _load_header->stubs_count());
 505     log_debug(aot, codecache, init)("  %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Nmethod], _load_header->nmethods_count());
 506     log_debug(aot, codecache, init)("  AOT code cache size: %u bytes", _load_header->cache_size());
 507 
 508     // Read strings
 509     load_strings();
 510   }
 511   if (_for_dump) {
 512     _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
 513     _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
 514     // Entries allocated at the end of buffer in reverse (as on stack).
 515     _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
 516     log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size());
 517   }
 518   _table = new AOTCodeAddressTable();
 519 }
 520 
 521 void AOTCodeCache::invalidate(AOTCodeEntry* entry) {
 522   // This could be concurent execution
 523   if (entry != nullptr && is_on()) { // Request could come after cache is closed.
 524     _cache->invalidate_entry(entry);
 525   }
 526 }
 527 
 528 void AOTCodeCache::init_early_stubs_table() {
 529   AOTCodeAddressTable* table = addr_table();
 530   if (table != nullptr) {
 531     table->init_early_stubs();
 532   }
 533 }
 534 
 535 void AOTCodeCache::init_shared_blobs_table() {
 536   AOTCodeAddressTable* table = addr_table();
 537   if (table != nullptr) {
 538     table->init_shared_blobs();
 539   }
 540 }
 541 
 542 void AOTCodeCache::init_stubs_table() {
 543   AOTCodeAddressTable* table = addr_table();
 544   if (table != nullptr) {
 545     table->init_stubs();
 546   }
 547 }
 548 
 549 void AOTCodeCache::init_early_c1_table() {
 550   AOTCodeAddressTable* table = addr_table();
 551   if (table != nullptr) {
 552     table->init_early_c1();
 553   }
 554 }
 555 
 556 void AOTCodeCache::init_c1_table() {
 557   AOTCodeAddressTable* table = addr_table();
 558   if (table != nullptr) {
 559     table->init_c1();
 560   }
 561 }
 562 
 563 void AOTCodeCache::init_c2_table() {
 564   AOTCodeAddressTable* table = addr_table();
 565   if (table != nullptr) {
 566     table->init_c2();
 567   }
 568 }
 569 
 570 AOTCodeCache::~AOTCodeCache() {
 571   if (_closing) {
 572     return; // Already closed
 573   }
 574   // Stop any further access to cache.
 575   // Checked on entry to load_nmethod() and store_nmethod().
 576   _closing = true;
 577   if (_for_use) {
 578     // Wait for all load_nmethod() finish.
 579     wait_for_no_nmethod_readers();
 580   }
 581   // Prevent writing code into cache while we are closing it.
 582   // This lock held by ciEnv::register_method() which calls store_nmethod().
 583   MutexLocker ml(Compile_lock);
 584   if (for_dump()) { // Finalize cache
 585     finish_write();
 586   }
 587   _load_buffer = nullptr;
 588   if (_C_store_buffer != nullptr) {
 589     FREE_C_HEAP_ARRAY(char, _C_store_buffer);
 590     _C_store_buffer = nullptr;
 591     _store_buffer = nullptr;
 592   }
 593   if (_table != nullptr) {
 594     MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
 595     delete _table;
 596     _table = nullptr;
 597   }
 598 }
 599 
 600 void AOTCodeCache::Config::record(uint cpu_features_offset) {
 601   _flags = 0;
 602 #ifdef ASSERT
 603   _flags |= debugVM;
 604 #endif
 605   if (UseCompressedOops) {
 606     _flags |= compressedOops;
 607   }
 608   if (UseCompressedClassPointers) {
 609     _flags |= compressedClassPointers;
 610   }
 611   if (UseTLAB) {
 612     _flags |= useTLAB;
 613   }
 614   if (JavaAssertions::systemClassDefault()) {
 615     _flags |= systemClassAssertions;
 616   }
 617   if (JavaAssertions::userClassDefault()) {
 618     _flags |= userClassAssertions;
 619   }
 620   if (EnableContended) {
 621     _flags |= enableContendedPadding;
 622   }
 623   if (RestrictContended) {
 624     _flags |= restrictContendedPadding;
 625   }
 626   if (PreserveFramePointer) {
 627     _flags |= preserveFramePointer;
 628   }
 629   _codeCacheSize         = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
 630   _compressedOopShift    = CompressedOops::shift();
 631   _compressedOopBase     = CompressedOops::base();
 632   _compressedKlassShift  = CompressedKlassPointers::shift();
 633   _compressedKlassBase   = CompressedKlassPointers::base();
 634   _contendedPaddingWidth = ContendedPaddingWidth;
 635   _objectAlignment       = ObjectAlignmentInBytes;
 636   _gcCardSize            = GCCardSizeInBytes;
 637   _gc                    = (uint)Universe::heap()->kind();
 638   _cpu_features_offset   = cpu_features_offset;
 639 }
 640 
 641 bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
 642   // First checks affect all cached AOT code
 643 #ifdef ASSERT
 644   if ((_flags & debugVM) == 0) {
 645     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
 646     return false;
 647   }
 648 #else
 649   if ((_flags & debugVM) != 0) {
 650     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
 651     return false;
 652   }
 653 #endif
 654 
 655   size_t codeCacheSize = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
 656   if (_codeCacheSize != codeCacheSize) {
 657     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CodeCache size = %dKb vs current %dKb", (int)(_codeCacheSize/K), (int)(codeCacheSize/K));
 658     return false;
 659   }
 660 
 661   CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
 662   if (aot_gc != Universe::heap()->kind()) {
 663     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
 664     return false;
 665   }
 666 
 667   if (_gcCardSize != (uint)GCCardSizeInBytes) {
 668     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with GCCardSizeInBytes = %d vs current %d", _gcCardSize, GCCardSizeInBytes);
 669     return false;
 670   }
 671 
 672   if (_objectAlignment != (uint)ObjectAlignmentInBytes) {
 673     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ObjectAlignmentInBytes = %d vs current %d", _objectAlignment, ObjectAlignmentInBytes);
 674     return false;
 675   }
 676 
 677   if (((_flags & enableContendedPadding) != 0) != EnableContended) {
 678     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with EnableContended = %s vs current %s", (EnableContended ? "false" : "true"), (EnableContended ? "true" : "false"));
 679     return false;
 680   }
 681   if (((_flags & restrictContendedPadding) != 0) != RestrictContended) {
 682     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with RestrictContended = %s vs current %s", (RestrictContended ? "false" : "true"), (RestrictContended ? "true" : "false"));
 683     return false;
 684   }
 685   if (_contendedPaddingWidth != (uint)ContendedPaddingWidth) {
 686     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ContendedPaddingWidth = %d vs current %d", _contendedPaddingWidth, ContendedPaddingWidth);
 687     return false;
 688   }
 689 
 690   if (((_flags & preserveFramePointer) != 0) != PreserveFramePointer) {
 691     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with PreserveFramePointer = %s vs current %s", (PreserveFramePointer ? "false" : "true"), (PreserveFramePointer ? "true" : "false"));
 692     return false;
 693   }
 694 
 695   if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
 696     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s vs current %s", (UseCompressedClassPointers ? "false" : "true"), (UseCompressedClassPointers ? "true" : "false"));
 697     return false;
 698   }
 699   if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
 700     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
 701     return false;
 702   }
 703   if ((_compressedKlassBase == nullptr || CompressedKlassPointers::base() == nullptr) && (_compressedKlassBase != CompressedKlassPointers::base())) {
 704     log_debug(aot, codecache, init)("AOT Code Cache disabled: incompatible CompressedKlassPointers::base(): %p vs current %p", _compressedKlassBase, CompressedKlassPointers::base());
 705     return false;
 706   }
 707 
 708   if (((_flags & compressedOops) != 0) != UseCompressedOops) {
 709     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s vs current %s", (UseCompressedOops ? "false" : "true"), (UseCompressedOops ? "true" : "false"));
 710     return false;
 711   }
 712   if (_compressedOopShift != (uint)CompressedOops::shift()) {
 713     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
 714     return false;
 715   }


 716   if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
 717     log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
 718     return false;
 719   }
 720 
 721   LogStreamHandle(Debug, aot, codecache, init) log;
 722   if (log.is_enabled()) {
 723     log.print_cr("Available CPU features: %s", VM_Version::features_string());
 724   }
 725 
 726   uint offset = _cpu_features_offset;
 727   uint cpu_features_size = *(uint *)cache->addr(offset);
 728   assert(cpu_features_size == (uint)VM_Version::cpu_features_size(), "must be");
 729   offset += sizeof(uint);
 730 
 731   void* cached_cpu_features_buffer = (void *)cache->addr(offset);
 732   if (log.is_enabled()) {
 733     ResourceMark rm; // required for stringStream::as_string()
 734     stringStream ss;
 735     VM_Version::get_cpu_features_name(cached_cpu_features_buffer, ss);
 736     log.print_cr("CPU features recorded in AOTCodeCache: %s", ss.as_string());
 737   }
 738 
 739   if (AOTCodeCPUFeatureCheck && !VM_Version::supports_features(cached_cpu_features_buffer)) {
 740     if (log.is_enabled()) {
 741       ResourceMark rm; // required for stringStream::as_string()
 742       stringStream ss;
 743       VM_Version::get_missing_features_name(cached_cpu_features_buffer, ss);
 744       log.print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
 745     }
 746     return false;
 747   }
 748 
 749   // Next affects only AOT nmethod
 750   if (((_flags & systemClassAssertions) != 0) != JavaAssertions::systemClassDefault()) {
 751     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::systemClassDefault() = %s vs current %s", (JavaAssertions::systemClassDefault() ? "disabled" : "enabled"), (JavaAssertions::systemClassDefault() ? "enabled" : "disabled"));
 752      FLAG_SET_ERGO(AOTCodeCaching, false);
 753   }
 754   if (((_flags & userClassAssertions) != 0) != JavaAssertions::userClassDefault()) {
 755     log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::userClassDefault() = %s vs current %s", (JavaAssertions::userClassDefault() ? "disabled" : "enabled"), (JavaAssertions::userClassDefault() ? "enabled" : "disabled"));
 756     FLAG_SET_ERGO(AOTCodeCaching, false);
 757   }
 758 
 759   return true;
 760 }
 761 
 762 bool AOTCodeCache::Header::verify(uint load_size) const {
 763   if (_version != AOT_CODE_VERSION) {
 764     log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
 765     return false;
 766   }
 767   if (load_size < _cache_size) {
 768     log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
 769     return false;
 770   }
 771   return true;
 772 }
 773 
 774 volatile int AOTCodeCache::_nmethod_readers = 0;
 775 
 776 AOTCodeCache* AOTCodeCache::open_for_use() {
 777   if (AOTCodeCache::is_on_for_use()) {
 778     return AOTCodeCache::cache();
 779   }
 780   return nullptr;
 781 }
 782 
 783 AOTCodeCache* AOTCodeCache::open_for_dump() {
 784   if (AOTCodeCache::is_on_for_dump()) {
 785     AOTCodeCache* cache = AOTCodeCache::cache();
 786     cache->clear_lookup_failed(); // Reset bit
 787     return cache;
 788   }
 789   return nullptr;
 790 }
 791 
 792 bool AOTCodeCache::is_address_in_aot_cache(address p) {
 793   AOTCodeCache* cache = open_for_use();
 794   if (cache == nullptr) {
 795     return false;
 796   }
 797   if ((p >= (address)cache->cache_buffer()) &&
 798       (p < (address)(cache->cache_buffer() + cache->load_size()))) {
 799     return true;
 800   }
 801   return false;
 802 }
 803 
 804 static void copy_bytes(const char* from, address to, uint size) {
 805   assert((int)size > 0, "sanity");
 806   memcpy(to, from, size);
 807   log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
 808 }
 809 
 810 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry, CompileTask* task) {
 811   _cache = cache;
 812   _entry = entry;
 813   _load_buffer = cache->cache_buffer();
 814   _read_position = 0;
 815   if (task != nullptr) {
 816     _compile_id = task->compile_id();
 817     _comp_level = task->comp_level();
 818     _preload    = task->preload();
 819   } else {
 820     _compile_id = 0;
 821     _comp_level = 0;
 822     _preload    = false;
 823   }
 824   _lookup_failed = false;
 825 }
 826 
 827 void AOTCodeReader::set_read_position(uint pos) {
 828   if (pos == _read_position) {
 829     return;
 830   }
 831   assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
 832   _read_position = pos;
 833 }
 834 
 835 bool AOTCodeCache::set_write_position(uint pos) {
 836   if (pos == _write_position) {
 837     return true;
 838   }
 839   if (_store_size < _write_position) {
 840     _store_size = _write_position; // Adjust during write
 841   }
 842   assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
 843   _write_position = pos;

 886   if (nbytes == 0) {
 887     return 0;
 888   }
 889   uint new_position = _write_position + nbytes;
 890   if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
 891     log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.",
 892                                 nbytes, _write_position);
 893     set_failed();
 894     report_store_failure();
 895     return 0;
 896   }
 897   copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
 898   log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
 899   _write_position += nbytes;
 900   if (_store_size < _write_position) {
 901     _store_size = _write_position;
 902   }
 903   return nbytes;
 904 }
 905 
 906 AOTCodeEntry* AOTCodeCache::find_code_entry(const methodHandle& method, uint comp_level) {
 907   assert(is_using_code(), "AOT code caching should be enabled");
 908   if (!method->in_aot_cache()) {
 909     return nullptr;
 910   }
 911   switch (comp_level) {
 912     case CompLevel_simple:
 913       if ((DisableAOTCode & (1 << 0)) != 0) {
 914         return nullptr;
 915       }
 916       break;
 917     case CompLevel_limited_profile:
 918       if ((DisableAOTCode & (1 << 1)) != 0) {
 919         return nullptr;
 920       }
 921       break;
 922     case CompLevel_full_optimization:
 923       if ((DisableAOTCode & (1 << 2)) != 0) {
 924         return nullptr;
 925       }
 926       break;
 927 
 928     default: return nullptr; // Level 1, 2, and 4 only
 929   }
 930   TraceTime t1("Total time to find AOT code", &_t_totalFind, enable_timers(), false);
 931   if (is_on() && _cache->cache_buffer() != nullptr) {
 932     uint id = AOTCacheAccess::convert_method_to_offset(method());
 933     AOTCodeEntry* entry = _cache->find_entry(AOTCodeEntry::Nmethod, id, comp_level);
 934     if (entry == nullptr) {
 935       LogStreamHandle(Info, aot, codecache, nmethod) log;
 936       if (log.is_enabled()) {
 937         ResourceMark rm;
 938         const char* target_name = method->name_and_sig_as_C_string();
 939         log.print("Missing entry for '%s' (comp_level %d, id: " UINT32_FORMAT_X_0 ")", target_name, (uint)comp_level, id);
 940       }
 941 #ifdef ASSERT
 942     } else {
 943       ResourceMark rm;
 944       assert(method() == entry->method(), "AOTCodeCache: saved nmethod's method %p (name: %s id: " UINT32_FORMAT_X_0
 945              ") is different from the method %p (name: %s, id: " UINT32_FORMAT_X_0 " being looked up" ,
 946              entry->method(), entry->method()->name_and_sig_as_C_string(), entry->id(), method(), method()->name_and_sig_as_C_string(), id);
 947 #endif
 948     }
 949 
 950     DirectiveSet* directives = DirectivesStack::getMatchingDirective(method, nullptr);
 951     if (directives->IgnorePrecompiledOption) {
 952       LogStreamHandle(Info, aot, codecache, compilation) log;
 953       if (log.is_enabled()) {
 954         log.print("Ignore AOT code entry on level %d for ", comp_level);
 955         method->print_value_on(&log);
 956       }
 957       return nullptr;
 958     }
 959 
 960     return entry;
 961   }
 962   return nullptr;
 963 }
 964 
 965 Method* AOTCodeEntry::method() {
 966   assert(_kind == Nmethod, "invalid kind %d", _kind);
 967   assert(AOTCodeCache::is_on_for_use(), "must be");
 968   return AOTCacheAccess::convert_offset_to_method(_id);
 969 }
 970 
 971 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
 972   return (void*)(cache->add_entry());
 973 }
 974 
 975 static bool check_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level, AOTCodeEntry* entry) {
 976   if (entry->kind() == kind) {
 977     assert(entry->id() == id, "sanity");
 978     if (kind != AOTCodeEntry::Nmethod || // addapters and stubs have only one version
 979         // Look only for normal AOT code entry, preload code is handled separately
 980         (!entry->not_entrant() && !entry->has_clinit_barriers() && (entry->comp_level() == comp_level))) {
 981       return true; // Found
 982     }
 983   }
 984   return false;
 985 }
 986 
 987 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level) {
 988   assert(_for_use, "sanity");
 989   uint count = _load_header->entries_count();
 990   if (_load_entries == nullptr) {
 991     // Read it
 992     _search_entries = (uint*)addr(_load_header->search_table_offset()); // [id, index]
 993     _load_entries = (AOTCodeEntry*)addr(_load_header->entries_offset());
 994     log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
 995   }
 996   // Binary search
 997   int l = 0;
 998   int h = count - 1;
 999   while (l <= h) {
1000     int mid = (l + h) >> 1;
1001     int ix = mid * 2;
1002     uint is = _search_entries[ix];
1003     if (is == id) {
1004       int index = _search_entries[ix + 1];
1005       AOTCodeEntry* entry = &(_load_entries[index]);
1006       if (check_entry(kind, id, comp_level, entry)) {
1007         return entry; // Found
1008       }
1009       // Leaner search around
1010       for (int i = mid - 1; i >= l; i--) { // search back
1011         ix = i * 2;
1012         is = _search_entries[ix];
1013         if (is != id) {
1014           break;
1015         }
1016         index = _search_entries[ix + 1];
1017         AOTCodeEntry* entry = &(_load_entries[index]);
1018         if (check_entry(kind, id, comp_level, entry)) {
1019           return entry; // Found
1020         }
1021       }
1022       for (int i = mid + 1; i <= h; i++) { // search forward
1023         ix = i * 2;
1024         is = _search_entries[ix];
1025         if (is != id) {
1026           break;
1027         }
1028         index = _search_entries[ix + 1];
1029         AOTCodeEntry* entry = &(_load_entries[index]);
1030         if (check_entry(kind, id, comp_level, entry)) {
1031           return entry; // Found
1032         }
1033       }
1034       break; // No match found
1035     } else if (is < id) {
1036       l = mid + 1;
1037     } else {
1038       h = mid - 1;
1039     }
1040   }
1041   return nullptr;
1042 }
1043 
1044 void AOTCodeCache::invalidate_entry(AOTCodeEntry* entry) {
1045   assert(entry!= nullptr, "all entries should be read already");
1046   if (entry->not_entrant()) {
1047     return; // Someone invalidated it already
1048   }
1049 #ifdef ASSERT
1050   assert(_load_entries != nullptr, "sanity");
1051   {
1052     uint name_offset = entry->offset() + entry->name_offset();
1053     const char* name = _load_buffer + name_offset;;
1054     uint level       = entry->comp_level();
1055     uint comp_id     = entry->comp_id();
1056     bool for_preload = entry->for_preload();
1057     bool clinit_brs  = entry->has_clinit_barriers();
1058     log_info(aot, codecache, nmethod)("Invalidating entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1059                                       name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1060   }
1061   assert(entry->is_loaded() || entry->for_preload(), "invalidate only AOT code in use or a preload code");
1062   bool found = false;
1063   uint i = 0;
1064   uint count = 0;
1065   if (entry->for_preload()) {
1066     count = _load_header->preload_entries_count();
1067     AOTCodeEntry* preload_entry = (AOTCodeEntry*)addr(_load_header->preload_entries_offset());
1068     for (; i < count; i++) {
1069       if (entry == &preload_entry[i]) {
1070         break;
1071       }
1072     }
1073   } else {
1074     count = _load_header->entries_count();
1075     for(; i < count; i++) {
1076       if (entry == &(_load_entries[i])) {
1077         break;
1078       }
1079     }
1080   }
1081   found = (i < count);
1082   assert(found, "entry should exist");
1083 #endif
1084   entry->set_not_entrant();
1085   uint name_offset = entry->offset() + entry->name_offset();
1086   const char* name = _load_buffer + name_offset;;
1087   uint level       = entry->comp_level();
1088   uint comp_id     = entry->comp_id();
1089   bool for_preload = entry->for_preload();
1090   bool clinit_brs  = entry->has_clinit_barriers();
1091   log_info(aot, codecache, nmethod)("Invalidated entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1092                                     name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1093 
1094   if (!for_preload && (entry->comp_level() == CompLevel_full_optimization)) {
1095     // Invalidate preload code if normal AOT C2 code is invalidated,
1096     // most likely because some dependencies changed during run.
1097     // We can still use normal AOT code if preload code is
1098     // invalidated - normal AOT code has less restrictions.
1099     Method* method = entry->method();
1100     AOTCodeEntry* preload_entry = method->aot_code_entry();
1101     if (preload_entry != nullptr) {
1102       assert(preload_entry->for_preload(), "expecting only such entries here");
1103       invalidate_entry(preload_entry);
1104     }
1105   }
1106 }
1107 
1108 static int uint_cmp(const void *i, const void *j) {
1109   uint a = *(uint *)i;
1110   uint b = *(uint *)j;
1111   return a > b ? 1 : a < b ? -1 : 0;
1112 }
1113 
1114 void AOTCodeCache::store_cpu_features(char*& buffer, uint buffer_size) {
1115   uint* size_ptr = (uint *)buffer;
1116   *size_ptr = buffer_size;
1117   buffer += sizeof(uint);
1118 
1119   VM_Version::store_cpu_features(buffer);
1120   log_debug(aot, codecache, exit)("CPU features recorded in AOTCodeCache: %s", VM_Version::features_string());
1121   buffer += buffer_size;
1122   buffer = align_up(buffer, DATA_ALIGNMENT);
1123 }
1124 
1125 bool AOTCodeCache::finish_write() {
1126   if (!align_write()) {
1127     return false;
1128   }
1129   uint strings_offset = _write_position;
1130   int strings_count = store_strings();
1131   if (strings_count < 0) {
1132     return false;
1133   }
1134   if (!align_write()) {
1135     return false;
1136   }
1137   uint strings_size = _write_position - strings_offset;
1138 
1139   uint code_count = _store_entries_cnt;
1140   if (code_count > 0) {
1141     _aot_code_directory = CachedCodeDirectory::create();
1142     assert(_aot_code_directory != nullptr, "Sanity check");
1143 
1144     uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT);



1145     uint search_count = code_count * 2;
1146     uint search_size = search_count * sizeof(uint);
1147     uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
1148     // _write_position should include code and strings
1149     uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
1150     uint cpu_features_size = VM_Version::cpu_features_size();
1151     uint total_cpu_features_size = sizeof(uint) + cpu_features_size; // sizeof(uint) to store cpu_features_size
1152     uint total_size = _write_position + header_size + code_alignment +
1153                       search_size + entries_size +
1154                       align_up(total_cpu_features_size, DATA_ALIGNMENT);
1155     assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size());
1156 


1157     // Allocate in AOT Cache buffer
1158     char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT);
1159     char* start = align_up(buffer, DATA_ALIGNMENT);
1160     char* current = start + header_size; // Skip header
1161 
1162     uint cpu_features_offset = current - start;
1163     store_cpu_features(current, cpu_features_size);
1164     assert(is_aligned(current, DATA_ALIGNMENT), "sanity check");
1165     assert(current < start + total_size, "sanity check");
1166 
1167     // Create ordered search table for entries [id, index];
1168     uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
1169 
1170     AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
1171     AOTCodeStats stats;



1172     uint max_size = 0;
1173     // AOTCodeEntry entries were allocated in reverse in store buffer.
1174     // Process them in reverse order to cache first code first.
1175 
1176     // Store AOTCodeEntry-s for preload code
1177     current = align_up(current, DATA_ALIGNMENT);
1178     uint preload_entries_cnt = 0;
1179     uint preload_entries_offset = current - start;
1180     AOTCodeEntry* preload_entries = (AOTCodeEntry*)current;
1181     for (int i = code_count - 1; i >= 0; i--) {
1182       AOTCodeEntry* entry = &entries_address[i];
1183       if (entry->load_fail()) {
1184         continue;
1185       }
1186       if (entry->for_preload()) {
1187         if (entry->not_entrant()) {
1188           // Skip not entrant preload code:
1189           // we can't pre-load code which may have failing dependencies.
1190           log_info(aot, codecache, exit)("Skip not entrant preload code comp_id: %d, comp_level: %d, hash: " UINT32_FORMAT_X_0 "%s",
1191                                          entry->comp_id(), entry->comp_level(), entry->id(), (entry->has_clinit_barriers() ? ", has clinit barriers" : ""));
1192         } else {
1193           copy_bytes((const char*)entry, (address)current, sizeof(AOTCodeEntry));
1194           stats.collect_entry_stats(entry);
1195           current += sizeof(AOTCodeEntry);
1196           preload_entries_cnt++;
1197         }
1198       }
1199     }
1200 
1201     // Now write the data for preload AOTCodeEntry
1202     for (int i = 0; i < (int)preload_entries_cnt; i++) {
1203       AOTCodeEntry* entry = &preload_entries[i];
1204       uint size = align_up(entry->size(), DATA_ALIGNMENT);
1205       if (size > max_size) {
1206         max_size = size;
1207       }
1208       copy_bytes((_store_buffer + entry->offset()), (address)current, size);
1209       entry->set_offset(current - start); // New offset
1210       current += size;
1211     }
1212 
1213     current = align_up(current, DATA_ALIGNMENT);
1214     uint entries_count = 0;
1215     uint new_entries_offset = current - start;
1216     AOTCodeEntry* code_entries = (AOTCodeEntry*)current;
1217     // Now scan normal entries
1218     for (int i = code_count - 1; i >= 0; i--) {
1219       AOTCodeEntry* entry = &entries_address[i];
1220       if (entry->load_fail() || entry->for_preload()) {
1221         continue;
1222       }
1223       if (entry->not_entrant()) {
1224         log_info(aot, codecache, exit)("Not entrant new entry comp_id: %d, comp_level: %d, hash: " UINT32_FORMAT_X_0 "%s",
1225                                        entry->comp_id(), entry->comp_level(), entry->id(), (entry->has_clinit_barriers() ? ", has clinit barriers" : ""));
1226         entry->set_entrant(); // Reset
1227       }
1228       copy_bytes((const char*)entry, (address)current, sizeof(AOTCodeEntry));
1229       stats.collect_entry_stats(entry);
1230       current += sizeof(AOTCodeEntry);
1231       search[entries_count*2 + 0] = entry->id();
1232       search[entries_count*2 + 1] = entries_count;
1233       entries_count++;
1234     }
1235 
1236     // Now write the data for normal AOTCodeEntry
1237     for (int i = 0; i < (int)entries_count; i++) {
1238       AOTCodeEntry* entry = &code_entries[i];
1239       uint size = align_up(entry->size(), DATA_ALIGNMENT);
1240       if (size > max_size) {
1241         max_size = size;
1242       }
1243       copy_bytes((_store_buffer + entry->offset()), (address)current, size);
1244       entry->set_offset(current - start); // New offset
1245       current += size;
1246     }
1247 
1248     if (preload_entries_cnt == 0 && entries_count == 0) {
1249       log_info(aot, codecache, exit)("AOT Code Cache was not created: no entries");
1250       FREE_C_HEAP_ARRAY(uint, search);
1251       return true; // Nothing to write
1252     }
1253     uint total_entries_cnt = preload_entries_cnt + entries_count;
1254     assert(total_entries_cnt <= code_count, "%d > %d", total_entries_cnt, code_count);
1255     // Write strings
1256     if (strings_count > 0) {
1257       copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
1258       strings_offset = (current - start); // New offset
1259       current += strings_size;
1260     }
1261 
1262     uint search_table_offset = current - start;
1263     // Sort and store search table
1264     qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
1265     search_size = 2 * entries_count * sizeof(uint);
1266     copy_bytes((const char*)search, (address)current, search_size);
1267     FREE_C_HEAP_ARRAY(uint, search);
1268     current += search_size;
1269 
1270     log_stats_on_exit(stats);
1271 


1272     uint size = (current - start);
1273     assert(size <= total_size, "%d > %d", size , total_size);





1274     log_debug(aot, codecache, exit)("  AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
1275 
1276     // Finalize header
1277     AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
1278     header->init(size, (uint)strings_count, strings_offset,
1279                  entries_count, search_table_offset, new_entries_offset,
1280                  preload_entries_cnt, preload_entries_offset,
1281                  stats.entry_count(AOTCodeEntry::Adapter), stats.entry_count(AOTCodeEntry::SharedBlob),
1282                  stats.entry_count(AOTCodeEntry::C1Blob), stats.entry_count(AOTCodeEntry::C2Blob),
1283                  stats.entry_count(AOTCodeEntry::Stub), cpu_features_offset);
1284 
1285     log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", total_entries_cnt);
1286 
1287     _aot_code_directory->set_aot_code_data(size, start);
1288   }
1289   return true;
1290 }
1291 
1292 //------------------Store/Load AOT code ----------------------
1293 
1294 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1295   AOTCodeCache* cache = open_for_dump();
1296   if (cache == nullptr) {
1297     return false;
1298   }
1299   assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1300 
1301   if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
1302     return false;
1303   }
1304   if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
1305     return false;
1306   }
1307   log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);

1342     return false;
1343   }
1344   CodeBlob::archive_blob(&blob, archive_buffer);
1345 
1346   uint reloc_data_size = blob.relocation_size();
1347   n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size);
1348   if (n != reloc_data_size) {
1349     return false;
1350   }
1351 
1352   bool has_oop_maps = false;
1353   if (blob.oop_maps() != nullptr) {
1354     if (!cache->write_oop_map_set(blob)) {
1355       return false;
1356     }
1357     has_oop_maps = true;
1358   }
1359 
1360 #ifndef PRODUCT
1361   // Write asm remarks
1362   if (!cache->write_asm_remarks(blob.asm_remarks(), /* use_string_table */ true)) {
1363     return false;
1364   }
1365   if (!cache->write_dbg_strings(blob.dbg_strings(), /* use_string_table */ true)) {
1366     return false;
1367   }
1368 #endif /* PRODUCT */
1369 
1370   if (!cache->write_relocations(blob)) {
1371     if (!cache->failed()) {
1372       // We may miss an address in AOT table - skip this code blob.
1373       cache->set_write_position(entry_position);
1374     }
1375     return false;
1376   }
1377 
1378   uint entry_size = cache->_write_position - entry_position;
1379   AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
1380                                                 entry_position, entry_size, name_offset, name_size,
1381                                                 blob_offset, has_oop_maps, blob.content_begin());
1382   log_debug(aot, codecache, stubs)("Wrote code blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1383   return true;
1384 }
1385 

1391 
1392 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1393   AOTCodeCache* cache = open_for_use();
1394   if (cache == nullptr) {
1395     return nullptr;
1396   }
1397   assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1398 
1399   if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
1400     return nullptr;
1401   }
1402   if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
1403     return nullptr;
1404   }
1405   log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1406 
1407   AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
1408   if (entry == nullptr) {
1409     return nullptr;
1410   }
1411   AOTCodeReader reader(cache, entry, nullptr);
1412   CodeBlob* blob = reader.compile_code_blob(name);
1413 
1414   log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
1415                                    (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
1416   return blob;
1417 }
1418 
1419 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
1420   assert(AOTCodeEntry::is_blob(entry_kind),
1421          "wrong entry kind for blob id %s", StubInfo::name(id));
1422   return load_code_blob(entry_kind, (uint)id, StubInfo::name(id));
1423 }
1424 
1425 CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
1426   uint entry_position = _entry->offset();
1427 
1428   // Read name
1429   uint name_offset = entry_position + _entry->name_offset();
1430   uint name_size = _entry->name_size(); // Includes '/0'
1431   const char* stored_name = addr(name_offset);
1432 
1433   if (strncmp(stored_name, name, (name_size - 1)) != 0) {
1434     log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'",
1435                                        stored_name, name);
1436     set_lookup_failed(); // Skip this blob
1437     return nullptr;
1438   }
1439 
1440   // Read archived code blob
1441   uint offset = entry_position + _entry->code_offset();
1442   CodeBlob* archived_blob = (CodeBlob*)addr(offset);
1443   offset += archived_blob->size();
1444 
1445   address reloc_data = (address)addr(offset);
1446   offset += archived_blob->relocation_size();
1447   set_read_position(offset);
1448 
1449   ImmutableOopMapSet* oop_maps = nullptr;
1450   if (_entry->has_oop_maps()) {
1451     oop_maps = read_oop_map_set();
1452   }
1453 
1454   CodeBlob* code_blob = CodeBlob::create(archived_blob,
1455                                          stored_name,
1456                                          reloc_data,
1457                                          oop_maps
1458                                         );
1459   if (code_blob == nullptr) { // no space left in CodeCache
1460     return nullptr;
1461   }
1462 
1463 #ifndef PRODUCT
1464   code_blob->asm_remarks().init();
1465   read_asm_remarks(code_blob->asm_remarks(), /* use_string_table */ true);
1466   code_blob->dbg_strings().init();
1467   read_dbg_strings(code_blob->dbg_strings(), /* use_string_table */ true);
1468 #endif // PRODUCT
1469 
1470   fix_relocations(code_blob);
1471 
1472 #ifdef ASSERT
1473   LogStreamHandle(Trace, aot, codecache, stubs) log;
1474   if (log.is_enabled()) {
1475     FlagSetting fs(PrintRelocations, true);
1476     code_blob->print_on(&log);
1477   }
1478 #endif
1479   return code_blob;
1480 }
1481 
1482 bool AOTCodeCache::store_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1483   if (!is_dumping_stub()) {
1484     return false;
1485   }
1486   AOTCodeCache* cache = open_for_dump();
1487   if (cache == nullptr) {
1488     return false;
1489   }
1490   log_info(aot, codecache, stubs)("Writing stub '%s' id:%d to AOT Code Cache", name, (int)id);
1491   if (!cache->align_write()) {
1492     return false;
1493   }
1494 #ifdef ASSERT
1495   CodeSection* cs = cgen->assembler()->code_section();
1496   if (cs->has_locs()) {
1497     uint reloc_count = cs->locs_count();
1498     tty->print_cr("======== write stubs code section relocations [%d]:", reloc_count);
1499     // Collect additional data
1500     RelocIterator iter(cs);
1501     while (iter.next()) {
1502       switch (iter.type()) {
1503         case relocInfo::none:
1504           break;
1505         default: {
1506           iter.print_current_on(tty);
1507           fatal("stub's relocation %d unimplemented", (int)iter.type());
1508           break;











1509         }


1510       }
1511     }
1512   }
1513 #endif
1514   uint entry_position = cache->_write_position;
1515 
1516   // Write code
1517   uint code_offset = 0;
1518   uint code_size = cgen->assembler()->pc() - start;
1519   uint n = cache->write_bytes(start, code_size);
1520   if (n != code_size) {
1521     return false;
1522   }
1523   // Write name
1524   uint name_offset = cache->_write_position - entry_position;
1525   uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1526   n = cache->write_bytes(name, name_size);
1527   if (n != name_size) {
1528     return false;
1529   }
1530   uint entry_size = cache->_write_position - entry_position;
1531   AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
1532                                                 code_offset, code_size,
1533                                                 AOTCodeEntry::Stub, (uint32_t)id);
1534   log_info(aot, codecache, stubs)("Wrote stub '%s' id:%d to AOT Code Cache", name, (int)id);
1535   return true;
1536 }
1537 
1538 bool AOTCodeCache::load_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1539   if (!is_using_stub()) {
1540     return false;
1541   }
1542   assert(start == cgen->assembler()->pc(), "wrong buffer");
1543   AOTCodeCache* cache = open_for_use();
1544   if (cache == nullptr) {
1545     return false;
1546   }
1547   AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Stub, (uint)id);
1548   if (entry == nullptr) {
1549     return false;
1550   }
1551   uint entry_position = entry->offset();
1552   // Read name
1553   uint name_offset = entry->name_offset() + entry_position;
1554   uint name_size   = entry->name_size(); // Includes '/0'
1555   const char* saved_name = cache->addr(name_offset);
1556   if (strncmp(name, saved_name, (name_size - 1)) != 0) {
1557     log_warning(aot, codecache)("Saved stub's name '%s' is different from '%s' for id:%d", saved_name, name, (int)id);
1558     cache->set_failed();
1559     report_load_failure();
1560     return false;
1561   }
1562   log_info(aot, codecache, stubs)("Reading stub '%s' id:%d from AOT Code Cache", name, (int)id);
1563   // Read code
1564   uint code_offset = entry->code_offset() + entry_position;
1565   uint code_size   = entry->code_size();
1566   copy_bytes(cache->addr(code_offset), start, code_size);
1567   cgen->assembler()->code_section()->set_end(start + code_size);
1568   log_info(aot, codecache, stubs)("Read stub '%s' id:%d from AOT Code Cache", name, (int)id);
1569   return true;
1570 }
1571 
1572 AOTCodeEntry* AOTCodeCache::store_nmethod(nmethod* nm, AbstractCompiler* compiler, bool for_preload) {
1573   if (!is_dumping_code()) {
1574     return nullptr;
1575   }
1576   assert(CDSConfig::is_dumping_aot_code(), "should be called only when allowed");
1577   AOTCodeCache* cache = open_for_dump();
1578   precond(cache != nullptr);
1579   precond(!nm->is_osr_method()); // AOT compilation is requested only during AOT cache assembly phase
1580   if (!compiler->is_c1() && !compiler->is_c2()) {
1581     // Only c1 and c2 compilers
1582     return nullptr;
1583   }
1584   int comp_level = nm->comp_level();
1585   if (comp_level == CompLevel_full_profile) {
1586     // Do not cache C1 compiles with full profile i.e. tier3
1587     return nullptr;
1588   }
1589   assert(comp_level == CompLevel_simple || comp_level == CompLevel_limited_profile || comp_level == CompLevel_full_optimization, "must be");
1590 
1591   TraceTime t1("Total time to store AOT code", &_t_totalStore, enable_timers(), false);
1592   AOTCodeEntry* entry = nullptr;
1593   entry = cache->write_nmethod(nm, for_preload);
1594   if (entry == nullptr) {
1595     log_info(aot, codecache, nmethod)("%d (L%d): nmethod store attempt failed", nm->compile_id(), comp_level);
1596   }
1597   return entry;
1598 }
1599 
1600 AOTCodeEntry* AOTCodeCache::write_nmethod(nmethod* nm, bool for_preload) {
1601   AOTCodeCache* cache = open_for_dump();
1602   assert(cache != nullptr, "sanity check");
1603   assert(!nm->has_clinit_barriers() || (ClassInitBarrierMode > 0), "sanity");
1604   uint comp_id = nm->compile_id();
1605   uint comp_level = nm->comp_level();
1606   Method* method = nm->method();
1607   if (!AOTCacheAccess::can_generate_aot_code(method)) {
1608     ResourceMark rm;
1609     log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' for AOT%s compile: not in AOT cache", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), (for_preload ? " preload" : ""));
1610     assert(AOTCacheAccess::can_generate_aot_code(method), "sanity");
1611     return nullptr;
1612   }
1613   InstanceKlass* holder = method->method_holder();
1614   bool builtin_loader = holder->class_loader_data()->is_builtin_class_loader_data();
1615   if (!builtin_loader) {
1616     ResourceMark rm;
1617     log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' loaded by custom class loader %s", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), holder->class_loader_data()->loader_name());
1618     assert(builtin_loader, "sanity");
1619     return nullptr;
1620   }
1621 
1622   _for_preload = for_preload;
1623   _has_clinit_barriers = nm->has_clinit_barriers();
1624 
1625   if (!align_write()) {
1626     return nullptr;
1627   }
1628 
1629   uint entry_position = _write_position;
1630 
1631   // Write name
1632   uint name_offset = 0;
1633   uint name_size   = 0;
1634   uint id = 0;
1635   uint n;
1636   {
1637     ResourceMark rm;
1638     const char* name = method->name_and_sig_as_C_string();
1639     log_info(aot, codecache, nmethod)("%d (L%d): Writing nmethod '%s' (comp level: %d, %s) to AOT Code Cache",
1640                                       comp_id, (int)comp_level, name, comp_level,
1641                                       (nm->has_clinit_barriers() ? ", has clinit barriers" : ""));
1642 
1643     LogStreamHandle(Info, aot, codecache, loader) log;
1644     if (log.is_enabled()) {
1645       oop loader = holder->class_loader();
1646       oop domain = holder->protection_domain();
1647       log.print("Holder: ");
1648       holder->print_value_on(&log);
1649       log.print(" loader: ");
1650       if (loader == nullptr) {
1651         log.print("nullptr");
1652       } else {
1653         loader->print_value_on(&log);
1654       }
1655       log.print(" domain: ");
1656       if (domain == nullptr) {
1657         log.print("nullptr");
1658       } else {
1659         domain->print_value_on(&log);
1660       }
1661       log.cr();
1662     }
1663     name_offset = _write_position  - entry_position;
1664     name_size   = (uint)strlen(name) + 1; // Includes '/0'
1665     n = write_bytes(name, name_size);
1666     if (n != name_size) {
1667       return nullptr;
1668     }
1669   }
1670   id = AOTCacheAccess::delta_from_base_address((address)nm->method());
1671 
1672   // Write CodeBlob
1673   if (!cache->align_write()) {
1674     return nullptr;
1675   }
1676   uint blob_offset = cache->_write_position - entry_position;
1677   address archive_buffer = cache->reserve_bytes(nm->size());
1678   if (archive_buffer == nullptr) {
1679     return nullptr;
1680   }
1681   CodeBlob::archive_blob(nm, archive_buffer);
1682 
1683   uint reloc_data_size = nm->relocation_size();
1684   n = write_bytes((address)nm->relocation_begin(), reloc_data_size);
1685   if (n != reloc_data_size) {
1686     return nullptr;
1687   }
1688 
1689   // Write oops and metadata present in the nmethod's data region
1690   if (!write_oops(nm)) {
1691     if (lookup_failed() && !failed()) {
1692       // Skip this method and reposition file
1693       set_write_position(entry_position);
1694     }
1695     return nullptr;
1696   }
1697   if (!write_metadata(nm)) {
1698     if (lookup_failed() && !failed()) {
1699       // Skip this method and reposition file
1700       set_write_position(entry_position);
1701     }
1702     return nullptr;
1703   }
1704 
1705   bool has_oop_maps = false;
1706   if (nm->oop_maps() != nullptr) {
1707     if (!cache->write_oop_map_set(*nm)) {
1708       return nullptr;
1709     }
1710     has_oop_maps = true;
1711   }
1712 
1713   uint immutable_data_size = nm->immutable_data_size();
1714   n = write_bytes(nm->immutable_data_begin(), immutable_data_size);
1715   if (n != immutable_data_size) {
1716     return nullptr;
1717   }
1718 
1719   JavaThread* thread = JavaThread::current();
1720   HandleMark hm(thread);
1721   GrowableArray<Handle> oop_list;
1722   GrowableArray<Metadata*> metadata_list;
1723 
1724   nm->create_reloc_immediates_list(thread, oop_list, metadata_list);
1725   if (!write_nmethod_reloc_immediates(oop_list, metadata_list)) {
1726     if (lookup_failed() && !failed()) {
1727       // Skip this method and reposition file
1728       set_write_position(entry_position);
1729     }
1730     return nullptr;
1731   }
1732 
1733   if (!write_relocations(*nm, &oop_list, &metadata_list)) {
1734     return nullptr;
1735   }
1736 
1737 #ifndef PRODUCT
1738   if (!cache->write_asm_remarks(nm->asm_remarks(), /* use_string_table */ false)) {
1739     return nullptr;
1740   }
1741   if (!cache->write_dbg_strings(nm->dbg_strings(), /* use_string_table */ false)) {
1742     return nullptr;
1743   }
1744 #endif /* PRODUCT */
1745 
1746   uint entry_size = _write_position - entry_position;
1747   AOTCodeEntry* entry = new (this) AOTCodeEntry(AOTCodeEntry::Nmethod, id,
1748                                                 entry_position, entry_size,
1749                                                 name_offset, name_size,
1750                                                 blob_offset, has_oop_maps,
1751                                                 nm->content_begin(), comp_level, comp_id,
1752                                                 nm->has_clinit_barriers(), for_preload);
1753 #ifdef ASSERT
1754   if (nm->has_clinit_barriers() || for_preload) {
1755     assert(for_preload, "sanity");
1756   }
1757 #endif
1758   {
1759     ResourceMark rm;
1760     const char* name = nm->method()->name_and_sig_as_C_string();
1761     log_info(aot, codecache, nmethod)("%d (L%d): Wrote nmethod '%s'%s to AOT Code Cache",
1762                            comp_id, (int)comp_level, name, (for_preload ? " (for preload)" : ""));
1763   }
1764   if (VerifyAOTCode) {
1765     return nullptr;
1766   }
1767   return entry;
1768 }
1769 
1770 bool AOTCodeCache::load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level) {
1771   if (!is_using_code()) {
1772     return false;
1773   }
1774   AOTCodeCache* cache = open_for_use();
1775   if (cache == nullptr) {
1776     return false;
1777   }
1778   assert(entry_bci == InvocationEntryBci, "unexpected entry_bci=%d", entry_bci);
1779   TraceTime t1("Total time to load AOT code", &_t_totalLoad, enable_timers(), false);
1780   CompileTask* task = env->task();
1781   task->mark_aot_load_start(os::elapsed_counter());
1782   AOTCodeEntry* entry = task->aot_code_entry();
1783   bool preload = task->preload();
1784   assert(entry != nullptr, "sanity");
1785   if (log_is_enabled(Info, aot, codecache, nmethod)) {
1786     VM_ENTRY_MARK;
1787     ResourceMark rm;
1788     methodHandle method(THREAD, target->get_Method());
1789     const char* target_name = method->name_and_sig_as_C_string();
1790     uint id = AOTCacheAccess::convert_method_to_offset(method());
1791     bool clinit_brs = entry->has_clinit_barriers();
1792     log_info(aot, codecache, nmethod)("%d (L%d): %s nmethod '%s' (id: " UINT32_FORMAT_X_0 "%s)",
1793                                       task->compile_id(), task->comp_level(), (preload ? "Preloading" : "Reading"),
1794                                       target_name, id, (clinit_brs ? ", has clinit barriers" : ""));
1795   }
1796   ReadingMark rdmk;
1797   if (rdmk.failed()) {
1798     // Cache is closed, cannot touch anything.
1799     return false;
1800   }
1801 
1802   AOTCodeReader reader(cache, entry, task);
1803   bool success = reader.compile_nmethod(env, target, compiler);
1804   if (success) {
1805     task->set_num_inlined_bytecodes(entry->num_inlined_bytecodes());
1806   } else {
1807     entry->set_load_fail();
1808     entry->set_not_entrant();
1809   }
1810   task->mark_aot_load_finish(os::elapsed_counter());
1811   return success;
1812 }
1813 
1814 bool AOTCodeReader::compile_nmethod(ciEnv* env, ciMethod* target, AbstractCompiler* compiler) {
1815   CompileTask* task = env->task();
1816   AOTCodeEntry* aot_code_entry = (AOTCodeEntry*)_entry;
1817   nmethod* nm = nullptr;
1818 
1819   uint entry_position = aot_code_entry->offset();
1820   uint archived_nm_offset = entry_position + aot_code_entry->code_offset();
1821   nmethod* archived_nm = (nmethod*)addr(archived_nm_offset);
1822   set_read_position(archived_nm_offset + archived_nm->size());
1823 
1824   OopRecorder* oop_recorder = new OopRecorder(env->arena());
1825   env->set_oop_recorder(oop_recorder);
1826 
1827   uint offset;
1828 
1829   offset = read_position();
1830   address reloc_data = (address)addr(offset);
1831   offset += archived_nm->relocation_size();
1832   set_read_position(offset);
1833 
1834   // Read oops and metadata
1835   VM_ENTRY_MARK
1836   GrowableArray<Handle> oop_list;
1837   GrowableArray<Metadata*> metadata_list;
1838 
1839   if (!read_oop_metadata_list(THREAD, target, oop_list, metadata_list, oop_recorder)) {
1840    return false;
1841   }
1842 
1843   ImmutableOopMapSet* oopmaps = read_oop_map_set();
1844 
1845   offset = read_position();
1846   address immutable_data = (address)addr(offset);
1847   offset += archived_nm->immutable_data_size();
1848   set_read_position(offset);
1849 
1850   GrowableArray<Handle> reloc_immediate_oop_list;
1851   GrowableArray<Metadata*> reloc_immediate_metadata_list;
1852   if (!read_oop_metadata_list(THREAD, target, reloc_immediate_oop_list, reloc_immediate_metadata_list, nullptr)) {
1853    return false;
1854   }
1855 
1856   // Read Dependencies (compressed already)
1857   Dependencies* dependencies = new Dependencies(env);
1858   dependencies->set_content(immutable_data, archived_nm->dependencies_size());
1859   env->set_dependencies(dependencies);
1860 
1861   const char* name = addr(entry_position + aot_code_entry->name_offset());
1862 
1863   if (VerifyAOTCode) {
1864     return false;
1865   }
1866 
1867   TraceTime t1("Total time to register AOT nmethod", &_t_totalRegister, enable_timers(), false);
1868   nm = env->register_aot_method(THREAD,
1869                                 target,
1870                                 compiler,
1871                                 archived_nm,
1872                                 reloc_data,
1873                                 oop_list,
1874                                 metadata_list,
1875                                 oopmaps,
1876                                 immutable_data,
1877                                 reloc_immediate_oop_list,
1878                                 reloc_immediate_metadata_list,
1879                                 this);
1880   bool success = task->is_success();
1881   if (success) {
1882     log_info(aot, codecache, nmethod)("%d (L%d): Read nmethod '%s' from AOT Code Cache", compile_id(), comp_level(), name);
1883 #ifdef ASSERT
1884     LogStreamHandle(Debug, aot, codecache, nmethod) log;
1885     if (log.is_enabled()) {
1886       FlagSetting fs(PrintRelocations, true);
1887       nm->print_on(&log);
1888       nm->decode2(&log);
1889     }
1890 #endif
1891   }
1892 
1893   return success;
1894 }
1895 
1896 bool skip_preload(methodHandle mh) {
1897   if (!mh->method_holder()->is_loaded()) {
1898     return true;
1899   }
1900   DirectiveSet* directives = DirectivesStack::getMatchingDirective(mh, nullptr);
1901   if (directives->DontPreloadOption) {
1902     LogStreamHandle(Info, aot, codecache, init) log;
1903     if (log.is_enabled()) {
1904       log.print("Exclude preloading code for ");
1905       mh->print_value_on(&log);
1906     }
1907     return true;
1908   }
1909   return false;
1910 }
1911 
1912 void AOTCodeCache::preload_code(JavaThread* thread) {
1913   if (!is_using_code()) {
1914     return;
1915   }
1916   if ((DisableAOTCode & (1 << 3)) != 0) {
1917     return; // no preloaded code (level 5);
1918   }
1919   _cache->preload_aot_code(thread);
1920 }
1921 
1922 void AOTCodeCache::preload_aot_code(TRAPS) {
1923   if (CompilationPolicy::compiler_count(CompLevel_full_optimization) == 0) {
1924     // Since we reuse the CompilerBroker API to install AOT code, we're required to have a JIT compiler for the
1925     // level we want (that is CompLevel_full_optimization).
1926     return;
1927   }
1928   TraceTime t1("Total time to preload AOT code", &_t_totalPreload, enable_timers(), false);
1929   assert(_for_use, "sanity");
1930   uint count = _load_header->entries_count();
1931   uint preload_entries_count = _load_header->preload_entries_count();
1932   if (preload_entries_count > 0) {
1933     log_info(aot, codecache, init)("Load %d preload entries from AOT Code Cache", preload_entries_count);
1934     AOTCodeEntry* preload_entry = (AOTCodeEntry*)addr(_load_header->preload_entries_offset());
1935     uint count = MIN2(preload_entries_count, AOTCodePreloadStop);
1936     for (uint i = AOTCodePreloadStart; i < count; i++) {
1937       AOTCodeEntry* entry = &preload_entry[i];
1938       if (entry->not_entrant()) {
1939         continue;
1940       }
1941       methodHandle mh(THREAD, entry->method());
1942       assert((mh.not_null() && AOTMetaspace::in_aot_cache((address)mh())), "sanity");
1943       if (skip_preload(mh)) {
1944         continue; // Exclude preloading for this method
1945       }
1946       assert(mh->method_holder()->is_loaded(), "");
1947       if (!mh->method_holder()->is_linked()) {
1948         assert(!HAS_PENDING_EXCEPTION, "");
1949         mh->method_holder()->link_class(THREAD);
1950         if (HAS_PENDING_EXCEPTION) {
1951           LogStreamHandle(Info, aot, codecache) log;
1952           if (log.is_enabled()) {
1953             ResourceMark rm;
1954             log.print("Linkage failed for %s: ", mh->method_holder()->external_name());
1955             THREAD->pending_exception()->print_value_on(&log);
1956             if (log_is_enabled(Debug, aot, codecache)) {
1957               THREAD->pending_exception()->print_on(&log);
1958             }
1959           }
1960           CLEAR_PENDING_EXCEPTION;
1961         }
1962       }
1963       if (mh->aot_code_entry() != nullptr) {
1964         // Second C2 compilation of the same method could happen for
1965         // different reasons without marking first entry as not entrant.
1966         continue; // Keep old entry to avoid issues
1967       }
1968       mh->set_aot_code_entry(entry);
1969       CompileBroker::compile_method(mh, InvocationEntryBci, CompLevel_full_optimization, 0, false, CompileTask::Reason_Preload, CHECK);
1970     }
1971   }
1972 }
1973 
1974 // ------------ process code and data --------------
1975 
1976 // Can't use -1. It is valid value for jump to iteself destination
1977 // used by static call stub: see NativeJump::jump_destination().
1978 #define BAD_ADDRESS_ID -2
1979 
1980 bool AOTCodeCache::write_relocations(CodeBlob& code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
1981   GrowableArray<uint> reloc_data;
1982   RelocIterator iter(&code_blob);
1983   LogStreamHandle(Trace, aot, codecache, reloc) log;
1984   while (iter.next()) {
1985     int idx = reloc_data.append(0); // default value
1986     switch (iter.type()) {
1987       case relocInfo::none:
1988       break;
1989       case relocInfo::oop_type: {
1990         oop_Relocation* r = (oop_Relocation*)iter.reloc();
1991         if (r->oop_is_immediate()) {
1992           assert(oop_list != nullptr, "sanity check");
1993           // store index of oop in the reloc immediate oop list
1994           Handle h(JavaThread::current(), r->oop_value());
1995           int oop_idx = oop_list->find(h);
1996           assert(oop_idx != -1, "sanity check");
1997           reloc_data.at_put(idx, (uint)oop_idx);
1998         }
1999         break;
2000       }
2001       case relocInfo::metadata_type: {
2002         metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2003         if (r->metadata_is_immediate()) {
2004           assert(metadata_list != nullptr, "sanity check");
2005           // store index of metadata in the reloc immediate metadata list
2006           int metadata_idx = metadata_list->find(r->metadata_value());
2007           assert(metadata_idx != -1, "sanity check");
2008           reloc_data.at_put(idx, (uint)metadata_idx);
2009         }
2010         break;
2011       }
2012       case relocInfo::virtual_call_type:  // Fall through. They all call resolve_*_call blobs.
2013       case relocInfo::opt_virtual_call_type:
2014       case relocInfo::static_call_type: {
2015         CallRelocation* r = (CallRelocation*)iter.reloc();
2016         address dest = r->destination();
2017         if (dest == r->addr()) { // possible call via trampoline on Aarch64
2018           dest = (address)-1;    // do nothing in this case when loading this relocation
2019         }
2020         int id = _table->id_for_address(dest, iter, &code_blob);
2021         if (id == BAD_ADDRESS_ID) {
2022           return false;
2023         }
2024         reloc_data.at_put(idx, id);
2025         break;
2026       }
2027       case relocInfo::trampoline_stub_type: {
2028         address dest = ((trampoline_stub_Relocation*)iter.reloc())->destination();
2029         int id = _table->id_for_address(dest, iter, &code_blob);
2030         if (id == BAD_ADDRESS_ID) {
2031           return false;
2032         }
2033         reloc_data.at_put(idx, id);
2034         break;
2035       }
2036       case relocInfo::static_stub_type:
2037         break;
2038       case relocInfo::runtime_call_type: {
2039         // Record offset of runtime destination
2040         CallRelocation* r = (CallRelocation*)iter.reloc();
2041         address dest = r->destination();
2042         if (dest == r->addr()) { // possible call via trampoline on Aarch64
2043           dest = (address)-1;    // do nothing in this case when loading this relocation
2044         }
2045         int id = _table->id_for_address(dest, iter, &code_blob);
2046         if (id == BAD_ADDRESS_ID) {
2047           return false;
2048         }
2049         reloc_data.at_put(idx, id);
2050         break;
2051       }
2052       case relocInfo::runtime_call_w_cp_type:
2053         log_debug(aot, codecache, reloc)("runtime_call_w_cp_type relocation is not implemented");
2054         return false;
2055       case relocInfo::external_word_type: {
2056         // Record offset of runtime target
2057         address target = ((external_word_Relocation*)iter.reloc())->target();
2058         int id = _table->id_for_address(target, iter, &code_blob);
2059         if (id == BAD_ADDRESS_ID) {
2060           return false;
2061         }
2062         reloc_data.at_put(idx, id);
2063         break;
2064       }
2065       case relocInfo::internal_word_type:
2066         break;
2067       case relocInfo::section_word_type:
2068         break;
2069       case relocInfo::poll_type:
2070         break;
2071       case relocInfo::poll_return_type:
2072         break;
2073       case relocInfo::post_call_nop_type:
2074         break;
2075       case relocInfo::entry_guard_type:
2076         break;
2077       default:
2078         log_debug(aot, codecache, reloc)("relocation %d unimplemented", (int)iter.type());
2079         return false;
2080         break;
2081     }
2082     if (log.is_enabled()) {
2083       iter.print_current_on(&log);
2084     }
2085   }
2086 
2087   // Write additional relocation data: uint per relocation
2088   // Write the count first
2089   int count = reloc_data.length();
2090   write_bytes(&count, sizeof(int));
2091   for (GrowableArrayIterator<uint> iter = reloc_data.begin();
2092        iter != reloc_data.end(); ++iter) {
2093     uint value = *iter;
2094     int n = write_bytes(&value, sizeof(uint));
2095     if (n != sizeof(uint)) {
2096       return false;
2097     }
2098   }
2099   return true;
2100 }
2101 
2102 void AOTCodeReader::fix_relocations(CodeBlob* code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
2103   LogStreamHandle(Trace, aot, reloc) log;
2104   uint offset = read_position();
2105   int count = *(int*)addr(offset);
2106   offset += sizeof(int);
2107   if (log.is_enabled()) {
2108     log.print_cr("======== extra relocations count=%d", count);
2109   }
2110   uint* reloc_data = (uint*)addr(offset);
2111   offset += (count * sizeof(uint));
2112   set_read_position(offset);
2113 
2114   RelocIterator iter(code_blob);
2115   int j = 0;
2116   while (iter.next()) {
2117     switch (iter.type()) {
2118       case relocInfo::none:
2119         break;
2120       case relocInfo::oop_type: {
2121         assert(code_blob->is_nmethod(), "sanity check");
2122         oop_Relocation* r = (oop_Relocation*)iter.reloc();
2123         if (r->oop_is_immediate()) {
2124           assert(oop_list != nullptr, "sanity check");
2125           Handle h = oop_list->at(reloc_data[j]);
2126           r->set_value(cast_from_oop<address>(h()));
2127         } else {
2128           r->fix_oop_relocation();
2129         }
2130         break;
2131       }
2132       case relocInfo::metadata_type: {
2133         assert(code_blob->is_nmethod(), "sanity check");
2134         metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2135         Metadata* m;
2136         if (r->metadata_is_immediate()) {
2137           assert(metadata_list != nullptr, "sanity check");
2138           m = metadata_list->at(reloc_data[j]);
2139         } else {
2140           // Get already updated value from nmethod.
2141           int index = r->metadata_index();
2142           m = code_blob->as_nmethod()->metadata_at(index);
2143         }
2144         r->set_value((address)m);
2145         break;
2146       }
2147       case relocInfo::virtual_call_type:   // Fall through. They all call resolve_*_call blobs.
2148       case relocInfo::opt_virtual_call_type:
2149       case relocInfo::static_call_type: {
2150         address dest = _cache->address_for_id(reloc_data[j]);
2151         if (dest != (address)-1) {
2152           ((CallRelocation*)iter.reloc())->set_destination(dest);
2153         }
2154         break;
2155       }
2156       case relocInfo::trampoline_stub_type: {
2157         address dest = _cache->address_for_id(reloc_data[j]);
2158         if (dest != (address)-1) {
2159           ((trampoline_stub_Relocation*)iter.reloc())->set_destination(dest);
2160         }
2161         break;
2162       }
2163       case relocInfo::static_stub_type:
2164         break;
2165       case relocInfo::runtime_call_type: {
2166         address dest = _cache->address_for_id(reloc_data[j]);
2167         if (dest != (address)-1) {
2168           ((CallRelocation*)iter.reloc())->set_destination(dest);
2169         }
2170         break;
2171       }
2172       case relocInfo::runtime_call_w_cp_type:
2173         // this relocation should not be in cache (see write_relocations)
2174         assert(false, "runtime_call_w_cp_type relocation is not implemented");
2175         break;
2176       case relocInfo::external_word_type: {
2177         address target = _cache->address_for_id(reloc_data[j]);
2178         // Add external address to global table
2179         int index = ExternalsRecorder::find_index(target);
2180         // Update index in relocation
2181         Relocation::add_jint(iter.data(), index);
2182         external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
2183         assert(reloc->target() == target, "sanity");
2184         reloc->set_value(target); // Patch address in the code
2185         break;
2186       }
2187       case relocInfo::internal_word_type: {
2188         internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
2189         r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2190         break;
2191       }
2192       case relocInfo::section_word_type: {
2193         section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
2194         r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2195         break;
2196       }
2197       case relocInfo::poll_type:
2198         break;
2199       case relocInfo::poll_return_type:
2200         break;
2201       case relocInfo::post_call_nop_type:
2202         break;
2203       case relocInfo::entry_guard_type:
2204         break;
2205       default:
2206         assert(false,"relocation %d unimplemented", (int)iter.type());
2207         break;
2208     }
2209     if (log.is_enabled()) {
2210       iter.print_current_on(&log);
2211     }
2212     j++;
2213   }
2214   assert(j == count, "sanity");
2215 }
2216 
2217 bool AOTCodeCache::write_nmethod_reloc_immediates(GrowableArray<Handle>& oop_list, GrowableArray<Metadata*>& metadata_list) {
2218   int count = oop_list.length();
2219   if (!write_bytes(&count, sizeof(int))) {
2220     return false;
2221   }
2222   for (GrowableArrayIterator<Handle> iter = oop_list.begin();
2223        iter != oop_list.end(); ++iter) {
2224     Handle h = *iter;
2225     if (!write_oop(h())) {
2226       return false;
2227     }
2228   }
2229 
2230   count = metadata_list.length();
2231   if (!write_bytes(&count, sizeof(int))) {
2232     return false;
2233   }
2234   for (GrowableArrayIterator<Metadata*> iter = metadata_list.begin();
2235        iter != metadata_list.end(); ++iter) {
2236     Metadata* m = *iter;
2237     if (!write_metadata(m)) {
2238       return false;
2239     }
2240   }
2241   return true;
2242 }
2243 
2244 bool AOTCodeCache::write_metadata(nmethod* nm) {
2245   int count = nm->metadata_count()-1;
2246   if (!write_bytes(&count, sizeof(int))) {
2247     return false;
2248   }
2249   for (Metadata** p = nm->metadata_begin(); p < nm->metadata_end(); p++) {
2250     if (!write_metadata(*p)) {
2251       return false;
2252     }
2253   }
2254   return true;
2255 }
2256 
2257 bool AOTCodeCache::write_metadata(Metadata* m) {
2258   uint n = 0;
2259   if (m == nullptr) {
2260     DataKind kind = DataKind::Null;
2261     n = write_bytes(&kind, sizeof(int));
2262     if (n != sizeof(int)) {
2263       return false;
2264     }
2265   } else if (m == (Metadata*)Universe::non_oop_word()) {
2266     DataKind kind = DataKind::No_Data;
2267     n = write_bytes(&kind, sizeof(int));
2268     if (n != sizeof(int)) {
2269       return false;
2270     }
2271   } else if (m->is_klass()) {
2272     if (!write_klass((Klass*)m)) {
2273       return false;
2274     }
2275   } else if (m->is_method()) {
2276     if (!write_method((Method*)m)) {
2277       return false;
2278     }
2279   } else if (m->is_methodCounters()) {
2280     DataKind kind = DataKind::MethodCnts;
2281     n = write_bytes(&kind, sizeof(int));
2282     if (n != sizeof(int)) {
2283       return false;
2284     }
2285     if (!write_method(((MethodCounters*)m)->method())) {
2286       return false;
2287     }
2288     log_debug(aot, codecache, metadata)("%d (L%d): Write MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2289   } else { // Not supported
2290     fatal("metadata : " INTPTR_FORMAT " unimplemented", p2i(m));
2291     return false;
2292   }
2293   return true;
2294 }
2295 
2296 Metadata* AOTCodeReader::read_metadata(const methodHandle& comp_method) {
2297   uint code_offset = read_position();
2298   Metadata* m = nullptr;
2299   DataKind kind = *(DataKind*)addr(code_offset);
2300   code_offset += sizeof(DataKind);
2301   set_read_position(code_offset);
2302   if (kind == DataKind::Null) {
2303     m = (Metadata*)nullptr;
2304   } else if (kind == DataKind::No_Data) {
2305     m = (Metadata*)Universe::non_oop_word();
2306   } else if (kind == DataKind::Klass) {
2307     m = (Metadata*)read_klass(comp_method);
2308   } else if (kind == DataKind::Method) {
2309     m = (Metadata*)read_method(comp_method);
2310   } else if (kind == DataKind::MethodCnts) {
2311     kind = *(DataKind*)addr(code_offset);
2312     code_offset += sizeof(DataKind);
2313     set_read_position(code_offset);
2314     m = (Metadata*)read_method(comp_method);
2315     if (m != nullptr) {
2316       Method* method = (Method*)m;
2317       m = method->get_method_counters(Thread::current());
2318       if (m == nullptr) {
2319         set_lookup_failed();
2320         log_debug(aot, codecache, metadata)("%d (L%d): Failed to get MethodCounters", compile_id(), comp_level());
2321       } else {
2322         log_debug(aot, codecache, metadata)("%d (L%d): Read MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2323       }
2324     }
2325   } else {
2326     set_lookup_failed();
2327     log_debug(aot, codecache, metadata)("%d (L%d): Unknown metadata's kind: %d", compile_id(), comp_level(), (int)kind);
2328   }
2329   return m;
2330 }
2331 
2332 bool AOTCodeCache::write_method(Method* method) {
2333   ResourceMark rm; // To method's name printing
2334   if (AOTCacheAccess::can_generate_aot_code(method)) {
2335     DataKind kind = DataKind::Method;
2336     uint n = write_bytes(&kind, sizeof(int));
2337     if (n != sizeof(int)) {
2338       return false;
2339     }
2340     uint method_offset = AOTCacheAccess::delta_from_base_address((address)method);
2341     n = write_bytes(&method_offset, sizeof(uint));
2342     if (n != sizeof(uint)) {
2343       return false;
2344     }
2345     log_debug(aot, codecache, metadata)("%d (L%d): Wrote method: %s @ 0x%08x",
2346              compile_id(), comp_level(), method->name_and_sig_as_C_string(), method_offset);
2347     return true;
2348   }
2349   log_debug(aot, codecache, metadata)("%d (L%d): Method is not archived: %s",
2350               compile_id(), comp_level(), method->name_and_sig_as_C_string());
2351   set_lookup_failed();
2352   return false;
2353 }
2354 
2355 Method* AOTCodeReader::read_method(const methodHandle& comp_method) {
2356   uint code_offset = read_position();
2357   uint method_offset = *(uint*)addr(code_offset);
2358   code_offset += sizeof(uint);
2359   set_read_position(code_offset);
2360   Method* m = AOTCacheAccess::convert_offset_to_method(method_offset);
2361   if (!AOTMetaspace::in_aot_cache((address)m)) {
2362     // Something changed in CDS
2363     set_lookup_failed();
2364     log_debug(aot, codecache, metadata)("Lookup failed for shared method: " INTPTR_FORMAT " is not in CDS ", p2i((address)m));
2365     return nullptr;
2366   }
2367   assert(m->is_method(), "sanity");
2368   ResourceMark rm;
2369   Klass* k = m->method_holder();
2370   if (!k->is_instance_klass()) {
2371     set_lookup_failed();
2372     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not instance klass",
2373                   compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2374     return nullptr;
2375   } else if (!AOTMetaspace::in_aot_cache((address)k)) {
2376     set_lookup_failed();
2377     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not in CDS",
2378                   compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2379     return nullptr;
2380   } else if (!InstanceKlass::cast(k)->is_loaded()) {
2381     set_lookup_failed();
2382     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not loaded",
2383                   compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2384     return nullptr;
2385   } else if (!InstanceKlass::cast(k)->is_linked()) {
2386     set_lookup_failed();
2387     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not linked%s",
2388                   compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name(), (_preload ? " for code preload" : ""));
2389     return nullptr;
2390   }
2391   log_debug(aot, codecache, metadata)("%d (L%d): Shared method lookup: %s",
2392                 compile_id(), comp_level(), m->name_and_sig_as_C_string());
2393   return m;
2394 }
2395 
2396 bool AOTCodeCache::write_klass(Klass* klass) {
2397   uint array_dim = 0;
2398   if (klass->is_objArray_klass()) {
2399     array_dim = ObjArrayKlass::cast(klass)->dimension();
2400     klass     = ObjArrayKlass::cast(klass)->bottom_klass(); // overwrites klass
2401   }
2402   uint init_state = 0;
2403   bool can_write = true;
2404   if (klass->is_instance_klass()) {
2405     InstanceKlass* ik = InstanceKlass::cast(klass);
2406     init_state = (ik->is_initialized() ? 1 : 0);
2407     can_write = AOTCacheAccess::can_generate_aot_code_for(ik);
2408   } else {
2409     can_write = AOTCacheAccess::can_generate_aot_code(klass);
2410   }
2411   ResourceMark rm;
2412   uint state = (array_dim << 1) | (init_state & 1);
2413   if (can_write) {
2414     DataKind kind = DataKind::Klass;
2415     uint n = write_bytes(&kind, sizeof(int));
2416     if (n != sizeof(int)) {
2417       return false;
2418     }
2419     // Record state of instance klass initialization and array dimentions.
2420     n = write_bytes(&state, sizeof(int));
2421     if (n != sizeof(int)) {
2422       return false;
2423     }
2424     uint klass_offset = AOTCacheAccess::delta_from_base_address((address)klass);
2425     n = write_bytes(&klass_offset, sizeof(uint));
2426     if (n != sizeof(uint)) {
2427       return false;
2428     }
2429     log_debug(aot, codecache, metadata)("%d (L%d): Registered klass: %s%s%s @ 0x%08x",
2430              compile_id(), comp_level(), klass->external_name(),
2431              (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2432              (array_dim > 0 ? " (object array)" : ""), klass_offset);
2433     return true;
2434   }
2435   log_debug(aot, codecache, metadata)("%d (L%d): Klassis not archived: %s%s%s",
2436               compile_id(), comp_level(), klass->external_name(),
2437               (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2438               (array_dim > 0 ? " (object array)" : ""));
2439   set_lookup_failed();
2440   return false;
2441 }
2442 
2443 Klass* AOTCodeReader::read_klass(const methodHandle& comp_method) {
2444   uint code_offset = read_position();
2445   uint state = *(uint*)addr(code_offset);
2446   uint init_state = (state  & 1);
2447   uint array_dim  = (state >> 1);
2448   code_offset += sizeof(int);
2449   uint klass_offset = *(uint*)addr(code_offset);
2450   code_offset += sizeof(uint);
2451   set_read_position(code_offset);
2452   Klass* k = AOTCacheAccess::convert_offset_to_klass(klass_offset);
2453   if (!AOTMetaspace::in_aot_cache((address)k)) {
2454     // Something changed in CDS
2455     set_lookup_failed();
2456     log_debug(aot, codecache, metadata)("Lookup failed for shared klass: " INTPTR_FORMAT " is not in CDS ", p2i((address)k));
2457     return nullptr;
2458   }
2459   assert(k->is_klass(), "sanity");
2460   ResourceMark rm;
2461   if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_loaded()) {
2462     set_lookup_failed();
2463     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not loaded",
2464                      compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2465     return nullptr;
2466   } else
2467   // Allow not initialized klass which was uninitialized during code caching or for preload
2468   if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_initialized() && (init_state == 1) && !_preload) {
2469     set_lookup_failed();
2470     log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not initialized",
2471                      compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2472     return nullptr;
2473   }
2474   if (array_dim > 0) {
2475     assert(k->is_instance_klass() || k->is_typeArray_klass(), "sanity check");
2476     Klass* ak = k->array_klass_or_null(array_dim);
2477     // FIXME: what would it take to create an array class on the fly?
2478 //    Klass* ak = k->array_klass(dim, JavaThread::current());
2479 //    guarantee(JavaThread::current()->pending_exception() == nullptr, "");
2480     if (ak == nullptr) {
2481       set_lookup_failed();
2482       log_debug(aot, codecache, metadata)("%d (L%d): %d-dimension array klass lookup failed: %s",
2483                        compile_id(), comp_level(), array_dim, k->external_name());
2484     }
2485     log_debug(aot, codecache, metadata)("%d (L%d): Klass lookup: %s (object array)", compile_id(), comp_level(), k->external_name());
2486     return ak;
2487   } else {
2488     log_debug(aot, codecache, metadata)("%d (L%d): Shared klass lookup: %s",
2489                   compile_id(), comp_level(), k->external_name());
2490     return k;
2491   }
2492 }
2493 
2494 bool AOTCodeCache::write_oop(jobject& jo) {
2495   oop obj = JNIHandles::resolve(jo);
2496   return write_oop(obj);
2497 }
2498 
2499 bool AOTCodeCache::write_oop(oop obj) {
2500   DataKind kind;
2501   uint n = 0;
2502   if (obj == nullptr) {
2503     kind = DataKind::Null;
2504     n = write_bytes(&kind, sizeof(int));
2505     if (n != sizeof(int)) {
2506       return false;
2507     }
2508   } else if (cast_from_oop<void *>(obj) == Universe::non_oop_word()) {
2509     kind = DataKind::No_Data;
2510     n = write_bytes(&kind, sizeof(int));
2511     if (n != sizeof(int)) {
2512       return false;
2513     }
2514   } else if (java_lang_Class::is_instance(obj)) {
2515     if (java_lang_Class::is_primitive(obj)) {
2516       int bt = (int)java_lang_Class::primitive_type(obj);
2517       kind = DataKind::Primitive;
2518       n = write_bytes(&kind, sizeof(int));
2519       if (n != sizeof(int)) {
2520         return false;
2521       }
2522       n = write_bytes(&bt, sizeof(int));
2523       if (n != sizeof(int)) {
2524         return false;
2525       }
2526       log_debug(aot, codecache, oops)("%d (L%d): Write primitive type klass: %s", compile_id(), comp_level(), type2name((BasicType)bt));
2527     } else {
2528       Klass* klass = java_lang_Class::as_Klass(obj);
2529       if (!write_klass(klass)) {
2530         return false;
2531       }
2532     }
2533   } else if (java_lang_String::is_instance(obj)) { // herere
2534     int k = AOTCacheAccess::get_archived_object_permanent_index(obj);  // k >= 0 means obj is a "permanent heap object"
2535     ResourceMark rm;
2536     size_t length_sz = 0;
2537     const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2538     if (k >= 0) {
2539       kind = DataKind::String;
2540       n = write_bytes(&kind, sizeof(int));
2541       if (n != sizeof(int)) {
2542         return false;
2543       }
2544       n = write_bytes(&k, sizeof(int));
2545       if (n != sizeof(int)) {
2546         return false;
2547       }
2548       log_debug(aot, codecache, oops)("%d (L%d): Write String object: " PTR_FORMAT " : %s", compile_id(), comp_level(), p2i(obj), string);
2549       return true;
2550     }
2551     // Not archived String object - bailout
2552     set_lookup_failed();
2553     log_debug(aot, codecache, oops)("%d (L%d): Not archived String object: " PTR_FORMAT " : %s",
2554                                       compile_id(), comp_level(), p2i(obj), string);
2555     return false;
2556   } else if (java_lang_Module::is_instance(obj)) {
2557     fatal("Module object unimplemented");
2558   } else if (java_lang_ClassLoader::is_instance(obj)) {
2559     if (obj == SystemDictionary::java_system_loader()) {
2560       kind = DataKind::SysLoader;
2561       log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_system_loader", compile_id(), comp_level());
2562     } else if (obj == SystemDictionary::java_platform_loader()) {
2563       kind = DataKind::PlaLoader;
2564       log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_platform_loader", compile_id(), comp_level());
2565     } else {
2566       ResourceMark rm;
2567       set_lookup_failed();
2568       log_debug(aot, codecache, oops)("%d (L%d): Not supported Class Loader: " PTR_FORMAT " : %s",
2569                                       compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2570       return false;
2571     }
2572     n = write_bytes(&kind, sizeof(int));
2573     if (n != sizeof(int)) {
2574       return false;
2575     }
2576   } else { // herere
2577     ResourceMark rm;
2578     int k = AOTCacheAccess::get_archived_object_permanent_index(obj);  // k >= 0 means obj is a "permanent heap object"
2579     if (k >= 0) {
2580       kind = DataKind::MH_Oop;
2581       n = write_bytes(&kind, sizeof(int));
2582       if (n != sizeof(int)) {
2583         return false;
2584       }
2585       n = write_bytes(&k, sizeof(int));
2586       if (n != sizeof(int)) {
2587         return false;
2588       }
2589       log_debug(aot, codecache, oops)("%d (L%d): Write MH object: " PTR_FORMAT " : %s",
2590                               compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2591       return true;
2592     }
2593     // Not archived Java object - bailout
2594     set_lookup_failed();
2595     log_debug(aot, codecache, oops)("%d (L%d): Not archived Java object: " PTR_FORMAT " : %s",
2596                               compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2597     return false;
2598   }
2599   return true;
2600 }
2601 
2602 oop AOTCodeReader::read_oop(JavaThread* thread, const methodHandle& comp_method) {
2603   uint code_offset = read_position();
2604   oop obj = nullptr;
2605   DataKind kind = *(DataKind*)addr(code_offset);
2606   code_offset += sizeof(DataKind);
2607   set_read_position(code_offset);
2608   if (kind == DataKind::Null) {
2609     return nullptr;
2610   } else if (kind == DataKind::No_Data) {
2611     return cast_to_oop(Universe::non_oop_word());
2612   } else if (kind == DataKind::Klass) {
2613     Klass* k = read_klass(comp_method);
2614     if (k == nullptr) {
2615       return nullptr;
2616     }
2617     obj = k->java_mirror();
2618     if (obj == nullptr) {
2619       set_lookup_failed();
2620       log_debug(aot, codecache, oops)("Lookup failed for java_mirror of klass %s", k->external_name());
2621       return nullptr;
2622     }
2623   } else if (kind == DataKind::Primitive) {
2624     code_offset = read_position();
2625     int t = *(int*)addr(code_offset);
2626     code_offset += sizeof(int);
2627     set_read_position(code_offset);
2628     BasicType bt = (BasicType)t;
2629     obj = java_lang_Class::primitive_mirror(bt);
2630     log_debug(aot, codecache, oops)("%d (L%d): Read primitive type klass: %s", compile_id(), comp_level(), type2name(bt));
2631   } else if (kind == DataKind::String) {
2632     code_offset = read_position();
2633     int k = *(int*)addr(code_offset);
2634     code_offset += sizeof(int);
2635     set_read_position(code_offset);
2636     obj = AOTCacheAccess::get_archived_object(k);
2637     if (obj == nullptr) {
2638       set_lookup_failed();
2639       log_debug(aot, codecache, oops)("Lookup failed for String object");
2640       return nullptr;
2641     }
2642     assert(java_lang_String::is_instance(obj), "must be string");
2643 
2644     ResourceMark rm;
2645     size_t length_sz = 0;
2646     const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2647     log_debug(aot, codecache, oops)("%d (L%d): Read String object: %s", compile_id(), comp_level(), string);
2648   } else if (kind == DataKind::SysLoader) {
2649     obj = SystemDictionary::java_system_loader();
2650     log_debug(aot, codecache, oops)("%d (L%d): Read java_system_loader", compile_id(), comp_level());
2651   } else if (kind == DataKind::PlaLoader) {
2652     obj = SystemDictionary::java_platform_loader();
2653     log_debug(aot, codecache, oops)("%d (L%d): Read java_platform_loader", compile_id(), comp_level());
2654   } else if (kind == DataKind::MH_Oop) {
2655     code_offset = read_position();
2656     int k = *(int*)addr(code_offset);
2657     code_offset += sizeof(int);
2658     set_read_position(code_offset);
2659     obj = AOTCacheAccess::get_archived_object(k);
2660     if (obj == nullptr) {
2661       set_lookup_failed();
2662       log_debug(aot, codecache, oops)("Lookup failed for MH object");
2663       return nullptr;
2664     }
2665     ResourceMark rm;
2666     log_debug(aot, codecache, oops)("%d (L%d): Read MH object: " PTR_FORMAT " : %s",
2667                               compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2668   } else {
2669     set_lookup_failed();
2670     log_debug(aot, codecache, oops)("%d (L%d): Unknown oop's kind: %d",
2671                      compile_id(), comp_level(), (int)kind);
2672     return nullptr;
2673   }
2674   return obj;
2675 }
2676 
2677 bool AOTCodeReader::read_oop_metadata_list(JavaThread* thread, ciMethod* target, GrowableArray<Handle> &oop_list, GrowableArray<Metadata*> &metadata_list, OopRecorder* oop_recorder) {
2678   methodHandle comp_method(JavaThread::current(), target->get_Method());
2679   JavaThread* current = JavaThread::current();
2680   uint offset = read_position();
2681   int count = *(int *)addr(offset);
2682   offset += sizeof(int);
2683   set_read_position(offset);
2684   for (int i = 0; i < count; i++) {
2685     oop obj = read_oop(current, comp_method);
2686     if (lookup_failed()) {
2687       return false;
2688     }
2689     Handle h(thread, obj);
2690     oop_list.append(h);
2691     if (oop_recorder != nullptr) {
2692       jobject jo = JNIHandles::make_local(thread, obj);
2693       if (oop_recorder->is_real(jo)) {
2694         oop_recorder->find_index(jo);
2695       } else {
2696         oop_recorder->allocate_oop_index(jo);
2697       }
2698     }
2699     LogStreamHandle(Debug, aot, codecache, oops) log;
2700     if (log.is_enabled()) {
2701       log.print("%d: " INTPTR_FORMAT " ", i, p2i(obj));
2702       if (obj == Universe::non_oop_word()) {
2703         log.print("non-oop word");
2704       } else if (obj == nullptr) {
2705         log.print("nullptr-oop");
2706       } else {
2707         obj->print_value_on(&log);
2708       }
2709       log.cr();
2710     }
2711   }
2712 
2713   offset = read_position();
2714   count = *(int *)addr(offset);
2715   offset += sizeof(int);
2716   set_read_position(offset);
2717   for (int i = 0; i < count; i++) {
2718     Metadata* m = read_metadata(comp_method);
2719     if (lookup_failed()) {
2720       return false;
2721     }
2722     metadata_list.append(m);
2723     if (oop_recorder != nullptr) {
2724       if (oop_recorder->is_real(m)) {
2725         oop_recorder->find_index(m);
2726       } else {
2727         oop_recorder->allocate_metadata_index(m);
2728       }





2729     }
2730     LogTarget(Debug, aot, codecache, metadata) log;
2731     if (log.is_enabled()) {
2732       LogStream ls(log);
2733       ls.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
2734       if (m == (Metadata*)Universe::non_oop_word()) {
2735         ls.print("non-metadata word");
2736       } else if (m == nullptr) {
2737         ls.print("nullptr-oop");
2738       } else {
2739         Metadata::print_value_on_maybe_null(&ls, m);
2740       }
2741       ls.cr();
2742     }

2743   }
2744   return true;
2745 }
2746 
2747 bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
2748   ImmutableOopMapSet* oopmaps = cb.oop_maps();
2749   int oopmaps_size = oopmaps->nr_of_bytes();
2750   if (!write_bytes(&oopmaps_size, sizeof(int))) {
2751     return false;
2752   }
2753   uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
2754   if (n != (uint)oopmaps->nr_of_bytes()) {
2755     return false;
2756   }
2757   return true;
2758 }
2759 
2760 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
2761   uint offset = read_position();
2762   int size = *(int *)addr(offset);
2763   offset += sizeof(int);
2764   ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
2765   offset += size;
2766   set_read_position(offset);
2767   return oopmaps;
2768 }
2769 
2770 bool AOTCodeCache::write_oops(nmethod* nm) {
2771   int count = nm->oops_count()-1;
2772   if (!write_bytes(&count, sizeof(int))) {
2773     return false;
2774   }
2775   for (oop* p = nm->oops_begin(); p < nm->oops_end(); p++) {
2776     if (!write_oop(*p)) {
2777       return false;
2778     }
2779   }
2780   return true;
2781 }
2782 
2783 #ifndef PRODUCT
2784 bool AOTCodeCache::write_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2785   // Write asm remarks
2786   uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2787   if (count_ptr == nullptr) {
2788     return false;
2789   }
2790   uint count = 0;
2791   bool result = asm_remarks.iterate([&] (uint offset, const char* str) -> bool {
2792     log_trace(aot, codecache, stubs)("asm remark offset=%d, str='%s'", offset, str);
2793     uint n = write_bytes(&offset, sizeof(uint));
2794     if (n != sizeof(uint)) {
2795       return false;
2796     }
2797     if (use_string_table) {
2798       const char* cstr = add_C_string(str);
2799       int id = _table->id_for_C_string((address)cstr);
2800       assert(id != -1, "asm remark string '%s' not found in AOTCodeAddressTable", str);
2801       n = write_bytes(&id, sizeof(int));
2802       if (n != sizeof(int)) {
2803         return false;
2804       }
2805     } else {
2806       n = write_bytes(str, (uint)strlen(str) + 1);
2807       if (n != strlen(str) + 1) {
2808         return false;
2809       }
2810     }
2811     count += 1;
2812     return true;
2813   });
2814   *count_ptr = count;
2815   return result;
2816 }
2817 
2818 void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2819   // Read asm remarks
2820   uint offset = read_position();
2821   uint count = *(uint *)addr(offset);
2822   offset += sizeof(uint);
2823   for (uint i = 0; i < count; i++) {
2824     uint remark_offset = *(uint *)addr(offset);
2825     offset += sizeof(uint);
2826     const char* remark = nullptr;
2827     if (use_string_table) {
2828       int remark_string_id = *(uint *)addr(offset);
2829       offset += sizeof(int);
2830       remark = (const char*)_cache->address_for_C_string(remark_string_id);
2831     } else {
2832       remark = (const char*)addr(offset);
2833       offset += (uint)strlen(remark)+1;
2834     }
2835     asm_remarks.insert(remark_offset, remark);
2836   }
2837   set_read_position(offset);
2838 }
2839 
2840 bool AOTCodeCache::write_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2841   // Write dbg strings
2842   uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2843   if (count_ptr == nullptr) {
2844     return false;
2845   }
2846   uint count = 0;
2847   bool result = dbg_strings.iterate([&] (const char* str) -> bool {
2848     log_trace(aot, codecache, stubs)("dbg string=%s", str);
2849     if (use_string_table) {
2850       const char* cstr = add_C_string(str);
2851       int id = _table->id_for_C_string((address)cstr);
2852       assert(id != -1, "db string '%s' not found in AOTCodeAddressTable", str);
2853       uint n = write_bytes(&id, sizeof(int));
2854       if (n != sizeof(int)) {
2855         return false;
2856       }
2857     } else {
2858       uint n = write_bytes(str, (uint)strlen(str) + 1);
2859       if (n != strlen(str) + 1) {
2860         return false;
2861       }
2862     }
2863     count += 1;
2864     return true;
2865   });
2866   *count_ptr = count;
2867   return result;
2868 }
2869 
2870 void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2871   // Read dbg strings
2872   uint offset = read_position();
2873   uint count = *(uint *)addr(offset);
2874   offset += sizeof(uint);
2875   for (uint i = 0; i < count; i++) {
2876     const char* str = nullptr;
2877     if (use_string_table) {
2878       int string_id = *(uint *)addr(offset);
2879       offset += sizeof(int);
2880       str = (const char*)_cache->address_for_C_string(string_id);
2881     } else {
2882       str = (const char*)addr(offset);
2883       offset += (uint)strlen(str)+1;
2884     }
2885     dbg_strings.insert(str);
2886   }
2887   set_read_position(offset);
2888 }
2889 #endif // PRODUCT
2890 
2891 //======================= AOTCodeAddressTable ===============
2892 
2893 // address table ids for generated routines, external addresses and C
2894 // string addresses are partitioned into positive integer ranges
2895 // defined by the following positive base and max values
2896 // i.e. [_extrs_base, _extrs_base + _extrs_max -1],
2897 //      [_stubs_base, _stubs_base + _stubs_max -1],
2898 //      ...
2899 //      [_c_str_base, _c_str_base + _c_str_max -1],
2900 #define _extrs_max 140
2901 #define _stubs_max 210
2902 #define _shared_blobs_max 25
2903 #define _C1_blobs_max 50
2904 #define _C2_blobs_max 25
2905 #define _blobs_max (_shared_blobs_max+_C1_blobs_max+_C2_blobs_max)

2906 #define _all_max (_extrs_max+_stubs_max+_blobs_max)
2907 
2908 #define _extrs_base 0
2909 #define _stubs_base (_extrs_base + _extrs_max)
2910 #define _shared_blobs_base (_stubs_base + _stubs_max)
2911 #define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max)
2912 #define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max)
2913 #define _blobs_end  (_shared_blobs_base + _blobs_max)
2914 #if (_C2_blobs_base >= _all_max)
2915 #error AOTCodeAddressTable ranges need adjusting
2916 #endif
2917 
2918 #define SET_ADDRESS(type, addr)                           \
2919   {                                                       \
2920     type##_addr[type##_length++] = (address) (addr);      \
2921     assert(type##_length <= type##_max, "increase size"); \
2922   }
2923 
2924 static bool initializing_extrs = false;
2925 
2926 void AOTCodeAddressTable::init_extrs() {
2927   if (_extrs_complete || initializing_extrs) return; // Done already
2928 
2929   assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting");
2930 
2931   initializing_extrs = true;
2932   _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
2933 
2934   _extrs_length = 0;
2935 
2936   // Record addresses of VM runtime methods
2937   SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite);
2938   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method);
2939   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract);
2940   SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss);



2941   {
2942     // Required by Shared blobs
2943     SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info);
2944     SET_ADDRESS(_extrs, Deoptimization::unpack_frames);
2945     SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception);
2946     SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C);
2947     SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C);
2948     SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C);
2949     SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2950     SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError);
2951     SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError);
2952     SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError);
2953     SET_ADDRESS(_extrs, SharedRuntime::throw_NullPointerException_at_call);
2954     SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2955     SET_ADDRESS(_extrs, CompressedOops::base_addr());
2956     SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr());
2957   }
2958   {
2959     // Required by initial stubs
2960     SET_ADDRESS(_extrs, StubRoutines::crc_table_addr());
2961 #if defined(AMD64)
2962     SET_ADDRESS(_extrs, StubRoutines::crc32c_table_addr());
2963 #endif
2964   }
2965 
2966 #ifdef COMPILER1
2967   {
2968     // Required by C1 blobs
2969     SET_ADDRESS(_extrs, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc));
2970     SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address);
2971     SET_ADDRESS(_extrs, SharedRuntime::register_finalizer);
2972     SET_ADDRESS(_extrs, Runtime1::is_instance_of);
2973     SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc);
2974     SET_ADDRESS(_extrs, Runtime1::check_abort_on_vm_exception);
2975     SET_ADDRESS(_extrs, Runtime1::new_instance);
2976     SET_ADDRESS(_extrs, Runtime1::counter_overflow);
2977     SET_ADDRESS(_extrs, Runtime1::new_type_array);
2978     SET_ADDRESS(_extrs, Runtime1::new_object_array);
2979     SET_ADDRESS(_extrs, Runtime1::new_multi_array);
2980     SET_ADDRESS(_extrs, Runtime1::throw_range_check_exception);
2981     SET_ADDRESS(_extrs, Runtime1::throw_index_exception);
2982     SET_ADDRESS(_extrs, Runtime1::throw_div0_exception);
2983     SET_ADDRESS(_extrs, Runtime1::throw_null_pointer_exception);
2984     SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception);
2985     SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception);
2986     SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error);

2987     SET_ADDRESS(_extrs, Runtime1::monitorenter);
2988     SET_ADDRESS(_extrs, Runtime1::monitorexit);
2989     SET_ADDRESS(_extrs, Runtime1::deoptimize);
2990     SET_ADDRESS(_extrs, Runtime1::access_field_patching);
2991     SET_ADDRESS(_extrs, Runtime1::move_klass_patching);
2992     SET_ADDRESS(_extrs, Runtime1::move_mirror_patching);
2993     SET_ADDRESS(_extrs, Runtime1::move_appendix_patching);
2994     SET_ADDRESS(_extrs, Runtime1::predicate_failed_trap);
2995     SET_ADDRESS(_extrs, Runtime1::unimplemented_entry);
2996     SET_ADDRESS(_extrs, Runtime1::trace_block_entry);
2997 #ifdef X86
2998     SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool);
2999     SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool);
3000     SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool);
3001     SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool);
3002 #endif
3003 #ifndef PRODUCT
3004     SET_ADDRESS(_extrs, os::breakpoint);
3005 #endif
3006   }
3007 #endif // COMPILER1
3008 
3009 #ifdef COMPILER2
3010   {
3011     // Required by C2 blobs
3012     SET_ADDRESS(_extrs, Deoptimization::uncommon_trap);
3013     SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C);
3014     SET_ADDRESS(_extrs, OptoRuntime::new_instance_C);
3015     SET_ADDRESS(_extrs, OptoRuntime::new_array_C);
3016     SET_ADDRESS(_extrs, OptoRuntime::new_array_nozero_C);
3017     SET_ADDRESS(_extrs, OptoRuntime::multianewarray2_C);
3018     SET_ADDRESS(_extrs, OptoRuntime::multianewarray3_C);
3019     SET_ADDRESS(_extrs, OptoRuntime::multianewarray4_C);
3020     SET_ADDRESS(_extrs, OptoRuntime::multianewarray5_C);
3021     SET_ADDRESS(_extrs, OptoRuntime::multianewarrayN_C);
3022 #if INCLUDE_JVMTI
3023     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_start);
3024     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_end);
3025     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_mount);
3026     SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_unmount);
3027 #endif
3028     SET_ADDRESS(_extrs, OptoRuntime::complete_monitor_locking_C);
3029     SET_ADDRESS(_extrs, OptoRuntime::monitor_notify_C);
3030     SET_ADDRESS(_extrs, OptoRuntime::monitor_notifyAll_C);
3031     SET_ADDRESS(_extrs, OptoRuntime::rethrow_C);
3032     SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C);
3033     SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C);
3034     SET_ADDRESS(_extrs, OptoRuntime::class_init_barrier_C);
3035 #if defined(AMD64)
3036     // Use by C2 intinsic
3037     SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31());
3038 #endif
3039   }
3040 #endif // COMPILER2

3041 #if INCLUDE_G1GC
3042   SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry);
3043 #endif
3044 
3045 #if INCLUDE_SHENANDOAHGC
3046   SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_oop);
3047   SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
3048   SET_ADDRESS(_extrs, ShenandoahRuntime::clone_barrier);
3049   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong);
3050   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong_narrow);
3051   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak);
3052   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak_narrow);
3053   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom);
3054   SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
3055   SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre);
3056 #endif
3057 
3058 #if INCLUDE_ZGC
3059   SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
3060 #if defined(AMD64)
3061   SET_ADDRESS(_extrs, &ZPointerLoadShift);
3062 #endif
3063 #endif // INCLUDE_ZGC
3064 
3065   SET_ADDRESS(_extrs, SharedRuntime::log_jni_monitor_still_held);
3066   SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry);
3067   SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages);
3068   SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit);
3069 
3070   SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C);
3071   SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone);
3072 #if defined(AMD64) && !defined(ZERO)
3073   SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply);
3074   SET_ADDRESS(_extrs, SharedRuntime::montgomery_square);
3075 #endif // AMD64
3076   SET_ADDRESS(_extrs, SharedRuntime::d2f);
3077   SET_ADDRESS(_extrs, SharedRuntime::d2i);
3078   SET_ADDRESS(_extrs, SharedRuntime::d2l);
3079   SET_ADDRESS(_extrs, SharedRuntime::dcos);
3080   SET_ADDRESS(_extrs, SharedRuntime::dexp);
3081   SET_ADDRESS(_extrs, SharedRuntime::dlog);
3082   SET_ADDRESS(_extrs, SharedRuntime::dlog10);
3083   SET_ADDRESS(_extrs, SharedRuntime::dpow);
3084   SET_ADDRESS(_extrs, SharedRuntime::dsin);
3085   SET_ADDRESS(_extrs, SharedRuntime::dtan);
3086   SET_ADDRESS(_extrs, SharedRuntime::f2i);
3087   SET_ADDRESS(_extrs, SharedRuntime::f2l);
3088 #ifndef ZERO
3089   SET_ADDRESS(_extrs, SharedRuntime::drem);
3090   SET_ADDRESS(_extrs, SharedRuntime::frem);
3091 #endif
3092   SET_ADDRESS(_extrs, SharedRuntime::l2d);
3093   SET_ADDRESS(_extrs, SharedRuntime::l2f);
3094   SET_ADDRESS(_extrs, SharedRuntime::ldiv);
3095   SET_ADDRESS(_extrs, SharedRuntime::lmul);
3096   SET_ADDRESS(_extrs, SharedRuntime::lrem);
3097 
3098   SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset());
3099   SET_ADDRESS(_extrs, Thread::current);
3100 
3101   SET_ADDRESS(_extrs, os::javaTimeMillis);
3102   SET_ADDRESS(_extrs, os::javaTimeNanos);
3103   // For JFR
3104   SET_ADDRESS(_extrs, os::elapsed_counter);
3105 #if defined(X86) && !defined(ZERO)
3106   SET_ADDRESS(_extrs, Rdtsc::elapsed_counter);
3107 #endif
3108 
3109 #if INCLUDE_JVMTI
3110   SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc);
3111   SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events);
3112 #endif /* INCLUDE_JVMTI */
3113 
3114 #ifndef PRODUCT
3115   SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr);
3116   SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure);
3117 #endif
3118 
3119 #ifndef ZERO
3120 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
3121   SET_ADDRESS(_extrs, MacroAssembler::debug64);
3122 #endif
3123 #if defined(AARCH64)
3124   SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper);
3125 #endif
3126 #endif // ZERO
3127 
3128   // addresses of fields in AOT runtime constants area
3129   address* p = AOTRuntimeConstants::field_addresses_list();
3130   while (*p != nullptr) {
3131     SET_ADDRESS(_extrs, *p++);
3132   }
3133 
3134   _extrs_complete = true;
3135   log_info(aot, codecache, init)("External addresses recorded");
3136 }
3137 
3138 static bool initializing_early_stubs = false;
3139 
3140 void AOTCodeAddressTable::init_early_stubs() {
3141   if (_complete || initializing_early_stubs) return; // Done already
3142   initializing_early_stubs = true;
3143   _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
3144   _stubs_length = 0;
3145   SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry());
3146 
3147   {
3148     // Required by C1 blobs
3149 #if defined(AMD64) && !defined(ZERO)
3150     SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip());
3151     SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup());
3152 #endif // AMD64
3153   }
3154 
3155   _early_stubs_complete = true;
3156   log_info(aot, codecache, init)("Early stubs recorded");
3157 }
3158 
3159 static bool initializing_shared_blobs = false;
3160 
3161 void AOTCodeAddressTable::init_shared_blobs() {
3162   if (_complete || initializing_shared_blobs) return; // Done already
3163   initializing_shared_blobs = true;
3164   address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode);
3165 
3166   // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel
3167   _shared_blobs_addr = blobs_addr;
3168   _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max;// C1 blobs addresses stored after shared blobs
3169   _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs
3170 
3171   _shared_blobs_length = 0;
3172   _C1_blobs_length = 0;
3173   _C2_blobs_length = 0;
3174 
3175   // clear the address table
3176   memset(blobs_addr, 0, sizeof(address)* _blobs_max);
3177 
3178   // Record addresses of generated code blobs
3179   SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub());
3180   SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub());
3181   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack());
3182   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception());
3183   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution());
3184   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls());
3185   SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub());
3186   SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_virtual_call_stub());
3187   SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_static_call_stub());
3188   SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->entry_point());
3189   SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point());
3190   SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point());
3191 #ifdef COMPILER2
3192   // polling_page_vectors_safepoint_handler_blob can be nullptr if AVX feature is not present or is disabled
3193   if (SharedRuntime::polling_page_vectors_safepoint_handler_blob() != nullptr) {
3194     SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point());
3195   }
3196 #endif
3197 #if INCLUDE_JVMCI
3198   if (EnableJVMCI) {
3199     SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap());
3200     SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap());
3201   }
3202 #endif
3203   SET_ADDRESS(_shared_blobs, SharedRuntime::throw_AbstractMethodError_entry());
3204   SET_ADDRESS(_shared_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry());
3205   SET_ADDRESS(_shared_blobs, SharedRuntime::throw_NullPointerException_at_call_entry());
3206   SET_ADDRESS(_shared_blobs, SharedRuntime::throw_StackOverflowError_entry());
3207   SET_ADDRESS(_shared_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry());
3208 
3209   assert(_shared_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _shared_blobs_length);
3210   _shared_blobs_complete = true;
3211   log_info(aot, codecache, init)("All shared blobs recorded");
3212 }
3213 
3214 static bool initializing_stubs = false;
3215 void AOTCodeAddressTable::init_stubs() {
3216   if (_complete || initializing_stubs) return; // Done already
3217   assert(_early_stubs_complete, "early stubs whould be initialized");
3218   initializing_stubs = true;
3219 
3220   // Stubs
3221   SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier());
3222   SET_ADDRESS(_stubs, StubRoutines::atomic_xchg_entry());
3223   SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_entry());
3224   SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_long_entry());
3225   SET_ADDRESS(_stubs, StubRoutines::atomic_add_entry());
3226   SET_ADDRESS(_stubs, StubRoutines::fence_entry());
3227 
3228   SET_ADDRESS(_stubs, StubRoutines::cont_thaw());
3229   SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrier());
3230   SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrierExc());
3231 
3232   JFR_ONLY(SET_ADDRESS(_stubs, SharedRuntime::jfr_write_checkpoint());)
3233 
3234   SET_ADDRESS(_stubs, StubRoutines::jbyte_arraycopy());
3235   SET_ADDRESS(_stubs, StubRoutines::jshort_arraycopy());
3236   SET_ADDRESS(_stubs, StubRoutines::jint_arraycopy());
3237   SET_ADDRESS(_stubs, StubRoutines::jlong_arraycopy());
3238   SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy);
3239   SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy_uninit);
3240 
3241   SET_ADDRESS(_stubs, StubRoutines::jbyte_disjoint_arraycopy());
3242   SET_ADDRESS(_stubs, StubRoutines::jshort_disjoint_arraycopy());
3243   SET_ADDRESS(_stubs, StubRoutines::jint_disjoint_arraycopy());
3244   SET_ADDRESS(_stubs, StubRoutines::jlong_disjoint_arraycopy());
3245   SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy);
3246   SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy_uninit);
3247 
3248   SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_arraycopy());
3249   SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_arraycopy());
3250   SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_arraycopy());
3251   SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_arraycopy());
3252   SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy);
3253   SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy_uninit);
3254 
3255   SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_disjoint_arraycopy());
3256   SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_disjoint_arraycopy());
3257   SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_disjoint_arraycopy());
3258   SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_disjoint_arraycopy());
3259   SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy);
3260   SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit);
3261 
3262   SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy);
3263   SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy_uninit);
3264 
3265   SET_ADDRESS(_stubs, StubRoutines::unsafe_arraycopy());
3266   SET_ADDRESS(_stubs, StubRoutines::generic_arraycopy());
3267 
3268   SET_ADDRESS(_stubs, StubRoutines::jbyte_fill());
3269   SET_ADDRESS(_stubs, StubRoutines::jshort_fill());
3270   SET_ADDRESS(_stubs, StubRoutines::jint_fill());
3271   SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_fill());
3272   SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_fill());
3273   SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_fill());
3274 
3275   SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback());
3276   SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback_sync());
3277 
3278   SET_ADDRESS(_stubs, StubRoutines::aescrypt_encryptBlock());
3279   SET_ADDRESS(_stubs, StubRoutines::aescrypt_decryptBlock());
3280   SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_encryptAESCrypt());
3281   SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_decryptAESCrypt());
3282   SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_encryptAESCrypt());
3283   SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_decryptAESCrypt());
3284   SET_ADDRESS(_stubs, StubRoutines::poly1305_processBlocks());
3285   SET_ADDRESS(_stubs, StubRoutines::counterMode_AESCrypt());
3286   SET_ADDRESS(_stubs, StubRoutines::ghash_processBlocks());
3287   SET_ADDRESS(_stubs, StubRoutines::chacha20Block());
3288   SET_ADDRESS(_stubs, StubRoutines::base64_encodeBlock());
3289   SET_ADDRESS(_stubs, StubRoutines::base64_decodeBlock());
3290   SET_ADDRESS(_stubs, StubRoutines::md5_implCompress());
3291   SET_ADDRESS(_stubs, StubRoutines::md5_implCompressMB());
3292   SET_ADDRESS(_stubs, StubRoutines::sha1_implCompress());
3293   SET_ADDRESS(_stubs, StubRoutines::sha1_implCompressMB());
3294   SET_ADDRESS(_stubs, StubRoutines::sha256_implCompress());
3295   SET_ADDRESS(_stubs, StubRoutines::sha256_implCompressMB());
3296   SET_ADDRESS(_stubs, StubRoutines::sha512_implCompress());
3297   SET_ADDRESS(_stubs, StubRoutines::sha512_implCompressMB());
3298   SET_ADDRESS(_stubs, StubRoutines::sha3_implCompress());
3299   SET_ADDRESS(_stubs, StubRoutines::sha3_implCompressMB());
3300   SET_ADDRESS(_stubs, StubRoutines::double_keccak());
3301   SET_ADDRESS(_stubs, StubRoutines::intpoly_assign());
3302   SET_ADDRESS(_stubs, StubRoutines::intpoly_montgomeryMult_P256());
3303   SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostNtt());
3304   SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostInverseNtt());
3305   SET_ADDRESS(_stubs, StubRoutines::dilithiumNttMult());
3306   SET_ADDRESS(_stubs, StubRoutines::dilithiumMontMulByConstant());
3307   SET_ADDRESS(_stubs, StubRoutines::dilithiumDecomposePoly());
3308 
3309   SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32());
3310   SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32C());
3311   SET_ADDRESS(_stubs, StubRoutines::updateBytesAdler32());
3312 
3313   SET_ADDRESS(_stubs, StubRoutines::multiplyToLen());
3314   SET_ADDRESS(_stubs, StubRoutines::squareToLen());
3315   SET_ADDRESS(_stubs, StubRoutines::mulAdd());
3316   SET_ADDRESS(_stubs, StubRoutines::montgomeryMultiply());
3317   SET_ADDRESS(_stubs, StubRoutines::montgomerySquare());
3318   SET_ADDRESS(_stubs, StubRoutines::bigIntegerRightShift());
3319   SET_ADDRESS(_stubs, StubRoutines::bigIntegerLeftShift());
3320   SET_ADDRESS(_stubs, StubRoutines::galoisCounterMode_AESCrypt());
3321 
3322   SET_ADDRESS(_stubs, StubRoutines::vectorizedMismatch());
3323 
3324   SET_ADDRESS(_stubs, StubRoutines::unsafe_setmemory());
3325 
3326   SET_ADDRESS(_stubs, StubRoutines::dexp());
3327   SET_ADDRESS(_stubs, StubRoutines::dlog());
3328   SET_ADDRESS(_stubs, StubRoutines::dlog10());
3329   SET_ADDRESS(_stubs, StubRoutines::dpow());
3330   SET_ADDRESS(_stubs, StubRoutines::dsin());
3331   SET_ADDRESS(_stubs, StubRoutines::dcos());
3332   SET_ADDRESS(_stubs, StubRoutines::dlibm_reduce_pi04l());
3333   SET_ADDRESS(_stubs, StubRoutines::dlibm_sin_cos_huge());
3334   SET_ADDRESS(_stubs, StubRoutines::dlibm_tan_cot_huge());
3335   SET_ADDRESS(_stubs, StubRoutines::dtan());
3336 
3337   SET_ADDRESS(_stubs, StubRoutines::f2hf_adr());
3338   SET_ADDRESS(_stubs, StubRoutines::hf2f_adr());
3339 
3340   for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
3341     SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_stub(slot));
3342   }
3343   SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_slow_path_stub());
3344 
3345 #if defined(AMD64) && !defined(ZERO)
3346   SET_ADDRESS(_stubs, StubRoutines::x86::d2i_fixup());
3347   SET_ADDRESS(_stubs, StubRoutines::x86::f2i_fixup());
3348   SET_ADDRESS(_stubs, StubRoutines::x86::f2l_fixup());
3349   SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_mask());
3350   SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_flip());
3351   SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_mask());
3352   SET_ADDRESS(_stubs, StubRoutines::x86::vector_popcount_lut());
3353   SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_mask());
3354   SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_flip());
3355   SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_mask());
3356   SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_flip());
3357   SET_ADDRESS(_stubs, StubRoutines::x86::vector_int_shuffle_mask());
3358   SET_ADDRESS(_stubs, StubRoutines::x86::vector_byte_shuffle_mask());
3359   SET_ADDRESS(_stubs, StubRoutines::x86::vector_short_shuffle_mask());
3360   SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_shuffle_mask());
3361   SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_sign_mask());
3362   SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_int());
3363   SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_short());
3364   SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_long());
3365   // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
3366   // See C2_MacroAssembler::load_iota_indices().
3367   for (int i = 0; i < 6; i++) {
3368     SET_ADDRESS(_stubs, StubRoutines::x86::vector_iota_indices() + i * 64);
3369   }
3370 #endif
3371 #if defined(AARCH64) && !defined(ZERO)
3372   SET_ADDRESS(_stubs, StubRoutines::aarch64::zero_blocks());
3373   SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives());
3374   SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives_long());
3375   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_array_equals());
3376   SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LL());
3377   SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UU());
3378   SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LU());
3379   SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UL());
3380   SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ul());
3381   SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ll());
3382   SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_uu());
3383   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_byte_array_inflate());
3384   SET_ADDRESS(_stubs, StubRoutines::aarch64::spin_wait());
3385 
3386   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BOOLEAN));
3387   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BYTE));
3388   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_SHORT));
3389   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_CHAR));
3390   SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_INT));
3391 #endif
3392 
3393   _complete = true;
3394   log_info(aot, codecache, init)("Stubs recorded");
3395 }
3396 
3397 void AOTCodeAddressTable::init_early_c1() {
3398 #ifdef COMPILER1
3399   // Runtime1 Blobs
3400   StubId id = StubInfo::stub_base(StubGroup::C1);
3401   // include forward_exception in range we publish
3402   StubId limit = StubInfo::next(StubId::c1_forward_exception_id);
3403   for (; id != limit; id = StubInfo::next(id)) {
3404     if (Runtime1::blob_for(id) == nullptr) {
3405       log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3406       continue;
3407     }
3408     if (Runtime1::entry_for(id) == nullptr) {
3409       log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3410       continue;
3411     }
3412     address entry = Runtime1::entry_for(id);
3413     SET_ADDRESS(_C1_blobs, entry);
3414   }
3415 #endif // COMPILER1
3416   assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3417   _early_c1_complete = true;
3418 }
3419 
3420 void AOTCodeAddressTable::init_c1() {
3421 #ifdef COMPILER1
3422   // Runtime1 Blobs
3423   assert(_early_c1_complete, "early C1 blobs should be initialized");
3424   StubId id = StubInfo::next(StubId::c1_forward_exception_id);
3425   StubId limit = StubInfo::next(StubInfo::stub_max(StubGroup::C1));
3426   for (; id != limit; id = StubInfo::next(id)) {
3427     if (Runtime1::blob_for(id) == nullptr) {
3428       log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3429       continue;
3430     }
3431     if (Runtime1::entry_for(id) == nullptr) {
3432       log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3433       continue;
3434     }
3435     address entry = Runtime1::entry_for(id);
3436     SET_ADDRESS(_C1_blobs, entry);
3437   }
3438 #if INCLUDE_G1GC
3439   if (UseG1GC) {
3440     G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3441     address entry = bs->pre_barrier_c1_runtime_code_blob()->code_begin();
3442     SET_ADDRESS(_C1_blobs, entry);
3443   }
3444 #endif // INCLUDE_G1GC
3445 #if INCLUDE_ZGC
3446   if (UseZGC) {
3447     ZBarrierSetC1* bs = (ZBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3448     SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_oop_field_preloaded_runtime_stub);
3449     SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_weak_oop_field_preloaded_runtime_stub);
3450     SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_with_healing);
3451     SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_without_healing);
3452   }
3453 #endif // INCLUDE_ZGC
3454 #if INCLUDE_SHENANDOAHGC
3455   if (UseShenandoahGC) {
3456     ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3457     SET_ADDRESS(_C1_blobs, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
3458     SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_rt_code_blob()->code_begin());
3459     SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin());
3460     SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_weak_rt_code_blob()->code_begin());
3461     SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_phantom_rt_code_blob()->code_begin());
3462   }
3463 #endif // INCLUDE_SHENANDOAHGC
3464 #endif // COMPILER1
3465 
3466   assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3467   _c1_complete = true;
3468   log_info(aot, codecache, init)("Runtime1 Blobs recorded");
3469 }
3470 
3471 void AOTCodeAddressTable::init_c2() {
3472 #ifdef COMPILER2
3473   // OptoRuntime Blobs
3474   SET_ADDRESS(_C2_blobs, OptoRuntime::uncommon_trap_blob()->entry_point());
3475   SET_ADDRESS(_C2_blobs, OptoRuntime::exception_blob()->entry_point());
3476   SET_ADDRESS(_C2_blobs, OptoRuntime::new_instance_Java());
3477   SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_Java());
3478   SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_nozero_Java());
3479   SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray2_Java());
3480   SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray3_Java());
3481   SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray4_Java());
3482   SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray5_Java());
3483   SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarrayN_Java());
3484   SET_ADDRESS(_C2_blobs, OptoRuntime::vtable_must_compile_stub());
3485   SET_ADDRESS(_C2_blobs, OptoRuntime::complete_monitor_locking_Java());
3486   SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notify_Java());
3487   SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notifyAll_Java());
3488   SET_ADDRESS(_C2_blobs, OptoRuntime::rethrow_stub());
3489   SET_ADDRESS(_C2_blobs, OptoRuntime::slow_arraycopy_Java());
3490   SET_ADDRESS(_C2_blobs, OptoRuntime::register_finalizer_Java());
3491   SET_ADDRESS(_C2_blobs, OptoRuntime::class_init_barrier_Java());
3492 #if INCLUDE_JVMTI
3493   SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_start());
3494   SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_end());
3495   SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_mount());
3496   SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_unmount());
3497 #endif /* INCLUDE_JVMTI */
3498 #endif
3499 
3500   assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length);
3501   _c2_complete = true;
3502   log_info(aot, codecache, init)("OptoRuntime Blobs recorded");
3503 }
3504 #undef SET_ADDRESS
3505 
3506 AOTCodeAddressTable::~AOTCodeAddressTable() {
3507   if (_extrs_addr != nullptr) {
3508     FREE_C_HEAP_ARRAY(address, _extrs_addr);
3509   }
3510   if (_stubs_addr != nullptr) {
3511     FREE_C_HEAP_ARRAY(address, _stubs_addr);
3512   }
3513   if (_shared_blobs_addr != nullptr) {
3514     FREE_C_HEAP_ARRAY(address, _shared_blobs_addr);
3515   }
3516 }
3517 
3518 #ifdef PRODUCT
3519 #define MAX_STR_COUNT 200
3520 #else
3521 #define MAX_STR_COUNT 500
3522 #endif
3523 #define _c_str_max  MAX_STR_COUNT
3524 static const int _c_str_base = _all_max;
3525 
3526 static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings
3527 static const char* _C_strings[MAX_STR_COUNT]    = {nullptr}; // Our duplicates
3528 static int _C_strings_count = 0;
3529 static int _C_strings_s[MAX_STR_COUNT] = {0};
3530 static int _C_strings_id[MAX_STR_COUNT] = {0};
3531 static int _C_strings_used = 0;
3532 
3533 void AOTCodeCache::load_strings() {
3534   uint strings_count  = _load_header->strings_count();
3535   if (strings_count == 0) {
3536     return;
3537   }
3538   uint strings_offset = _load_header->strings_offset();
3539   uint* string_lengths = (uint*)addr(strings_offset);
3540   strings_offset += (strings_count * sizeof(uint));
3541   uint strings_size = _load_header->search_table_offset() - strings_offset;
3542   // We have to keep cached strings longer than _cache buffer
3543   // because they are refernced from compiled code which may
3544   // still be executed on VM exit after _cache is freed.
3545   char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
3546   memcpy(p, addr(strings_offset), strings_size);
3547   _C_strings_buf = p;
3548   assert(strings_count <= MAX_STR_COUNT, "sanity");
3549   for (uint i = 0; i < strings_count; i++) {
3550     _C_strings[i] = p;
3551     uint len = string_lengths[i];
3552     _C_strings_s[i] = i;
3553     _C_strings_id[i] = i;
3554     p += len;
3555   }
3556   assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
3557   _C_strings_count = strings_count;
3558   _C_strings_used  = strings_count;
3559   log_debug(aot, codecache, init)("  Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset);
3560 }
3561 

3663   assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3664   if (idx == -1) {
3665     return (address)-1;
3666   }
3667   uint id = (uint)idx;
3668   // special case for symbols based relative to os::init
3669   if (id > (_c_str_base + _c_str_max)) {
3670     return (address)os::init + idx;
3671   }
3672   if (idx < 0) {
3673     fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3674     return nullptr;
3675   }
3676   // no need to compare unsigned id against 0
3677   if (/* id >= _extrs_base && */ id < _extrs_length) {
3678     return _extrs_addr[id - _extrs_base];
3679   }
3680   if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3681     return _stubs_addr[id - _stubs_base];
3682   }
3683   if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3684     return _stubs_addr[id - _stubs_base];
3685   }
3686   if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) {
3687     return _shared_blobs_addr[id - _shared_blobs_base];
3688   }
3689   if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3690     return _C1_blobs_addr[id - _C1_blobs_base];
3691   }
3692   if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3693     return _C1_blobs_addr[id - _C1_blobs_base];
3694   }
3695   if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) {
3696     return _C2_blobs_addr[id - _C2_blobs_base];
3697   }
3698   if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
3699     return address_for_C_string(id - _c_str_base);
3700   }
3701   fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3702   return nullptr;
3703 }
3704 
3705 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* blob) {
3706   assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3707   int id = -1;
3708   if (addr == (address)-1) { // Static call stub has jump to itself
3709     return id;
3710   }
3711   // Check card_table_base address first since it can point to any address
3712   BarrierSet* bs = BarrierSet::barrier_set();
3713   guarantee(!bs->is_a(BarrierSet::CardTableBarrierSet) || addr != ci_card_table_address_as<address>(), "sanity");
3714 
3715   // Seach for C string
3716   id = id_for_C_string(addr);
3717   if (id >= 0) {
3718     return id + _c_str_base;
3719   }
3720   if (StubRoutines::contains(addr)) {
3721     // Search in stubs
3722     id = search_address(addr, _stubs_addr, _stubs_length);
3723     if (id == BAD_ADDRESS_ID) {
3724       StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
3725       if (desc == nullptr) {
3726         desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset);
3727       }
3728       const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
3729       assert(false, "Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
3730     } else {
3731       return _stubs_base + id;
3732     }
3733   } else {
3734     CodeBlob* cb = CodeCache::find_blob(addr);
3735     if (cb != nullptr) {

3736       int id_base = _shared_blobs_base;
3737       // Search in code blobs
3738       id = search_address(addr, _shared_blobs_addr, _shared_blobs_length);
3739       if (id == BAD_ADDRESS_ID) {
3740         id_base = _C1_blobs_base;
3741         // search C1 blobs
3742         id = search_address(addr, _C1_blobs_addr, _C1_blobs_length);
3743       }
3744       if (id == BAD_ADDRESS_ID) {
3745         id_base = _C2_blobs_base;
3746         // search C2 blobs
3747         id = search_address(addr, _C2_blobs_addr, _C2_blobs_length);
3748       }
3749       if (id == BAD_ADDRESS_ID) {
3750         assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name());
3751       } else {
3752         return id_base + id;
3753       }
3754     } else {
3755       // Search in runtime functions
3756       id = search_address(addr, _extrs_addr, _extrs_length);
3757       if (id == BAD_ADDRESS_ID) {
3758         ResourceMark rm;
3759         const int buflen = 1024;
3760         char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
3761         int offset = 0;
3762         if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
3763           if (offset > 0) {
3764             // Could be address of C string
3765             uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
3766             CompileTask* task = ciEnv::current()->task();
3767             uint compile_id = 0;
3768             uint comp_level =0;
3769             if (task != nullptr) { // this could be called from compiler runtime initialization (compiler blobs)
3770               compile_id = task->compile_id();
3771               comp_level = task->comp_level();
3772             }
3773             log_debug(aot, codecache)("%d (L%d): Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
3774                           compile_id, comp_level, p2i(addr), dist, (const char*)addr);
3775             assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
3776             return dist;
3777           }

3778           reloc.print_current_on(tty);
3779           blob->print_on(tty);
3780           blob->print_code_on(tty);
3781           assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);

3782         } else {

3783           reloc.print_current_on(tty);
3784           blob->print_on(tty);
3785           blob->print_code_on(tty);
3786           os::find(addr, tty);
3787           assert(false, "Address " INTPTR_FORMAT " for <unknown>/('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr);

3788         }
3789       } else {
3790         return _extrs_base + id;
3791       }
3792     }
3793   }
3794   return id;
3795 }
3796 
3797 #undef _extrs_max
3798 #undef _stubs_max
3799 #undef _shared_blobs_max
3800 #undef _C1_blobs_max
3801 #undef _C2_blobs_max
3802 #undef _blobs_max
3803 #undef _extrs_base
3804 #undef _stubs_base
3805 #undef _shared_blobs_base
3806 #undef _C1_blobs_base
3807 #undef _C2_blobs_base
3808 #undef _blobs_end
3809 
3810 AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
3811 
3812 void AOTRuntimeConstants::initialize_from_runtime() {
3813   BarrierSet* bs = BarrierSet::barrier_set();
3814   if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
3815     CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
3816     _aot_runtime_constants._card_table_address = ci_card_table_address_as<address>();
3817     _aot_runtime_constants._grain_shift = ctbs->grain_shift();
3818   }
3819 }
3820 
3821 address AOTRuntimeConstants::_field_addresses_list[] = {
3822   card_table_address(),
3823   grain_shift_address(),
3824   nullptr
3825 };
3826 
3827 
3828 void AOTCodeCache::wait_for_no_nmethod_readers() {
3829   while (true) {
3830     int cur = AtomicAccess::load(&_nmethod_readers);
3831     int upd = -(cur + 1);
3832     if (cur >= 0 && AtomicAccess::cmpxchg(&_nmethod_readers, cur, upd) == cur) {
3833       // Success, no new readers should appear.
3834       break;
3835     }
3836   }
3837 
3838   // Now wait for all readers to leave.
3839   SpinYield w;
3840   while (AtomicAccess::load(&_nmethod_readers) != -1) {
3841     w.wait();
3842   }
3843 }
3844 
3845 AOTCodeCache::ReadingMark::ReadingMark() {
3846   while (true) {
3847     int cur = AtomicAccess::load(&_nmethod_readers);
3848     if (cur < 0) {
3849       // Cache is already closed, cannot proceed.
3850       _failed = true;
3851       return;
3852     }
3853     if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3854       // Successfully recorded ourselves as entered.
3855       _failed = false;
3856       return;
3857     }
3858   }
3859 }
3860 
3861 AOTCodeCache::ReadingMark::~ReadingMark() {
3862   if (_failed) {
3863     return;
3864   }
3865   while (true) {
3866     int cur = AtomicAccess::load(&_nmethod_readers);
3867     if (cur > 0) {
3868       // Cache is open, we are counting down towards 0.
3869       if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur - 1) == cur) {
3870         return;
3871       }
3872     } else {
3873       // Cache is closed, we are counting up towards -1.
3874       if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3875         return;
3876       }
3877     }
3878   }
3879 }
3880 
3881 void AOTCodeCache::print_timers_on(outputStream* st) {
3882   if (is_using_code()) {
3883     st->print_cr ("    AOT Code Preload Time:   %7.3f s", _t_totalPreload.seconds());
3884     st->print_cr ("    AOT Code Load Time:   %7.3f s", _t_totalLoad.seconds());
3885     st->print_cr ("      nmethod register:     %7.3f s", _t_totalRegister.seconds());
3886     st->print_cr ("      find AOT code entry:  %7.3f s", _t_totalFind.seconds());
3887   }
3888   if (is_dumping_code()) {
3889     st->print_cr ("    AOT Code Store Time:  %7.3f s", _t_totalStore.seconds());
3890   }
3891 }
3892 
3893 AOTCodeStats AOTCodeStats::add_aot_code_stats(AOTCodeStats stats1, AOTCodeStats stats2) {
3894   AOTCodeStats result;
3895   for (int kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3896     result.ccstats._kind_cnt[kind] = stats1.entry_count(kind) + stats2.entry_count(kind);
3897   }
3898 
3899   for (int lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3900     result.ccstats._nmethod_cnt[lvl] = stats1.nmethod_count(lvl) + stats2.nmethod_count(lvl);
3901   }
3902   result.ccstats._clinit_barriers_cnt = stats1.clinit_barriers_count() + stats2.clinit_barriers_count();
3903   return result;
3904 }
3905 
3906 void AOTCodeCache::log_stats_on_exit(AOTCodeStats& stats) {
3907   LogStreamHandle(Debug, aot, codecache, exit) log;
3908   if (log.is_enabled()) {
3909     for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3910       log.print_cr("  %s: total=%u", aot_code_entry_kind_name[kind], stats.entry_count(kind));
3911       if (kind == AOTCodeEntry::Nmethod) {
3912         for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3913           log.print_cr("    Tier %d: total=%u", lvl, stats.nmethod_count(lvl));
3914         }
3915       }
3916     }
3917   }
3918 }
3919 
3920 static void print_helper1(outputStream* st, const char* name, int count) {
3921   if (count > 0) {
3922     st->print(" %s=%d", name, count);
3923   }
3924 }
3925 
3926 void AOTCodeCache::print_statistics_on(outputStream* st) {
3927   AOTCodeCache* cache = open_for_use();
3928   if (cache != nullptr) {
3929     ReadingMark rdmk;
3930     if (rdmk.failed()) {
3931       // Cache is closed, cannot touch anything.
3932       return;
3933     }
3934     AOTCodeStats stats;
3935 
3936     uint preload_count = cache->_load_header->preload_entries_count();
3937     AOTCodeEntry* preload_entries = (AOTCodeEntry*)cache->addr(cache->_load_header->preload_entries_offset());
3938     for (uint i = 0; i < preload_count; i++) {
3939       stats.collect_all_stats(&preload_entries[i]);
3940     }
3941 
3942     uint count = cache->_load_header->entries_count();
3943     AOTCodeEntry* load_entries = (AOTCodeEntry*)cache->addr(cache->_load_header->entries_offset());
3944     for (uint i = 0; i < count; i++) {
3945       stats.collect_all_stats(&load_entries[i]);
3946     }
3947 
3948     for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3949       if (stats.entry_count(kind) > 0) {
3950         st->print("  %s:", aot_code_entry_kind_name[kind]);
3951         print_helper1(st, "total", stats.entry_count(kind));
3952         print_helper1(st, "loaded", stats.entry_loaded_count(kind));
3953         print_helper1(st, "invalidated", stats.entry_invalidated_count(kind));
3954         print_helper1(st, "failed", stats.entry_load_failed_count(kind));
3955         st->cr();
3956       }
3957       if (kind == AOTCodeEntry::Nmethod) {
3958         for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3959           if (stats.nmethod_count(lvl) > 0) {
3960             st->print("    AOT Code T%d", lvl);
3961             print_helper1(st, "total", stats.nmethod_count(lvl));
3962             print_helper1(st, "loaded", stats.nmethod_loaded_count(lvl));
3963             print_helper1(st, "invalidated", stats.nmethod_invalidated_count(lvl));
3964             print_helper1(st, "failed", stats.nmethod_load_failed_count(lvl));
3965             if (lvl == AOTCompLevel_count-1) {
3966               print_helper1(st, "has_clinit_barriers", stats.clinit_barriers_count());
3967             }
3968             st->cr();
3969           }
3970         }
3971       }
3972     }
3973     LogStreamHandle(Debug, aot, codecache, init) log;
3974     if (log.is_enabled()) {
3975       AOTCodeCache::print_unused_entries_on(&log);
3976     }
3977     LogStreamHandle(Trace, aot, codecache) aot_info;
3978     // need a lock to traverse the code cache
3979     if (aot_info.is_enabled()) {
3980       MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag);
3981       NMethodIterator iter(NMethodIterator::all);
3982       while (iter.next()) {
3983         nmethod* nm = iter.method();
3984         if (nm->is_in_use() && !nm->is_native_method() && !nm->is_osr_method()) {
3985           aot_info.print("%5d:%c%c%c%d:", nm->compile_id(),
3986                          (nm->method()->in_aot_cache() ? 'S' : ' '),
3987                          (nm->is_aot() ? 'A' : ' '),
3988                          (nm->preloaded() ? 'P' : ' '),
3989                          nm->comp_level());
3990           print_helper(nm, &aot_info);
3991           aot_info.print(": ");
3992           CompileTask::print(&aot_info, nm, nullptr, true /*short_form*/);
3993           LogStreamHandle(Trace, aot, codecache) aot_debug;
3994           if (aot_debug.is_enabled()) {
3995             MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), nm->method()));
3996             if (mtd != nullptr) {
3997               mtd->iterate_compiles([&](CompileTrainingData* ctd) {
3998                 aot_debug.print("     CTD: "); ctd->print_on(&aot_debug); aot_debug.cr();
3999               });
4000             }
4001           }
4002         }
4003       }
4004     }
4005   }
4006 }
4007 
4008 void AOTCodeEntry::print(outputStream* st) const {
4009   st->print_cr(" AOT Code Cache entry " INTPTR_FORMAT " [kind: %d, id: " UINT32_FORMAT_X_0 ", offset: %d, size: %d, comp_level: %d, comp_id: %d, %s%s%s%s]",
4010                p2i(this), (int)_kind, _id, _offset, _size, _comp_level, _comp_id,
4011                (_not_entrant? "not_entrant" : "entrant"),
4012                (_loaded ? ", loaded" : ""),
4013                (_has_clinit_barriers ? ", has_clinit_barriers" : ""),
4014                (_for_preload ? ", for_preload" : ""));
4015 }
4016 
4017 // This is called after initialize() but before init2()
4018 // and _cache is not set yet.
4019 void AOTCodeCache::print_on(outputStream* st) {
4020   if (opened_cache != nullptr && opened_cache->for_use()) {
4021     ReadingMark rdmk;
4022     if (rdmk.failed()) {
4023       // Cache is closed, cannot touch anything.
4024       return;
4025     }
4026 
4027     st->print_cr("\nAOT Code Cache Preload entries");
4028 
4029     uint preload_count = opened_cache->_load_header->preload_entries_count();
4030     AOTCodeEntry* preload_entries = (AOTCodeEntry*)opened_cache->addr(opened_cache->_load_header->preload_entries_offset());
4031     for (uint i = 0; i < preload_count; i++) {
4032       AOTCodeEntry* entry = &preload_entries[i];
4033 
4034       uint entry_position = entry->offset();
4035       uint name_offset = entry->name_offset() + entry_position;
4036       const char* saved_name = opened_cache->addr(name_offset);
4037 
4038       st->print_cr("%4u: %10s Id:%u L%u size=%u '%s' %s%s%s",
4039                    i, aot_code_entry_kind_name[entry->kind()], entry->id(), entry->comp_level(),
4040                    entry->size(),  saved_name,
4041                    entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
4042                    entry->is_loaded()           ? " loaded"              : "",
4043                    entry->not_entrant()         ? " not_entrant"         : "");
4044 
4045       st->print_raw("         ");
4046       AOTCodeReader reader(opened_cache, entry, nullptr);
4047       reader.print_on(st);
4048     }
4049 
4050     st->print_cr("\nAOT Code Cache entries");
4051 
4052     uint count = opened_cache->_load_header->entries_count();
4053     uint* search_entries = (uint*)opened_cache->addr(opened_cache->_load_header->search_table_offset()); // [id, index]
4054     AOTCodeEntry* load_entries = (AOTCodeEntry*)opened_cache->addr(opened_cache->_load_header->entries_offset());
4055 
4056     for (uint i = 0; i < count; i++) {

4057       int index = search_entries[2*i + 1];
4058       AOTCodeEntry* entry = &(load_entries[index]);
4059 
4060       uint entry_position = entry->offset();
4061       uint name_offset = entry->name_offset() + entry_position;
4062       const char* saved_name = opened_cache->addr(name_offset);
4063 
4064       st->print_cr("%4u: %10s idx:%4u Id:%u L%u size=%u '%s' %s%s%s%s",
4065                    i, aot_code_entry_kind_name[entry->kind()], index, entry->id(), entry->comp_level(),
4066                    entry->size(),  saved_name,
4067                    entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
4068                    entry->for_preload()         ? " for_preload"         : "",
4069                    entry->is_loaded()           ? " loaded"              : "",
4070                    entry->not_entrant()         ? " not_entrant"         : "");
4071 
4072       st->print_raw("         ");
4073       AOTCodeReader reader(opened_cache, entry, nullptr);
4074       reader.print_on(st);
4075     }
4076   }
4077 }
4078 
4079 void AOTCodeCache::print_unused_entries_on(outputStream* st) {
4080   LogStreamHandle(Info, aot, codecache, init) info;
4081   if (info.is_enabled()) {
4082     AOTCodeCache::iterate([&](AOTCodeEntry* entry) {
4083       if (entry->is_nmethod() && !entry->is_loaded()) {
4084         MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), entry->method()));
4085         if (mtd != nullptr) {
4086           if (mtd->has_holder()) {
4087             if (mtd->holder()->method_holder()->is_initialized()) {
4088               ResourceMark rm;
4089               mtd->iterate_compiles([&](CompileTrainingData* ctd) {
4090                 if ((uint)ctd->level() == entry->comp_level()) {
4091                   if (ctd->init_deps_left_acquire() == 0) {
4092                     nmethod* nm = mtd->holder()->code();
4093                     if (nm == nullptr) {
4094                       if (mtd->holder()->queued_for_compilation()) {
4095                         return; // scheduled for compilation
4096                       }
4097                     } else if ((uint)nm->comp_level() >= entry->comp_level()) {
4098                       return; // already online compiled and superseded by a more optimal method
4099                     }
4100                     info.print("AOT Code Cache entry not loaded: ");
4101                     ctd->print_on(&info);
4102                     info.cr();
4103                   }
4104                 }
4105               });
4106             } else {
4107               // not yet initialized
4108             }
4109           } else {
4110             info.print("AOT Code Cache entry doesn't have a holder: ");
4111             mtd->print_on(&info);
4112             info.cr();
4113           }
4114         }
4115       }
4116     });
4117   }
4118 }
4119 
4120 void AOTCodeReader::print_on(outputStream* st) {
4121   uint entry_position = _entry->offset();
4122   set_read_position(entry_position);
4123 
4124   // Read name
4125   uint name_offset = entry_position + _entry->name_offset();
4126   uint name_size = _entry->name_size(); // Includes '/0'
4127   const char* name = addr(name_offset);
4128 
4129   st->print_cr("  name: %s", name);
4130 }
4131 
< prev index next >