1 /*
2 * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25
26 #include "asm/macroAssembler.hpp"
27 #include "cds/aotCacheAccess.hpp"
28 #include "cds/aotMetaspace.hpp"
29 #include "cds/cds_globals.hpp"
30 #include "cds/cdsConfig.hpp"
31 #include "cds/heapShared.hpp"
32 #include "ci/ciConstant.hpp"
33 #include "ci/ciEnv.hpp"
34 #include "ci/ciField.hpp"
35 #include "ci/ciMethod.hpp"
36 #include "ci/ciMethodData.hpp"
37 #include "ci/ciObject.hpp"
38 #include "ci/ciUtilities.inline.hpp"
39 #include "classfile/javaAssertions.hpp"
40 #include "classfile/stringTable.hpp"
41 #include "classfile/symbolTable.hpp"
42 #include "classfile/systemDictionary.hpp"
43 #include "classfile/vmClasses.hpp"
44 #include "classfile/vmIntrinsics.hpp"
45 #include "code/aotCodeCache.hpp"
46 #include "code/codeBlob.hpp"
47 #include "code/codeCache.hpp"
48 #include "code/oopRecorder.inline.hpp"
49 #include "compiler/abstractCompiler.hpp"
50 #include "compiler/compilationPolicy.hpp"
51 #include "compiler/compileBroker.hpp"
52 #include "compiler/compileTask.hpp"
53 #include "gc/g1/g1BarrierSetRuntime.hpp"
54 #include "gc/shared/gcConfig.hpp"
55 #include "logging/logStream.hpp"
56 #include "memory/memoryReserver.hpp"
57 #include "memory/universe.hpp"
58 #include "oops/klass.inline.hpp"
59 #include "oops/method.inline.hpp"
60 #include "oops/trainingData.hpp"
61 #include "prims/jvmtiThreadState.hpp"
62 #include "runtime/atomicAccess.hpp"
63 #include "runtime/deoptimization.hpp"
64 #include "runtime/flags/flagSetting.hpp"
65 #include "runtime/globals_extension.hpp"
66 #include "runtime/handles.inline.hpp"
67 #include "runtime/java.hpp"
68 #include "runtime/jniHandles.inline.hpp"
69 #include "runtime/mutexLocker.hpp"
70 #include "runtime/os.inline.hpp"
71 #include "runtime/sharedRuntime.hpp"
72 #include "runtime/stubCodeGenerator.hpp"
73 #include "runtime/stubRoutines.hpp"
74 #include "runtime/threadIdentifier.hpp"
75 #include "runtime/timerTrace.hpp"
76 #include "utilities/copy.hpp"
77 #include "utilities/formatBuffer.hpp"
78 #include "utilities/ostream.hpp"
79 #include "utilities/spinYield.hpp"
80 #ifdef COMPILER1
81 #include "c1/c1_LIRAssembler.hpp"
82 #include "c1/c1_Runtime1.hpp"
83 #include "gc/g1/c1/g1BarrierSetC1.hpp"
84 #include "gc/shared/c1/barrierSetC1.hpp"
85 #if INCLUDE_SHENANDOAHGC
86 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
87 #endif // INCLUDE_SHENANDOAHGC
88 #include "gc/z/c1/zBarrierSetC1.hpp"
89 #endif // COMPILER1
90 #ifdef COMPILER2
91 #include "opto/runtime.hpp"
92 #endif
93 #if INCLUDE_JVMCI
94 #include "jvmci/jvmci.hpp"
95 #endif
96 #if INCLUDE_G1GC
97 #include "gc/g1/g1BarrierSetRuntime.hpp"
98 #endif
99 #if INCLUDE_SHENANDOAHGC
100 #include "gc/shenandoah/shenandoahRuntime.hpp"
101 #endif
102 #if INCLUDE_ZGC
103 #include "gc/z/zBarrierSetRuntime.hpp"
104 #endif
105 #if defined(X86) && !defined(ZERO)
106 #include "rdtsc_x86.hpp"
107 #endif
108
109 #include <errno.h>
110 #include <sys/stat.h>
111
112 const char* aot_code_entry_kind_name[] = {
113 #define DECL_KIND_STRING(kind) XSTR(kind),
114 DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
115 #undef DECL_KIND_STRING
116 };
117
118 static elapsedTimer _t_totalLoad;
119 static elapsedTimer _t_totalPreload;
120 static elapsedTimer _t_totalRegister;
121 static elapsedTimer _t_totalFind;
122 static elapsedTimer _t_totalStore;
123
124 static bool enable_timers() {
125 return CITime || log_is_enabled(Info, init);
126 }
127
128 static void report_load_failure() {
129 if (AbortVMOnAOTCodeFailure) {
130 vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
131 }
132 log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
133 AOTCodeCache::disable_caching();
134 }
135
136 static void report_store_failure() {
137 if (AbortVMOnAOTCodeFailure) {
138 tty->print_cr("Unable to create AOT Code Cache.");
139 vm_abort(false);
140 }
141 log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
142 AOTCodeCache::disable_caching();
143 }
144
145 // The sequence of AOT code caching flags and parametters settings.
146 //
147 // 1. The initial AOT code caching flags setting is done
148 // during call to CDSConfig::check_vm_args_consistency().
149 //
150 // 2. The earliest AOT code state check done in compilationPolicy_init()
151 // where we set number of compiler threads for AOT assembly phase.
152 //
153 // 3. We determine presence of AOT code in AOT Cache in
154 // AOTMetaspace::open_static_archive() which is calles
155 // after compilationPolicy_init() but before codeCache_init().
156 //
157 // 4. AOTCodeCache::initialize() is called during universe_init()
158 // and does final AOT state and flags settings.
159 //
160 // 5. Finally AOTCodeCache::init2() is called after universe_init()
161 // when all GC settings are finalized.
162
163 // Next methods determine which action we do with AOT code depending
164 // on phase of AOT process: assembly or production.
165
166 bool AOTCodeCache::is_dumping_adapter() {
167 return AOTAdapterCaching && is_on_for_dump();
168 }
169
170 bool AOTCodeCache::is_using_adapter() {
171 return AOTAdapterCaching && is_on_for_use();
172 }
173
174 bool AOTCodeCache::is_dumping_stub() {
175 return AOTStubCaching && is_on_for_dump();
176 }
177
178 bool AOTCodeCache::is_using_stub() {
179 return AOTStubCaching && is_on_for_use();
180 }
181
182 bool AOTCodeCache::is_dumping_code() {
183 return AOTCodeCaching && is_on_for_dump();
184 }
185
186 bool AOTCodeCache::is_using_code() {
187 return AOTCodeCaching && is_on_for_use();
188 }
189
190 // This is used before AOTCodeCahe is initialized
191 // but after AOT (CDS) Cache flags consistency is checked.
192 bool AOTCodeCache::maybe_dumping_code() {
193 return AOTCodeCaching && CDSConfig::is_dumping_final_static_archive();
194 }
195
196 // Next methods could be called regardless of AOT code cache status.
197 // Initially they are called during AOT flags parsing and finilized
198 // in AOTCodeCache::initialize().
199 void AOTCodeCache::enable_caching() {
200 FLAG_SET_ERGO_IF_DEFAULT(AOTCodeCaching, true);
201 FLAG_SET_ERGO_IF_DEFAULT(AOTStubCaching, true);
202 FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true);
203 }
204
205 void AOTCodeCache::disable_caching() {
206 FLAG_SET_ERGO(AOTCodeCaching, false);
207 FLAG_SET_ERGO(AOTStubCaching, false);
208 FLAG_SET_ERGO(AOTAdapterCaching, false);
209 }
210
211 bool AOTCodeCache::is_caching_enabled() {
212 return AOTCodeCaching || AOTStubCaching || AOTAdapterCaching;
213 }
214
215 static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
216 assert(AOTCodeEntry::is_valid_entry_kind(kind), "invalid AOTCodeEntry kind %d", (int)kind);
217 // There can be a conflict of id between an Adapter and *Blob, but that should not cause any functional issue
218 // becasue both id and kind are used to find an entry, and that combination should be unique
219 if (kind == AOTCodeEntry::Adapter) {
220 return id;
221 } else if (kind == AOTCodeEntry::SharedBlob) {
222 assert(StubInfo::is_shared(static_cast<BlobId>(id)), "not a shared blob id %d", id);
223 return id;
224 } else if (kind == AOTCodeEntry::C1Blob) {
225 assert(StubInfo::is_c1(static_cast<BlobId>(id)), "not a c1 blob id %d", id);
226 return id;
227 } else {
228 // kind must be AOTCodeEntry::C2Blob
229 assert(StubInfo::is_c2(static_cast<BlobId>(id)), "not a c2 blob id %d", id);
230 return id;
231 }
232 }
233
234 static uint _max_aot_code_size = 0;
235 uint AOTCodeCache::max_aot_code_size() {
236 return _max_aot_code_size;
237 }
238
239 bool AOTCodeCache::is_code_load_thread_on() {
240 return UseAOTCodeLoadThread && AOTCodeCaching;
241 }
242
243 bool AOTCodeCache::allow_const_field(ciConstant& value) {
244 ciEnv* env = CURRENT_ENV;
245 precond(env != nullptr);
246 assert(!env->is_precompile() || is_dumping_code(), "AOT compilation should be enabled");
247 return !env->is_precompile() // Restrict only when we generate AOT code
248 // Can not trust primitive too || !is_reference_type(value.basic_type())
249 // May disable this too for now || is_reference_type(value.basic_type()) && value.as_object()->should_be_constant()
250 ;
251 }
252
253 // It is called from AOTMetaspace::initialize_shared_spaces()
254 // which is called from universe_init().
255 // At this point all AOT class linking seetings are finilized
256 // and AOT cache is open so we can map AOT code region.
257 void AOTCodeCache::initialize() {
258 if (!is_caching_enabled()) {
259 log_info(aot, codecache, init)("AOT Code Cache is not used: disabled.");
260 return;
261 }
262 #if defined(ZERO) || !(defined(AMD64) || defined(AARCH64))
263 log_info(aot, codecache, init)("AOT Code Cache is not supported on this platform.");
264 disable_caching();
265 return;
266 #else
267 assert(!FLAG_IS_DEFAULT(AOTCache), "AOTCache should be specified");
268
269 // Disable stubs caching until JDK-8357398 is fixed.
270 FLAG_SET_ERGO(AOTStubCaching, false);
271
272 if (VerifyOops) {
273 // Disable AOT stubs caching when VerifyOops flag is on.
274 // Verify oops code generated a lot of C strings which overflow
275 // AOT C string table (which has fixed size).
276 // AOT C string table will be reworked later to handle such cases.
277 //
278 // Note: AOT adapters are not affected - they don't have oop operations.
279 log_info(aot, codecache, init)("AOT Stubs Caching is not supported with VerifyOops.");
280 FLAG_SET_ERGO(AOTStubCaching, false);
281 }
282
283 bool is_dumping = false;
284 bool is_using = false;
285 if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) {
286 is_dumping = is_caching_enabled();
287 } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) {
288 is_using = is_caching_enabled();
289 }
290 if (ClassInitBarrierMode > 0 && !(is_dumping && AOTCodeCaching)) {
291 log_info(aot, codecache, init)("Set ClassInitBarrierMode to 0 because AOT Code dumping is off.");
292 FLAG_SET_ERGO(ClassInitBarrierMode, 0);
293 }
294 if (!(is_dumping || is_using)) {
295 log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used.");
296 disable_caching();
297 return; // AOT code caching disabled on command line
298 }
299 // Reserve AOT Cache region when we dumping AOT code.
300 _max_aot_code_size = AOTCodeMaxSize;
301 if (is_dumping && !FLAG_IS_DEFAULT(AOTCodeMaxSize)) {
302 if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) {
303 _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity());
304 log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K));
305 }
306 }
307 size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0;
308 if (is_using && aot_code_size == 0) {
309 log_info(aot, codecache, init)("AOT Code Cache is empty");
310 disable_caching();
311 return;
312 }
313 if (!open_cache(is_dumping, is_using)) {
314 if (is_using) {
315 report_load_failure();
316 } else {
317 report_store_failure();
318 }
319 return;
320 }
321 if (is_dumping) {
322 FLAG_SET_DEFAULT(FoldStableValues, false);
323 FLAG_SET_DEFAULT(ForceUnreachable, true);
324 }
325 FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
326 #endif // defined(AMD64) || defined(AARCH64)
327 }
328
329 static AOTCodeCache* opened_cache = nullptr; // Use this until we verify the cache
330 AOTCodeCache* AOTCodeCache::_cache = nullptr;
331 DEBUG_ONLY( bool AOTCodeCache::_passed_init2 = false; )
332
333 // It is called after universe_init() when all GC settings are finalized.
334 void AOTCodeCache::init2() {
335 DEBUG_ONLY( _passed_init2 = true; )
336 if (opened_cache == nullptr) {
337 return;
338 }
339 // After Universe initialized
340 if (!opened_cache->verify_config_on_use()) { // Check on AOT code loading
341 delete opened_cache;
342 opened_cache = nullptr;
343 report_load_failure();
344 return;
345 }
346
347 // initialize aot runtime constants as appropriate to this runtime
348 AOTRuntimeConstants::initialize_from_runtime();
349
350 // initialize the table of external routines and initial stubs so we can save
351 // generated code blobs that reference them
352 AOTCodeAddressTable* table = opened_cache->_table;
353 assert(table != nullptr, "should be initialized already");
354 table->init_extrs();
355
356 // Now cache and address table are ready for AOT code generation
357 _cache = opened_cache;
358
359 // Set ClassInitBarrierMode after all checks since it affects code generation
360 if (is_dumping_code()) {
361 FLAG_SET_ERGO_IF_DEFAULT(ClassInitBarrierMode, 1);
362 } else {
363 FLAG_SET_ERGO(ClassInitBarrierMode, 0);
364 }
365 }
366
367 bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
368 opened_cache = new AOTCodeCache(is_dumping, is_using);
369 if (opened_cache->failed()) {
370 delete opened_cache;
371 opened_cache = nullptr;
372 return false;
373 }
374 return true;
375 }
376
377 static void print_helper(nmethod* nm, outputStream* st) {
378 AOTCodeCache::iterate([&](AOTCodeEntry* e) {
379 if (e->method() == nm->method()) {
380 ResourceMark rm;
381 stringStream ss;
382 ss.print("A%s%d", (e->for_preload() ? "P" : ""), e->comp_level());
383 ss.print("[%s%s%s]",
384 (e->is_loaded() ? "L" : ""),
385 (e->load_fail() ? "F" : ""),
386 (e->not_entrant() ? "I" : ""));
387 ss.print("#%d", e->comp_id());
388
389 st->print(" %s", ss.freeze());
390 }
391 });
392 }
393
394 void AOTCodeCache::close() {
395 if (is_on()) {
396 delete _cache; // Free memory
397 _cache = nullptr;
398 opened_cache = nullptr;
399 }
400 }
401
402 class CachedCodeDirectory : public CachedCodeDirectoryInternal {
403 public:
404 uint _aot_code_size;
405 char* _aot_code_data;
406
407 void set_aot_code_data(uint size, char* aot_data) {
408 _aot_code_size = size;
409 AOTCacheAccess::set_pointer(&_aot_code_data, aot_data);
410 }
411
412 static CachedCodeDirectory* create();
413 };
414
415 // Storing AOT code in the AOT code region (ac) of AOT Cache:
416 //
417 // [1] Use CachedCodeDirectory to keep track of all of data related to AOT code.
418 // E.g., you can build a hashtable to record what methods have been archived.
419 //
420 // [2] Memory for all data for AOT code, including CachedCodeDirectory, should be
421 // allocated using AOTCacheAccess::allocate_aot_code_region().
422 //
423 // [3] CachedCodeDirectory must be the very first allocation.
424 //
425 // [4] Two kinds of pointer can be stored:
426 // - A pointer p that points to metadata. AOTCacheAccess::can_generate_aot_code(p) must return true.
427 // - A pointer to a buffer returned by AOTCacheAccess::allocate_aot_code_region().
428 // (It's OK to point to an interior location within this buffer).
429 // Such pointers must be stored using AOTCacheAccess::set_pointer()
430 //
431 // The buffers allocated by AOTCacheAccess::allocate_aot_code_region() are in a contiguous region. At runtime, this
432 // region is mapped to the process address space. All the pointers in this buffer are relocated as necessary
433 // (e.g., to account for the runtime location of the CodeCache).
434 //
435 // This is always at the very beginning of the mmaped CDS "ac" (AOT code) region
436 static CachedCodeDirectory* _aot_code_directory = nullptr;
437
438 CachedCodeDirectory* CachedCodeDirectory::create() {
439 assert(AOTCacheAccess::is_aot_code_region_empty(), "must be");
440 CachedCodeDirectory* dir = (CachedCodeDirectory*)AOTCacheAccess::allocate_aot_code_region(sizeof(CachedCodeDirectory));
441 dir->dumptime_init_internal();
442 return dir;
443 }
444
445 #define DATA_ALIGNMENT HeapWordSize
446
447 AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
448 _load_header(nullptr),
449 _load_buffer(nullptr),
450 _store_buffer(nullptr),
451 _C_store_buffer(nullptr),
452 _write_position(0),
453 _load_size(0),
454 _store_size(0),
455 _for_use(is_using),
456 _for_dump(is_dumping),
457 _closing(false),
458 _failed(false),
459 _lookup_failed(false),
460 _for_preload(false),
461 _has_clinit_barriers(false),
462 _table(nullptr),
463 _load_entries(nullptr),
464 _search_entries(nullptr),
465 _store_entries(nullptr),
466 _C_strings_buf(nullptr),
467 _store_entries_cnt(0),
468 _compile_id(0),
469 _comp_level(0)
470 {
471 // Read header at the begining of cache
472 if (_for_use) {
473 // Read cache
474 size_t load_size = AOTCacheAccess::get_aot_code_region_size();
475 ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode);
476 if (!rs.is_reserved()) {
477 log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size);
478 set_failed();
479 return;
480 }
481 if (!AOTCacheAccess::map_aot_code_region(rs)) {
482 log_warning(aot, codecache, init)("Failed to read/mmap AOT code region (ac) into AOT Code Cache");
483 set_failed();
484 return;
485 }
486 _aot_code_directory = (CachedCodeDirectory*)rs.base();
487 _aot_code_directory->runtime_init_internal();
488
489 _load_size = _aot_code_directory->_aot_code_size;
490 _load_buffer = _aot_code_directory->_aot_code_data;
491 assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
492 log_info(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " from AOT Code Cache", _load_size, p2i(_load_buffer));
493
494 _load_header = (Header*)addr(0);
495 if (!_load_header->verify(_load_size)) {
496 set_failed();
497 return;
498 }
499 log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count());
500 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Adapter], _load_header->adapters_count());
501 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::SharedBlob], _load_header->shared_blobs_count());
502 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::C1Blob], _load_header->C1_blobs_count());
503 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::C2Blob], _load_header->C2_blobs_count());
504 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Stub], _load_header->stubs_count());
505 log_debug(aot, codecache, init)(" %s: total=%u", aot_code_entry_kind_name[AOTCodeEntry::Nmethod], _load_header->nmethods_count());
506 log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", _load_header->cache_size());
507
508 // Read strings
509 load_strings();
510 }
511 if (_for_dump) {
512 _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
513 _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
514 // Entries allocated at the end of buffer in reverse (as on stack).
515 _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
516 log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size());
517 }
518 _table = new AOTCodeAddressTable();
519 }
520
521 void AOTCodeCache::invalidate(AOTCodeEntry* entry) {
522 // This could be concurent execution
523 if (entry != nullptr && is_on()) { // Request could come after cache is closed.
524 _cache->invalidate_entry(entry);
525 }
526 }
527
528 void AOTCodeCache::init_early_stubs_table() {
529 AOTCodeAddressTable* table = addr_table();
530 if (table != nullptr) {
531 table->init_early_stubs();
532 }
533 }
534
535 void AOTCodeCache::init_shared_blobs_table() {
536 AOTCodeAddressTable* table = addr_table();
537 if (table != nullptr) {
538 table->init_shared_blobs();
539 }
540 }
541
542 void AOTCodeCache::init_stubs_table() {
543 AOTCodeAddressTable* table = addr_table();
544 if (table != nullptr) {
545 table->init_stubs();
546 }
547 }
548
549 void AOTCodeCache::init_early_c1_table() {
550 AOTCodeAddressTable* table = addr_table();
551 if (table != nullptr) {
552 table->init_early_c1();
553 }
554 }
555
556 void AOTCodeCache::init_c1_table() {
557 AOTCodeAddressTable* table = addr_table();
558 if (table != nullptr) {
559 table->init_c1();
560 }
561 }
562
563 void AOTCodeCache::init_c2_table() {
564 AOTCodeAddressTable* table = addr_table();
565 if (table != nullptr) {
566 table->init_c2();
567 }
568 }
569
570 AOTCodeCache::~AOTCodeCache() {
571 if (_closing) {
572 return; // Already closed
573 }
574 // Stop any further access to cache.
575 // Checked on entry to load_nmethod() and store_nmethod().
576 _closing = true;
577 if (_for_use) {
578 // Wait for all load_nmethod() finish.
579 wait_for_no_nmethod_readers();
580 }
581 // Prevent writing code into cache while we are closing it.
582 // This lock held by ciEnv::register_method() which calls store_nmethod().
583 MutexLocker ml(Compile_lock);
584 if (for_dump()) { // Finalize cache
585 finish_write();
586 }
587 _load_buffer = nullptr;
588 if (_C_store_buffer != nullptr) {
589 FREE_C_HEAP_ARRAY(char, _C_store_buffer);
590 _C_store_buffer = nullptr;
591 _store_buffer = nullptr;
592 }
593 if (_table != nullptr) {
594 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
595 delete _table;
596 _table = nullptr;
597 }
598 }
599
600 void AOTCodeCache::Config::record(uint cpu_features_offset) {
601 _flags = 0;
602 #ifdef ASSERT
603 _flags |= debugVM;
604 #endif
605 if (UseCompressedOops) {
606 _flags |= compressedOops;
607 }
608 if (UseCompressedClassPointers) {
609 _flags |= compressedClassPointers;
610 }
611 if (UseTLAB) {
612 _flags |= useTLAB;
613 }
614 if (JavaAssertions::systemClassDefault()) {
615 _flags |= systemClassAssertions;
616 }
617 if (JavaAssertions::userClassDefault()) {
618 _flags |= userClassAssertions;
619 }
620 if (EnableContended) {
621 _flags |= enableContendedPadding;
622 }
623 if (RestrictContended) {
624 _flags |= restrictContendedPadding;
625 }
626 if (PreserveFramePointer) {
627 _flags |= preserveFramePointer;
628 }
629 _codeCacheSize = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
630 _compressedOopShift = CompressedOops::shift();
631 _compressedOopBase = CompressedOops::base();
632 _compressedKlassShift = CompressedKlassPointers::shift();
633 _compressedKlassBase = CompressedKlassPointers::base();
634 _contendedPaddingWidth = ContendedPaddingWidth;
635 _objectAlignment = ObjectAlignmentInBytes;
636 _gcCardSize = GCCardSizeInBytes;
637 _gc = (uint)Universe::heap()->kind();
638 _cpu_features_offset = cpu_features_offset;
639 }
640
641 bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
642 // First checks affect all cached AOT code
643 #ifdef ASSERT
644 if ((_flags & debugVM) == 0) {
645 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
646 return false;
647 }
648 #else
649 if ((_flags & debugVM) != 0) {
650 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
651 return false;
652 }
653 #endif
654
655 size_t codeCacheSize = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
656 if (_codeCacheSize != codeCacheSize) {
657 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CodeCache size = %dKb vs current %dKb", (int)(_codeCacheSize/K), (int)(codeCacheSize/K));
658 return false;
659 }
660
661 CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
662 if (aot_gc != Universe::heap()->kind()) {
663 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
664 return false;
665 }
666
667 if (_gcCardSize != (uint)GCCardSizeInBytes) {
668 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with GCCardSizeInBytes = %d vs current %d", _gcCardSize, GCCardSizeInBytes);
669 return false;
670 }
671
672 if (_objectAlignment != (uint)ObjectAlignmentInBytes) {
673 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ObjectAlignmentInBytes = %d vs current %d", _objectAlignment, ObjectAlignmentInBytes);
674 return false;
675 }
676
677 if (((_flags & enableContendedPadding) != 0) != EnableContended) {
678 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with EnableContended = %s vs current %s", (EnableContended ? "false" : "true"), (EnableContended ? "true" : "false"));
679 return false;
680 }
681 if (((_flags & restrictContendedPadding) != 0) != RestrictContended) {
682 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with RestrictContended = %s vs current %s", (RestrictContended ? "false" : "true"), (RestrictContended ? "true" : "false"));
683 return false;
684 }
685 if (_contendedPaddingWidth != (uint)ContendedPaddingWidth) {
686 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ContendedPaddingWidth = %d vs current %d", _contendedPaddingWidth, ContendedPaddingWidth);
687 return false;
688 }
689
690 if (((_flags & preserveFramePointer) != 0) != PreserveFramePointer) {
691 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with PreserveFramePointer = %s vs current %s", (PreserveFramePointer ? "false" : "true"), (PreserveFramePointer ? "true" : "false"));
692 return false;
693 }
694
695 if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
696 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s vs current %s", (UseCompressedClassPointers ? "false" : "true"), (UseCompressedClassPointers ? "true" : "false"));
697 return false;
698 }
699 if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
700 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
701 return false;
702 }
703 if ((_compressedKlassBase == nullptr || CompressedKlassPointers::base() == nullptr) && (_compressedKlassBase != CompressedKlassPointers::base())) {
704 log_debug(aot, codecache, init)("AOT Code Cache disabled: incompatible CompressedKlassPointers::base(): %p vs current %p", _compressedKlassBase, CompressedKlassPointers::base());
705 return false;
706 }
707
708 if (((_flags & compressedOops) != 0) != UseCompressedOops) {
709 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s vs current %s", (UseCompressedOops ? "false" : "true"), (UseCompressedOops ? "true" : "false"));
710 return false;
711 }
712 if (_compressedOopShift != (uint)CompressedOops::shift()) {
713 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
714 return false;
715 }
716 if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
717 log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
718 return false;
719 }
720
721 LogStreamHandle(Debug, aot, codecache, init) log;
722 if (log.is_enabled()) {
723 log.print_cr("Available CPU features: %s", VM_Version::features_string());
724 }
725
726 uint offset = _cpu_features_offset;
727 uint cpu_features_size = *(uint *)cache->addr(offset);
728 assert(cpu_features_size == (uint)VM_Version::cpu_features_size(), "must be");
729 offset += sizeof(uint);
730
731 void* cached_cpu_features_buffer = (void *)cache->addr(offset);
732 if (log.is_enabled()) {
733 ResourceMark rm; // required for stringStream::as_string()
734 stringStream ss;
735 VM_Version::get_cpu_features_name(cached_cpu_features_buffer, ss);
736 log.print_cr("CPU features recorded in AOTCodeCache: %s", ss.as_string());
737 }
738
739 if (AOTCodeCPUFeatureCheck && !VM_Version::supports_features(cached_cpu_features_buffer)) {
740 if (log.is_enabled()) {
741 ResourceMark rm; // required for stringStream::as_string()
742 stringStream ss;
743 VM_Version::get_missing_features_name(cached_cpu_features_buffer, ss);
744 log.print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
745 }
746 return false;
747 }
748
749 // Next affects only AOT nmethod
750 if (((_flags & systemClassAssertions) != 0) != JavaAssertions::systemClassDefault()) {
751 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::systemClassDefault() = %s vs current %s", (JavaAssertions::systemClassDefault() ? "disabled" : "enabled"), (JavaAssertions::systemClassDefault() ? "enabled" : "disabled"));
752 FLAG_SET_ERGO(AOTCodeCaching, false);
753 }
754 if (((_flags & userClassAssertions) != 0) != JavaAssertions::userClassDefault()) {
755 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::userClassDefault() = %s vs current %s", (JavaAssertions::userClassDefault() ? "disabled" : "enabled"), (JavaAssertions::userClassDefault() ? "enabled" : "disabled"));
756 FLAG_SET_ERGO(AOTCodeCaching, false);
757 }
758
759 return true;
760 }
761
762 bool AOTCodeCache::Header::verify(uint load_size) const {
763 if (_version != AOT_CODE_VERSION) {
764 log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
765 return false;
766 }
767 if (load_size < _cache_size) {
768 log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
769 return false;
770 }
771 return true;
772 }
773
774 volatile int AOTCodeCache::_nmethod_readers = 0;
775
776 AOTCodeCache* AOTCodeCache::open_for_use() {
777 if (AOTCodeCache::is_on_for_use()) {
778 return AOTCodeCache::cache();
779 }
780 return nullptr;
781 }
782
783 AOTCodeCache* AOTCodeCache::open_for_dump() {
784 if (AOTCodeCache::is_on_for_dump()) {
785 AOTCodeCache* cache = AOTCodeCache::cache();
786 cache->clear_lookup_failed(); // Reset bit
787 return cache;
788 }
789 return nullptr;
790 }
791
792 bool AOTCodeCache::is_address_in_aot_cache(address p) {
793 AOTCodeCache* cache = open_for_use();
794 if (cache == nullptr) {
795 return false;
796 }
797 if ((p >= (address)cache->cache_buffer()) &&
798 (p < (address)(cache->cache_buffer() + cache->load_size()))) {
799 return true;
800 }
801 return false;
802 }
803
804 static void copy_bytes(const char* from, address to, uint size) {
805 assert((int)size > 0, "sanity");
806 memcpy(to, from, size);
807 log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
808 }
809
810 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry, CompileTask* task) {
811 _cache = cache;
812 _entry = entry;
813 _load_buffer = cache->cache_buffer();
814 _read_position = 0;
815 if (task != nullptr) {
816 _compile_id = task->compile_id();
817 _comp_level = task->comp_level();
818 _preload = task->preload();
819 } else {
820 _compile_id = 0;
821 _comp_level = 0;
822 _preload = false;
823 }
824 _lookup_failed = false;
825 }
826
827 void AOTCodeReader::set_read_position(uint pos) {
828 if (pos == _read_position) {
829 return;
830 }
831 assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
832 _read_position = pos;
833 }
834
835 bool AOTCodeCache::set_write_position(uint pos) {
836 if (pos == _write_position) {
837 return true;
838 }
839 if (_store_size < _write_position) {
840 _store_size = _write_position; // Adjust during write
841 }
842 assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
843 _write_position = pos;
844 return true;
845 }
846
847 static char align_buffer[256] = { 0 };
848
849 bool AOTCodeCache::align_write() {
850 // We are not executing code from cache - we copy it by bytes first.
851 // No need for big alignment (or at all).
852 uint padding = DATA_ALIGNMENT - (_write_position & (DATA_ALIGNMENT - 1));
853 if (padding == DATA_ALIGNMENT) {
854 return true;
855 }
856 uint n = write_bytes((const void*)&align_buffer, padding);
857 if (n != padding) {
858 return false;
859 }
860 log_trace(aot, codecache)("Adjust write alignment in AOT Code Cache");
861 return true;
862 }
863
864 // Check to see if AOT code cache has required space to store "nbytes" of data
865 address AOTCodeCache::reserve_bytes(uint nbytes) {
866 assert(for_dump(), "Code Cache file is not created");
867 uint new_position = _write_position + nbytes;
868 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
869 log_warning(aot,codecache)("Failed to ensure %d bytes at offset %d in AOT Code Cache. Increase AOTCodeMaxSize.",
870 nbytes, _write_position);
871 set_failed();
872 report_store_failure();
873 return nullptr;
874 }
875 address buffer = (address)(_store_buffer + _write_position);
876 log_trace(aot, codecache)("Reserved %d bytes at offset %d in AOT Code Cache", nbytes, _write_position);
877 _write_position += nbytes;
878 if (_store_size < _write_position) {
879 _store_size = _write_position;
880 }
881 return buffer;
882 }
883
884 uint AOTCodeCache::write_bytes(const void* buffer, uint nbytes) {
885 assert(for_dump(), "Code Cache file is not created");
886 if (nbytes == 0) {
887 return 0;
888 }
889 uint new_position = _write_position + nbytes;
890 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
891 log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.",
892 nbytes, _write_position);
893 set_failed();
894 report_store_failure();
895 return 0;
896 }
897 copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
898 log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
899 _write_position += nbytes;
900 if (_store_size < _write_position) {
901 _store_size = _write_position;
902 }
903 return nbytes;
904 }
905
906 AOTCodeEntry* AOTCodeCache::find_code_entry(const methodHandle& method, uint comp_level) {
907 assert(is_using_code(), "AOT code caching should be enabled");
908 if (!method->in_aot_cache()) {
909 return nullptr;
910 }
911 switch (comp_level) {
912 case CompLevel_simple:
913 if ((DisableAOTCode & (1 << 0)) != 0) {
914 return nullptr;
915 }
916 break;
917 case CompLevel_limited_profile:
918 if ((DisableAOTCode & (1 << 1)) != 0) {
919 return nullptr;
920 }
921 break;
922 case CompLevel_full_optimization:
923 if ((DisableAOTCode & (1 << 2)) != 0) {
924 return nullptr;
925 }
926 break;
927
928 default: return nullptr; // Level 1, 2, and 4 only
929 }
930 TraceTime t1("Total time to find AOT code", &_t_totalFind, enable_timers(), false);
931 if (is_on() && _cache->cache_buffer() != nullptr) {
932 uint id = AOTCacheAccess::convert_method_to_offset(method());
933 AOTCodeEntry* entry = _cache->find_entry(AOTCodeEntry::Nmethod, id, comp_level);
934 if (entry == nullptr) {
935 LogStreamHandle(Info, aot, codecache, nmethod) log;
936 if (log.is_enabled()) {
937 ResourceMark rm;
938 const char* target_name = method->name_and_sig_as_C_string();
939 log.print("Missing entry for '%s' (comp_level %d, id: " UINT32_FORMAT_X_0 ")", target_name, (uint)comp_level, id);
940 }
941 #ifdef ASSERT
942 } else {
943 ResourceMark rm;
944 assert(method() == entry->method(), "AOTCodeCache: saved nmethod's method %p (name: %s id: " UINT32_FORMAT_X_0
945 ") is different from the method %p (name: %s, id: " UINT32_FORMAT_X_0 " being looked up" ,
946 entry->method(), entry->method()->name_and_sig_as_C_string(), entry->id(), method(), method()->name_and_sig_as_C_string(), id);
947 #endif
948 }
949
950 DirectiveSet* directives = DirectivesStack::getMatchingDirective(method, nullptr);
951 if (directives->IgnorePrecompiledOption) {
952 LogStreamHandle(Info, aot, codecache, compilation) log;
953 if (log.is_enabled()) {
954 log.print("Ignore AOT code entry on level %d for ", comp_level);
955 method->print_value_on(&log);
956 }
957 return nullptr;
958 }
959
960 return entry;
961 }
962 return nullptr;
963 }
964
965 Method* AOTCodeEntry::method() {
966 assert(_kind == Nmethod, "invalid kind %d", _kind);
967 assert(AOTCodeCache::is_on_for_use(), "must be");
968 return AOTCacheAccess::convert_offset_to_method(_id);
969 }
970
971 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
972 return (void*)(cache->add_entry());
973 }
974
975 static bool check_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level, AOTCodeEntry* entry) {
976 if (entry->kind() == kind) {
977 assert(entry->id() == id, "sanity");
978 if (kind != AOTCodeEntry::Nmethod || // addapters and stubs have only one version
979 // Look only for normal AOT code entry, preload code is handled separately
980 (!entry->not_entrant() && !entry->has_clinit_barriers() && (entry->comp_level() == comp_level))) {
981 return true; // Found
982 }
983 }
984 return false;
985 }
986
987 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level) {
988 assert(_for_use, "sanity");
989 uint count = _load_header->entries_count();
990 if (_load_entries == nullptr) {
991 // Read it
992 _search_entries = (uint*)addr(_load_header->search_table_offset()); // [id, index]
993 _load_entries = (AOTCodeEntry*)addr(_load_header->entries_offset());
994 log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
995 }
996 // Binary search
997 int l = 0;
998 int h = count - 1;
999 while (l <= h) {
1000 int mid = (l + h) >> 1;
1001 int ix = mid * 2;
1002 uint is = _search_entries[ix];
1003 if (is == id) {
1004 int index = _search_entries[ix + 1];
1005 AOTCodeEntry* entry = &(_load_entries[index]);
1006 if (check_entry(kind, id, comp_level, entry)) {
1007 return entry; // Found
1008 }
1009 // Leaner search around
1010 for (int i = mid - 1; i >= l; i--) { // search back
1011 ix = i * 2;
1012 is = _search_entries[ix];
1013 if (is != id) {
1014 break;
1015 }
1016 index = _search_entries[ix + 1];
1017 AOTCodeEntry* entry = &(_load_entries[index]);
1018 if (check_entry(kind, id, comp_level, entry)) {
1019 return entry; // Found
1020 }
1021 }
1022 for (int i = mid + 1; i <= h; i++) { // search forward
1023 ix = i * 2;
1024 is = _search_entries[ix];
1025 if (is != id) {
1026 break;
1027 }
1028 index = _search_entries[ix + 1];
1029 AOTCodeEntry* entry = &(_load_entries[index]);
1030 if (check_entry(kind, id, comp_level, entry)) {
1031 return entry; // Found
1032 }
1033 }
1034 break; // No match found
1035 } else if (is < id) {
1036 l = mid + 1;
1037 } else {
1038 h = mid - 1;
1039 }
1040 }
1041 return nullptr;
1042 }
1043
1044 void AOTCodeCache::invalidate_entry(AOTCodeEntry* entry) {
1045 assert(entry!= nullptr, "all entries should be read already");
1046 if (entry->not_entrant()) {
1047 return; // Someone invalidated it already
1048 }
1049 #ifdef ASSERT
1050 assert(_load_entries != nullptr, "sanity");
1051 {
1052 uint name_offset = entry->offset() + entry->name_offset();
1053 const char* name = _load_buffer + name_offset;;
1054 uint level = entry->comp_level();
1055 uint comp_id = entry->comp_id();
1056 bool for_preload = entry->for_preload();
1057 bool clinit_brs = entry->has_clinit_barriers();
1058 log_info(aot, codecache, nmethod)("Invalidating entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1059 name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1060 }
1061 assert(entry->is_loaded() || entry->for_preload(), "invalidate only AOT code in use or a preload code");
1062 bool found = false;
1063 uint i = 0;
1064 uint count = 0;
1065 if (entry->for_preload()) {
1066 count = _load_header->preload_entries_count();
1067 AOTCodeEntry* preload_entry = (AOTCodeEntry*)addr(_load_header->preload_entries_offset());
1068 for (; i < count; i++) {
1069 if (entry == &preload_entry[i]) {
1070 break;
1071 }
1072 }
1073 } else {
1074 count = _load_header->entries_count();
1075 for(; i < count; i++) {
1076 if (entry == &(_load_entries[i])) {
1077 break;
1078 }
1079 }
1080 }
1081 found = (i < count);
1082 assert(found, "entry should exist");
1083 #endif
1084 entry->set_not_entrant();
1085 uint name_offset = entry->offset() + entry->name_offset();
1086 const char* name = _load_buffer + name_offset;;
1087 uint level = entry->comp_level();
1088 uint comp_id = entry->comp_id();
1089 bool for_preload = entry->for_preload();
1090 bool clinit_brs = entry->has_clinit_barriers();
1091 log_info(aot, codecache, nmethod)("Invalidated entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1092 name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1093
1094 if (!for_preload && (entry->comp_level() == CompLevel_full_optimization)) {
1095 // Invalidate preload code if normal AOT C2 code is invalidated,
1096 // most likely because some dependencies changed during run.
1097 // We can still use normal AOT code if preload code is
1098 // invalidated - normal AOT code has less restrictions.
1099 Method* method = entry->method();
1100 AOTCodeEntry* preload_entry = method->aot_code_entry();
1101 if (preload_entry != nullptr) {
1102 assert(preload_entry->for_preload(), "expecting only such entries here");
1103 invalidate_entry(preload_entry);
1104 }
1105 }
1106 }
1107
1108 static int uint_cmp(const void *i, const void *j) {
1109 uint a = *(uint *)i;
1110 uint b = *(uint *)j;
1111 return a > b ? 1 : a < b ? -1 : 0;
1112 }
1113
1114 void AOTCodeCache::store_cpu_features(char*& buffer, uint buffer_size) {
1115 uint* size_ptr = (uint *)buffer;
1116 *size_ptr = buffer_size;
1117 buffer += sizeof(uint);
1118
1119 VM_Version::store_cpu_features(buffer);
1120 log_debug(aot, codecache, exit)("CPU features recorded in AOTCodeCache: %s", VM_Version::features_string());
1121 buffer += buffer_size;
1122 buffer = align_up(buffer, DATA_ALIGNMENT);
1123 }
1124
1125 bool AOTCodeCache::finish_write() {
1126 if (!align_write()) {
1127 return false;
1128 }
1129 uint strings_offset = _write_position;
1130 int strings_count = store_strings();
1131 if (strings_count < 0) {
1132 return false;
1133 }
1134 if (!align_write()) {
1135 return false;
1136 }
1137 uint strings_size = _write_position - strings_offset;
1138
1139 uint code_count = _store_entries_cnt;
1140 if (code_count > 0) {
1141 _aot_code_directory = CachedCodeDirectory::create();
1142 assert(_aot_code_directory != nullptr, "Sanity check");
1143
1144 uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT);
1145 uint search_count = code_count * 2;
1146 uint search_size = search_count * sizeof(uint);
1147 uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
1148 // _write_position should include code and strings
1149 uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
1150 uint cpu_features_size = VM_Version::cpu_features_size();
1151 uint total_cpu_features_size = sizeof(uint) + cpu_features_size; // sizeof(uint) to store cpu_features_size
1152 uint total_size = _write_position + header_size + code_alignment +
1153 search_size + entries_size +
1154 align_up(total_cpu_features_size, DATA_ALIGNMENT);
1155 assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size());
1156
1157 // Allocate in AOT Cache buffer
1158 char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT);
1159 char* start = align_up(buffer, DATA_ALIGNMENT);
1160 char* current = start + header_size; // Skip header
1161
1162 uint cpu_features_offset = current - start;
1163 store_cpu_features(current, cpu_features_size);
1164 assert(is_aligned(current, DATA_ALIGNMENT), "sanity check");
1165 assert(current < start + total_size, "sanity check");
1166
1167 // Create ordered search table for entries [id, index];
1168 uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
1169
1170 AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
1171 AOTCodeStats stats;
1172 uint max_size = 0;
1173 // AOTCodeEntry entries were allocated in reverse in store buffer.
1174 // Process them in reverse order to cache first code first.
1175
1176 // Store AOTCodeEntry-s for preload code
1177 current = align_up(current, DATA_ALIGNMENT);
1178 uint preload_entries_cnt = 0;
1179 uint preload_entries_offset = current - start;
1180 AOTCodeEntry* preload_entries = (AOTCodeEntry*)current;
1181 for (int i = code_count - 1; i >= 0; i--) {
1182 AOTCodeEntry* entry = &entries_address[i];
1183 if (entry->load_fail()) {
1184 continue;
1185 }
1186 if (entry->for_preload()) {
1187 if (entry->not_entrant()) {
1188 // Skip not entrant preload code:
1189 // we can't pre-load code which may have failing dependencies.
1190 log_info(aot, codecache, exit)("Skip not entrant preload code comp_id: %d, comp_level: %d, hash: " UINT32_FORMAT_X_0 "%s",
1191 entry->comp_id(), entry->comp_level(), entry->id(), (entry->has_clinit_barriers() ? ", has clinit barriers" : ""));
1192 } else {
1193 copy_bytes((const char*)entry, (address)current, sizeof(AOTCodeEntry));
1194 stats.collect_entry_stats(entry);
1195 current += sizeof(AOTCodeEntry);
1196 preload_entries_cnt++;
1197 }
1198 }
1199 }
1200
1201 // Now write the data for preload AOTCodeEntry
1202 for (int i = 0; i < (int)preload_entries_cnt; i++) {
1203 AOTCodeEntry* entry = &preload_entries[i];
1204 uint size = align_up(entry->size(), DATA_ALIGNMENT);
1205 if (size > max_size) {
1206 max_size = size;
1207 }
1208 copy_bytes((_store_buffer + entry->offset()), (address)current, size);
1209 entry->set_offset(current - start); // New offset
1210 current += size;
1211 }
1212
1213 current = align_up(current, DATA_ALIGNMENT);
1214 uint entries_count = 0;
1215 uint new_entries_offset = current - start;
1216 AOTCodeEntry* code_entries = (AOTCodeEntry*)current;
1217 // Now scan normal entries
1218 for (int i = code_count - 1; i >= 0; i--) {
1219 AOTCodeEntry* entry = &entries_address[i];
1220 if (entry->load_fail() || entry->for_preload()) {
1221 continue;
1222 }
1223 if (entry->not_entrant()) {
1224 log_info(aot, codecache, exit)("Not entrant new entry comp_id: %d, comp_level: %d, hash: " UINT32_FORMAT_X_0 "%s",
1225 entry->comp_id(), entry->comp_level(), entry->id(), (entry->has_clinit_barriers() ? ", has clinit barriers" : ""));
1226 entry->set_entrant(); // Reset
1227 }
1228 copy_bytes((const char*)entry, (address)current, sizeof(AOTCodeEntry));
1229 stats.collect_entry_stats(entry);
1230 current += sizeof(AOTCodeEntry);
1231 search[entries_count*2 + 0] = entry->id();
1232 search[entries_count*2 + 1] = entries_count;
1233 entries_count++;
1234 }
1235
1236 // Now write the data for normal AOTCodeEntry
1237 for (int i = 0; i < (int)entries_count; i++) {
1238 AOTCodeEntry* entry = &code_entries[i];
1239 uint size = align_up(entry->size(), DATA_ALIGNMENT);
1240 if (size > max_size) {
1241 max_size = size;
1242 }
1243 copy_bytes((_store_buffer + entry->offset()), (address)current, size);
1244 entry->set_offset(current - start); // New offset
1245 current += size;
1246 }
1247
1248 if (preload_entries_cnt == 0 && entries_count == 0) {
1249 log_info(aot, codecache, exit)("AOT Code Cache was not created: no entries");
1250 FREE_C_HEAP_ARRAY(uint, search);
1251 return true; // Nothing to write
1252 }
1253 uint total_entries_cnt = preload_entries_cnt + entries_count;
1254 assert(total_entries_cnt <= code_count, "%d > %d", total_entries_cnt, code_count);
1255 // Write strings
1256 if (strings_count > 0) {
1257 copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
1258 strings_offset = (current - start); // New offset
1259 current += strings_size;
1260 }
1261
1262 uint search_table_offset = current - start;
1263 // Sort and store search table
1264 qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
1265 search_size = 2 * entries_count * sizeof(uint);
1266 copy_bytes((const char*)search, (address)current, search_size);
1267 FREE_C_HEAP_ARRAY(uint, search);
1268 current += search_size;
1269
1270 log_stats_on_exit(stats);
1271
1272 uint size = (current - start);
1273 assert(size <= total_size, "%d > %d", size , total_size);
1274 log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
1275
1276 // Finalize header
1277 AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
1278 header->init(size, (uint)strings_count, strings_offset,
1279 entries_count, search_table_offset, new_entries_offset,
1280 preload_entries_cnt, preload_entries_offset,
1281 stats.entry_count(AOTCodeEntry::Adapter), stats.entry_count(AOTCodeEntry::SharedBlob),
1282 stats.entry_count(AOTCodeEntry::C1Blob), stats.entry_count(AOTCodeEntry::C2Blob),
1283 stats.entry_count(AOTCodeEntry::Stub), cpu_features_offset);
1284
1285 log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", total_entries_cnt);
1286
1287 _aot_code_directory->set_aot_code_data(size, start);
1288 }
1289 return true;
1290 }
1291
1292 //------------------Store/Load AOT code ----------------------
1293
1294 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1295 AOTCodeCache* cache = open_for_dump();
1296 if (cache == nullptr) {
1297 return false;
1298 }
1299 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1300
1301 if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
1302 return false;
1303 }
1304 if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
1305 return false;
1306 }
1307 log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1308
1309 #ifdef ASSERT
1310 LogStreamHandle(Trace, aot, codecache, stubs) log;
1311 if (log.is_enabled()) {
1312 FlagSetting fs(PrintRelocations, true);
1313 blob.print_on(&log);
1314 }
1315 #endif
1316 // we need to take a lock to prevent race between compiler threads generating AOT code
1317 // and the main thread generating adapter
1318 MutexLocker ml(Compile_lock);
1319 if (!is_on()) {
1320 return false; // AOT code cache was already dumped and closed.
1321 }
1322 if (!cache->align_write()) {
1323 return false;
1324 }
1325 uint entry_position = cache->_write_position;
1326
1327 // Write name
1328 uint name_offset = cache->_write_position - entry_position;
1329 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1330 uint n = cache->write_bytes(name, name_size);
1331 if (n != name_size) {
1332 return false;
1333 }
1334
1335 // Write CodeBlob
1336 if (!cache->align_write()) {
1337 return false;
1338 }
1339 uint blob_offset = cache->_write_position - entry_position;
1340 address archive_buffer = cache->reserve_bytes(blob.size());
1341 if (archive_buffer == nullptr) {
1342 return false;
1343 }
1344 CodeBlob::archive_blob(&blob, archive_buffer);
1345
1346 uint reloc_data_size = blob.relocation_size();
1347 n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size);
1348 if (n != reloc_data_size) {
1349 return false;
1350 }
1351
1352 bool has_oop_maps = false;
1353 if (blob.oop_maps() != nullptr) {
1354 if (!cache->write_oop_map_set(blob)) {
1355 return false;
1356 }
1357 has_oop_maps = true;
1358 }
1359
1360 #ifndef PRODUCT
1361 // Write asm remarks
1362 if (!cache->write_asm_remarks(blob.asm_remarks(), /* use_string_table */ true)) {
1363 return false;
1364 }
1365 if (!cache->write_dbg_strings(blob.dbg_strings(), /* use_string_table */ true)) {
1366 return false;
1367 }
1368 #endif /* PRODUCT */
1369
1370 if (!cache->write_relocations(blob)) {
1371 if (!cache->failed()) {
1372 // We may miss an address in AOT table - skip this code blob.
1373 cache->set_write_position(entry_position);
1374 }
1375 return false;
1376 }
1377
1378 uint entry_size = cache->_write_position - entry_position;
1379 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
1380 entry_position, entry_size, name_offset, name_size,
1381 blob_offset, has_oop_maps, blob.content_begin());
1382 log_debug(aot, codecache, stubs)("Wrote code blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1383 return true;
1384 }
1385
1386 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) {
1387 assert(AOTCodeEntry::is_blob(entry_kind),
1388 "wrong entry kind for blob id %s", StubInfo::name(id));
1389 return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id));
1390 }
1391
1392 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1393 AOTCodeCache* cache = open_for_use();
1394 if (cache == nullptr) {
1395 return nullptr;
1396 }
1397 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1398
1399 if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
1400 return nullptr;
1401 }
1402 if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
1403 return nullptr;
1404 }
1405 log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1406
1407 AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
1408 if (entry == nullptr) {
1409 return nullptr;
1410 }
1411 AOTCodeReader reader(cache, entry, nullptr);
1412 CodeBlob* blob = reader.compile_code_blob(name);
1413
1414 log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
1415 (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
1416 return blob;
1417 }
1418
1419 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
1420 assert(AOTCodeEntry::is_blob(entry_kind),
1421 "wrong entry kind for blob id %s", StubInfo::name(id));
1422 return load_code_blob(entry_kind, (uint)id, StubInfo::name(id));
1423 }
1424
1425 CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
1426 uint entry_position = _entry->offset();
1427
1428 // Read name
1429 uint name_offset = entry_position + _entry->name_offset();
1430 uint name_size = _entry->name_size(); // Includes '/0'
1431 const char* stored_name = addr(name_offset);
1432
1433 if (strncmp(stored_name, name, (name_size - 1)) != 0) {
1434 log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'",
1435 stored_name, name);
1436 set_lookup_failed(); // Skip this blob
1437 return nullptr;
1438 }
1439
1440 // Read archived code blob
1441 uint offset = entry_position + _entry->code_offset();
1442 CodeBlob* archived_blob = (CodeBlob*)addr(offset);
1443 offset += archived_blob->size();
1444
1445 address reloc_data = (address)addr(offset);
1446 offset += archived_blob->relocation_size();
1447 set_read_position(offset);
1448
1449 ImmutableOopMapSet* oop_maps = nullptr;
1450 if (_entry->has_oop_maps()) {
1451 oop_maps = read_oop_map_set();
1452 }
1453
1454 CodeBlob* code_blob = CodeBlob::create(archived_blob,
1455 stored_name,
1456 reloc_data,
1457 oop_maps
1458 );
1459 if (code_blob == nullptr) { // no space left in CodeCache
1460 return nullptr;
1461 }
1462
1463 #ifndef PRODUCT
1464 code_blob->asm_remarks().init();
1465 read_asm_remarks(code_blob->asm_remarks(), /* use_string_table */ true);
1466 code_blob->dbg_strings().init();
1467 read_dbg_strings(code_blob->dbg_strings(), /* use_string_table */ true);
1468 #endif // PRODUCT
1469
1470 fix_relocations(code_blob);
1471
1472 #ifdef ASSERT
1473 LogStreamHandle(Trace, aot, codecache, stubs) log;
1474 if (log.is_enabled()) {
1475 FlagSetting fs(PrintRelocations, true);
1476 code_blob->print_on(&log);
1477 }
1478 #endif
1479 return code_blob;
1480 }
1481
1482 bool AOTCodeCache::store_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1483 if (!is_dumping_stub()) {
1484 return false;
1485 }
1486 AOTCodeCache* cache = open_for_dump();
1487 if (cache == nullptr) {
1488 return false;
1489 }
1490 log_info(aot, codecache, stubs)("Writing stub '%s' id:%d to AOT Code Cache", name, (int)id);
1491 if (!cache->align_write()) {
1492 return false;
1493 }
1494 #ifdef ASSERT
1495 CodeSection* cs = cgen->assembler()->code_section();
1496 if (cs->has_locs()) {
1497 uint reloc_count = cs->locs_count();
1498 tty->print_cr("======== write stubs code section relocations [%d]:", reloc_count);
1499 // Collect additional data
1500 RelocIterator iter(cs);
1501 while (iter.next()) {
1502 switch (iter.type()) {
1503 case relocInfo::none:
1504 break;
1505 default: {
1506 iter.print_current_on(tty);
1507 fatal("stub's relocation %d unimplemented", (int)iter.type());
1508 break;
1509 }
1510 }
1511 }
1512 }
1513 #endif
1514 uint entry_position = cache->_write_position;
1515
1516 // Write code
1517 uint code_offset = 0;
1518 uint code_size = cgen->assembler()->pc() - start;
1519 uint n = cache->write_bytes(start, code_size);
1520 if (n != code_size) {
1521 return false;
1522 }
1523 // Write name
1524 uint name_offset = cache->_write_position - entry_position;
1525 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1526 n = cache->write_bytes(name, name_size);
1527 if (n != name_size) {
1528 return false;
1529 }
1530 uint entry_size = cache->_write_position - entry_position;
1531 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
1532 code_offset, code_size,
1533 AOTCodeEntry::Stub, (uint32_t)id);
1534 log_info(aot, codecache, stubs)("Wrote stub '%s' id:%d to AOT Code Cache", name, (int)id);
1535 return true;
1536 }
1537
1538 bool AOTCodeCache::load_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1539 if (!is_using_stub()) {
1540 return false;
1541 }
1542 assert(start == cgen->assembler()->pc(), "wrong buffer");
1543 AOTCodeCache* cache = open_for_use();
1544 if (cache == nullptr) {
1545 return false;
1546 }
1547 AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Stub, (uint)id);
1548 if (entry == nullptr) {
1549 return false;
1550 }
1551 uint entry_position = entry->offset();
1552 // Read name
1553 uint name_offset = entry->name_offset() + entry_position;
1554 uint name_size = entry->name_size(); // Includes '/0'
1555 const char* saved_name = cache->addr(name_offset);
1556 if (strncmp(name, saved_name, (name_size - 1)) != 0) {
1557 log_warning(aot, codecache)("Saved stub's name '%s' is different from '%s' for id:%d", saved_name, name, (int)id);
1558 cache->set_failed();
1559 report_load_failure();
1560 return false;
1561 }
1562 log_info(aot, codecache, stubs)("Reading stub '%s' id:%d from AOT Code Cache", name, (int)id);
1563 // Read code
1564 uint code_offset = entry->code_offset() + entry_position;
1565 uint code_size = entry->code_size();
1566 copy_bytes(cache->addr(code_offset), start, code_size);
1567 cgen->assembler()->code_section()->set_end(start + code_size);
1568 log_info(aot, codecache, stubs)("Read stub '%s' id:%d from AOT Code Cache", name, (int)id);
1569 return true;
1570 }
1571
1572 AOTCodeEntry* AOTCodeCache::store_nmethod(nmethod* nm, AbstractCompiler* compiler, bool for_preload) {
1573 if (!is_dumping_code()) {
1574 return nullptr;
1575 }
1576 assert(CDSConfig::is_dumping_aot_code(), "should be called only when allowed");
1577 AOTCodeCache* cache = open_for_dump();
1578 precond(cache != nullptr);
1579 precond(!nm->is_osr_method()); // AOT compilation is requested only during AOT cache assembly phase
1580 if (!compiler->is_c1() && !compiler->is_c2()) {
1581 // Only c1 and c2 compilers
1582 return nullptr;
1583 }
1584 int comp_level = nm->comp_level();
1585 if (comp_level == CompLevel_full_profile) {
1586 // Do not cache C1 compiles with full profile i.e. tier3
1587 return nullptr;
1588 }
1589 assert(comp_level == CompLevel_simple || comp_level == CompLevel_limited_profile || comp_level == CompLevel_full_optimization, "must be");
1590
1591 TraceTime t1("Total time to store AOT code", &_t_totalStore, enable_timers(), false);
1592 AOTCodeEntry* entry = nullptr;
1593 entry = cache->write_nmethod(nm, for_preload);
1594 if (entry == nullptr) {
1595 log_info(aot, codecache, nmethod)("%d (L%d): nmethod store attempt failed", nm->compile_id(), comp_level);
1596 }
1597 return entry;
1598 }
1599
1600 AOTCodeEntry* AOTCodeCache::write_nmethod(nmethod* nm, bool for_preload) {
1601 AOTCodeCache* cache = open_for_dump();
1602 assert(cache != nullptr, "sanity check");
1603 assert(!nm->has_clinit_barriers() || (ClassInitBarrierMode > 0), "sanity");
1604 uint comp_id = nm->compile_id();
1605 uint comp_level = nm->comp_level();
1606 Method* method = nm->method();
1607 if (!AOTCacheAccess::can_generate_aot_code(method)) {
1608 ResourceMark rm;
1609 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' for AOT%s compile: not in AOT cache", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), (for_preload ? " preload" : ""));
1610 assert(AOTCacheAccess::can_generate_aot_code(method), "sanity");
1611 return nullptr;
1612 }
1613 InstanceKlass* holder = method->method_holder();
1614 bool builtin_loader = holder->class_loader_data()->is_builtin_class_loader_data();
1615 if (!builtin_loader) {
1616 ResourceMark rm;
1617 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' loaded by custom class loader %s", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), holder->class_loader_data()->loader_name());
1618 assert(builtin_loader, "sanity");
1619 return nullptr;
1620 }
1621
1622 _for_preload = for_preload;
1623 _has_clinit_barriers = nm->has_clinit_barriers();
1624
1625 if (!align_write()) {
1626 return nullptr;
1627 }
1628
1629 uint entry_position = _write_position;
1630
1631 // Write name
1632 uint name_offset = 0;
1633 uint name_size = 0;
1634 uint id = 0;
1635 uint n;
1636 {
1637 ResourceMark rm;
1638 const char* name = method->name_and_sig_as_C_string();
1639 log_info(aot, codecache, nmethod)("%d (L%d): Writing nmethod '%s' (comp level: %d, %s) to AOT Code Cache",
1640 comp_id, (int)comp_level, name, comp_level,
1641 (nm->has_clinit_barriers() ? ", has clinit barriers" : ""));
1642
1643 LogStreamHandle(Info, aot, codecache, loader) log;
1644 if (log.is_enabled()) {
1645 oop loader = holder->class_loader();
1646 oop domain = holder->protection_domain();
1647 log.print("Holder: ");
1648 holder->print_value_on(&log);
1649 log.print(" loader: ");
1650 if (loader == nullptr) {
1651 log.print("nullptr");
1652 } else {
1653 loader->print_value_on(&log);
1654 }
1655 log.print(" domain: ");
1656 if (domain == nullptr) {
1657 log.print("nullptr");
1658 } else {
1659 domain->print_value_on(&log);
1660 }
1661 log.cr();
1662 }
1663 name_offset = _write_position - entry_position;
1664 name_size = (uint)strlen(name) + 1; // Includes '/0'
1665 n = write_bytes(name, name_size);
1666 if (n != name_size) {
1667 return nullptr;
1668 }
1669 }
1670 id = AOTCacheAccess::delta_from_base_address((address)nm->method());
1671
1672 // Write CodeBlob
1673 if (!cache->align_write()) {
1674 return nullptr;
1675 }
1676 uint blob_offset = cache->_write_position - entry_position;
1677 address archive_buffer = cache->reserve_bytes(nm->size());
1678 if (archive_buffer == nullptr) {
1679 return nullptr;
1680 }
1681 CodeBlob::archive_blob(nm, archive_buffer);
1682
1683 uint reloc_data_size = nm->relocation_size();
1684 n = write_bytes((address)nm->relocation_begin(), reloc_data_size);
1685 if (n != reloc_data_size) {
1686 return nullptr;
1687 }
1688
1689 // Write oops and metadata present in the nmethod's data region
1690 if (!write_oops(nm)) {
1691 if (lookup_failed() && !failed()) {
1692 // Skip this method and reposition file
1693 set_write_position(entry_position);
1694 }
1695 return nullptr;
1696 }
1697 if (!write_metadata(nm)) {
1698 if (lookup_failed() && !failed()) {
1699 // Skip this method and reposition file
1700 set_write_position(entry_position);
1701 }
1702 return nullptr;
1703 }
1704
1705 bool has_oop_maps = false;
1706 if (nm->oop_maps() != nullptr) {
1707 if (!cache->write_oop_map_set(*nm)) {
1708 return nullptr;
1709 }
1710 has_oop_maps = true;
1711 }
1712
1713 uint immutable_data_size = nm->immutable_data_size();
1714 n = write_bytes(nm->immutable_data_begin(), immutable_data_size);
1715 if (n != immutable_data_size) {
1716 return nullptr;
1717 }
1718
1719 JavaThread* thread = JavaThread::current();
1720 HandleMark hm(thread);
1721 GrowableArray<Handle> oop_list;
1722 GrowableArray<Metadata*> metadata_list;
1723
1724 nm->create_reloc_immediates_list(thread, oop_list, metadata_list);
1725 if (!write_nmethod_reloc_immediates(oop_list, metadata_list)) {
1726 if (lookup_failed() && !failed()) {
1727 // Skip this method and reposition file
1728 set_write_position(entry_position);
1729 }
1730 return nullptr;
1731 }
1732
1733 if (!write_relocations(*nm, &oop_list, &metadata_list)) {
1734 return nullptr;
1735 }
1736
1737 #ifndef PRODUCT
1738 if (!cache->write_asm_remarks(nm->asm_remarks(), /* use_string_table */ false)) {
1739 return nullptr;
1740 }
1741 if (!cache->write_dbg_strings(nm->dbg_strings(), /* use_string_table */ false)) {
1742 return nullptr;
1743 }
1744 #endif /* PRODUCT */
1745
1746 uint entry_size = _write_position - entry_position;
1747 AOTCodeEntry* entry = new (this) AOTCodeEntry(AOTCodeEntry::Nmethod, id,
1748 entry_position, entry_size,
1749 name_offset, name_size,
1750 blob_offset, has_oop_maps,
1751 nm->content_begin(), comp_level, comp_id,
1752 nm->has_clinit_barriers(), for_preload);
1753 #ifdef ASSERT
1754 if (nm->has_clinit_barriers() || for_preload) {
1755 assert(for_preload, "sanity");
1756 }
1757 #endif
1758 {
1759 ResourceMark rm;
1760 const char* name = nm->method()->name_and_sig_as_C_string();
1761 log_info(aot, codecache, nmethod)("%d (L%d): Wrote nmethod '%s'%s to AOT Code Cache",
1762 comp_id, (int)comp_level, name, (for_preload ? " (for preload)" : ""));
1763 }
1764 if (VerifyAOTCode) {
1765 return nullptr;
1766 }
1767 return entry;
1768 }
1769
1770 bool AOTCodeCache::load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level) {
1771 if (!is_using_code()) {
1772 return false;
1773 }
1774 AOTCodeCache* cache = open_for_use();
1775 if (cache == nullptr) {
1776 return false;
1777 }
1778 assert(entry_bci == InvocationEntryBci, "unexpected entry_bci=%d", entry_bci);
1779 TraceTime t1("Total time to load AOT code", &_t_totalLoad, enable_timers(), false);
1780 CompileTask* task = env->task();
1781 task->mark_aot_load_start(os::elapsed_counter());
1782 AOTCodeEntry* entry = task->aot_code_entry();
1783 bool preload = task->preload();
1784 assert(entry != nullptr, "sanity");
1785 if (log_is_enabled(Info, aot, codecache, nmethod)) {
1786 VM_ENTRY_MARK;
1787 ResourceMark rm;
1788 methodHandle method(THREAD, target->get_Method());
1789 const char* target_name = method->name_and_sig_as_C_string();
1790 uint id = AOTCacheAccess::convert_method_to_offset(method());
1791 bool clinit_brs = entry->has_clinit_barriers();
1792 log_info(aot, codecache, nmethod)("%d (L%d): %s nmethod '%s' (id: " UINT32_FORMAT_X_0 "%s)",
1793 task->compile_id(), task->comp_level(), (preload ? "Preloading" : "Reading"),
1794 target_name, id, (clinit_brs ? ", has clinit barriers" : ""));
1795 }
1796 ReadingMark rdmk;
1797 if (rdmk.failed()) {
1798 // Cache is closed, cannot touch anything.
1799 return false;
1800 }
1801
1802 AOTCodeReader reader(cache, entry, task);
1803 bool success = reader.compile_nmethod(env, target, compiler);
1804 if (success) {
1805 task->set_num_inlined_bytecodes(entry->num_inlined_bytecodes());
1806 } else {
1807 entry->set_load_fail();
1808 entry->set_not_entrant();
1809 }
1810 task->mark_aot_load_finish(os::elapsed_counter());
1811 return success;
1812 }
1813
1814 bool AOTCodeReader::compile_nmethod(ciEnv* env, ciMethod* target, AbstractCompiler* compiler) {
1815 CompileTask* task = env->task();
1816 AOTCodeEntry* aot_code_entry = (AOTCodeEntry*)_entry;
1817 nmethod* nm = nullptr;
1818
1819 uint entry_position = aot_code_entry->offset();
1820 uint archived_nm_offset = entry_position + aot_code_entry->code_offset();
1821 nmethod* archived_nm = (nmethod*)addr(archived_nm_offset);
1822 set_read_position(archived_nm_offset + archived_nm->size());
1823
1824 OopRecorder* oop_recorder = new OopRecorder(env->arena());
1825 env->set_oop_recorder(oop_recorder);
1826
1827 uint offset;
1828
1829 offset = read_position();
1830 address reloc_data = (address)addr(offset);
1831 offset += archived_nm->relocation_size();
1832 set_read_position(offset);
1833
1834 // Read oops and metadata
1835 VM_ENTRY_MARK
1836 GrowableArray<Handle> oop_list;
1837 GrowableArray<Metadata*> metadata_list;
1838
1839 if (!read_oop_metadata_list(THREAD, target, oop_list, metadata_list, oop_recorder)) {
1840 return false;
1841 }
1842
1843 ImmutableOopMapSet* oopmaps = read_oop_map_set();
1844
1845 offset = read_position();
1846 address immutable_data = (address)addr(offset);
1847 offset += archived_nm->immutable_data_size();
1848 set_read_position(offset);
1849
1850 GrowableArray<Handle> reloc_immediate_oop_list;
1851 GrowableArray<Metadata*> reloc_immediate_metadata_list;
1852 if (!read_oop_metadata_list(THREAD, target, reloc_immediate_oop_list, reloc_immediate_metadata_list, nullptr)) {
1853 return false;
1854 }
1855
1856 // Read Dependencies (compressed already)
1857 Dependencies* dependencies = new Dependencies(env);
1858 dependencies->set_content(immutable_data, archived_nm->dependencies_size());
1859 env->set_dependencies(dependencies);
1860
1861 const char* name = addr(entry_position + aot_code_entry->name_offset());
1862
1863 if (VerifyAOTCode) {
1864 return false;
1865 }
1866
1867 TraceTime t1("Total time to register AOT nmethod", &_t_totalRegister, enable_timers(), false);
1868 nm = env->register_aot_method(THREAD,
1869 target,
1870 compiler,
1871 archived_nm,
1872 reloc_data,
1873 oop_list,
1874 metadata_list,
1875 oopmaps,
1876 immutable_data,
1877 reloc_immediate_oop_list,
1878 reloc_immediate_metadata_list,
1879 this);
1880 bool success = task->is_success();
1881 if (success) {
1882 log_info(aot, codecache, nmethod)("%d (L%d): Read nmethod '%s' from AOT Code Cache", compile_id(), comp_level(), name);
1883 #ifdef ASSERT
1884 LogStreamHandle(Debug, aot, codecache, nmethod) log;
1885 if (log.is_enabled()) {
1886 FlagSetting fs(PrintRelocations, true);
1887 nm->print_on(&log);
1888 nm->decode2(&log);
1889 }
1890 #endif
1891 }
1892
1893 return success;
1894 }
1895
1896 bool skip_preload(methodHandle mh) {
1897 if (!mh->method_holder()->is_loaded()) {
1898 return true;
1899 }
1900 DirectiveSet* directives = DirectivesStack::getMatchingDirective(mh, nullptr);
1901 if (directives->DontPreloadOption) {
1902 LogStreamHandle(Info, aot, codecache, init) log;
1903 if (log.is_enabled()) {
1904 log.print("Exclude preloading code for ");
1905 mh->print_value_on(&log);
1906 }
1907 return true;
1908 }
1909 return false;
1910 }
1911
1912 void AOTCodeCache::preload_code(JavaThread* thread) {
1913 if (!is_using_code()) {
1914 return;
1915 }
1916 if ((DisableAOTCode & (1 << 3)) != 0) {
1917 return; // no preloaded code (level 5);
1918 }
1919 _cache->preload_aot_code(thread);
1920 }
1921
1922 void AOTCodeCache::preload_aot_code(TRAPS) {
1923 if (CompilationPolicy::compiler_count(CompLevel_full_optimization) == 0) {
1924 // Since we reuse the CompilerBroker API to install AOT code, we're required to have a JIT compiler for the
1925 // level we want (that is CompLevel_full_optimization).
1926 return;
1927 }
1928 TraceTime t1("Total time to preload AOT code", &_t_totalPreload, enable_timers(), false);
1929 assert(_for_use, "sanity");
1930 uint count = _load_header->entries_count();
1931 uint preload_entries_count = _load_header->preload_entries_count();
1932 if (preload_entries_count > 0) {
1933 log_info(aot, codecache, init)("Load %d preload entries from AOT Code Cache", preload_entries_count);
1934 AOTCodeEntry* preload_entry = (AOTCodeEntry*)addr(_load_header->preload_entries_offset());
1935 uint count = MIN2(preload_entries_count, AOTCodePreloadStop);
1936 for (uint i = AOTCodePreloadStart; i < count; i++) {
1937 AOTCodeEntry* entry = &preload_entry[i];
1938 if (entry->not_entrant()) {
1939 continue;
1940 }
1941 methodHandle mh(THREAD, entry->method());
1942 assert((mh.not_null() && AOTMetaspace::in_aot_cache((address)mh())), "sanity");
1943 if (skip_preload(mh)) {
1944 continue; // Exclude preloading for this method
1945 }
1946 assert(mh->method_holder()->is_loaded(), "");
1947 if (!mh->method_holder()->is_linked()) {
1948 assert(!HAS_PENDING_EXCEPTION, "");
1949 mh->method_holder()->link_class(THREAD);
1950 if (HAS_PENDING_EXCEPTION) {
1951 LogStreamHandle(Info, aot, codecache) log;
1952 if (log.is_enabled()) {
1953 ResourceMark rm;
1954 log.print("Linkage failed for %s: ", mh->method_holder()->external_name());
1955 THREAD->pending_exception()->print_value_on(&log);
1956 if (log_is_enabled(Debug, aot, codecache)) {
1957 THREAD->pending_exception()->print_on(&log);
1958 }
1959 }
1960 CLEAR_PENDING_EXCEPTION;
1961 }
1962 }
1963 if (mh->aot_code_entry() != nullptr) {
1964 // Second C2 compilation of the same method could happen for
1965 // different reasons without marking first entry as not entrant.
1966 continue; // Keep old entry to avoid issues
1967 }
1968 mh->set_aot_code_entry(entry);
1969 CompileBroker::compile_method(mh, InvocationEntryBci, CompLevel_full_optimization, 0, false, CompileTask::Reason_Preload, CHECK);
1970 }
1971 }
1972 }
1973
1974 // ------------ process code and data --------------
1975
1976 // Can't use -1. It is valid value for jump to iteself destination
1977 // used by static call stub: see NativeJump::jump_destination().
1978 #define BAD_ADDRESS_ID -2
1979
1980 bool AOTCodeCache::write_relocations(CodeBlob& code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
1981 GrowableArray<uint> reloc_data;
1982 RelocIterator iter(&code_blob);
1983 LogStreamHandle(Trace, aot, codecache, reloc) log;
1984 while (iter.next()) {
1985 int idx = reloc_data.append(0); // default value
1986 switch (iter.type()) {
1987 case relocInfo::none:
1988 break;
1989 case relocInfo::oop_type: {
1990 oop_Relocation* r = (oop_Relocation*)iter.reloc();
1991 if (r->oop_is_immediate()) {
1992 assert(oop_list != nullptr, "sanity check");
1993 // store index of oop in the reloc immediate oop list
1994 Handle h(JavaThread::current(), r->oop_value());
1995 int oop_idx = oop_list->find(h);
1996 assert(oop_idx != -1, "sanity check");
1997 reloc_data.at_put(idx, (uint)oop_idx);
1998 }
1999 break;
2000 }
2001 case relocInfo::metadata_type: {
2002 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2003 if (r->metadata_is_immediate()) {
2004 assert(metadata_list != nullptr, "sanity check");
2005 // store index of metadata in the reloc immediate metadata list
2006 int metadata_idx = metadata_list->find(r->metadata_value());
2007 assert(metadata_idx != -1, "sanity check");
2008 reloc_data.at_put(idx, (uint)metadata_idx);
2009 }
2010 break;
2011 }
2012 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
2013 case relocInfo::opt_virtual_call_type:
2014 case relocInfo::static_call_type: {
2015 CallRelocation* r = (CallRelocation*)iter.reloc();
2016 address dest = r->destination();
2017 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2018 dest = (address)-1; // do nothing in this case when loading this relocation
2019 }
2020 int id = _table->id_for_address(dest, iter, &code_blob);
2021 if (id == BAD_ADDRESS_ID) {
2022 return false;
2023 }
2024 reloc_data.at_put(idx, id);
2025 break;
2026 }
2027 case relocInfo::trampoline_stub_type: {
2028 address dest = ((trampoline_stub_Relocation*)iter.reloc())->destination();
2029 int id = _table->id_for_address(dest, iter, &code_blob);
2030 if (id == BAD_ADDRESS_ID) {
2031 return false;
2032 }
2033 reloc_data.at_put(idx, id);
2034 break;
2035 }
2036 case relocInfo::static_stub_type:
2037 break;
2038 case relocInfo::runtime_call_type: {
2039 // Record offset of runtime destination
2040 CallRelocation* r = (CallRelocation*)iter.reloc();
2041 address dest = r->destination();
2042 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2043 dest = (address)-1; // do nothing in this case when loading this relocation
2044 }
2045 int id = _table->id_for_address(dest, iter, &code_blob);
2046 if (id == BAD_ADDRESS_ID) {
2047 return false;
2048 }
2049 reloc_data.at_put(idx, id);
2050 break;
2051 }
2052 case relocInfo::runtime_call_w_cp_type:
2053 log_debug(aot, codecache, reloc)("runtime_call_w_cp_type relocation is not implemented");
2054 return false;
2055 case relocInfo::external_word_type: {
2056 // Record offset of runtime target
2057 address target = ((external_word_Relocation*)iter.reloc())->target();
2058 int id = _table->id_for_address(target, iter, &code_blob);
2059 if (id == BAD_ADDRESS_ID) {
2060 return false;
2061 }
2062 reloc_data.at_put(idx, id);
2063 break;
2064 }
2065 case relocInfo::internal_word_type:
2066 break;
2067 case relocInfo::section_word_type:
2068 break;
2069 case relocInfo::poll_type:
2070 break;
2071 case relocInfo::poll_return_type:
2072 break;
2073 case relocInfo::post_call_nop_type:
2074 break;
2075 case relocInfo::entry_guard_type:
2076 break;
2077 default:
2078 log_debug(aot, codecache, reloc)("relocation %d unimplemented", (int)iter.type());
2079 return false;
2080 break;
2081 }
2082 if (log.is_enabled()) {
2083 iter.print_current_on(&log);
2084 }
2085 }
2086
2087 // Write additional relocation data: uint per relocation
2088 // Write the count first
2089 int count = reloc_data.length();
2090 write_bytes(&count, sizeof(int));
2091 for (GrowableArrayIterator<uint> iter = reloc_data.begin();
2092 iter != reloc_data.end(); ++iter) {
2093 uint value = *iter;
2094 int n = write_bytes(&value, sizeof(uint));
2095 if (n != sizeof(uint)) {
2096 return false;
2097 }
2098 }
2099 return true;
2100 }
2101
2102 void AOTCodeReader::fix_relocations(CodeBlob* code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
2103 LogStreamHandle(Trace, aot, reloc) log;
2104 uint offset = read_position();
2105 int count = *(int*)addr(offset);
2106 offset += sizeof(int);
2107 if (log.is_enabled()) {
2108 log.print_cr("======== extra relocations count=%d", count);
2109 }
2110 uint* reloc_data = (uint*)addr(offset);
2111 offset += (count * sizeof(uint));
2112 set_read_position(offset);
2113
2114 RelocIterator iter(code_blob);
2115 int j = 0;
2116 while (iter.next()) {
2117 switch (iter.type()) {
2118 case relocInfo::none:
2119 break;
2120 case relocInfo::oop_type: {
2121 assert(code_blob->is_nmethod(), "sanity check");
2122 oop_Relocation* r = (oop_Relocation*)iter.reloc();
2123 if (r->oop_is_immediate()) {
2124 assert(oop_list != nullptr, "sanity check");
2125 Handle h = oop_list->at(reloc_data[j]);
2126 r->set_value(cast_from_oop<address>(h()));
2127 } else {
2128 r->fix_oop_relocation();
2129 }
2130 break;
2131 }
2132 case relocInfo::metadata_type: {
2133 assert(code_blob->is_nmethod(), "sanity check");
2134 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2135 Metadata* m;
2136 if (r->metadata_is_immediate()) {
2137 assert(metadata_list != nullptr, "sanity check");
2138 m = metadata_list->at(reloc_data[j]);
2139 } else {
2140 // Get already updated value from nmethod.
2141 int index = r->metadata_index();
2142 m = code_blob->as_nmethod()->metadata_at(index);
2143 }
2144 r->set_value((address)m);
2145 break;
2146 }
2147 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
2148 case relocInfo::opt_virtual_call_type:
2149 case relocInfo::static_call_type: {
2150 address dest = _cache->address_for_id(reloc_data[j]);
2151 if (dest != (address)-1) {
2152 ((CallRelocation*)iter.reloc())->set_destination(dest);
2153 }
2154 break;
2155 }
2156 case relocInfo::trampoline_stub_type: {
2157 address dest = _cache->address_for_id(reloc_data[j]);
2158 if (dest != (address)-1) {
2159 ((trampoline_stub_Relocation*)iter.reloc())->set_destination(dest);
2160 }
2161 break;
2162 }
2163 case relocInfo::static_stub_type:
2164 break;
2165 case relocInfo::runtime_call_type: {
2166 address dest = _cache->address_for_id(reloc_data[j]);
2167 if (dest != (address)-1) {
2168 ((CallRelocation*)iter.reloc())->set_destination(dest);
2169 }
2170 break;
2171 }
2172 case relocInfo::runtime_call_w_cp_type:
2173 // this relocation should not be in cache (see write_relocations)
2174 assert(false, "runtime_call_w_cp_type relocation is not implemented");
2175 break;
2176 case relocInfo::external_word_type: {
2177 address target = _cache->address_for_id(reloc_data[j]);
2178 // Add external address to global table
2179 int index = ExternalsRecorder::find_index(target);
2180 // Update index in relocation
2181 Relocation::add_jint(iter.data(), index);
2182 external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
2183 assert(reloc->target() == target, "sanity");
2184 reloc->set_value(target); // Patch address in the code
2185 break;
2186 }
2187 case relocInfo::internal_word_type: {
2188 internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
2189 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2190 break;
2191 }
2192 case relocInfo::section_word_type: {
2193 section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
2194 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2195 break;
2196 }
2197 case relocInfo::poll_type:
2198 break;
2199 case relocInfo::poll_return_type:
2200 break;
2201 case relocInfo::post_call_nop_type:
2202 break;
2203 case relocInfo::entry_guard_type:
2204 break;
2205 default:
2206 assert(false,"relocation %d unimplemented", (int)iter.type());
2207 break;
2208 }
2209 if (log.is_enabled()) {
2210 iter.print_current_on(&log);
2211 }
2212 j++;
2213 }
2214 assert(j == count, "sanity");
2215 }
2216
2217 bool AOTCodeCache::write_nmethod_reloc_immediates(GrowableArray<Handle>& oop_list, GrowableArray<Metadata*>& metadata_list) {
2218 int count = oop_list.length();
2219 if (!write_bytes(&count, sizeof(int))) {
2220 return false;
2221 }
2222 for (GrowableArrayIterator<Handle> iter = oop_list.begin();
2223 iter != oop_list.end(); ++iter) {
2224 Handle h = *iter;
2225 if (!write_oop(h())) {
2226 return false;
2227 }
2228 }
2229
2230 count = metadata_list.length();
2231 if (!write_bytes(&count, sizeof(int))) {
2232 return false;
2233 }
2234 for (GrowableArrayIterator<Metadata*> iter = metadata_list.begin();
2235 iter != metadata_list.end(); ++iter) {
2236 Metadata* m = *iter;
2237 if (!write_metadata(m)) {
2238 return false;
2239 }
2240 }
2241 return true;
2242 }
2243
2244 bool AOTCodeCache::write_metadata(nmethod* nm) {
2245 int count = nm->metadata_count()-1;
2246 if (!write_bytes(&count, sizeof(int))) {
2247 return false;
2248 }
2249 for (Metadata** p = nm->metadata_begin(); p < nm->metadata_end(); p++) {
2250 if (!write_metadata(*p)) {
2251 return false;
2252 }
2253 }
2254 return true;
2255 }
2256
2257 bool AOTCodeCache::write_metadata(Metadata* m) {
2258 uint n = 0;
2259 if (m == nullptr) {
2260 DataKind kind = DataKind::Null;
2261 n = write_bytes(&kind, sizeof(int));
2262 if (n != sizeof(int)) {
2263 return false;
2264 }
2265 } else if (m == (Metadata*)Universe::non_oop_word()) {
2266 DataKind kind = DataKind::No_Data;
2267 n = write_bytes(&kind, sizeof(int));
2268 if (n != sizeof(int)) {
2269 return false;
2270 }
2271 } else if (m->is_klass()) {
2272 if (!write_klass((Klass*)m)) {
2273 return false;
2274 }
2275 } else if (m->is_method()) {
2276 if (!write_method((Method*)m)) {
2277 return false;
2278 }
2279 } else if (m->is_methodCounters()) {
2280 DataKind kind = DataKind::MethodCnts;
2281 n = write_bytes(&kind, sizeof(int));
2282 if (n != sizeof(int)) {
2283 return false;
2284 }
2285 if (!write_method(((MethodCounters*)m)->method())) {
2286 return false;
2287 }
2288 log_debug(aot, codecache, metadata)("%d (L%d): Write MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2289 } else { // Not supported
2290 fatal("metadata : " INTPTR_FORMAT " unimplemented", p2i(m));
2291 return false;
2292 }
2293 return true;
2294 }
2295
2296 Metadata* AOTCodeReader::read_metadata(const methodHandle& comp_method) {
2297 uint code_offset = read_position();
2298 Metadata* m = nullptr;
2299 DataKind kind = *(DataKind*)addr(code_offset);
2300 code_offset += sizeof(DataKind);
2301 set_read_position(code_offset);
2302 if (kind == DataKind::Null) {
2303 m = (Metadata*)nullptr;
2304 } else if (kind == DataKind::No_Data) {
2305 m = (Metadata*)Universe::non_oop_word();
2306 } else if (kind == DataKind::Klass) {
2307 m = (Metadata*)read_klass(comp_method);
2308 } else if (kind == DataKind::Method) {
2309 m = (Metadata*)read_method(comp_method);
2310 } else if (kind == DataKind::MethodCnts) {
2311 kind = *(DataKind*)addr(code_offset);
2312 code_offset += sizeof(DataKind);
2313 set_read_position(code_offset);
2314 m = (Metadata*)read_method(comp_method);
2315 if (m != nullptr) {
2316 Method* method = (Method*)m;
2317 m = method->get_method_counters(Thread::current());
2318 if (m == nullptr) {
2319 set_lookup_failed();
2320 log_debug(aot, codecache, metadata)("%d (L%d): Failed to get MethodCounters", compile_id(), comp_level());
2321 } else {
2322 log_debug(aot, codecache, metadata)("%d (L%d): Read MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2323 }
2324 }
2325 } else {
2326 set_lookup_failed();
2327 log_debug(aot, codecache, metadata)("%d (L%d): Unknown metadata's kind: %d", compile_id(), comp_level(), (int)kind);
2328 }
2329 return m;
2330 }
2331
2332 bool AOTCodeCache::write_method(Method* method) {
2333 ResourceMark rm; // To method's name printing
2334 if (AOTCacheAccess::can_generate_aot_code(method)) {
2335 DataKind kind = DataKind::Method;
2336 uint n = write_bytes(&kind, sizeof(int));
2337 if (n != sizeof(int)) {
2338 return false;
2339 }
2340 uint method_offset = AOTCacheAccess::delta_from_base_address((address)method);
2341 n = write_bytes(&method_offset, sizeof(uint));
2342 if (n != sizeof(uint)) {
2343 return false;
2344 }
2345 log_debug(aot, codecache, metadata)("%d (L%d): Wrote method: %s @ 0x%08x",
2346 compile_id(), comp_level(), method->name_and_sig_as_C_string(), method_offset);
2347 return true;
2348 }
2349 log_debug(aot, codecache, metadata)("%d (L%d): Method is not archived: %s",
2350 compile_id(), comp_level(), method->name_and_sig_as_C_string());
2351 set_lookup_failed();
2352 return false;
2353 }
2354
2355 Method* AOTCodeReader::read_method(const methodHandle& comp_method) {
2356 uint code_offset = read_position();
2357 uint method_offset = *(uint*)addr(code_offset);
2358 code_offset += sizeof(uint);
2359 set_read_position(code_offset);
2360 Method* m = AOTCacheAccess::convert_offset_to_method(method_offset);
2361 if (!AOTMetaspace::in_aot_cache((address)m)) {
2362 // Something changed in CDS
2363 set_lookup_failed();
2364 log_debug(aot, codecache, metadata)("Lookup failed for shared method: " INTPTR_FORMAT " is not in CDS ", p2i((address)m));
2365 return nullptr;
2366 }
2367 assert(m->is_method(), "sanity");
2368 ResourceMark rm;
2369 Klass* k = m->method_holder();
2370 if (!k->is_instance_klass()) {
2371 set_lookup_failed();
2372 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not instance klass",
2373 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2374 return nullptr;
2375 } else if (!AOTMetaspace::in_aot_cache((address)k)) {
2376 set_lookup_failed();
2377 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not in CDS",
2378 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2379 return nullptr;
2380 } else if (!InstanceKlass::cast(k)->is_loaded()) {
2381 set_lookup_failed();
2382 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not loaded",
2383 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2384 return nullptr;
2385 } else if (!InstanceKlass::cast(k)->is_linked()) {
2386 set_lookup_failed();
2387 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not linked%s",
2388 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name(), (_preload ? " for code preload" : ""));
2389 return nullptr;
2390 }
2391 log_debug(aot, codecache, metadata)("%d (L%d): Shared method lookup: %s",
2392 compile_id(), comp_level(), m->name_and_sig_as_C_string());
2393 return m;
2394 }
2395
2396 bool AOTCodeCache::write_klass(Klass* klass) {
2397 uint array_dim = 0;
2398 if (klass->is_objArray_klass()) {
2399 array_dim = ObjArrayKlass::cast(klass)->dimension();
2400 klass = ObjArrayKlass::cast(klass)->bottom_klass(); // overwrites klass
2401 }
2402 uint init_state = 0;
2403 bool can_write = true;
2404 if (klass->is_instance_klass()) {
2405 InstanceKlass* ik = InstanceKlass::cast(klass);
2406 init_state = (ik->is_initialized() ? 1 : 0);
2407 can_write = AOTCacheAccess::can_generate_aot_code_for(ik);
2408 } else {
2409 can_write = AOTCacheAccess::can_generate_aot_code(klass);
2410 }
2411 ResourceMark rm;
2412 uint state = (array_dim << 1) | (init_state & 1);
2413 if (can_write) {
2414 DataKind kind = DataKind::Klass;
2415 uint n = write_bytes(&kind, sizeof(int));
2416 if (n != sizeof(int)) {
2417 return false;
2418 }
2419 // Record state of instance klass initialization and array dimentions.
2420 n = write_bytes(&state, sizeof(int));
2421 if (n != sizeof(int)) {
2422 return false;
2423 }
2424 uint klass_offset = AOTCacheAccess::delta_from_base_address((address)klass);
2425 n = write_bytes(&klass_offset, sizeof(uint));
2426 if (n != sizeof(uint)) {
2427 return false;
2428 }
2429 log_debug(aot, codecache, metadata)("%d (L%d): Registered klass: %s%s%s @ 0x%08x",
2430 compile_id(), comp_level(), klass->external_name(),
2431 (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2432 (array_dim > 0 ? " (object array)" : ""), klass_offset);
2433 return true;
2434 }
2435 log_debug(aot, codecache, metadata)("%d (L%d): Klassis not archived: %s%s%s",
2436 compile_id(), comp_level(), klass->external_name(),
2437 (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2438 (array_dim > 0 ? " (object array)" : ""));
2439 set_lookup_failed();
2440 return false;
2441 }
2442
2443 Klass* AOTCodeReader::read_klass(const methodHandle& comp_method) {
2444 uint code_offset = read_position();
2445 uint state = *(uint*)addr(code_offset);
2446 uint init_state = (state & 1);
2447 uint array_dim = (state >> 1);
2448 code_offset += sizeof(int);
2449 uint klass_offset = *(uint*)addr(code_offset);
2450 code_offset += sizeof(uint);
2451 set_read_position(code_offset);
2452 Klass* k = AOTCacheAccess::convert_offset_to_klass(klass_offset);
2453 if (!AOTMetaspace::in_aot_cache((address)k)) {
2454 // Something changed in CDS
2455 set_lookup_failed();
2456 log_debug(aot, codecache, metadata)("Lookup failed for shared klass: " INTPTR_FORMAT " is not in CDS ", p2i((address)k));
2457 return nullptr;
2458 }
2459 assert(k->is_klass(), "sanity");
2460 ResourceMark rm;
2461 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_loaded()) {
2462 set_lookup_failed();
2463 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not loaded",
2464 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2465 return nullptr;
2466 } else
2467 // Allow not initialized klass which was uninitialized during code caching or for preload
2468 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_initialized() && (init_state == 1) && !_preload) {
2469 set_lookup_failed();
2470 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not initialized",
2471 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2472 return nullptr;
2473 }
2474 if (array_dim > 0) {
2475 assert(k->is_instance_klass() || k->is_typeArray_klass(), "sanity check");
2476 Klass* ak = k->array_klass_or_null(array_dim);
2477 // FIXME: what would it take to create an array class on the fly?
2478 // Klass* ak = k->array_klass(dim, JavaThread::current());
2479 // guarantee(JavaThread::current()->pending_exception() == nullptr, "");
2480 if (ak == nullptr) {
2481 set_lookup_failed();
2482 log_debug(aot, codecache, metadata)("%d (L%d): %d-dimension array klass lookup failed: %s",
2483 compile_id(), comp_level(), array_dim, k->external_name());
2484 }
2485 log_debug(aot, codecache, metadata)("%d (L%d): Klass lookup: %s (object array)", compile_id(), comp_level(), k->external_name());
2486 return ak;
2487 } else {
2488 log_debug(aot, codecache, metadata)("%d (L%d): Shared klass lookup: %s",
2489 compile_id(), comp_level(), k->external_name());
2490 return k;
2491 }
2492 }
2493
2494 bool AOTCodeCache::write_oop(jobject& jo) {
2495 oop obj = JNIHandles::resolve(jo);
2496 return write_oop(obj);
2497 }
2498
2499 bool AOTCodeCache::write_oop(oop obj) {
2500 DataKind kind;
2501 uint n = 0;
2502 if (obj == nullptr) {
2503 kind = DataKind::Null;
2504 n = write_bytes(&kind, sizeof(int));
2505 if (n != sizeof(int)) {
2506 return false;
2507 }
2508 } else if (cast_from_oop<void *>(obj) == Universe::non_oop_word()) {
2509 kind = DataKind::No_Data;
2510 n = write_bytes(&kind, sizeof(int));
2511 if (n != sizeof(int)) {
2512 return false;
2513 }
2514 } else if (java_lang_Class::is_instance(obj)) {
2515 if (java_lang_Class::is_primitive(obj)) {
2516 int bt = (int)java_lang_Class::primitive_type(obj);
2517 kind = DataKind::Primitive;
2518 n = write_bytes(&kind, sizeof(int));
2519 if (n != sizeof(int)) {
2520 return false;
2521 }
2522 n = write_bytes(&bt, sizeof(int));
2523 if (n != sizeof(int)) {
2524 return false;
2525 }
2526 log_debug(aot, codecache, oops)("%d (L%d): Write primitive type klass: %s", compile_id(), comp_level(), type2name((BasicType)bt));
2527 } else {
2528 Klass* klass = java_lang_Class::as_Klass(obj);
2529 if (!write_klass(klass)) {
2530 return false;
2531 }
2532 }
2533 } else if (java_lang_String::is_instance(obj)) { // herere
2534 int k = AOTCacheAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2535 ResourceMark rm;
2536 size_t length_sz = 0;
2537 const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2538 if (k >= 0) {
2539 kind = DataKind::String;
2540 n = write_bytes(&kind, sizeof(int));
2541 if (n != sizeof(int)) {
2542 return false;
2543 }
2544 n = write_bytes(&k, sizeof(int));
2545 if (n != sizeof(int)) {
2546 return false;
2547 }
2548 log_debug(aot, codecache, oops)("%d (L%d): Write String object: " PTR_FORMAT " : %s", compile_id(), comp_level(), p2i(obj), string);
2549 return true;
2550 }
2551 // Not archived String object - bailout
2552 set_lookup_failed();
2553 log_debug(aot, codecache, oops)("%d (L%d): Not archived String object: " PTR_FORMAT " : %s",
2554 compile_id(), comp_level(), p2i(obj), string);
2555 return false;
2556 } else if (java_lang_Module::is_instance(obj)) {
2557 fatal("Module object unimplemented");
2558 } else if (java_lang_ClassLoader::is_instance(obj)) {
2559 if (obj == SystemDictionary::java_system_loader()) {
2560 kind = DataKind::SysLoader;
2561 log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_system_loader", compile_id(), comp_level());
2562 } else if (obj == SystemDictionary::java_platform_loader()) {
2563 kind = DataKind::PlaLoader;
2564 log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_platform_loader", compile_id(), comp_level());
2565 } else {
2566 ResourceMark rm;
2567 set_lookup_failed();
2568 log_debug(aot, codecache, oops)("%d (L%d): Not supported Class Loader: " PTR_FORMAT " : %s",
2569 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2570 return false;
2571 }
2572 n = write_bytes(&kind, sizeof(int));
2573 if (n != sizeof(int)) {
2574 return false;
2575 }
2576 } else { // herere
2577 ResourceMark rm;
2578 int k = AOTCacheAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2579 if (k >= 0) {
2580 kind = DataKind::MH_Oop;
2581 n = write_bytes(&kind, sizeof(int));
2582 if (n != sizeof(int)) {
2583 return false;
2584 }
2585 n = write_bytes(&k, sizeof(int));
2586 if (n != sizeof(int)) {
2587 return false;
2588 }
2589 log_debug(aot, codecache, oops)("%d (L%d): Write MH object: " PTR_FORMAT " : %s",
2590 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2591 return true;
2592 }
2593 // Not archived Java object - bailout
2594 set_lookup_failed();
2595 log_debug(aot, codecache, oops)("%d (L%d): Not archived Java object: " PTR_FORMAT " : %s",
2596 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2597 return false;
2598 }
2599 return true;
2600 }
2601
2602 oop AOTCodeReader::read_oop(JavaThread* thread, const methodHandle& comp_method) {
2603 uint code_offset = read_position();
2604 oop obj = nullptr;
2605 DataKind kind = *(DataKind*)addr(code_offset);
2606 code_offset += sizeof(DataKind);
2607 set_read_position(code_offset);
2608 if (kind == DataKind::Null) {
2609 return nullptr;
2610 } else if (kind == DataKind::No_Data) {
2611 return cast_to_oop(Universe::non_oop_word());
2612 } else if (kind == DataKind::Klass) {
2613 Klass* k = read_klass(comp_method);
2614 if (k == nullptr) {
2615 return nullptr;
2616 }
2617 obj = k->java_mirror();
2618 if (obj == nullptr) {
2619 set_lookup_failed();
2620 log_debug(aot, codecache, oops)("Lookup failed for java_mirror of klass %s", k->external_name());
2621 return nullptr;
2622 }
2623 } else if (kind == DataKind::Primitive) {
2624 code_offset = read_position();
2625 int t = *(int*)addr(code_offset);
2626 code_offset += sizeof(int);
2627 set_read_position(code_offset);
2628 BasicType bt = (BasicType)t;
2629 obj = java_lang_Class::primitive_mirror(bt);
2630 log_debug(aot, codecache, oops)("%d (L%d): Read primitive type klass: %s", compile_id(), comp_level(), type2name(bt));
2631 } else if (kind == DataKind::String) {
2632 code_offset = read_position();
2633 int k = *(int*)addr(code_offset);
2634 code_offset += sizeof(int);
2635 set_read_position(code_offset);
2636 obj = AOTCacheAccess::get_archived_object(k);
2637 if (obj == nullptr) {
2638 set_lookup_failed();
2639 log_debug(aot, codecache, oops)("Lookup failed for String object");
2640 return nullptr;
2641 }
2642 assert(java_lang_String::is_instance(obj), "must be string");
2643
2644 ResourceMark rm;
2645 size_t length_sz = 0;
2646 const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2647 log_debug(aot, codecache, oops)("%d (L%d): Read String object: %s", compile_id(), comp_level(), string);
2648 } else if (kind == DataKind::SysLoader) {
2649 obj = SystemDictionary::java_system_loader();
2650 log_debug(aot, codecache, oops)("%d (L%d): Read java_system_loader", compile_id(), comp_level());
2651 } else if (kind == DataKind::PlaLoader) {
2652 obj = SystemDictionary::java_platform_loader();
2653 log_debug(aot, codecache, oops)("%d (L%d): Read java_platform_loader", compile_id(), comp_level());
2654 } else if (kind == DataKind::MH_Oop) {
2655 code_offset = read_position();
2656 int k = *(int*)addr(code_offset);
2657 code_offset += sizeof(int);
2658 set_read_position(code_offset);
2659 obj = AOTCacheAccess::get_archived_object(k);
2660 if (obj == nullptr) {
2661 set_lookup_failed();
2662 log_debug(aot, codecache, oops)("Lookup failed for MH object");
2663 return nullptr;
2664 }
2665 ResourceMark rm;
2666 log_debug(aot, codecache, oops)("%d (L%d): Read MH object: " PTR_FORMAT " : %s",
2667 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2668 } else {
2669 set_lookup_failed();
2670 log_debug(aot, codecache, oops)("%d (L%d): Unknown oop's kind: %d",
2671 compile_id(), comp_level(), (int)kind);
2672 return nullptr;
2673 }
2674 return obj;
2675 }
2676
2677 bool AOTCodeReader::read_oop_metadata_list(JavaThread* thread, ciMethod* target, GrowableArray<Handle> &oop_list, GrowableArray<Metadata*> &metadata_list, OopRecorder* oop_recorder) {
2678 methodHandle comp_method(JavaThread::current(), target->get_Method());
2679 JavaThread* current = JavaThread::current();
2680 uint offset = read_position();
2681 int count = *(int *)addr(offset);
2682 offset += sizeof(int);
2683 set_read_position(offset);
2684 for (int i = 0; i < count; i++) {
2685 oop obj = read_oop(current, comp_method);
2686 if (lookup_failed()) {
2687 return false;
2688 }
2689 Handle h(thread, obj);
2690 oop_list.append(h);
2691 if (oop_recorder != nullptr) {
2692 jobject jo = JNIHandles::make_local(thread, obj);
2693 if (oop_recorder->is_real(jo)) {
2694 oop_recorder->find_index(jo);
2695 } else {
2696 oop_recorder->allocate_oop_index(jo);
2697 }
2698 }
2699 LogStreamHandle(Debug, aot, codecache, oops) log;
2700 if (log.is_enabled()) {
2701 log.print("%d: " INTPTR_FORMAT " ", i, p2i(obj));
2702 if (obj == Universe::non_oop_word()) {
2703 log.print("non-oop word");
2704 } else if (obj == nullptr) {
2705 log.print("nullptr-oop");
2706 } else {
2707 obj->print_value_on(&log);
2708 }
2709 log.cr();
2710 }
2711 }
2712
2713 offset = read_position();
2714 count = *(int *)addr(offset);
2715 offset += sizeof(int);
2716 set_read_position(offset);
2717 for (int i = 0; i < count; i++) {
2718 Metadata* m = read_metadata(comp_method);
2719 if (lookup_failed()) {
2720 return false;
2721 }
2722 metadata_list.append(m);
2723 if (oop_recorder != nullptr) {
2724 if (oop_recorder->is_real(m)) {
2725 oop_recorder->find_index(m);
2726 } else {
2727 oop_recorder->allocate_metadata_index(m);
2728 }
2729 }
2730 LogTarget(Debug, aot, codecache, metadata) log;
2731 if (log.is_enabled()) {
2732 LogStream ls(log);
2733 ls.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
2734 if (m == (Metadata*)Universe::non_oop_word()) {
2735 ls.print("non-metadata word");
2736 } else if (m == nullptr) {
2737 ls.print("nullptr-oop");
2738 } else {
2739 Metadata::print_value_on_maybe_null(&ls, m);
2740 }
2741 ls.cr();
2742 }
2743 }
2744 return true;
2745 }
2746
2747 bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
2748 ImmutableOopMapSet* oopmaps = cb.oop_maps();
2749 int oopmaps_size = oopmaps->nr_of_bytes();
2750 if (!write_bytes(&oopmaps_size, sizeof(int))) {
2751 return false;
2752 }
2753 uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
2754 if (n != (uint)oopmaps->nr_of_bytes()) {
2755 return false;
2756 }
2757 return true;
2758 }
2759
2760 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
2761 uint offset = read_position();
2762 int size = *(int *)addr(offset);
2763 offset += sizeof(int);
2764 ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
2765 offset += size;
2766 set_read_position(offset);
2767 return oopmaps;
2768 }
2769
2770 bool AOTCodeCache::write_oops(nmethod* nm) {
2771 int count = nm->oops_count()-1;
2772 if (!write_bytes(&count, sizeof(int))) {
2773 return false;
2774 }
2775 for (oop* p = nm->oops_begin(); p < nm->oops_end(); p++) {
2776 if (!write_oop(*p)) {
2777 return false;
2778 }
2779 }
2780 return true;
2781 }
2782
2783 #ifndef PRODUCT
2784 bool AOTCodeCache::write_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2785 // Write asm remarks
2786 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2787 if (count_ptr == nullptr) {
2788 return false;
2789 }
2790 uint count = 0;
2791 bool result = asm_remarks.iterate([&] (uint offset, const char* str) -> bool {
2792 log_trace(aot, codecache, stubs)("asm remark offset=%d, str='%s'", offset, str);
2793 uint n = write_bytes(&offset, sizeof(uint));
2794 if (n != sizeof(uint)) {
2795 return false;
2796 }
2797 if (use_string_table) {
2798 const char* cstr = add_C_string(str);
2799 int id = _table->id_for_C_string((address)cstr);
2800 assert(id != -1, "asm remark string '%s' not found in AOTCodeAddressTable", str);
2801 n = write_bytes(&id, sizeof(int));
2802 if (n != sizeof(int)) {
2803 return false;
2804 }
2805 } else {
2806 n = write_bytes(str, (uint)strlen(str) + 1);
2807 if (n != strlen(str) + 1) {
2808 return false;
2809 }
2810 }
2811 count += 1;
2812 return true;
2813 });
2814 *count_ptr = count;
2815 return result;
2816 }
2817
2818 void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2819 // Read asm remarks
2820 uint offset = read_position();
2821 uint count = *(uint *)addr(offset);
2822 offset += sizeof(uint);
2823 for (uint i = 0; i < count; i++) {
2824 uint remark_offset = *(uint *)addr(offset);
2825 offset += sizeof(uint);
2826 const char* remark = nullptr;
2827 if (use_string_table) {
2828 int remark_string_id = *(uint *)addr(offset);
2829 offset += sizeof(int);
2830 remark = (const char*)_cache->address_for_C_string(remark_string_id);
2831 } else {
2832 remark = (const char*)addr(offset);
2833 offset += (uint)strlen(remark)+1;
2834 }
2835 asm_remarks.insert(remark_offset, remark);
2836 }
2837 set_read_position(offset);
2838 }
2839
2840 bool AOTCodeCache::write_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2841 // Write dbg strings
2842 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2843 if (count_ptr == nullptr) {
2844 return false;
2845 }
2846 uint count = 0;
2847 bool result = dbg_strings.iterate([&] (const char* str) -> bool {
2848 log_trace(aot, codecache, stubs)("dbg string=%s", str);
2849 if (use_string_table) {
2850 const char* cstr = add_C_string(str);
2851 int id = _table->id_for_C_string((address)cstr);
2852 assert(id != -1, "db string '%s' not found in AOTCodeAddressTable", str);
2853 uint n = write_bytes(&id, sizeof(int));
2854 if (n != sizeof(int)) {
2855 return false;
2856 }
2857 } else {
2858 uint n = write_bytes(str, (uint)strlen(str) + 1);
2859 if (n != strlen(str) + 1) {
2860 return false;
2861 }
2862 }
2863 count += 1;
2864 return true;
2865 });
2866 *count_ptr = count;
2867 return result;
2868 }
2869
2870 void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2871 // Read dbg strings
2872 uint offset = read_position();
2873 uint count = *(uint *)addr(offset);
2874 offset += sizeof(uint);
2875 for (uint i = 0; i < count; i++) {
2876 const char* str = nullptr;
2877 if (use_string_table) {
2878 int string_id = *(uint *)addr(offset);
2879 offset += sizeof(int);
2880 str = (const char*)_cache->address_for_C_string(string_id);
2881 } else {
2882 str = (const char*)addr(offset);
2883 offset += (uint)strlen(str)+1;
2884 }
2885 dbg_strings.insert(str);
2886 }
2887 set_read_position(offset);
2888 }
2889 #endif // PRODUCT
2890
2891 //======================= AOTCodeAddressTable ===============
2892
2893 // address table ids for generated routines, external addresses and C
2894 // string addresses are partitioned into positive integer ranges
2895 // defined by the following positive base and max values
2896 // i.e. [_extrs_base, _extrs_base + _extrs_max -1],
2897 // [_stubs_base, _stubs_base + _stubs_max -1],
2898 // ...
2899 // [_c_str_base, _c_str_base + _c_str_max -1],
2900 #define _extrs_max 140
2901 #define _stubs_max 210
2902 #define _shared_blobs_max 25
2903 #define _C1_blobs_max 50
2904 #define _C2_blobs_max 25
2905 #define _blobs_max (_shared_blobs_max+_C1_blobs_max+_C2_blobs_max)
2906 #define _all_max (_extrs_max+_stubs_max+_blobs_max)
2907
2908 #define _extrs_base 0
2909 #define _stubs_base (_extrs_base + _extrs_max)
2910 #define _shared_blobs_base (_stubs_base + _stubs_max)
2911 #define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max)
2912 #define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max)
2913 #define _blobs_end (_shared_blobs_base + _blobs_max)
2914 #if (_C2_blobs_base >= _all_max)
2915 #error AOTCodeAddressTable ranges need adjusting
2916 #endif
2917
2918 #define SET_ADDRESS(type, addr) \
2919 { \
2920 type##_addr[type##_length++] = (address) (addr); \
2921 assert(type##_length <= type##_max, "increase size"); \
2922 }
2923
2924 static bool initializing_extrs = false;
2925
2926 void AOTCodeAddressTable::init_extrs() {
2927 if (_extrs_complete || initializing_extrs) return; // Done already
2928
2929 assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting");
2930
2931 initializing_extrs = true;
2932 _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
2933
2934 _extrs_length = 0;
2935
2936 // Record addresses of VM runtime methods
2937 SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite);
2938 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method);
2939 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract);
2940 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss);
2941 {
2942 // Required by Shared blobs
2943 SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info);
2944 SET_ADDRESS(_extrs, Deoptimization::unpack_frames);
2945 SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception);
2946 SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C);
2947 SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C);
2948 SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C);
2949 SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2950 SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError);
2951 SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError);
2952 SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError);
2953 SET_ADDRESS(_extrs, SharedRuntime::throw_NullPointerException_at_call);
2954 SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2955 SET_ADDRESS(_extrs, CompressedOops::base_addr());
2956 SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr());
2957 }
2958 {
2959 // Required by initial stubs
2960 SET_ADDRESS(_extrs, StubRoutines::crc_table_addr());
2961 #if defined(AMD64)
2962 SET_ADDRESS(_extrs, StubRoutines::crc32c_table_addr());
2963 #endif
2964 }
2965
2966 #ifdef COMPILER1
2967 {
2968 // Required by C1 blobs
2969 SET_ADDRESS(_extrs, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc));
2970 SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address);
2971 SET_ADDRESS(_extrs, SharedRuntime::register_finalizer);
2972 SET_ADDRESS(_extrs, Runtime1::is_instance_of);
2973 SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc);
2974 SET_ADDRESS(_extrs, Runtime1::check_abort_on_vm_exception);
2975 SET_ADDRESS(_extrs, Runtime1::new_instance);
2976 SET_ADDRESS(_extrs, Runtime1::counter_overflow);
2977 SET_ADDRESS(_extrs, Runtime1::new_type_array);
2978 SET_ADDRESS(_extrs, Runtime1::new_object_array);
2979 SET_ADDRESS(_extrs, Runtime1::new_multi_array);
2980 SET_ADDRESS(_extrs, Runtime1::throw_range_check_exception);
2981 SET_ADDRESS(_extrs, Runtime1::throw_index_exception);
2982 SET_ADDRESS(_extrs, Runtime1::throw_div0_exception);
2983 SET_ADDRESS(_extrs, Runtime1::throw_null_pointer_exception);
2984 SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception);
2985 SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception);
2986 SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error);
2987 SET_ADDRESS(_extrs, Runtime1::monitorenter);
2988 SET_ADDRESS(_extrs, Runtime1::monitorexit);
2989 SET_ADDRESS(_extrs, Runtime1::deoptimize);
2990 SET_ADDRESS(_extrs, Runtime1::access_field_patching);
2991 SET_ADDRESS(_extrs, Runtime1::move_klass_patching);
2992 SET_ADDRESS(_extrs, Runtime1::move_mirror_patching);
2993 SET_ADDRESS(_extrs, Runtime1::move_appendix_patching);
2994 SET_ADDRESS(_extrs, Runtime1::predicate_failed_trap);
2995 SET_ADDRESS(_extrs, Runtime1::unimplemented_entry);
2996 SET_ADDRESS(_extrs, Runtime1::trace_block_entry);
2997 #ifdef X86
2998 SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool);
2999 SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool);
3000 SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool);
3001 SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool);
3002 #endif
3003 #ifndef PRODUCT
3004 SET_ADDRESS(_extrs, os::breakpoint);
3005 #endif
3006 }
3007 #endif // COMPILER1
3008
3009 #ifdef COMPILER2
3010 {
3011 // Required by C2 blobs
3012 SET_ADDRESS(_extrs, Deoptimization::uncommon_trap);
3013 SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C);
3014 SET_ADDRESS(_extrs, OptoRuntime::new_instance_C);
3015 SET_ADDRESS(_extrs, OptoRuntime::new_array_C);
3016 SET_ADDRESS(_extrs, OptoRuntime::new_array_nozero_C);
3017 SET_ADDRESS(_extrs, OptoRuntime::multianewarray2_C);
3018 SET_ADDRESS(_extrs, OptoRuntime::multianewarray3_C);
3019 SET_ADDRESS(_extrs, OptoRuntime::multianewarray4_C);
3020 SET_ADDRESS(_extrs, OptoRuntime::multianewarray5_C);
3021 SET_ADDRESS(_extrs, OptoRuntime::multianewarrayN_C);
3022 #if INCLUDE_JVMTI
3023 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_start);
3024 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_end);
3025 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_mount);
3026 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_unmount);
3027 #endif
3028 SET_ADDRESS(_extrs, OptoRuntime::complete_monitor_locking_C);
3029 SET_ADDRESS(_extrs, OptoRuntime::monitor_notify_C);
3030 SET_ADDRESS(_extrs, OptoRuntime::monitor_notifyAll_C);
3031 SET_ADDRESS(_extrs, OptoRuntime::rethrow_C);
3032 SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C);
3033 SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C);
3034 SET_ADDRESS(_extrs, OptoRuntime::class_init_barrier_C);
3035 #if defined(AMD64)
3036 // Use by C2 intinsic
3037 SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31());
3038 #endif
3039 }
3040 #endif // COMPILER2
3041 #if INCLUDE_G1GC
3042 SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry);
3043 #endif
3044
3045 #if INCLUDE_SHENANDOAHGC
3046 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_oop);
3047 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
3048 SET_ADDRESS(_extrs, ShenandoahRuntime::clone_barrier);
3049 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong);
3050 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong_narrow);
3051 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak);
3052 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak_narrow);
3053 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom);
3054 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
3055 SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre);
3056 #endif
3057
3058 #if INCLUDE_ZGC
3059 SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
3060 #if defined(AMD64)
3061 SET_ADDRESS(_extrs, &ZPointerLoadShift);
3062 #endif
3063 #endif // INCLUDE_ZGC
3064
3065 SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry);
3066 SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages);
3067 SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit);
3068
3069 SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C);
3070 SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone);
3071 #if defined(AMD64) && !defined(ZERO)
3072 SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply);
3073 SET_ADDRESS(_extrs, SharedRuntime::montgomery_square);
3074 #endif // AMD64
3075 SET_ADDRESS(_extrs, SharedRuntime::d2f);
3076 SET_ADDRESS(_extrs, SharedRuntime::d2i);
3077 SET_ADDRESS(_extrs, SharedRuntime::d2l);
3078 SET_ADDRESS(_extrs, SharedRuntime::dcos);
3079 SET_ADDRESS(_extrs, SharedRuntime::dexp);
3080 SET_ADDRESS(_extrs, SharedRuntime::dlog);
3081 SET_ADDRESS(_extrs, SharedRuntime::dlog10);
3082 SET_ADDRESS(_extrs, SharedRuntime::dpow);
3083 SET_ADDRESS(_extrs, SharedRuntime::dsin);
3084 SET_ADDRESS(_extrs, SharedRuntime::dtan);
3085 SET_ADDRESS(_extrs, SharedRuntime::f2i);
3086 SET_ADDRESS(_extrs, SharedRuntime::f2l);
3087 #ifndef ZERO
3088 SET_ADDRESS(_extrs, SharedRuntime::drem);
3089 SET_ADDRESS(_extrs, SharedRuntime::frem);
3090 #endif
3091 SET_ADDRESS(_extrs, SharedRuntime::l2d);
3092 SET_ADDRESS(_extrs, SharedRuntime::l2f);
3093 SET_ADDRESS(_extrs, SharedRuntime::ldiv);
3094 SET_ADDRESS(_extrs, SharedRuntime::lmul);
3095 SET_ADDRESS(_extrs, SharedRuntime::lrem);
3096
3097 SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset());
3098 SET_ADDRESS(_extrs, Thread::current);
3099
3100 SET_ADDRESS(_extrs, os::javaTimeMillis);
3101 SET_ADDRESS(_extrs, os::javaTimeNanos);
3102 // For JFR
3103 SET_ADDRESS(_extrs, os::elapsed_counter);
3104 #if defined(X86) && !defined(ZERO)
3105 SET_ADDRESS(_extrs, Rdtsc::elapsed_counter);
3106 #endif
3107
3108 #if INCLUDE_JVMTI
3109 SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc);
3110 SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events);
3111 #endif /* INCLUDE_JVMTI */
3112
3113 #ifndef PRODUCT
3114 SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr);
3115 SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure);
3116 #endif
3117
3118 #ifndef ZERO
3119 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
3120 SET_ADDRESS(_extrs, MacroAssembler::debug64);
3121 #endif
3122 #if defined(AARCH64)
3123 SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper);
3124 #endif
3125 #endif // ZERO
3126
3127 // addresses of fields in AOT runtime constants area
3128 address* p = AOTRuntimeConstants::field_addresses_list();
3129 while (*p != nullptr) {
3130 SET_ADDRESS(_extrs, *p++);
3131 }
3132
3133 _extrs_complete = true;
3134 log_info(aot, codecache, init)("External addresses recorded");
3135 }
3136
3137 static bool initializing_early_stubs = false;
3138
3139 void AOTCodeAddressTable::init_early_stubs() {
3140 if (_complete || initializing_early_stubs) return; // Done already
3141 initializing_early_stubs = true;
3142 _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
3143 _stubs_length = 0;
3144 SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry());
3145
3146 {
3147 // Required by C1 blobs
3148 #if defined(AMD64) && !defined(ZERO)
3149 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip());
3150 SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup());
3151 #endif // AMD64
3152 }
3153
3154 _early_stubs_complete = true;
3155 log_info(aot, codecache, init)("Early stubs recorded");
3156 }
3157
3158 static bool initializing_shared_blobs = false;
3159
3160 void AOTCodeAddressTable::init_shared_blobs() {
3161 if (_complete || initializing_shared_blobs) return; // Done already
3162 initializing_shared_blobs = true;
3163 address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode);
3164
3165 // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel
3166 _shared_blobs_addr = blobs_addr;
3167 _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max;// C1 blobs addresses stored after shared blobs
3168 _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs
3169
3170 _shared_blobs_length = 0;
3171 _C1_blobs_length = 0;
3172 _C2_blobs_length = 0;
3173
3174 // clear the address table
3175 memset(blobs_addr, 0, sizeof(address)* _blobs_max);
3176
3177 // Record addresses of generated code blobs
3178 SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub());
3179 SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub());
3180 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack());
3181 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception());
3182 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution());
3183 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls());
3184 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub());
3185 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_virtual_call_stub());
3186 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_static_call_stub());
3187 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->entry_point());
3188 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point());
3189 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point());
3190 #ifdef COMPILER2
3191 // polling_page_vectors_safepoint_handler_blob can be nullptr if AVX feature is not present or is disabled
3192 if (SharedRuntime::polling_page_vectors_safepoint_handler_blob() != nullptr) {
3193 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point());
3194 }
3195 #endif
3196 #if INCLUDE_JVMCI
3197 if (EnableJVMCI) {
3198 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap());
3199 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap());
3200 }
3201 #endif
3202 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_AbstractMethodError_entry());
3203 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry());
3204 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_NullPointerException_at_call_entry());
3205 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_StackOverflowError_entry());
3206 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry());
3207
3208 assert(_shared_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _shared_blobs_length);
3209 _shared_blobs_complete = true;
3210 log_info(aot, codecache, init)("All shared blobs recorded");
3211 }
3212
3213 static bool initializing_stubs = false;
3214 void AOTCodeAddressTable::init_stubs() {
3215 if (_complete || initializing_stubs) return; // Done already
3216 assert(_early_stubs_complete, "early stubs whould be initialized");
3217 initializing_stubs = true;
3218
3219 // Stubs
3220 SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier());
3221 SET_ADDRESS(_stubs, StubRoutines::atomic_xchg_entry());
3222 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_entry());
3223 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_long_entry());
3224 SET_ADDRESS(_stubs, StubRoutines::atomic_add_entry());
3225 SET_ADDRESS(_stubs, StubRoutines::fence_entry());
3226
3227 SET_ADDRESS(_stubs, StubRoutines::cont_thaw());
3228 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrier());
3229 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrierExc());
3230
3231 JFR_ONLY(SET_ADDRESS(_stubs, SharedRuntime::jfr_write_checkpoint());)
3232
3233 SET_ADDRESS(_stubs, StubRoutines::jbyte_arraycopy());
3234 SET_ADDRESS(_stubs, StubRoutines::jshort_arraycopy());
3235 SET_ADDRESS(_stubs, StubRoutines::jint_arraycopy());
3236 SET_ADDRESS(_stubs, StubRoutines::jlong_arraycopy());
3237 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy);
3238 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy_uninit);
3239
3240 SET_ADDRESS(_stubs, StubRoutines::jbyte_disjoint_arraycopy());
3241 SET_ADDRESS(_stubs, StubRoutines::jshort_disjoint_arraycopy());
3242 SET_ADDRESS(_stubs, StubRoutines::jint_disjoint_arraycopy());
3243 SET_ADDRESS(_stubs, StubRoutines::jlong_disjoint_arraycopy());
3244 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy);
3245 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy_uninit);
3246
3247 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_arraycopy());
3248 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_arraycopy());
3249 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_arraycopy());
3250 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_arraycopy());
3251 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy);
3252 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy_uninit);
3253
3254 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_disjoint_arraycopy());
3255 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_disjoint_arraycopy());
3256 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_disjoint_arraycopy());
3257 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_disjoint_arraycopy());
3258 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy);
3259 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit);
3260
3261 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy);
3262 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy_uninit);
3263
3264 SET_ADDRESS(_stubs, StubRoutines::unsafe_arraycopy());
3265 SET_ADDRESS(_stubs, StubRoutines::generic_arraycopy());
3266
3267 SET_ADDRESS(_stubs, StubRoutines::jbyte_fill());
3268 SET_ADDRESS(_stubs, StubRoutines::jshort_fill());
3269 SET_ADDRESS(_stubs, StubRoutines::jint_fill());
3270 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_fill());
3271 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_fill());
3272 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_fill());
3273
3274 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback());
3275 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback_sync());
3276
3277 SET_ADDRESS(_stubs, StubRoutines::aescrypt_encryptBlock());
3278 SET_ADDRESS(_stubs, StubRoutines::aescrypt_decryptBlock());
3279 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_encryptAESCrypt());
3280 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_decryptAESCrypt());
3281 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_encryptAESCrypt());
3282 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_decryptAESCrypt());
3283 SET_ADDRESS(_stubs, StubRoutines::poly1305_processBlocks());
3284 SET_ADDRESS(_stubs, StubRoutines::counterMode_AESCrypt());
3285 SET_ADDRESS(_stubs, StubRoutines::ghash_processBlocks());
3286 SET_ADDRESS(_stubs, StubRoutines::chacha20Block());
3287 SET_ADDRESS(_stubs, StubRoutines::base64_encodeBlock());
3288 SET_ADDRESS(_stubs, StubRoutines::base64_decodeBlock());
3289 SET_ADDRESS(_stubs, StubRoutines::md5_implCompress());
3290 SET_ADDRESS(_stubs, StubRoutines::md5_implCompressMB());
3291 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompress());
3292 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompressMB());
3293 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompress());
3294 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompressMB());
3295 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompress());
3296 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompressMB());
3297 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompress());
3298 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompressMB());
3299 SET_ADDRESS(_stubs, StubRoutines::double_keccak());
3300 SET_ADDRESS(_stubs, StubRoutines::intpoly_assign());
3301 SET_ADDRESS(_stubs, StubRoutines::intpoly_montgomeryMult_P256());
3302 SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostNtt());
3303 SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostInverseNtt());
3304 SET_ADDRESS(_stubs, StubRoutines::dilithiumNttMult());
3305 SET_ADDRESS(_stubs, StubRoutines::dilithiumMontMulByConstant());
3306 SET_ADDRESS(_stubs, StubRoutines::dilithiumDecomposePoly());
3307
3308 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32());
3309 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32C());
3310 SET_ADDRESS(_stubs, StubRoutines::updateBytesAdler32());
3311
3312 SET_ADDRESS(_stubs, StubRoutines::multiplyToLen());
3313 SET_ADDRESS(_stubs, StubRoutines::squareToLen());
3314 SET_ADDRESS(_stubs, StubRoutines::mulAdd());
3315 SET_ADDRESS(_stubs, StubRoutines::montgomeryMultiply());
3316 SET_ADDRESS(_stubs, StubRoutines::montgomerySquare());
3317 SET_ADDRESS(_stubs, StubRoutines::bigIntegerRightShift());
3318 SET_ADDRESS(_stubs, StubRoutines::bigIntegerLeftShift());
3319 SET_ADDRESS(_stubs, StubRoutines::galoisCounterMode_AESCrypt());
3320
3321 SET_ADDRESS(_stubs, StubRoutines::vectorizedMismatch());
3322
3323 SET_ADDRESS(_stubs, StubRoutines::unsafe_setmemory());
3324
3325 SET_ADDRESS(_stubs, StubRoutines::dexp());
3326 SET_ADDRESS(_stubs, StubRoutines::dlog());
3327 SET_ADDRESS(_stubs, StubRoutines::dlog10());
3328 SET_ADDRESS(_stubs, StubRoutines::dpow());
3329 SET_ADDRESS(_stubs, StubRoutines::dsin());
3330 SET_ADDRESS(_stubs, StubRoutines::dcos());
3331 SET_ADDRESS(_stubs, StubRoutines::dlibm_reduce_pi04l());
3332 SET_ADDRESS(_stubs, StubRoutines::dlibm_sin_cos_huge());
3333 SET_ADDRESS(_stubs, StubRoutines::dlibm_tan_cot_huge());
3334 SET_ADDRESS(_stubs, StubRoutines::dtan());
3335
3336 SET_ADDRESS(_stubs, StubRoutines::f2hf_adr());
3337 SET_ADDRESS(_stubs, StubRoutines::hf2f_adr());
3338
3339 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
3340 SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_stub(slot));
3341 }
3342 SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_slow_path_stub());
3343
3344 #if defined(AMD64) && !defined(ZERO)
3345 SET_ADDRESS(_stubs, StubRoutines::x86::d2i_fixup());
3346 SET_ADDRESS(_stubs, StubRoutines::x86::f2i_fixup());
3347 SET_ADDRESS(_stubs, StubRoutines::x86::f2l_fixup());
3348 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_mask());
3349 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_flip());
3350 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_mask());
3351 SET_ADDRESS(_stubs, StubRoutines::x86::vector_popcount_lut());
3352 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_mask());
3353 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_flip());
3354 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_mask());
3355 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_flip());
3356 SET_ADDRESS(_stubs, StubRoutines::x86::vector_int_shuffle_mask());
3357 SET_ADDRESS(_stubs, StubRoutines::x86::vector_byte_shuffle_mask());
3358 SET_ADDRESS(_stubs, StubRoutines::x86::vector_short_shuffle_mask());
3359 SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_shuffle_mask());
3360 SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_sign_mask());
3361 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_int());
3362 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_short());
3363 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_long());
3364 // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
3365 // See C2_MacroAssembler::load_iota_indices().
3366 for (int i = 0; i < 6; i++) {
3367 SET_ADDRESS(_stubs, StubRoutines::x86::vector_iota_indices() + i * 64);
3368 }
3369 #endif
3370 #if defined(AARCH64) && !defined(ZERO)
3371 SET_ADDRESS(_stubs, StubRoutines::aarch64::zero_blocks());
3372 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives());
3373 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives_long());
3374 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_array_equals());
3375 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LL());
3376 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UU());
3377 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LU());
3378 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UL());
3379 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ul());
3380 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ll());
3381 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_uu());
3382 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_byte_array_inflate());
3383 SET_ADDRESS(_stubs, StubRoutines::aarch64::spin_wait());
3384
3385 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BOOLEAN));
3386 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BYTE));
3387 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_SHORT));
3388 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_CHAR));
3389 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_INT));
3390 #endif
3391
3392 _complete = true;
3393 log_info(aot, codecache, init)("Stubs recorded");
3394 }
3395
3396 void AOTCodeAddressTable::init_early_c1() {
3397 #ifdef COMPILER1
3398 // Runtime1 Blobs
3399 StubId id = StubInfo::stub_base(StubGroup::C1);
3400 // include forward_exception in range we publish
3401 StubId limit = StubInfo::next(StubId::c1_forward_exception_id);
3402 for (; id != limit; id = StubInfo::next(id)) {
3403 if (Runtime1::blob_for(id) == nullptr) {
3404 log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3405 continue;
3406 }
3407 if (Runtime1::entry_for(id) == nullptr) {
3408 log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3409 continue;
3410 }
3411 address entry = Runtime1::entry_for(id);
3412 SET_ADDRESS(_C1_blobs, entry);
3413 }
3414 #endif // COMPILER1
3415 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3416 _early_c1_complete = true;
3417 }
3418
3419 void AOTCodeAddressTable::init_c1() {
3420 #ifdef COMPILER1
3421 // Runtime1 Blobs
3422 assert(_early_c1_complete, "early C1 blobs should be initialized");
3423 StubId id = StubInfo::next(StubId::c1_forward_exception_id);
3424 StubId limit = StubInfo::next(StubInfo::stub_max(StubGroup::C1));
3425 for (; id != limit; id = StubInfo::next(id)) {
3426 if (Runtime1::blob_for(id) == nullptr) {
3427 log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3428 continue;
3429 }
3430 if (Runtime1::entry_for(id) == nullptr) {
3431 log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3432 continue;
3433 }
3434 address entry = Runtime1::entry_for(id);
3435 SET_ADDRESS(_C1_blobs, entry);
3436 }
3437 #if INCLUDE_G1GC
3438 if (UseG1GC) {
3439 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3440 address entry = bs->pre_barrier_c1_runtime_code_blob()->code_begin();
3441 SET_ADDRESS(_C1_blobs, entry);
3442 }
3443 #endif // INCLUDE_G1GC
3444 #if INCLUDE_ZGC
3445 if (UseZGC) {
3446 ZBarrierSetC1* bs = (ZBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3447 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_oop_field_preloaded_runtime_stub);
3448 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_weak_oop_field_preloaded_runtime_stub);
3449 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_with_healing);
3450 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_without_healing);
3451 }
3452 #endif // INCLUDE_ZGC
3453 #if INCLUDE_SHENANDOAHGC
3454 if (UseShenandoahGC) {
3455 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3456 SET_ADDRESS(_C1_blobs, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
3457 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_rt_code_blob()->code_begin());
3458 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin());
3459 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_weak_rt_code_blob()->code_begin());
3460 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_phantom_rt_code_blob()->code_begin());
3461 }
3462 #endif // INCLUDE_SHENANDOAHGC
3463 #endif // COMPILER1
3464
3465 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3466 _c1_complete = true;
3467 log_info(aot, codecache, init)("Runtime1 Blobs recorded");
3468 }
3469
3470 void AOTCodeAddressTable::init_c2() {
3471 #ifdef COMPILER2
3472 // OptoRuntime Blobs
3473 SET_ADDRESS(_C2_blobs, OptoRuntime::uncommon_trap_blob()->entry_point());
3474 SET_ADDRESS(_C2_blobs, OptoRuntime::exception_blob()->entry_point());
3475 SET_ADDRESS(_C2_blobs, OptoRuntime::new_instance_Java());
3476 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_Java());
3477 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_nozero_Java());
3478 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray2_Java());
3479 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray3_Java());
3480 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray4_Java());
3481 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray5_Java());
3482 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarrayN_Java());
3483 SET_ADDRESS(_C2_blobs, OptoRuntime::vtable_must_compile_stub());
3484 SET_ADDRESS(_C2_blobs, OptoRuntime::complete_monitor_locking_Java());
3485 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notify_Java());
3486 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notifyAll_Java());
3487 SET_ADDRESS(_C2_blobs, OptoRuntime::rethrow_stub());
3488 SET_ADDRESS(_C2_blobs, OptoRuntime::slow_arraycopy_Java());
3489 SET_ADDRESS(_C2_blobs, OptoRuntime::register_finalizer_Java());
3490 SET_ADDRESS(_C2_blobs, OptoRuntime::class_init_barrier_Java());
3491 #if INCLUDE_JVMTI
3492 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_start());
3493 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_end());
3494 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_mount());
3495 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_unmount());
3496 #endif /* INCLUDE_JVMTI */
3497 #endif
3498
3499 assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length);
3500 _c2_complete = true;
3501 log_info(aot, codecache, init)("OptoRuntime Blobs recorded");
3502 }
3503 #undef SET_ADDRESS
3504
3505 AOTCodeAddressTable::~AOTCodeAddressTable() {
3506 if (_extrs_addr != nullptr) {
3507 FREE_C_HEAP_ARRAY(address, _extrs_addr);
3508 }
3509 if (_stubs_addr != nullptr) {
3510 FREE_C_HEAP_ARRAY(address, _stubs_addr);
3511 }
3512 if (_shared_blobs_addr != nullptr) {
3513 FREE_C_HEAP_ARRAY(address, _shared_blobs_addr);
3514 }
3515 }
3516
3517 #ifdef PRODUCT
3518 #define MAX_STR_COUNT 200
3519 #else
3520 #define MAX_STR_COUNT 500
3521 #endif
3522 #define _c_str_max MAX_STR_COUNT
3523 static const int _c_str_base = _all_max;
3524
3525 static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings
3526 static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; // Our duplicates
3527 static int _C_strings_count = 0;
3528 static int _C_strings_s[MAX_STR_COUNT] = {0};
3529 static int _C_strings_id[MAX_STR_COUNT] = {0};
3530 static int _C_strings_used = 0;
3531
3532 void AOTCodeCache::load_strings() {
3533 uint strings_count = _load_header->strings_count();
3534 if (strings_count == 0) {
3535 return;
3536 }
3537 uint strings_offset = _load_header->strings_offset();
3538 uint* string_lengths = (uint*)addr(strings_offset);
3539 strings_offset += (strings_count * sizeof(uint));
3540 uint strings_size = _load_header->search_table_offset() - strings_offset;
3541 // We have to keep cached strings longer than _cache buffer
3542 // because they are refernced from compiled code which may
3543 // still be executed on VM exit after _cache is freed.
3544 char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
3545 memcpy(p, addr(strings_offset), strings_size);
3546 _C_strings_buf = p;
3547 assert(strings_count <= MAX_STR_COUNT, "sanity");
3548 for (uint i = 0; i < strings_count; i++) {
3549 _C_strings[i] = p;
3550 uint len = string_lengths[i];
3551 _C_strings_s[i] = i;
3552 _C_strings_id[i] = i;
3553 p += len;
3554 }
3555 assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
3556 _C_strings_count = strings_count;
3557 _C_strings_used = strings_count;
3558 log_debug(aot, codecache, init)(" Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset);
3559 }
3560
3561 int AOTCodeCache::store_strings() {
3562 if (_C_strings_used > 0) {
3563 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3564 uint offset = _write_position;
3565 uint length = 0;
3566 uint* lengths = (uint *)reserve_bytes(sizeof(uint) * _C_strings_used);
3567 if (lengths == nullptr) {
3568 return -1;
3569 }
3570 for (int i = 0; i < _C_strings_used; i++) {
3571 const char* str = _C_strings[_C_strings_s[i]];
3572 uint len = (uint)strlen(str) + 1;
3573 length += len;
3574 assert(len < 1000, "big string: %s", str);
3575 lengths[i] = len;
3576 uint n = write_bytes(str, len);
3577 if (n != len) {
3578 return -1;
3579 }
3580 }
3581 log_debug(aot, codecache, exit)(" Wrote %d C strings of total length %d at offset %d to AOT Code Cache",
3582 _C_strings_used, length, offset);
3583 }
3584 return _C_strings_used;
3585 }
3586
3587 const char* AOTCodeCache::add_C_string(const char* str) {
3588 if (is_on_for_dump() && str != nullptr) {
3589 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3590 AOTCodeAddressTable* table = addr_table();
3591 if (table != nullptr) {
3592 return table->add_C_string(str);
3593 }
3594 }
3595 return str;
3596 }
3597
3598 const char* AOTCodeAddressTable::add_C_string(const char* str) {
3599 if (_extrs_complete) {
3600 // Check previous strings address
3601 for (int i = 0; i < _C_strings_count; i++) {
3602 if (_C_strings_in[i] == str) {
3603 return _C_strings[i]; // Found previous one - return our duplicate
3604 } else if (strcmp(_C_strings[i], str) == 0) {
3605 return _C_strings[i];
3606 }
3607 }
3608 // Add new one
3609 if (_C_strings_count < MAX_STR_COUNT) {
3610 // Passed in string can be freed and used space become inaccessible.
3611 // Keep original address but duplicate string for future compare.
3612 _C_strings_id[_C_strings_count] = -1; // Init
3613 _C_strings_in[_C_strings_count] = str;
3614 const char* dup = os::strdup(str);
3615 _C_strings[_C_strings_count++] = dup;
3616 log_trace(aot, codecache, stringtable)("add_C_string: [%d] " INTPTR_FORMAT " '%s'", _C_strings_count, p2i(dup), dup);
3617 return dup;
3618 } else {
3619 assert(false, "Number of C strings >= MAX_STR_COUNT");
3620 }
3621 }
3622 return str;
3623 }
3624
3625 int AOTCodeAddressTable::id_for_C_string(address str) {
3626 if (str == nullptr) {
3627 return -1;
3628 }
3629 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3630 for (int i = 0; i < _C_strings_count; i++) {
3631 if (_C_strings[i] == (const char*)str) { // found
3632 int id = _C_strings_id[i];
3633 if (id >= 0) {
3634 assert(id < _C_strings_used, "%d >= %d", id , _C_strings_used);
3635 return id; // Found recorded
3636 }
3637 // Not found in recorded, add new
3638 id = _C_strings_used++;
3639 _C_strings_s[id] = i;
3640 _C_strings_id[i] = id;
3641 return id;
3642 }
3643 }
3644 return -1;
3645 }
3646
3647 address AOTCodeAddressTable::address_for_C_string(int idx) {
3648 assert(idx < _C_strings_count, "sanity");
3649 return (address)_C_strings[idx];
3650 }
3651
3652 static int search_address(address addr, address* table, uint length) {
3653 for (int i = 0; i < (int)length; i++) {
3654 if (table[i] == addr) {
3655 return i;
3656 }
3657 }
3658 return BAD_ADDRESS_ID;
3659 }
3660
3661 address AOTCodeAddressTable::address_for_id(int idx) {
3662 assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3663 if (idx == -1) {
3664 return (address)-1;
3665 }
3666 uint id = (uint)idx;
3667 // special case for symbols based relative to os::init
3668 if (id > (_c_str_base + _c_str_max)) {
3669 return (address)os::init + idx;
3670 }
3671 if (idx < 0) {
3672 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3673 return nullptr;
3674 }
3675 // no need to compare unsigned id against 0
3676 if (/* id >= _extrs_base && */ id < _extrs_length) {
3677 return _extrs_addr[id - _extrs_base];
3678 }
3679 if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3680 return _stubs_addr[id - _stubs_base];
3681 }
3682 if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3683 return _stubs_addr[id - _stubs_base];
3684 }
3685 if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) {
3686 return _shared_blobs_addr[id - _shared_blobs_base];
3687 }
3688 if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3689 return _C1_blobs_addr[id - _C1_blobs_base];
3690 }
3691 if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3692 return _C1_blobs_addr[id - _C1_blobs_base];
3693 }
3694 if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) {
3695 return _C2_blobs_addr[id - _C2_blobs_base];
3696 }
3697 if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
3698 return address_for_C_string(id - _c_str_base);
3699 }
3700 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3701 return nullptr;
3702 }
3703
3704 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* blob) {
3705 assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3706 int id = -1;
3707 if (addr == (address)-1) { // Static call stub has jump to itself
3708 return id;
3709 }
3710 // Check card_table_base address first since it can point to any address
3711 BarrierSet* bs = BarrierSet::barrier_set();
3712 guarantee(!bs->is_a(BarrierSet::CardTableBarrierSet) || addr != ci_card_table_address_as<address>(), "sanity");
3713
3714 // Seach for C string
3715 id = id_for_C_string(addr);
3716 if (id >= 0) {
3717 return id + _c_str_base;
3718 }
3719 if (StubRoutines::contains(addr)) {
3720 // Search in stubs
3721 id = search_address(addr, _stubs_addr, _stubs_length);
3722 if (id == BAD_ADDRESS_ID) {
3723 StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
3724 if (desc == nullptr) {
3725 desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset);
3726 }
3727 const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
3728 assert(false, "Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
3729 } else {
3730 return _stubs_base + id;
3731 }
3732 } else {
3733 CodeBlob* cb = CodeCache::find_blob(addr);
3734 if (cb != nullptr) {
3735 int id_base = _shared_blobs_base;
3736 // Search in code blobs
3737 id = search_address(addr, _shared_blobs_addr, _shared_blobs_length);
3738 if (id == BAD_ADDRESS_ID) {
3739 id_base = _C1_blobs_base;
3740 // search C1 blobs
3741 id = search_address(addr, _C1_blobs_addr, _C1_blobs_length);
3742 }
3743 if (id == BAD_ADDRESS_ID) {
3744 id_base = _C2_blobs_base;
3745 // search C2 blobs
3746 id = search_address(addr, _C2_blobs_addr, _C2_blobs_length);
3747 }
3748 if (id == BAD_ADDRESS_ID) {
3749 assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name());
3750 } else {
3751 return id_base + id;
3752 }
3753 } else {
3754 // Search in runtime functions
3755 id = search_address(addr, _extrs_addr, _extrs_length);
3756 if (id == BAD_ADDRESS_ID) {
3757 ResourceMark rm;
3758 const int buflen = 1024;
3759 char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
3760 int offset = 0;
3761 if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
3762 if (offset > 0) {
3763 // Could be address of C string
3764 uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
3765 CompileTask* task = ciEnv::current()->task();
3766 uint compile_id = 0;
3767 uint comp_level =0;
3768 if (task != nullptr) { // this could be called from compiler runtime initialization (compiler blobs)
3769 compile_id = task->compile_id();
3770 comp_level = task->comp_level();
3771 }
3772 log_debug(aot, codecache)("%d (L%d): Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
3773 compile_id, comp_level, p2i(addr), dist, (const char*)addr);
3774 assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
3775 return dist;
3776 }
3777 reloc.print_current_on(tty);
3778 blob->print_on(tty);
3779 blob->print_code_on(tty);
3780 assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);
3781 } else {
3782 reloc.print_current_on(tty);
3783 blob->print_on(tty);
3784 blob->print_code_on(tty);
3785 os::find(addr, tty);
3786 assert(false, "Address " INTPTR_FORMAT " for <unknown>/('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr);
3787 }
3788 } else {
3789 return _extrs_base + id;
3790 }
3791 }
3792 }
3793 return id;
3794 }
3795
3796 #undef _extrs_max
3797 #undef _stubs_max
3798 #undef _shared_blobs_max
3799 #undef _C1_blobs_max
3800 #undef _C2_blobs_max
3801 #undef _blobs_max
3802 #undef _extrs_base
3803 #undef _stubs_base
3804 #undef _shared_blobs_base
3805 #undef _C1_blobs_base
3806 #undef _C2_blobs_base
3807 #undef _blobs_end
3808
3809 AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
3810
3811 void AOTRuntimeConstants::initialize_from_runtime() {
3812 BarrierSet* bs = BarrierSet::barrier_set();
3813 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
3814 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
3815 _aot_runtime_constants._card_table_address = ci_card_table_address_as<address>();
3816 _aot_runtime_constants._grain_shift = ctbs->grain_shift();
3817 }
3818 }
3819
3820 address AOTRuntimeConstants::_field_addresses_list[] = {
3821 card_table_address(),
3822 grain_shift_address(),
3823 nullptr
3824 };
3825
3826
3827 void AOTCodeCache::wait_for_no_nmethod_readers() {
3828 while (true) {
3829 int cur = AtomicAccess::load(&_nmethod_readers);
3830 int upd = -(cur + 1);
3831 if (cur >= 0 && AtomicAccess::cmpxchg(&_nmethod_readers, cur, upd) == cur) {
3832 // Success, no new readers should appear.
3833 break;
3834 }
3835 }
3836
3837 // Now wait for all readers to leave.
3838 SpinYield w;
3839 while (AtomicAccess::load(&_nmethod_readers) != -1) {
3840 w.wait();
3841 }
3842 }
3843
3844 AOTCodeCache::ReadingMark::ReadingMark() {
3845 while (true) {
3846 int cur = AtomicAccess::load(&_nmethod_readers);
3847 if (cur < 0) {
3848 // Cache is already closed, cannot proceed.
3849 _failed = true;
3850 return;
3851 }
3852 if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3853 // Successfully recorded ourselves as entered.
3854 _failed = false;
3855 return;
3856 }
3857 }
3858 }
3859
3860 AOTCodeCache::ReadingMark::~ReadingMark() {
3861 if (_failed) {
3862 return;
3863 }
3864 while (true) {
3865 int cur = AtomicAccess::load(&_nmethod_readers);
3866 if (cur > 0) {
3867 // Cache is open, we are counting down towards 0.
3868 if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur - 1) == cur) {
3869 return;
3870 }
3871 } else {
3872 // Cache is closed, we are counting up towards -1.
3873 if (AtomicAccess::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3874 return;
3875 }
3876 }
3877 }
3878 }
3879
3880 void AOTCodeCache::print_timers_on(outputStream* st) {
3881 if (is_using_code()) {
3882 st->print_cr (" AOT Code Preload Time: %7.3f s", _t_totalPreload.seconds());
3883 st->print_cr (" AOT Code Load Time: %7.3f s", _t_totalLoad.seconds());
3884 st->print_cr (" nmethod register: %7.3f s", _t_totalRegister.seconds());
3885 st->print_cr (" find AOT code entry: %7.3f s", _t_totalFind.seconds());
3886 }
3887 if (is_dumping_code()) {
3888 st->print_cr (" AOT Code Store Time: %7.3f s", _t_totalStore.seconds());
3889 }
3890 }
3891
3892 AOTCodeStats AOTCodeStats::add_aot_code_stats(AOTCodeStats stats1, AOTCodeStats stats2) {
3893 AOTCodeStats result;
3894 for (int kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3895 result.ccstats._kind_cnt[kind] = stats1.entry_count(kind) + stats2.entry_count(kind);
3896 }
3897
3898 for (int lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3899 result.ccstats._nmethod_cnt[lvl] = stats1.nmethod_count(lvl) + stats2.nmethod_count(lvl);
3900 }
3901 result.ccstats._clinit_barriers_cnt = stats1.clinit_barriers_count() + stats2.clinit_barriers_count();
3902 return result;
3903 }
3904
3905 void AOTCodeCache::log_stats_on_exit(AOTCodeStats& stats) {
3906 LogStreamHandle(Debug, aot, codecache, exit) log;
3907 if (log.is_enabled()) {
3908 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3909 log.print_cr(" %s: total=%u", aot_code_entry_kind_name[kind], stats.entry_count(kind));
3910 if (kind == AOTCodeEntry::Nmethod) {
3911 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3912 log.print_cr(" Tier %d: total=%u", lvl, stats.nmethod_count(lvl));
3913 }
3914 }
3915 }
3916 }
3917 }
3918
3919 static void print_helper1(outputStream* st, const char* name, int count) {
3920 if (count > 0) {
3921 st->print(" %s=%d", name, count);
3922 }
3923 }
3924
3925 void AOTCodeCache::print_statistics_on(outputStream* st) {
3926 AOTCodeCache* cache = open_for_use();
3927 if (cache != nullptr) {
3928 ReadingMark rdmk;
3929 if (rdmk.failed()) {
3930 // Cache is closed, cannot touch anything.
3931 return;
3932 }
3933 AOTCodeStats stats;
3934
3935 uint preload_count = cache->_load_header->preload_entries_count();
3936 AOTCodeEntry* preload_entries = (AOTCodeEntry*)cache->addr(cache->_load_header->preload_entries_offset());
3937 for (uint i = 0; i < preload_count; i++) {
3938 stats.collect_all_stats(&preload_entries[i]);
3939 }
3940
3941 uint count = cache->_load_header->entries_count();
3942 AOTCodeEntry* load_entries = (AOTCodeEntry*)cache->addr(cache->_load_header->entries_offset());
3943 for (uint i = 0; i < count; i++) {
3944 stats.collect_all_stats(&load_entries[i]);
3945 }
3946
3947 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3948 if (stats.entry_count(kind) > 0) {
3949 st->print(" %s:", aot_code_entry_kind_name[kind]);
3950 print_helper1(st, "total", stats.entry_count(kind));
3951 print_helper1(st, "loaded", stats.entry_loaded_count(kind));
3952 print_helper1(st, "invalidated", stats.entry_invalidated_count(kind));
3953 print_helper1(st, "failed", stats.entry_load_failed_count(kind));
3954 st->cr();
3955 }
3956 if (kind == AOTCodeEntry::Nmethod) {
3957 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3958 if (stats.nmethod_count(lvl) > 0) {
3959 st->print(" AOT Code T%d", lvl);
3960 print_helper1(st, "total", stats.nmethod_count(lvl));
3961 print_helper1(st, "loaded", stats.nmethod_loaded_count(lvl));
3962 print_helper1(st, "invalidated", stats.nmethod_invalidated_count(lvl));
3963 print_helper1(st, "failed", stats.nmethod_load_failed_count(lvl));
3964 if (lvl == AOTCompLevel_count-1) {
3965 print_helper1(st, "has_clinit_barriers", stats.clinit_barriers_count());
3966 }
3967 st->cr();
3968 }
3969 }
3970 }
3971 }
3972 LogStreamHandle(Debug, aot, codecache, init) log;
3973 if (log.is_enabled()) {
3974 AOTCodeCache::print_unused_entries_on(&log);
3975 }
3976 LogStreamHandle(Trace, aot, codecache) aot_info;
3977 // need a lock to traverse the code cache
3978 if (aot_info.is_enabled()) {
3979 MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag);
3980 NMethodIterator iter(NMethodIterator::all);
3981 while (iter.next()) {
3982 nmethod* nm = iter.method();
3983 if (nm->is_in_use() && !nm->is_native_method() && !nm->is_osr_method()) {
3984 aot_info.print("%5d:%c%c%c%d:", nm->compile_id(),
3985 (nm->method()->in_aot_cache() ? 'S' : ' '),
3986 (nm->is_aot() ? 'A' : ' '),
3987 (nm->preloaded() ? 'P' : ' '),
3988 nm->comp_level());
3989 print_helper(nm, &aot_info);
3990 aot_info.print(": ");
3991 CompileTask::print(&aot_info, nm, nullptr, true /*short_form*/);
3992 LogStreamHandle(Trace, aot, codecache) aot_debug;
3993 if (aot_debug.is_enabled()) {
3994 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), nm->method()));
3995 if (mtd != nullptr) {
3996 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
3997 aot_debug.print(" CTD: "); ctd->print_on(&aot_debug); aot_debug.cr();
3998 });
3999 }
4000 }
4001 }
4002 }
4003 }
4004 }
4005 }
4006
4007 void AOTCodeEntry::print(outputStream* st) const {
4008 st->print_cr(" AOT Code Cache entry " INTPTR_FORMAT " [kind: %d, id: " UINT32_FORMAT_X_0 ", offset: %d, size: %d, comp_level: %d, comp_id: %d, %s%s%s%s]",
4009 p2i(this), (int)_kind, _id, _offset, _size, _comp_level, _comp_id,
4010 (_not_entrant? "not_entrant" : "entrant"),
4011 (_loaded ? ", loaded" : ""),
4012 (_has_clinit_barriers ? ", has_clinit_barriers" : ""),
4013 (_for_preload ? ", for_preload" : ""));
4014 }
4015
4016 // This is called after initialize() but before init2()
4017 // and _cache is not set yet.
4018 void AOTCodeCache::print_on(outputStream* st) {
4019 if (opened_cache != nullptr && opened_cache->for_use()) {
4020 ReadingMark rdmk;
4021 if (rdmk.failed()) {
4022 // Cache is closed, cannot touch anything.
4023 return;
4024 }
4025
4026 st->print_cr("\nAOT Code Cache Preload entries");
4027
4028 uint preload_count = opened_cache->_load_header->preload_entries_count();
4029 AOTCodeEntry* preload_entries = (AOTCodeEntry*)opened_cache->addr(opened_cache->_load_header->preload_entries_offset());
4030 for (uint i = 0; i < preload_count; i++) {
4031 AOTCodeEntry* entry = &preload_entries[i];
4032
4033 uint entry_position = entry->offset();
4034 uint name_offset = entry->name_offset() + entry_position;
4035 const char* saved_name = opened_cache->addr(name_offset);
4036
4037 st->print_cr("%4u: %10s Id:%u L%u size=%u '%s' %s%s%s",
4038 i, aot_code_entry_kind_name[entry->kind()], entry->id(), entry->comp_level(),
4039 entry->size(), saved_name,
4040 entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
4041 entry->is_loaded() ? " loaded" : "",
4042 entry->not_entrant() ? " not_entrant" : "");
4043
4044 st->print_raw(" ");
4045 AOTCodeReader reader(opened_cache, entry, nullptr);
4046 reader.print_on(st);
4047 }
4048
4049 st->print_cr("\nAOT Code Cache entries");
4050
4051 uint count = opened_cache->_load_header->entries_count();
4052 uint* search_entries = (uint*)opened_cache->addr(opened_cache->_load_header->search_table_offset()); // [id, index]
4053 AOTCodeEntry* load_entries = (AOTCodeEntry*)opened_cache->addr(opened_cache->_load_header->entries_offset());
4054
4055 for (uint i = 0; i < count; i++) {
4056 int index = search_entries[2*i + 1];
4057 AOTCodeEntry* entry = &(load_entries[index]);
4058
4059 uint entry_position = entry->offset();
4060 uint name_offset = entry->name_offset() + entry_position;
4061 const char* saved_name = opened_cache->addr(name_offset);
4062
4063 st->print_cr("%4u: %10s idx:%4u Id:%u L%u size=%u '%s' %s%s%s%s",
4064 i, aot_code_entry_kind_name[entry->kind()], index, entry->id(), entry->comp_level(),
4065 entry->size(), saved_name,
4066 entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
4067 entry->for_preload() ? " for_preload" : "",
4068 entry->is_loaded() ? " loaded" : "",
4069 entry->not_entrant() ? " not_entrant" : "");
4070
4071 st->print_raw(" ");
4072 AOTCodeReader reader(opened_cache, entry, nullptr);
4073 reader.print_on(st);
4074 }
4075 }
4076 }
4077
4078 void AOTCodeCache::print_unused_entries_on(outputStream* st) {
4079 LogStreamHandle(Info, aot, codecache, init) info;
4080 if (info.is_enabled()) {
4081 AOTCodeCache::iterate([&](AOTCodeEntry* entry) {
4082 if (entry->is_nmethod() && !entry->is_loaded()) {
4083 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), entry->method()));
4084 if (mtd != nullptr) {
4085 if (mtd->has_holder()) {
4086 if (mtd->holder()->method_holder()->is_initialized()) {
4087 ResourceMark rm;
4088 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
4089 if ((uint)ctd->level() == entry->comp_level()) {
4090 if (ctd->init_deps_left_acquire() == 0) {
4091 nmethod* nm = mtd->holder()->code();
4092 if (nm == nullptr) {
4093 if (mtd->holder()->queued_for_compilation()) {
4094 return; // scheduled for compilation
4095 }
4096 } else if ((uint)nm->comp_level() >= entry->comp_level()) {
4097 return; // already online compiled and superseded by a more optimal method
4098 }
4099 info.print("AOT Code Cache entry not loaded: ");
4100 ctd->print_on(&info);
4101 info.cr();
4102 }
4103 }
4104 });
4105 } else {
4106 // not yet initialized
4107 }
4108 } else {
4109 info.print("AOT Code Cache entry doesn't have a holder: ");
4110 mtd->print_on(&info);
4111 info.cr();
4112 }
4113 }
4114 }
4115 });
4116 }
4117 }
4118
4119 void AOTCodeReader::print_on(outputStream* st) {
4120 uint entry_position = _entry->offset();
4121 set_read_position(entry_position);
4122
4123 // Read name
4124 uint name_offset = entry_position + _entry->name_offset();
4125 uint name_size = _entry->name_size(); // Includes '/0'
4126 const char* name = addr(name_offset);
4127
4128 st->print_cr(" name: %s", name);
4129 }
4130