1 /*
2 * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25
26 #include "asm/macroAssembler.hpp"
27 #include "cds/aotCacheAccess.hpp"
28 #include "cds/aotMetaspace.hpp"
29 #include "cds/cds_globals.hpp"
30 #include "cds/cdsConfig.hpp"
31 #include "cds/heapShared.hpp"
32 #include "ci/ciConstant.hpp"
33 #include "ci/ciEnv.hpp"
34 #include "ci/ciField.hpp"
35 #include "ci/ciMethod.hpp"
36 #include "ci/ciMethodData.hpp"
37 #include "ci/ciObject.hpp"
38 #include "ci/ciUtilities.inline.hpp"
39 #include "classfile/javaAssertions.hpp"
40 #include "classfile/stringTable.hpp"
41 #include "classfile/symbolTable.hpp"
42 #include "classfile/systemDictionary.hpp"
43 #include "classfile/vmClasses.hpp"
44 #include "classfile/vmIntrinsics.hpp"
45 #include "code/aotCodeCache.hpp"
46 #include "code/codeBlob.hpp"
47 #include "code/codeCache.hpp"
48 #include "code/oopRecorder.inline.hpp"
49 #include "compiler/abstractCompiler.hpp"
50 #include "compiler/compilationPolicy.hpp"
51 #include "compiler/compileBroker.hpp"
52 #include "compiler/compileTask.hpp"
53 #include "gc/g1/g1BarrierSetRuntime.hpp"
54 #include "gc/shared/gcConfig.hpp"
55 #include "logging/logStream.hpp"
56 #include "memory/memoryReserver.hpp"
57 #include "memory/universe.hpp"
58 #include "oops/klass.inline.hpp"
59 #include "oops/method.inline.hpp"
60 #include "oops/trainingData.hpp"
61 #include "prims/jvmtiThreadState.hpp"
62 #include "runtime/atomic.hpp"
63 #include "runtime/deoptimization.hpp"
64 #include "runtime/flags/flagSetting.hpp"
65 #include "runtime/globals_extension.hpp"
66 #include "runtime/handles.inline.hpp"
67 #include "runtime/java.hpp"
68 #include "runtime/jniHandles.inline.hpp"
69 #include "runtime/mutexLocker.hpp"
70 #include "runtime/os.inline.hpp"
71 #include "runtime/sharedRuntime.hpp"
72 #include "runtime/stubCodeGenerator.hpp"
73 #include "runtime/stubRoutines.hpp"
74 #include "runtime/threadIdentifier.hpp"
75 #include "runtime/timerTrace.hpp"
76 #include "utilities/copy.hpp"
77 #include "utilities/formatBuffer.hpp"
78 #include "utilities/ostream.hpp"
79 #include "utilities/spinYield.hpp"
80 #ifdef COMPILER1
81 #include "c1/c1_LIRAssembler.hpp"
82 #include "c1/c1_Runtime1.hpp"
83 #include "gc/g1/c1/g1BarrierSetC1.hpp"
84 #include "gc/shared/c1/barrierSetC1.hpp"
85 #if INCLUDE_SHENANDOAHGC
86 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
87 #endif // INCLUDE_SHENANDOAHGC
88 #include "gc/z/c1/zBarrierSetC1.hpp"
89 #endif // COMPILER1
90 #ifdef COMPILER2
91 #include "opto/runtime.hpp"
92 #endif
93 #if INCLUDE_JVMCI
94 #include "jvmci/jvmci.hpp"
95 #endif
96 #if INCLUDE_G1GC
97 #include "gc/g1/g1BarrierSetRuntime.hpp"
98 #endif
99 #if INCLUDE_SHENANDOAHGC
100 #include "gc/shenandoah/shenandoahRuntime.hpp"
101 #endif
102 #if INCLUDE_ZGC
103 #include "gc/z/zBarrierSetRuntime.hpp"
104 #endif
105 #if defined(X86) && !defined(ZERO)
106 #include "rdtsc_x86.hpp"
107 #endif
108
109 #include <errno.h>
110 #include <sys/stat.h>
111
112 const char* aot_code_entry_kind_name[] = {
113 #define DECL_KIND_STRING(kind) XSTR(kind),
114 DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
115 #undef DECL_KIND_STRING
116 };
117
118 static elapsedTimer _t_totalLoad;
119 static elapsedTimer _t_totalRegister;
120 static elapsedTimer _t_totalFind;
121 static elapsedTimer _t_totalStore;
122
123 static bool enable_timers() {
124 return CITime || log_is_enabled(Info, init);
125 }
126
127 static void report_load_failure() {
128 if (AbortVMOnAOTCodeFailure) {
129 vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
130 }
131 log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
132 AOTCodeCache::disable_caching();
133 }
134
135 static void report_store_failure() {
136 if (AbortVMOnAOTCodeFailure) {
137 tty->print_cr("Unable to create AOT Code Cache.");
138 vm_abort(false);
139 }
140 log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
141 AOTCodeCache::disable_caching();
142 }
143
144 // The sequence of AOT code caching flags and parametters settings.
145 //
146 // 1. The initial AOT code caching flags setting is done
147 // during call to CDSConfig::check_vm_args_consistency().
148 //
149 // 2. The earliest AOT code state check done in compilationPolicy_init()
150 // where we set number of compiler threads for AOT assembly phase.
151 //
152 // 3. We determine presence of AOT code in AOT Cache in
153 // AOTMetaspace::open_static_archive() which is calles
154 // after compilationPolicy_init() but before codeCache_init().
155 //
156 // 4. AOTCodeCache::initialize() is called during universe_init()
157 // and does final AOT state and flags settings.
158 //
159 // 5. Finally AOTCodeCache::init2() is called after universe_init()
160 // when all GC settings are finalized.
161
162 // Next methods determine which action we do with AOT code depending
163 // on phase of AOT process: assembly or production.
164
165 bool AOTCodeCache::is_dumping_adapter() {
166 return AOTAdapterCaching && is_on_for_dump();
167 }
168
169 bool AOTCodeCache::is_using_adapter() {
170 return AOTAdapterCaching && is_on_for_use();
171 }
172
173 bool AOTCodeCache::is_dumping_stub() {
174 return AOTStubCaching && is_on_for_dump();
175 }
176
177 bool AOTCodeCache::is_using_stub() {
178 return AOTStubCaching && is_on_for_use();
179 }
180
181 bool AOTCodeCache::is_dumping_code() {
182 return AOTCodeCaching && is_on_for_dump();
183 }
184
185 bool AOTCodeCache::is_using_code() {
186 return AOTCodeCaching && is_on_for_use();
187 }
188
189 // This is used before AOTCodeCahe is initialized
190 // but after AOT (CDS) Cache flags consistency is checked.
191 bool AOTCodeCache::maybe_dumping_code() {
192 return AOTCodeCaching && CDSConfig::is_dumping_final_static_archive();
193 }
194
195 // Next methods could be called regardless of AOT code cache status.
196 // Initially they are called during AOT flags parsing and finilized
197 // in AOTCodeCache::initialize().
198 void AOTCodeCache::enable_caching() {
199 FLAG_SET_ERGO_IF_DEFAULT(AOTCodeCaching, true);
200 FLAG_SET_ERGO_IF_DEFAULT(AOTStubCaching, true);
201 FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true);
202 }
203
204 void AOTCodeCache::disable_caching() {
205 FLAG_SET_ERGO(AOTCodeCaching, false);
206 FLAG_SET_ERGO(AOTStubCaching, false);
207 FLAG_SET_ERGO(AOTAdapterCaching, false);
208 }
209
210 bool AOTCodeCache::is_caching_enabled() {
211 return AOTCodeCaching || AOTStubCaching || AOTAdapterCaching;
212 }
213
214 static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
215 assert(AOTCodeEntry::is_valid_entry_kind(kind), "invalid AOTCodeEntry kind %d", (int)kind);
216 // There can be a conflict of id between an Adapter and *Blob, but that should not cause any functional issue
217 // becasue both id and kind are used to find an entry, and that combination should be unique
218 if (kind == AOTCodeEntry::Adapter) {
219 return id;
220 } else if (kind == AOTCodeEntry::SharedBlob) {
221 assert(StubInfo::is_shared(static_cast<BlobId>(id)), "not a shared blob id %d", id);
222 return id;
223 } else if (kind == AOTCodeEntry::C1Blob) {
224 assert(StubInfo::is_c1(static_cast<BlobId>(id)), "not a c1 blob id %d", id);
225 return id;
226 } else {
227 // kind must be AOTCodeEntry::C2Blob
228 assert(StubInfo::is_c2(static_cast<BlobId>(id)), "not a c2 blob id %d", id);
229 return id;
230 }
231 }
232
233 static uint _max_aot_code_size = 0;
234 uint AOTCodeCache::max_aot_code_size() {
235 return _max_aot_code_size;
236 }
237
238 bool AOTCodeCache::is_code_load_thread_on() {
239 return UseAOTCodeLoadThread && AOTCodeCaching;
240 }
241
242 bool AOTCodeCache::allow_const_field(ciConstant& value) {
243 ciEnv* env = CURRENT_ENV;
244 precond(env != nullptr);
245 assert(!env->is_precompile() || is_dumping_code(), "AOT compilation should be enabled");
246 return !env->is_precompile() // Restrict only when we generate AOT code
247 // Can not trust primitive too || !is_reference_type(value.basic_type())
248 // May disable this too for now || is_reference_type(value.basic_type()) && value.as_object()->should_be_constant()
249 ;
250 }
251
252 // It is called from AOTMetaspace::initialize_shared_spaces()
253 // which is called from universe_init().
254 // At this point all AOT class linking seetings are finilized
255 // and AOT cache is open so we can map AOT code region.
256 void AOTCodeCache::initialize() {
257 if (!is_caching_enabled()) {
258 log_info(aot, codecache, init)("AOT Code Cache is not used: disabled.");
259 return;
260 }
261 #if defined(ZERO) || !(defined(AMD64) || defined(AARCH64))
262 log_info(aot, codecache, init)("AOT Code Cache is not supported on this platform.");
263 disable_caching();
264 return;
265 #else
266 assert(!FLAG_IS_DEFAULT(AOTCache), "AOTCache should be specified");
267
268 // Disable stubs caching until JDK-8357398 is fixed.
269 FLAG_SET_ERGO(AOTStubCaching, false);
270
271 if (VerifyOops) {
272 // Disable AOT stubs caching when VerifyOops flag is on.
273 // Verify oops code generated a lot of C strings which overflow
274 // AOT C string table (which has fixed size).
275 // AOT C string table will be reworked later to handle such cases.
276 //
277 // Note: AOT adapters are not affected - they don't have oop operations.
278 log_info(aot, codecache, init)("AOT Stubs Caching is not supported with VerifyOops.");
279 FLAG_SET_ERGO(AOTStubCaching, false);
280 }
281
282 bool is_dumping = false;
283 bool is_using = false;
284 if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) {
285 is_dumping = is_caching_enabled();
286 } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) {
287 is_using = is_caching_enabled();
288 }
289 if (ClassInitBarrierMode > 0 && !(is_dumping && AOTCodeCaching)) {
290 log_info(aot, codecache, init)("Set ClassInitBarrierMode to 0 because AOT Code dumping is off.");
291 FLAG_SET_ERGO(ClassInitBarrierMode, 0);
292 }
293 if (!(is_dumping || is_using)) {
294 log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used.");
295 disable_caching();
296 return; // AOT code caching disabled on command line
297 }
298 // Reserve AOT Cache region when we dumping AOT code.
299 _max_aot_code_size = AOTCodeMaxSize;
300 if (is_dumping && !FLAG_IS_DEFAULT(AOTCodeMaxSize)) {
301 if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) {
302 _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity());
303 log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K));
304 }
305 }
306 size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0;
307 if (is_using && aot_code_size == 0) {
308 log_info(aot, codecache, init)("AOT Code Cache is empty");
309 disable_caching();
310 return;
311 }
312 if (!open_cache(is_dumping, is_using)) {
313 if (is_using) {
314 report_load_failure();
315 } else {
316 report_store_failure();
317 }
318 return;
319 }
320 if (is_dumping) {
321 FLAG_SET_DEFAULT(FoldStableValues, false);
322 FLAG_SET_DEFAULT(ForceUnreachable, true);
323 }
324 FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
325 #endif // defined(AMD64) || defined(AARCH64)
326 }
327
328 static AOTCodeCache* opened_cache = nullptr; // Use this until we verify the cache
329 AOTCodeCache* AOTCodeCache::_cache = nullptr;
330 DEBUG_ONLY( bool AOTCodeCache::_passed_init2 = false; )
331
332 // It is called after universe_init() when all GC settings are finalized.
333 void AOTCodeCache::init2() {
334 DEBUG_ONLY( _passed_init2 = true; )
335 if (opened_cache == nullptr) {
336 return;
337 }
338 // After Universe initialized
339 BarrierSet* bs = BarrierSet::barrier_set();
340 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
341 address byte_map_base = ci_card_table_address_as<address>();
342 if (is_on_for_dump() && !external_word_Relocation::can_be_relocated(byte_map_base)) {
343 // Bail out since we can't encode card table base address with relocation
344 log_warning(aot, codecache, init)("Can't create AOT Code Cache because card table base address is not relocatable: " INTPTR_FORMAT, p2i(byte_map_base));
345 close();
346 report_load_failure();
347 return;
348 }
349 }
350 if (!opened_cache->verify_config_on_use()) { // Check on AOT code loading
351 delete opened_cache;
352 opened_cache = nullptr;
353 report_load_failure();
354 return;
355 }
356
357 // initialize aot runtime constants as appropriate to this runtime
358 AOTRuntimeConstants::initialize_from_runtime();
359
360 // initialize the table of external routines and initial stubs so we can save
361 // generated code blobs that reference them
362 AOTCodeAddressTable* table = opened_cache->_table;
363 assert(table != nullptr, "should be initialized already");
364 table->init_extrs();
365
366 // Now cache and address table are ready for AOT code generation
367 _cache = opened_cache;
368
369 // Set ClassInitBarrierMode after all checks since it affects code generation
370 if (is_dumping_code()) {
371 FLAG_SET_ERGO_IF_DEFAULT(ClassInitBarrierMode, 1);
372 } else {
373 FLAG_SET_ERGO(ClassInitBarrierMode, 0);
374 }
375 }
376
377 bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
378 opened_cache = new AOTCodeCache(is_dumping, is_using);
379 if (opened_cache->failed()) {
380 delete opened_cache;
381 opened_cache = nullptr;
382 return false;
383 }
384 return true;
385 }
386
387 static void print_helper(nmethod* nm, outputStream* st) {
388 AOTCodeCache::iterate([&](AOTCodeEntry* e) {
389 if (e->method() == nm->method()) {
390 ResourceMark rm;
391 stringStream ss;
392 ss.print("A%s%d", (e->for_preload() ? "P" : ""), e->comp_level());
393 ss.print("[%s%s%s]",
394 (e->is_loaded() ? "L" : ""),
395 (e->load_fail() ? "F" : ""),
396 (e->not_entrant() ? "I" : ""));
397 ss.print("#%d", e->comp_id());
398
399 st->print(" %s", ss.freeze());
400 }
401 });
402 }
403
404 void AOTCodeCache::close() {
405 if (is_on()) {
406 delete _cache; // Free memory
407 _cache = nullptr;
408 opened_cache = nullptr;
409 }
410 }
411
412 class CachedCodeDirectory : public CachedCodeDirectoryInternal {
413 public:
414 uint _aot_code_size;
415 char* _aot_code_data;
416
417 void set_aot_code_data(uint size, char* aot_data) {
418 _aot_code_size = size;
419 AOTCacheAccess::set_pointer(&_aot_code_data, aot_data);
420 }
421
422 static CachedCodeDirectory* create();
423 };
424
425 // Storing AOT code in the AOT code region (ac) of AOT Cache:
426 //
427 // [1] Use CachedCodeDirectory to keep track of all of data related to AOT code.
428 // E.g., you can build a hashtable to record what methods have been archived.
429 //
430 // [2] Memory for all data for AOT code, including CachedCodeDirectory, should be
431 // allocated using AOTCacheAccess::allocate_aot_code_region().
432 //
433 // [3] CachedCodeDirectory must be the very first allocation.
434 //
435 // [4] Two kinds of pointer can be stored:
436 // - A pointer p that points to metadata. AOTCacheAccess::can_generate_aot_code(p) must return true.
437 // - A pointer to a buffer returned by AOTCacheAccess::allocate_aot_code_region().
438 // (It's OK to point to an interior location within this buffer).
439 // Such pointers must be stored using AOTCacheAccess::set_pointer()
440 //
441 // The buffers allocated by AOTCacheAccess::allocate_aot_code_region() are in a contiguous region. At runtime, this
442 // region is mapped to the process address space. All the pointers in this buffer are relocated as necessary
443 // (e.g., to account for the runtime location of the CodeCache).
444 //
445 // This is always at the very beginning of the mmaped CDS "ac" (AOT code) region
446 static CachedCodeDirectory* _aot_code_directory = nullptr;
447
448 CachedCodeDirectory* CachedCodeDirectory::create() {
449 assert(AOTCacheAccess::is_aot_code_region_empty(), "must be");
450 CachedCodeDirectory* dir = (CachedCodeDirectory*)AOTCacheAccess::allocate_aot_code_region(sizeof(CachedCodeDirectory));
451 dir->dumptime_init_internal();
452 return dir;
453 }
454
455 #define DATA_ALIGNMENT HeapWordSize
456
457 AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
458 _load_header(nullptr),
459 _load_buffer(nullptr),
460 _store_buffer(nullptr),
461 _C_store_buffer(nullptr),
462 _write_position(0),
463 _load_size(0),
464 _store_size(0),
465 _for_use(is_using),
466 _for_dump(is_dumping),
467 _closing(false),
468 _failed(false),
469 _lookup_failed(false),
470 _for_preload(false),
471 _has_clinit_barriers(false),
472 _table(nullptr),
473 _load_entries(nullptr),
474 _search_entries(nullptr),
475 _store_entries(nullptr),
476 _C_strings_buf(nullptr),
477 _store_entries_cnt(0),
478 _compile_id(0),
479 _comp_level(0)
480 {
481 // Read header at the begining of cache
482 if (_for_use) {
483 // Read cache
484 size_t load_size = AOTCacheAccess::get_aot_code_region_size();
485 ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode);
486 if (!rs.is_reserved()) {
487 log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size);
488 set_failed();
489 return;
490 }
491 if (!AOTCacheAccess::map_aot_code_region(rs)) {
492 log_warning(aot, codecache, init)("Failed to read/mmap AOT code region (ac) into AOT Code Cache");
493 set_failed();
494 return;
495 }
496 _aot_code_directory = (CachedCodeDirectory*)rs.base();
497 _aot_code_directory->runtime_init_internal();
498
499 _load_size = _aot_code_directory->_aot_code_size;
500 _load_buffer = _aot_code_directory->_aot_code_data;
501 assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
502 log_info(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " from AOT Code Cache", _load_size, p2i(_load_buffer));
503
504 _load_header = (Header*)addr(0);
505 if (!_load_header->verify(_load_size)) {
506 set_failed();
507 return;
508 }
509 log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count());
510 log_debug(aot, codecache, init)(" Adapters: total=%u", _load_header->adapters_count());
511 log_debug(aot, codecache, init)(" Shared Blobs: total=%u", _load_header->shared_blobs_count());
512 log_debug(aot, codecache, init)(" C1 Blobs: total=%u", _load_header->C1_blobs_count());
513 log_debug(aot, codecache, init)(" C2 Blobs: total=%u", _load_header->C2_blobs_count());
514 log_debug(aot, codecache, init)(" Stubs: total=%u", _load_header->stubs_count());
515 log_debug(aot, codecache, init)(" Nmethods: total=%u", _load_header->nmethods_count());
516 log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", _load_header->cache_size());
517
518 // Read strings
519 load_strings();
520 }
521 if (_for_dump) {
522 _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
523 _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
524 // Entries allocated at the end of buffer in reverse (as on stack).
525 _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
526 log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size());
527 }
528 _table = new AOTCodeAddressTable();
529 }
530
531 void AOTCodeCache::invalidate(AOTCodeEntry* entry) {
532 // This could be concurent execution
533 if (entry != nullptr && is_on()) { // Request could come after cache is closed.
534 _cache->invalidate_entry(entry);
535 }
536 }
537
538 void AOTCodeCache::init_early_stubs_table() {
539 AOTCodeAddressTable* table = addr_table();
540 if (table != nullptr) {
541 table->init_early_stubs();
542 }
543 }
544
545 void AOTCodeCache::init_shared_blobs_table() {
546 AOTCodeAddressTable* table = addr_table();
547 if (table != nullptr) {
548 table->init_shared_blobs();
549 }
550 }
551
552 void AOTCodeCache::init_stubs_table() {
553 AOTCodeAddressTable* table = addr_table();
554 if (table != nullptr) {
555 table->init_stubs();
556 }
557 }
558
559 void AOTCodeCache::init_early_c1_table() {
560 AOTCodeAddressTable* table = addr_table();
561 if (table != nullptr) {
562 table->init_early_c1();
563 }
564 }
565
566 void AOTCodeCache::init_c1_table() {
567 AOTCodeAddressTable* table = addr_table();
568 if (table != nullptr) {
569 table->init_c1();
570 }
571 }
572
573 void AOTCodeCache::init_c2_table() {
574 AOTCodeAddressTable* table = addr_table();
575 if (table != nullptr) {
576 table->init_c2();
577 }
578 }
579
580 AOTCodeCache::~AOTCodeCache() {
581 if (_closing) {
582 return; // Already closed
583 }
584 // Stop any further access to cache.
585 // Checked on entry to load_nmethod() and store_nmethod().
586 _closing = true;
587 if (_for_use) {
588 // Wait for all load_nmethod() finish.
589 wait_for_no_nmethod_readers();
590 }
591 // Prevent writing code into cache while we are closing it.
592 // This lock held by ciEnv::register_method() which calls store_nmethod().
593 MutexLocker ml(Compile_lock);
594 if (for_dump()) { // Finalize cache
595 finish_write();
596 }
597 _load_buffer = nullptr;
598 if (_C_store_buffer != nullptr) {
599 FREE_C_HEAP_ARRAY(char, _C_store_buffer);
600 _C_store_buffer = nullptr;
601 _store_buffer = nullptr;
602 }
603 if (_table != nullptr) {
604 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
605 delete _table;
606 _table = nullptr;
607 }
608 }
609
610 void AOTCodeCache::Config::record(uint cpu_features_offset) {
611 _flags = 0;
612 #ifdef ASSERT
613 _flags |= debugVM;
614 #endif
615 if (UseCompressedOops) {
616 _flags |= compressedOops;
617 }
618 if (UseCompressedClassPointers) {
619 _flags |= compressedClassPointers;
620 }
621 if (UseTLAB) {
622 _flags |= useTLAB;
623 }
624 if (JavaAssertions::systemClassDefault()) {
625 _flags |= systemClassAssertions;
626 }
627 if (JavaAssertions::userClassDefault()) {
628 _flags |= userClassAssertions;
629 }
630 if (EnableContended) {
631 _flags |= enableContendedPadding;
632 }
633 if (RestrictContended) {
634 _flags |= restrictContendedPadding;
635 }
636 if (PreserveFramePointer) {
637 _flags |= preserveFramePointer;
638 }
639 _codeCacheSize = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
640 _compressedOopShift = CompressedOops::shift();
641 _compressedOopBase = CompressedOops::base();
642 _compressedKlassShift = CompressedKlassPointers::shift();
643 _compressedKlassBase = CompressedKlassPointers::base();
644 _contendedPaddingWidth = ContendedPaddingWidth;
645 _objectAlignment = ObjectAlignmentInBytes;
646 _gc = (uint)Universe::heap()->kind();
647 _cpu_features_offset = cpu_features_offset;
648 }
649
650 bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
651 // First checks affect all cached AOT code
652 #ifdef ASSERT
653 if ((_flags & debugVM) == 0) {
654 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
655 return false;
656 }
657 #else
658 if ((_flags & debugVM) != 0) {
659 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
660 return false;
661 }
662 #endif
663
664 size_t codeCacheSize = pointer_delta(CodeCache::high_bound(), CodeCache::low_bound(), 1);
665 if (_codeCacheSize != codeCacheSize) {
666 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CodeCache size = %dKb vs current %dKb", (int)(_codeCacheSize/K), (int)(codeCacheSize/K));
667 return false;
668 }
669
670 CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
671 if (aot_gc != Universe::heap()->kind()) {
672 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
673 return false;
674 }
675
676 if (_objectAlignment != (uint)ObjectAlignmentInBytes) {
677 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ObjectAlignmentInBytes = %d vs current %d", _objectAlignment, ObjectAlignmentInBytes);
678 return false;
679 }
680
681 if (((_flags & enableContendedPadding) != 0) != EnableContended) {
682 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with EnableContended = %s vs current %s", (EnableContended ? "false" : "true"), (EnableContended ? "true" : "false"));
683 return false;
684 }
685 if (((_flags & restrictContendedPadding) != 0) != RestrictContended) {
686 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with RestrictContended = %s vs current %s", (RestrictContended ? "false" : "true"), (RestrictContended ? "true" : "false"));
687 return false;
688 }
689 if (_contendedPaddingWidth != (uint)ContendedPaddingWidth) {
690 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ContendedPaddingWidth = %d vs current %d", _contendedPaddingWidth, ContendedPaddingWidth);
691 return false;
692 }
693
694 if (((_flags & preserveFramePointer) != 0) != PreserveFramePointer) {
695 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with PreserveFramePointer = %s vs current %s", (PreserveFramePointer ? "false" : "true"), (PreserveFramePointer ? "true" : "false"));
696 return false;
697 }
698
699 if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
700 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s vs current %s", (UseCompressedClassPointers ? "false" : "true"), (UseCompressedClassPointers ? "true" : "false"));
701 return false;
702 }
703 if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
704 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
705 return false;
706 }
707 if ((_compressedKlassBase == nullptr || CompressedKlassPointers::base() == nullptr) && (_compressedKlassBase != CompressedKlassPointers::base())) {
708 log_debug(aot, codecache, init)("AOT Code Cache disabled: incompatible CompressedKlassPointers::base(): %p vs current %p", _compressedKlassBase, CompressedKlassPointers::base());
709 return false;
710 }
711
712 if (((_flags & compressedOops) != 0) != UseCompressedOops) {
713 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s vs current %s", (UseCompressedOops ? "false" : "true"), (UseCompressedOops ? "true" : "false"));
714 return false;
715 }
716 if (_compressedOopShift != (uint)CompressedOops::shift()) {
717 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
718 return false;
719 }
720 if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
721 log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
722 return false;
723 }
724
725 LogStreamHandle(Debug, aot, codecache, init) log;
726 if (log.is_enabled()) {
727 log.print_cr("Available CPU features: %s", VM_Version::features_string());
728 }
729
730 uint offset = _cpu_features_offset;
731 uint cpu_features_size = *(uint *)cache->addr(offset);
732 assert(cpu_features_size == (uint)VM_Version::cpu_features_size(), "must be");
733 offset += sizeof(uint);
734
735 void* cached_cpu_features_buffer = (void *)cache->addr(offset);
736 if (log.is_enabled()) {
737 ResourceMark rm; // required for stringStream::as_string()
738 stringStream ss;
739 VM_Version::get_cpu_features_name(cached_cpu_features_buffer, ss);
740 log.print_cr("CPU features recorded in AOTCodeCache: %s", ss.as_string());
741 }
742
743 if (AOTCodeCPUFeatureCheck && !VM_Version::supports_features(cached_cpu_features_buffer)) {
744 if (log.is_enabled()) {
745 ResourceMark rm; // required for stringStream::as_string()
746 stringStream ss;
747 VM_Version::get_missing_features_name(cached_cpu_features_buffer, ss);
748 log.print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
749 }
750 return false;
751 }
752
753 // Next affects only AOT nmethod
754 if (((_flags & systemClassAssertions) != 0) != JavaAssertions::systemClassDefault()) {
755 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::systemClassDefault() = %s vs current %s", (JavaAssertions::systemClassDefault() ? "disabled" : "enabled"), (JavaAssertions::systemClassDefault() ? "enabled" : "disabled"));
756 FLAG_SET_ERGO(AOTCodeCaching, false);
757 }
758 if (((_flags & userClassAssertions) != 0) != JavaAssertions::userClassDefault()) {
759 log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::userClassDefault() = %s vs current %s", (JavaAssertions::userClassDefault() ? "disabled" : "enabled"), (JavaAssertions::userClassDefault() ? "enabled" : "disabled"));
760 FLAG_SET_ERGO(AOTCodeCaching, false);
761 }
762
763 return true;
764 }
765
766 bool AOTCodeCache::Header::verify(uint load_size) const {
767 if (_version != AOT_CODE_VERSION) {
768 log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
769 return false;
770 }
771 if (load_size < _cache_size) {
772 log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
773 return false;
774 }
775 return true;
776 }
777
778 volatile int AOTCodeCache::_nmethod_readers = 0;
779
780 AOTCodeCache* AOTCodeCache::open_for_use() {
781 if (AOTCodeCache::is_on_for_use()) {
782 return AOTCodeCache::cache();
783 }
784 return nullptr;
785 }
786
787 AOTCodeCache* AOTCodeCache::open_for_dump() {
788 if (AOTCodeCache::is_on_for_dump()) {
789 AOTCodeCache* cache = AOTCodeCache::cache();
790 cache->clear_lookup_failed(); // Reset bit
791 return cache;
792 }
793 return nullptr;
794 }
795
796 bool AOTCodeCache::is_address_in_aot_cache(address p) {
797 AOTCodeCache* cache = open_for_use();
798 if (cache == nullptr) {
799 return false;
800 }
801 if ((p >= (address)cache->cache_buffer()) &&
802 (p < (address)(cache->cache_buffer() + cache->load_size()))) {
803 return true;
804 }
805 return false;
806 }
807
808 static void copy_bytes(const char* from, address to, uint size) {
809 assert((int)size > 0, "sanity");
810 memcpy(to, from, size);
811 log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
812 }
813
814 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry, CompileTask* task) {
815 _cache = cache;
816 _entry = entry;
817 _load_buffer = cache->cache_buffer();
818 _read_position = 0;
819 if (task != nullptr) {
820 _compile_id = task->compile_id();
821 _comp_level = task->comp_level();
822 _preload = task->preload();
823 } else {
824 _compile_id = 0;
825 _comp_level = 0;
826 _preload = false;
827 }
828 _lookup_failed = false;
829 }
830
831 void AOTCodeReader::set_read_position(uint pos) {
832 if (pos == _read_position) {
833 return;
834 }
835 assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
836 _read_position = pos;
837 }
838
839 bool AOTCodeCache::set_write_position(uint pos) {
840 if (pos == _write_position) {
841 return true;
842 }
843 if (_store_size < _write_position) {
844 _store_size = _write_position; // Adjust during write
845 }
846 assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
847 _write_position = pos;
848 return true;
849 }
850
851 static char align_buffer[256] = { 0 };
852
853 bool AOTCodeCache::align_write() {
854 // We are not executing code from cache - we copy it by bytes first.
855 // No need for big alignment (or at all).
856 uint padding = DATA_ALIGNMENT - (_write_position & (DATA_ALIGNMENT - 1));
857 if (padding == DATA_ALIGNMENT) {
858 return true;
859 }
860 uint n = write_bytes((const void*)&align_buffer, padding);
861 if (n != padding) {
862 return false;
863 }
864 log_trace(aot, codecache)("Adjust write alignment in AOT Code Cache");
865 return true;
866 }
867
868 // Check to see if AOT code cache has required space to store "nbytes" of data
869 address AOTCodeCache::reserve_bytes(uint nbytes) {
870 assert(for_dump(), "Code Cache file is not created");
871 uint new_position = _write_position + nbytes;
872 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
873 log_warning(aot,codecache)("Failed to ensure %d bytes at offset %d in AOT Code Cache. Increase AOTCodeMaxSize.",
874 nbytes, _write_position);
875 set_failed();
876 report_store_failure();
877 return nullptr;
878 }
879 address buffer = (address)(_store_buffer + _write_position);
880 log_trace(aot, codecache)("Reserved %d bytes at offset %d in AOT Code Cache", nbytes, _write_position);
881 _write_position += nbytes;
882 if (_store_size < _write_position) {
883 _store_size = _write_position;
884 }
885 return buffer;
886 }
887
888 uint AOTCodeCache::write_bytes(const void* buffer, uint nbytes) {
889 assert(for_dump(), "Code Cache file is not created");
890 if (nbytes == 0) {
891 return 0;
892 }
893 uint new_position = _write_position + nbytes;
894 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
895 log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.",
896 nbytes, _write_position);
897 set_failed();
898 report_store_failure();
899 return 0;
900 }
901 copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
902 log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
903 _write_position += nbytes;
904 if (_store_size < _write_position) {
905 _store_size = _write_position;
906 }
907 return nbytes;
908 }
909
910 AOTCodeEntry* AOTCodeCache::find_code_entry(const methodHandle& method, uint comp_level) {
911 assert(is_using_code(), "AOT code caching should be enabled");
912 if (!method->in_aot_cache()) {
913 return nullptr;
914 }
915 switch (comp_level) {
916 case CompLevel_simple:
917 if ((DisableAOTCode & (1 << 0)) != 0) {
918 return nullptr;
919 }
920 break;
921 case CompLevel_limited_profile:
922 if ((DisableAOTCode & (1 << 1)) != 0) {
923 return nullptr;
924 }
925 break;
926 case CompLevel_full_optimization:
927 if ((DisableAOTCode & (1 << 2)) != 0) {
928 return nullptr;
929 }
930 break;
931
932 default: return nullptr; // Level 1, 2, and 4 only
933 }
934 TraceTime t1("Total time to find AOT code", &_t_totalFind, enable_timers(), false);
935 if (is_on() && _cache->cache_buffer() != nullptr) {
936 uint id = AOTCacheAccess::convert_method_to_offset(method());
937 AOTCodeEntry* entry = _cache->find_entry(AOTCodeEntry::Code, id, comp_level);
938 if (entry == nullptr) {
939 LogStreamHandle(Info, aot, codecache, nmethod) log;
940 if (log.is_enabled()) {
941 ResourceMark rm;
942 const char* target_name = method->name_and_sig_as_C_string();
943 log.print("Missing entry for '%s' (comp_level %d, id: " UINT32_FORMAT_X_0 ")", target_name, (uint)comp_level, id);
944 }
945 #ifdef ASSERT
946 } else {
947 ResourceMark rm;
948 assert(method() == entry->method(), "AOTCodeCache: saved nmethod's method %p (name: %s id: " UINT32_FORMAT_X_0
949 ") is different from the method %p (name: %s, id: " UINT32_FORMAT_X_0 " being looked up" ,
950 entry->method(), entry->method()->name_and_sig_as_C_string(), entry->id(), method(), method()->name_and_sig_as_C_string(), id);
951 #endif
952 }
953
954 DirectiveSet* directives = DirectivesStack::getMatchingDirective(method, nullptr);
955 if (directives->IgnorePrecompiledOption) {
956 LogStreamHandle(Info, aot, codecache, compilation) log;
957 if (log.is_enabled()) {
958 log.print("Ignore AOT code entry on level %d for ", comp_level);
959 method->print_value_on(&log);
960 }
961 return nullptr;
962 }
963
964 return entry;
965 }
966 return nullptr;
967 }
968
969 Method* AOTCodeEntry::method() {
970 assert(_kind == Code, "invalid kind %d", _kind);
971 assert(AOTCodeCache::is_on_for_use(), "must be");
972 return AOTCacheAccess::convert_offset_to_method(_id);
973 }
974
975 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
976 return (void*)(cache->add_entry());
977 }
978
979 static bool check_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level, AOTCodeEntry* entry) {
980 if (entry->kind() == kind) {
981 assert(entry->id() == id, "sanity");
982 if (kind != AOTCodeEntry::Code || // addapters and stubs have only one version
983 // Look only for normal AOT code entry, preload code is handled separately
984 (!entry->not_entrant() && !entry->has_clinit_barriers() && (entry->comp_level() == comp_level))) {
985 return true; // Found
986 }
987 }
988 return false;
989 }
990
991 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level) {
992 assert(_for_use, "sanity");
993 uint count = _load_header->entries_count();
994 if (_load_entries == nullptr) {
995 // Read it
996 _search_entries = (uint*)addr(_load_header->search_table_offset()); // [id, index]
997 _load_entries = (AOTCodeEntry*)addr(_load_header->entries_offset());
998 log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
999 }
1000 // Binary search
1001 int l = 0;
1002 int h = count - 1;
1003 while (l <= h) {
1004 int mid = (l + h) >> 1;
1005 int ix = mid * 2;
1006 uint is = _search_entries[ix];
1007 if (is == id) {
1008 int index = _search_entries[ix + 1];
1009 AOTCodeEntry* entry = &(_load_entries[index]);
1010 if (check_entry(kind, id, comp_level, entry)) {
1011 return entry; // Found
1012 }
1013 // Leaner search around
1014 for (int i = mid - 1; i >= l; i--) { // search back
1015 ix = i * 2;
1016 is = _search_entries[ix];
1017 if (is != id) {
1018 break;
1019 }
1020 index = _search_entries[ix + 1];
1021 AOTCodeEntry* entry = &(_load_entries[index]);
1022 if (check_entry(kind, id, comp_level, entry)) {
1023 return entry; // Found
1024 }
1025 }
1026 for (int i = mid + 1; i <= h; i++) { // search forward
1027 ix = i * 2;
1028 is = _search_entries[ix];
1029 if (is != id) {
1030 break;
1031 }
1032 index = _search_entries[ix + 1];
1033 AOTCodeEntry* entry = &(_load_entries[index]);
1034 if (check_entry(kind, id, comp_level, entry)) {
1035 return entry; // Found
1036 }
1037 }
1038 break; // No match found
1039 } else if (is < id) {
1040 l = mid + 1;
1041 } else {
1042 h = mid - 1;
1043 }
1044 }
1045 return nullptr;
1046 }
1047
1048 void AOTCodeCache::invalidate_entry(AOTCodeEntry* entry) {
1049 assert(entry!= nullptr, "all entries should be read already");
1050 if (entry->not_entrant()) {
1051 return; // Someone invalidated it already
1052 }
1053 #ifdef ASSERT
1054 assert(_load_entries != nullptr, "sanity");
1055 {
1056 uint name_offset = entry->offset() + entry->name_offset();
1057 const char* name = _load_buffer + name_offset;;
1058 uint level = entry->comp_level();
1059 uint comp_id = entry->comp_id();
1060 bool for_preload = entry->for_preload();
1061 bool clinit_brs = entry->has_clinit_barriers();
1062 log_info(aot, codecache, nmethod)("Invalidating entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1063 name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1064 }
1065 assert(entry->is_loaded() || entry->for_preload(), "invalidate only AOT code in use or a preload code");
1066 bool found = false;
1067 uint count = _load_header->entries_count();
1068 uint i = 0;
1069 for(; i < count; i++) {
1070 if (entry == &(_load_entries[i])) {
1071 break;
1072 }
1073 }
1074 found = (i < count);
1075 assert(found, "entry should exist");
1076 #endif
1077 entry->set_not_entrant();
1078 uint name_offset = entry->offset() + entry->name_offset();
1079 const char* name = _load_buffer + name_offset;;
1080 uint level = entry->comp_level();
1081 uint comp_id = entry->comp_id();
1082 bool for_preload = entry->for_preload();
1083 bool clinit_brs = entry->has_clinit_barriers();
1084 log_info(aot, codecache, nmethod)("Invalidated entry for '%s' (comp_id %d, comp_level %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
1085 name, comp_id, level, entry->id(), (for_preload ? "P" : "A"), (clinit_brs ? ", has clinit barriers" : ""));
1086
1087 if (!for_preload && (entry->comp_level() == CompLevel_full_optimization)) {
1088 // Invalidate preload code if normal AOT C2 code is invalidated,
1089 // most likely because some dependencies changed during run.
1090 // We can still use normal AOT code if preload code is
1091 // invalidated - normal AOT code has less restrictions.
1092 Method* method = entry->method();
1093 AOTCodeEntry* preload_entry = method->aot_code_entry();
1094 if (preload_entry != nullptr) {
1095 assert(preload_entry->for_preload(), "expecting only such entries here");
1096 invalidate_entry(preload_entry);
1097 }
1098 }
1099 }
1100
1101 static int uint_cmp(const void *i, const void *j) {
1102 uint a = *(uint *)i;
1103 uint b = *(uint *)j;
1104 return a > b ? 1 : a < b ? -1 : 0;
1105 }
1106
1107 void AOTCodeCache::store_cpu_features(char*& buffer, uint buffer_size) {
1108 uint* size_ptr = (uint *)buffer;
1109 *size_ptr = buffer_size;
1110 buffer += sizeof(uint);
1111
1112 VM_Version::store_cpu_features(buffer);
1113 log_debug(aot, codecache, exit)("CPU features recorded in AOTCodeCache: %s", VM_Version::features_string());
1114 buffer += buffer_size;
1115 buffer = align_up(buffer, DATA_ALIGNMENT);
1116 }
1117
1118 bool AOTCodeCache::finish_write() {
1119 if (!align_write()) {
1120 return false;
1121 }
1122 uint strings_offset = _write_position;
1123 int strings_count = store_strings();
1124 if (strings_count < 0) {
1125 return false;
1126 }
1127 if (!align_write()) {
1128 return false;
1129 }
1130 uint strings_size = _write_position - strings_offset;
1131
1132 uint entries_count = 0; // Number of entrant (useful) code entries
1133 uint entries_offset = _write_position;
1134
1135 uint code_count = _store_entries_cnt;
1136 if (code_count > 0) {
1137 _aot_code_directory = CachedCodeDirectory::create();
1138 assert(_aot_code_directory != nullptr, "Sanity check");
1139
1140 uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT);
1141 uint search_count = code_count * 2;
1142 uint search_size = search_count * sizeof(uint);
1143 uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
1144 uint preload_entries_cnt = 0;
1145 uint* preload_entries = NEW_C_HEAP_ARRAY(uint, code_count, mtCode);
1146 uint preload_entries_size = code_count * sizeof(uint);
1147 // _write_position should include code and strings
1148 uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
1149 uint cpu_features_size = VM_Version::cpu_features_size();
1150 uint total_cpu_features_size = sizeof(uint) + cpu_features_size; // sizeof(uint) to store cpu_features_size
1151 uint total_size = _write_position + header_size + code_alignment +
1152 search_size + preload_entries_size + entries_size +
1153 align_up(total_cpu_features_size, DATA_ALIGNMENT);
1154 assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size());
1155
1156 // Allocate in AOT Cache buffer
1157 char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT);
1158 char* start = align_up(buffer, DATA_ALIGNMENT);
1159 char* current = start + header_size; // Skip header
1160
1161 uint cpu_features_offset = current - start;
1162 store_cpu_features(current, cpu_features_size);
1163 assert(is_aligned(current, DATA_ALIGNMENT), "sanity check");
1164 assert(current < start + total_size, "sanity check");
1165
1166 // Create ordered search table for entries [id, index];
1167 uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
1168
1169 AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
1170 uint adapters_count = 0;
1171 uint shared_blobs_count = 0;
1172 uint C1_blobs_count = 0;
1173 uint C2_blobs_count = 0;
1174 uint stubs_count = 0;
1175 uint nmethods_count = 0;
1176 uint max_size = 0;
1177 // AOTCodeEntry entries were allocated in reverse in store buffer.
1178 // Process them in reverse order to cache first code first.
1179 for (int i = code_count - 1; i >= 0; i--) {
1180 AOTCodeEntry* entry = &entries_address[i];
1181 if (entry->load_fail()) {
1182 continue;
1183 }
1184 if (entry->not_entrant()) {
1185 log_info(aot, codecache, exit)("Not entrant new entry comp_id: %d, comp_level: %d, hash: " UINT32_FORMAT_X_0 "%s",
1186 entry->comp_id(), entry->comp_level(), entry->id(), (entry->has_clinit_barriers() ? ", has clinit barriers" : ""));
1187 if (entry->for_preload()) {
1188 // Skip not entrant preload code:
1189 // we can't pre-load code which may have failing dependencies.
1190 continue;
1191 }
1192 entry->set_entrant(); // Reset
1193 } else if (entry->for_preload()) {
1194 // record entrant first version code for pre-loading
1195 preload_entries[preload_entries_cnt++] = entries_count;
1196 }
1197 {
1198 uint size = align_up(entry->size(), DATA_ALIGNMENT);
1199 if (size > max_size) {
1200 max_size = size;
1201 }
1202 copy_bytes((_store_buffer + entry->offset()), (address)current, size);
1203 entry->set_offset(current - start); // New offset
1204 current += size;
1205 uint n = write_bytes(entry, sizeof(AOTCodeEntry));
1206 if (n != sizeof(AOTCodeEntry)) {
1207 FREE_C_HEAP_ARRAY(uint, search);
1208 return false;
1209 }
1210 search[entries_count*2 + 0] = entry->id();
1211 search[entries_count*2 + 1] = entries_count;
1212 entries_count++;
1213 AOTCodeEntry::Kind kind = entry->kind();
1214 if (kind == AOTCodeEntry::Adapter) {
1215 adapters_count++;
1216 } else if (kind == AOTCodeEntry::SharedBlob) {
1217 shared_blobs_count++;
1218 } else if (kind == AOTCodeEntry::C1Blob) {
1219 C1_blobs_count++;
1220 } else if (kind == AOTCodeEntry::C2Blob) {
1221 C2_blobs_count++;
1222 } else if (kind == AOTCodeEntry::Stub) {
1223 stubs_count++;
1224 } else {
1225 assert(kind == AOTCodeEntry::Code, "sanity");
1226 nmethods_count++;
1227 }
1228 }
1229 }
1230
1231 if (entries_count == 0) {
1232 log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires");
1233 FREE_C_HEAP_ARRAY(uint, search);
1234 return true; // Nothing to write
1235 }
1236 assert(entries_count <= code_count, "%d > %d", entries_count, code_count);
1237 // Write strings
1238 if (strings_count > 0) {
1239 copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
1240 strings_offset = (current - start); // New offset
1241 current += strings_size;
1242 }
1243 uint preload_entries_offset = (current - start);
1244 preload_entries_size = preload_entries_cnt * sizeof(uint);
1245 if (preload_entries_size > 0) {
1246 copy_bytes((const char*)preload_entries, (address)current, preload_entries_size);
1247 current += preload_entries_size;
1248 log_info(aot, codecache, exit)("Wrote %d preload entries to AOT Code Cache", preload_entries_cnt);
1249 }
1250 if (preload_entries != nullptr) {
1251 FREE_C_HEAP_ARRAY(uint, preload_entries);
1252 }
1253
1254 uint search_table_offset = current - start;
1255 // Sort and store search table
1256 qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
1257 search_size = 2 * entries_count * sizeof(uint);
1258 copy_bytes((const char*)search, (address)current, search_size);
1259 FREE_C_HEAP_ARRAY(uint, search);
1260 current += search_size;
1261
1262 // Write entries
1263 current = align_up(current, DATA_ALIGNMENT);
1264 uint new_entries_offset = current - start;
1265 entries_size = entries_count * sizeof(AOTCodeEntry); // New size
1266 copy_bytes((_store_buffer + entries_offset), (address)current, entries_size);
1267 current += entries_size;
1268
1269 log_stats_on_exit();
1270
1271 uint size = (current - start);
1272 assert(size <= total_size, "%d > %d", size , total_size);
1273 uint blobs_count = shared_blobs_count + C1_blobs_count + C2_blobs_count;
1274 assert(nmethods_count == (entries_count - (stubs_count + blobs_count + adapters_count)), "sanity");
1275 log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count);
1276 log_debug(aot, codecache, exit)(" Shared Blobs: total=%u", shared_blobs_count);
1277 log_debug(aot, codecache, exit)(" C1 Blobs: total=%u", C1_blobs_count);
1278 log_debug(aot, codecache, exit)(" C2 Blobs: total=%u", C2_blobs_count);
1279 log_debug(aot, codecache, exit)(" Stubs: total=%u", stubs_count);
1280 log_debug(aot, codecache, exit)(" Nmethods: total=%u", nmethods_count);
1281 log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
1282
1283 // Finalize header
1284 AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
1285 header->init(size, (uint)strings_count, strings_offset,
1286 entries_count, search_table_offset, new_entries_offset,
1287 preload_entries_cnt, preload_entries_offset,
1288 adapters_count, shared_blobs_count,
1289 C1_blobs_count, C2_blobs_count,
1290 stubs_count, cpu_features_offset);
1291
1292 log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count);
1293
1294 _aot_code_directory->set_aot_code_data(size, start);
1295 }
1296 return true;
1297 }
1298
1299 //------------------Store/Load AOT code ----------------------
1300
1301 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1302 AOTCodeCache* cache = open_for_dump();
1303 if (cache == nullptr) {
1304 return false;
1305 }
1306 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1307
1308 if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
1309 return false;
1310 }
1311 if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
1312 return false;
1313 }
1314 log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1315
1316 #ifdef ASSERT
1317 LogStreamHandle(Trace, aot, codecache, stubs) log;
1318 if (log.is_enabled()) {
1319 FlagSetting fs(PrintRelocations, true);
1320 blob.print_on(&log);
1321 }
1322 #endif
1323 // we need to take a lock to prevent race between compiler threads generating AOT code
1324 // and the main thread generating adapter
1325 MutexLocker ml(Compile_lock);
1326 if (!is_on()) {
1327 return false; // AOT code cache was already dumped and closed.
1328 }
1329 if (!cache->align_write()) {
1330 return false;
1331 }
1332 uint entry_position = cache->_write_position;
1333
1334 // Write name
1335 uint name_offset = cache->_write_position - entry_position;
1336 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1337 uint n = cache->write_bytes(name, name_size);
1338 if (n != name_size) {
1339 return false;
1340 }
1341
1342 // Write CodeBlob
1343 if (!cache->align_write()) {
1344 return false;
1345 }
1346 uint blob_offset = cache->_write_position - entry_position;
1347 address archive_buffer = cache->reserve_bytes(blob.size());
1348 if (archive_buffer == nullptr) {
1349 return false;
1350 }
1351 CodeBlob::archive_blob(&blob, archive_buffer);
1352
1353 uint reloc_data_size = blob.relocation_size();
1354 n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size);
1355 if (n != reloc_data_size) {
1356 return false;
1357 }
1358
1359 bool has_oop_maps = false;
1360 if (blob.oop_maps() != nullptr) {
1361 if (!cache->write_oop_map_set(blob)) {
1362 return false;
1363 }
1364 has_oop_maps = true;
1365 }
1366
1367 #ifndef PRODUCT
1368 // Write asm remarks
1369 if (!cache->write_asm_remarks(blob.asm_remarks(), /* use_string_table */ true)) {
1370 return false;
1371 }
1372 if (!cache->write_dbg_strings(blob.dbg_strings(), /* use_string_table */ true)) {
1373 return false;
1374 }
1375 #endif /* PRODUCT */
1376
1377 if (!cache->write_relocations(blob)) {
1378 if (!cache->failed()) {
1379 // We may miss an address in AOT table - skip this code blob.
1380 cache->set_write_position(entry_position);
1381 }
1382 return false;
1383 }
1384
1385 uint entry_size = cache->_write_position - entry_position;
1386 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
1387 entry_position, entry_size, name_offset, name_size,
1388 blob_offset, has_oop_maps, blob.content_begin());
1389 log_debug(aot, codecache, stubs)("Wrote code blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1390 return true;
1391 }
1392
1393 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) {
1394 assert(AOTCodeEntry::is_blob(entry_kind),
1395 "wrong entry kind for blob id %s", StubInfo::name(id));
1396 return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id));
1397 }
1398
1399 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1400 AOTCodeCache* cache = open_for_use();
1401 if (cache == nullptr) {
1402 return nullptr;
1403 }
1404 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1405
1406 if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
1407 return nullptr;
1408 }
1409 if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
1410 return nullptr;
1411 }
1412 log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1413
1414 AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
1415 if (entry == nullptr) {
1416 return nullptr;
1417 }
1418 AOTCodeReader reader(cache, entry, nullptr);
1419 CodeBlob* blob = reader.compile_code_blob(name);
1420
1421 log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
1422 (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
1423 return blob;
1424 }
1425
1426 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
1427 assert(AOTCodeEntry::is_blob(entry_kind),
1428 "wrong entry kind for blob id %s", StubInfo::name(id));
1429 return load_code_blob(entry_kind, (uint)id, StubInfo::name(id));
1430 }
1431
1432 CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
1433 uint entry_position = _entry->offset();
1434
1435 // Read name
1436 uint name_offset = entry_position + _entry->name_offset();
1437 uint name_size = _entry->name_size(); // Includes '/0'
1438 const char* stored_name = addr(name_offset);
1439
1440 if (strncmp(stored_name, name, (name_size - 1)) != 0) {
1441 log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'",
1442 stored_name, name);
1443 set_lookup_failed(); // Skip this blob
1444 return nullptr;
1445 }
1446
1447 // Read archived code blob
1448 uint offset = entry_position + _entry->code_offset();
1449 CodeBlob* archived_blob = (CodeBlob*)addr(offset);
1450 offset += archived_blob->size();
1451
1452 address reloc_data = (address)addr(offset);
1453 offset += archived_blob->relocation_size();
1454 set_read_position(offset);
1455
1456 ImmutableOopMapSet* oop_maps = nullptr;
1457 if (_entry->has_oop_maps()) {
1458 oop_maps = read_oop_map_set();
1459 }
1460
1461 CodeBlob* code_blob = CodeBlob::create(archived_blob,
1462 stored_name,
1463 reloc_data,
1464 oop_maps
1465 );
1466 if (code_blob == nullptr) { // no space left in CodeCache
1467 return nullptr;
1468 }
1469
1470 #ifndef PRODUCT
1471 code_blob->asm_remarks().init();
1472 read_asm_remarks(code_blob->asm_remarks(), /* use_string_table */ true);
1473 code_blob->dbg_strings().init();
1474 read_dbg_strings(code_blob->dbg_strings(), /* use_string_table */ true);
1475 #endif // PRODUCT
1476
1477 fix_relocations(code_blob);
1478
1479 #ifdef ASSERT
1480 LogStreamHandle(Trace, aot, codecache, stubs) log;
1481 if (log.is_enabled()) {
1482 FlagSetting fs(PrintRelocations, true);
1483 code_blob->print_on(&log);
1484 }
1485 #endif
1486 return code_blob;
1487 }
1488
1489 bool AOTCodeCache::store_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1490 if (!is_dumping_stub()) {
1491 return false;
1492 }
1493 AOTCodeCache* cache = open_for_dump();
1494 if (cache == nullptr) {
1495 return false;
1496 }
1497 log_info(aot, codecache, stubs)("Writing stub '%s' id:%d to AOT Code Cache", name, (int)id);
1498 if (!cache->align_write()) {
1499 return false;
1500 }
1501 #ifdef ASSERT
1502 CodeSection* cs = cgen->assembler()->code_section();
1503 if (cs->has_locs()) {
1504 uint reloc_count = cs->locs_count();
1505 tty->print_cr("======== write stubs code section relocations [%d]:", reloc_count);
1506 // Collect additional data
1507 RelocIterator iter(cs);
1508 while (iter.next()) {
1509 switch (iter.type()) {
1510 case relocInfo::none:
1511 break;
1512 default: {
1513 iter.print_current_on(tty);
1514 fatal("stub's relocation %d unimplemented", (int)iter.type());
1515 break;
1516 }
1517 }
1518 }
1519 }
1520 #endif
1521 uint entry_position = cache->_write_position;
1522
1523 // Write code
1524 uint code_offset = 0;
1525 uint code_size = cgen->assembler()->pc() - start;
1526 uint n = cache->write_bytes(start, code_size);
1527 if (n != code_size) {
1528 return false;
1529 }
1530 // Write name
1531 uint name_offset = cache->_write_position - entry_position;
1532 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1533 n = cache->write_bytes(name, name_size);
1534 if (n != name_size) {
1535 return false;
1536 }
1537 uint entry_size = cache->_write_position - entry_position;
1538 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
1539 code_offset, code_size,
1540 AOTCodeEntry::Stub, (uint32_t)id);
1541 log_info(aot, codecache, stubs)("Wrote stub '%s' id:%d to AOT Code Cache", name, (int)id);
1542 return true;
1543 }
1544
1545 bool AOTCodeCache::load_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1546 if (!is_using_stub()) {
1547 return false;
1548 }
1549 assert(start == cgen->assembler()->pc(), "wrong buffer");
1550 AOTCodeCache* cache = open_for_use();
1551 if (cache == nullptr) {
1552 return false;
1553 }
1554 AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Stub, (uint)id);
1555 if (entry == nullptr) {
1556 return false;
1557 }
1558 uint entry_position = entry->offset();
1559 // Read name
1560 uint name_offset = entry->name_offset() + entry_position;
1561 uint name_size = entry->name_size(); // Includes '/0'
1562 const char* saved_name = cache->addr(name_offset);
1563 if (strncmp(name, saved_name, (name_size - 1)) != 0) {
1564 log_warning(aot, codecache)("Saved stub's name '%s' is different from '%s' for id:%d", saved_name, name, (int)id);
1565 cache->set_failed();
1566 report_load_failure();
1567 return false;
1568 }
1569 log_info(aot, codecache, stubs)("Reading stub '%s' id:%d from AOT Code Cache", name, (int)id);
1570 // Read code
1571 uint code_offset = entry->code_offset() + entry_position;
1572 uint code_size = entry->code_size();
1573 copy_bytes(cache->addr(code_offset), start, code_size);
1574 cgen->assembler()->code_section()->set_end(start + code_size);
1575 log_info(aot, codecache, stubs)("Read stub '%s' id:%d from AOT Code Cache", name, (int)id);
1576 return true;
1577 }
1578
1579 AOTCodeEntry* AOTCodeCache::store_nmethod(nmethod* nm, AbstractCompiler* compiler, bool for_preload) {
1580 if (!is_dumping_code()) {
1581 return nullptr;
1582 }
1583 assert(CDSConfig::is_dumping_aot_code(), "should be called only when allowed");
1584 AOTCodeCache* cache = open_for_dump();
1585 precond(cache != nullptr);
1586 precond(!nm->is_osr_method()); // AOT compilation is requested only during AOT cache assembly phase
1587 if (!compiler->is_c1() && !compiler->is_c2()) {
1588 // Only c1 and c2 compilers
1589 return nullptr;
1590 }
1591 int comp_level = nm->comp_level();
1592 if (comp_level == CompLevel_full_profile) {
1593 // Do not cache C1 compiles with full profile i.e. tier3
1594 return nullptr;
1595 }
1596 assert(comp_level == CompLevel_simple || comp_level == CompLevel_limited_profile || comp_level == CompLevel_full_optimization, "must be");
1597
1598 TraceTime t1("Total time to store AOT code", &_t_totalStore, enable_timers(), false);
1599 AOTCodeEntry* entry = nullptr;
1600 entry = cache->write_nmethod(nm, for_preload);
1601 if (entry == nullptr) {
1602 log_info(aot, codecache, nmethod)("%d (L%d): nmethod store attempt failed", nm->compile_id(), comp_level);
1603 }
1604 return entry;
1605 }
1606
1607 AOTCodeEntry* AOTCodeCache::write_nmethod(nmethod* nm, bool for_preload) {
1608 AOTCodeCache* cache = open_for_dump();
1609 assert(cache != nullptr, "sanity check");
1610 assert(!nm->has_clinit_barriers() || (ClassInitBarrierMode > 0), "sanity");
1611 uint comp_id = nm->compile_id();
1612 uint comp_level = nm->comp_level();
1613 Method* method = nm->method();
1614 if (!AOTCacheAccess::can_generate_aot_code(method)) {
1615 ResourceMark rm;
1616 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' for AOT%s compile: not in AOT cache", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), (for_preload ? " preload" : ""));
1617 assert(AOTCacheAccess::can_generate_aot_code(method), "sanity");
1618 return nullptr;
1619 }
1620 InstanceKlass* holder = method->method_holder();
1621 bool builtin_loader = holder->class_loader_data()->is_builtin_class_loader_data();
1622 if (!builtin_loader) {
1623 ResourceMark rm;
1624 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' loaded by custom class loader %s", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), holder->class_loader_data()->loader_name());
1625 assert(builtin_loader, "sanity");
1626 return nullptr;
1627 }
1628
1629 _for_preload = for_preload;
1630 _has_clinit_barriers = nm->has_clinit_barriers();
1631
1632 if (!align_write()) {
1633 return nullptr;
1634 }
1635
1636 uint entry_position = _write_position;
1637
1638 // Write name
1639 uint name_offset = 0;
1640 uint name_size = 0;
1641 uint id = 0;
1642 uint n;
1643 {
1644 ResourceMark rm;
1645 const char* name = method->name_and_sig_as_C_string();
1646 log_info(aot, codecache, nmethod)("%d (L%d): Writing nmethod '%s' (comp level: %d, %s) to AOT Code Cache",
1647 comp_id, (int)comp_level, name, comp_level,
1648 (nm->has_clinit_barriers() ? ", has clinit barriers" : ""));
1649
1650 LogStreamHandle(Info, aot, codecache, loader) log;
1651 if (log.is_enabled()) {
1652 oop loader = holder->class_loader();
1653 oop domain = holder->protection_domain();
1654 log.print("Holder: ");
1655 holder->print_value_on(&log);
1656 log.print(" loader: ");
1657 if (loader == nullptr) {
1658 log.print("nullptr");
1659 } else {
1660 loader->print_value_on(&log);
1661 }
1662 log.print(" domain: ");
1663 if (domain == nullptr) {
1664 log.print("nullptr");
1665 } else {
1666 domain->print_value_on(&log);
1667 }
1668 log.cr();
1669 }
1670 name_offset = _write_position - entry_position;
1671 name_size = (uint)strlen(name) + 1; // Includes '/0'
1672 n = write_bytes(name, name_size);
1673 if (n != name_size) {
1674 return nullptr;
1675 }
1676 }
1677 id = AOTCacheAccess::delta_from_base_address((address)nm->method());
1678
1679 // Write CodeBlob
1680 if (!cache->align_write()) {
1681 return nullptr;
1682 }
1683 uint blob_offset = cache->_write_position - entry_position;
1684 address archive_buffer = cache->reserve_bytes(nm->size());
1685 if (archive_buffer == nullptr) {
1686 return nullptr;
1687 }
1688 CodeBlob::archive_blob(nm, archive_buffer);
1689
1690 uint reloc_data_size = nm->relocation_size();
1691 n = write_bytes((address)nm->relocation_begin(), reloc_data_size);
1692 if (n != reloc_data_size) {
1693 return nullptr;
1694 }
1695
1696 // Write oops and metadata present in the nmethod's data region
1697 if (!write_oops(nm)) {
1698 if (lookup_failed() && !failed()) {
1699 // Skip this method and reposition file
1700 set_write_position(entry_position);
1701 }
1702 return nullptr;
1703 }
1704 if (!write_metadata(nm)) {
1705 if (lookup_failed() && !failed()) {
1706 // Skip this method and reposition file
1707 set_write_position(entry_position);
1708 }
1709 return nullptr;
1710 }
1711
1712 bool has_oop_maps = false;
1713 if (nm->oop_maps() != nullptr) {
1714 if (!cache->write_oop_map_set(*nm)) {
1715 return nullptr;
1716 }
1717 has_oop_maps = true;
1718 }
1719
1720 uint immutable_data_size = nm->immutable_data_size();
1721 n = write_bytes(nm->immutable_data_begin(), immutable_data_size);
1722 if (n != immutable_data_size) {
1723 return nullptr;
1724 }
1725
1726 JavaThread* thread = JavaThread::current();
1727 HandleMark hm(thread);
1728 GrowableArray<Handle> oop_list;
1729 GrowableArray<Metadata*> metadata_list;
1730
1731 nm->create_reloc_immediates_list(thread, oop_list, metadata_list);
1732 if (!write_nmethod_reloc_immediates(oop_list, metadata_list)) {
1733 if (lookup_failed() && !failed()) {
1734 // Skip this method and reposition file
1735 set_write_position(entry_position);
1736 }
1737 return nullptr;
1738 }
1739
1740 if (!write_relocations(*nm, &oop_list, &metadata_list)) {
1741 return nullptr;
1742 }
1743
1744 #ifndef PRODUCT
1745 if (!cache->write_asm_remarks(nm->asm_remarks(), /* use_string_table */ false)) {
1746 return nullptr;
1747 }
1748 if (!cache->write_dbg_strings(nm->dbg_strings(), /* use_string_table */ false)) {
1749 return nullptr;
1750 }
1751 #endif /* PRODUCT */
1752
1753 uint entry_size = _write_position - entry_position;
1754 AOTCodeEntry* entry = new (this) AOTCodeEntry(AOTCodeEntry::Code, id,
1755 entry_position, entry_size,
1756 name_offset, name_size,
1757 blob_offset, has_oop_maps,
1758 nm->content_begin(), comp_level, comp_id,
1759 nm->has_clinit_barriers(), for_preload);
1760 #ifdef ASSERT
1761 if (nm->has_clinit_barriers() || for_preload) {
1762 assert(for_preload, "sanity");
1763 }
1764 #endif
1765 {
1766 ResourceMark rm;
1767 const char* name = nm->method()->name_and_sig_as_C_string();
1768 log_info(aot, codecache, nmethod)("%d (L%d): Wrote nmethod '%s'%s to AOT Code Cache",
1769 comp_id, (int)comp_level, name, (for_preload ? " (for preload)" : ""));
1770 }
1771 if (VerifyAOTCode) {
1772 return nullptr;
1773 }
1774 return entry;
1775 }
1776
1777 bool AOTCodeCache::load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level) {
1778 if (!is_using_code()) {
1779 return false;
1780 }
1781 AOTCodeCache* cache = open_for_use();
1782 if (cache == nullptr) {
1783 return false;
1784 }
1785 assert(entry_bci == InvocationEntryBci, "unexpected entry_bci=%d", entry_bci);
1786 TraceTime t1("Total time to load AOT code", &_t_totalLoad, enable_timers(), false);
1787 CompileTask* task = env->task();
1788 task->mark_aot_load_start(os::elapsed_counter());
1789 AOTCodeEntry* entry = task->aot_code_entry();
1790 bool preload = task->preload();
1791 assert(entry != nullptr, "sanity");
1792 if (log_is_enabled(Info, aot, codecache, nmethod)) {
1793 VM_ENTRY_MARK;
1794 ResourceMark rm;
1795 methodHandle method(THREAD, target->get_Method());
1796 const char* target_name = method->name_and_sig_as_C_string();
1797 uint id = AOTCacheAccess::convert_method_to_offset(method());
1798 bool clinit_brs = entry->has_clinit_barriers();
1799 log_info(aot, codecache, nmethod)("%d (L%d): %s nmethod '%s' (id: " UINT32_FORMAT_X_0 "%s)",
1800 task->compile_id(), task->comp_level(), (preload ? "Preloading" : "Reading"),
1801 target_name, id, (clinit_brs ? ", has clinit barriers" : ""));
1802 }
1803 ReadingMark rdmk;
1804 if (rdmk.failed()) {
1805 // Cache is closed, cannot touch anything.
1806 return false;
1807 }
1808
1809 AOTCodeReader reader(cache, entry, task);
1810 bool success = reader.compile_nmethod(env, target, compiler);
1811 if (success) {
1812 task->set_num_inlined_bytecodes(entry->num_inlined_bytecodes());
1813 } else {
1814 entry->set_load_fail();
1815 entry->set_not_entrant();
1816 }
1817 task->mark_aot_load_finish(os::elapsed_counter());
1818 return success;
1819 }
1820
1821 bool AOTCodeReader::compile_nmethod(ciEnv* env, ciMethod* target, AbstractCompiler* compiler) {
1822 CompileTask* task = env->task();
1823 AOTCodeEntry* aot_code_entry = (AOTCodeEntry*)_entry;
1824 nmethod* nm = nullptr;
1825
1826 uint entry_position = aot_code_entry->offset();
1827 uint archived_nm_offset = entry_position + aot_code_entry->code_offset();
1828 nmethod* archived_nm = (nmethod*)addr(archived_nm_offset);
1829 set_read_position(archived_nm_offset + archived_nm->size());
1830
1831 OopRecorder* oop_recorder = new OopRecorder(env->arena());
1832 env->set_oop_recorder(oop_recorder);
1833
1834 uint offset;
1835
1836 offset = read_position();
1837 address reloc_data = (address)addr(offset);
1838 offset += archived_nm->relocation_size();
1839 set_read_position(offset);
1840
1841 // Read oops and metadata
1842 VM_ENTRY_MARK
1843 GrowableArray<Handle> oop_list;
1844 GrowableArray<Metadata*> metadata_list;
1845
1846 if (!read_oop_metadata_list(THREAD, target, oop_list, metadata_list, oop_recorder)) {
1847 return false;
1848 }
1849
1850 ImmutableOopMapSet* oopmaps = read_oop_map_set();
1851
1852 offset = read_position();
1853 address immutable_data = (address)addr(offset);
1854 offset += archived_nm->immutable_data_size();
1855 set_read_position(offset);
1856
1857 GrowableArray<Handle> reloc_immediate_oop_list;
1858 GrowableArray<Metadata*> reloc_immediate_metadata_list;
1859 if (!read_oop_metadata_list(THREAD, target, reloc_immediate_oop_list, reloc_immediate_metadata_list, nullptr)) {
1860 return false;
1861 }
1862
1863 // Read Dependencies (compressed already)
1864 Dependencies* dependencies = new Dependencies(env);
1865 dependencies->set_content(immutable_data, archived_nm->dependencies_size());
1866 env->set_dependencies(dependencies);
1867
1868 const char* name = addr(entry_position + aot_code_entry->name_offset());
1869
1870 if (VerifyAOTCode) {
1871 return false;
1872 }
1873
1874 TraceTime t1("Total time to register AOT nmethod", &_t_totalRegister, enable_timers(), false);
1875 nm = env->register_aot_method(THREAD,
1876 target,
1877 compiler,
1878 archived_nm,
1879 reloc_data,
1880 oop_list,
1881 metadata_list,
1882 oopmaps,
1883 immutable_data,
1884 reloc_immediate_oop_list,
1885 reloc_immediate_metadata_list,
1886 this);
1887 bool success = task->is_success();
1888 if (success) {
1889 log_info(aot, codecache, nmethod)("%d (L%d): Read nmethod '%s' from AOT Code Cache", compile_id(), comp_level(), name);
1890 #ifdef ASSERT
1891 LogStreamHandle(Debug, aot, codecache, nmethod) log;
1892 if (log.is_enabled()) {
1893 FlagSetting fs(PrintRelocations, true);
1894 nm->print_on(&log);
1895 nm->decode2(&log);
1896 }
1897 #endif
1898 }
1899
1900 return success;
1901 }
1902
1903 bool skip_preload(methodHandle mh) {
1904 if (!mh->method_holder()->is_loaded()) {
1905 return true;
1906 }
1907 DirectiveSet* directives = DirectivesStack::getMatchingDirective(mh, nullptr);
1908 if (directives->DontPreloadOption) {
1909 LogStreamHandle(Info, aot, codecache, init) log;
1910 if (log.is_enabled()) {
1911 log.print("Exclude preloading code for ");
1912 mh->print_value_on(&log);
1913 }
1914 return true;
1915 }
1916 return false;
1917 }
1918
1919 void AOTCodeCache::preload_code(JavaThread* thread) {
1920 if (!is_using_code()) {
1921 return;
1922 }
1923 if ((DisableAOTCode & (1 << 3)) != 0) {
1924 return; // no preloaded code (level 5);
1925 }
1926 _cache->preload_aot_code(thread);
1927 }
1928
1929 void AOTCodeCache::preload_aot_code(TRAPS) {
1930 if (CompilationPolicy::compiler_count(CompLevel_full_optimization) == 0) {
1931 // Since we reuse the CompilerBroker API to install AOT code, we're required to have a JIT compiler for the
1932 // level we want (that is CompLevel_full_optimization).
1933 return;
1934 }
1935 assert(_for_use, "sanity");
1936 uint count = _load_header->entries_count();
1937 if (_load_entries == nullptr) {
1938 // Read it
1939 _search_entries = (uint*)addr(_load_header->search_table_offset()); // [id, index]
1940 _load_entries = (AOTCodeEntry*)addr(_load_header->entries_offset());
1941 log_info(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
1942 }
1943 uint preload_entries_count = _load_header->preload_entries_count();
1944 if (preload_entries_count > 0) {
1945 uint* entries_index = (uint*)addr(_load_header->preload_entries_offset());
1946 log_info(aot, codecache, init)("Load %d preload entries from AOT Code Cache", preload_entries_count);
1947 uint count = MIN2(preload_entries_count, AOTCodeLoadStop);
1948 for (uint i = AOTCodeLoadStart; i < count; i++) {
1949 uint index = entries_index[i];
1950 AOTCodeEntry* entry = &(_load_entries[index]);
1951 if (entry->not_entrant()) {
1952 continue;
1953 }
1954 methodHandle mh(THREAD, entry->method());
1955 assert((mh.not_null() && AOTMetaspace::in_aot_cache((address)mh())), "sanity");
1956 if (skip_preload(mh)) {
1957 continue; // Exclude preloading for this method
1958 }
1959 assert(mh->method_holder()->is_loaded(), "");
1960 if (!mh->method_holder()->is_linked()) {
1961 assert(!HAS_PENDING_EXCEPTION, "");
1962 mh->method_holder()->link_class(THREAD);
1963 if (HAS_PENDING_EXCEPTION) {
1964 LogStreamHandle(Info, aot, codecache) log;
1965 if (log.is_enabled()) {
1966 ResourceMark rm;
1967 log.print("Linkage failed for %s: ", mh->method_holder()->external_name());
1968 THREAD->pending_exception()->print_value_on(&log);
1969 if (log_is_enabled(Debug, aot, codecache)) {
1970 THREAD->pending_exception()->print_on(&log);
1971 }
1972 }
1973 CLEAR_PENDING_EXCEPTION;
1974 }
1975 }
1976 if (mh->aot_code_entry() != nullptr) {
1977 // Second C2 compilation of the same method could happen for
1978 // different reasons without marking first entry as not entrant.
1979 continue; // Keep old entry to avoid issues
1980 }
1981 mh->set_aot_code_entry(entry);
1982 CompileBroker::compile_method(mh, InvocationEntryBci, CompLevel_full_optimization, 0, false, CompileTask::Reason_Preload, CHECK);
1983 }
1984 }
1985 }
1986
1987 // ------------ process code and data --------------
1988
1989 // Can't use -1. It is valid value for jump to iteself destination
1990 // used by static call stub: see NativeJump::jump_destination().
1991 #define BAD_ADDRESS_ID -2
1992
1993 bool AOTCodeCache::write_relocations(CodeBlob& code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
1994 GrowableArray<uint> reloc_data;
1995 RelocIterator iter(&code_blob);
1996 LogStreamHandle(Trace, aot, codecache, reloc) log;
1997 while (iter.next()) {
1998 int idx = reloc_data.append(0); // default value
1999 switch (iter.type()) {
2000 case relocInfo::none:
2001 break;
2002 case relocInfo::oop_type: {
2003 oop_Relocation* r = (oop_Relocation*)iter.reloc();
2004 if (r->oop_is_immediate()) {
2005 assert(oop_list != nullptr, "sanity check");
2006 // store index of oop in the reloc immediate oop list
2007 Handle h(JavaThread::current(), r->oop_value());
2008 int oop_idx = oop_list->find(h);
2009 assert(oop_idx != -1, "sanity check");
2010 reloc_data.at_put(idx, (uint)oop_idx);
2011 }
2012 break;
2013 }
2014 case relocInfo::metadata_type: {
2015 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2016 if (r->metadata_is_immediate()) {
2017 assert(metadata_list != nullptr, "sanity check");
2018 // store index of metadata in the reloc immediate metadata list
2019 int metadata_idx = metadata_list->find(r->metadata_value());
2020 assert(metadata_idx != -1, "sanity check");
2021 reloc_data.at_put(idx, (uint)metadata_idx);
2022 }
2023 break;
2024 }
2025 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
2026 case relocInfo::opt_virtual_call_type:
2027 case relocInfo::static_call_type: {
2028 CallRelocation* r = (CallRelocation*)iter.reloc();
2029 address dest = r->destination();
2030 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2031 dest = (address)-1; // do nothing in this case when loading this relocation
2032 }
2033 int id = _table->id_for_address(dest, iter, &code_blob);
2034 if (id == BAD_ADDRESS_ID) {
2035 return false;
2036 }
2037 reloc_data.at_put(idx, id);
2038 break;
2039 }
2040 case relocInfo::trampoline_stub_type: {
2041 address dest = ((trampoline_stub_Relocation*)iter.reloc())->destination();
2042 int id = _table->id_for_address(dest, iter, &code_blob);
2043 if (id == BAD_ADDRESS_ID) {
2044 return false;
2045 }
2046 reloc_data.at_put(idx, id);
2047 break;
2048 }
2049 case relocInfo::static_stub_type:
2050 break;
2051 case relocInfo::runtime_call_type: {
2052 // Record offset of runtime destination
2053 CallRelocation* r = (CallRelocation*)iter.reloc();
2054 address dest = r->destination();
2055 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2056 dest = (address)-1; // do nothing in this case when loading this relocation
2057 }
2058 int id = _table->id_for_address(dest, iter, &code_blob);
2059 if (id == BAD_ADDRESS_ID) {
2060 return false;
2061 }
2062 reloc_data.at_put(idx, id);
2063 break;
2064 }
2065 case relocInfo::runtime_call_w_cp_type:
2066 log_debug(aot, codecache, reloc)("runtime_call_w_cp_type relocation is not implemented");
2067 return false;
2068 case relocInfo::external_word_type: {
2069 // Record offset of runtime target
2070 address target = ((external_word_Relocation*)iter.reloc())->target();
2071 int id = _table->id_for_address(target, iter, &code_blob);
2072 if (id == BAD_ADDRESS_ID) {
2073 return false;
2074 }
2075 reloc_data.at_put(idx, id);
2076 break;
2077 }
2078 case relocInfo::internal_word_type:
2079 break;
2080 case relocInfo::section_word_type:
2081 break;
2082 case relocInfo::poll_type:
2083 break;
2084 case relocInfo::poll_return_type:
2085 break;
2086 case relocInfo::post_call_nop_type:
2087 break;
2088 case relocInfo::entry_guard_type:
2089 break;
2090 default:
2091 log_debug(aot, codecache, reloc)("relocation %d unimplemented", (int)iter.type());
2092 return false;
2093 break;
2094 }
2095 if (log.is_enabled()) {
2096 iter.print_current_on(&log);
2097 }
2098 }
2099
2100 // Write additional relocation data: uint per relocation
2101 // Write the count first
2102 int count = reloc_data.length();
2103 write_bytes(&count, sizeof(int));
2104 for (GrowableArrayIterator<uint> iter = reloc_data.begin();
2105 iter != reloc_data.end(); ++iter) {
2106 uint value = *iter;
2107 int n = write_bytes(&value, sizeof(uint));
2108 if (n != sizeof(uint)) {
2109 return false;
2110 }
2111 }
2112 return true;
2113 }
2114
2115 void AOTCodeReader::fix_relocations(CodeBlob* code_blob, GrowableArray<Handle>* oop_list, GrowableArray<Metadata*>* metadata_list) {
2116 LogStreamHandle(Trace, aot, reloc) log;
2117 uint offset = read_position();
2118 int count = *(int*)addr(offset);
2119 offset += sizeof(int);
2120 if (log.is_enabled()) {
2121 log.print_cr("======== extra relocations count=%d", count);
2122 }
2123 uint* reloc_data = (uint*)addr(offset);
2124 offset += (count * sizeof(uint));
2125 set_read_position(offset);
2126
2127 RelocIterator iter(code_blob);
2128 int j = 0;
2129 while (iter.next()) {
2130 switch (iter.type()) {
2131 case relocInfo::none:
2132 break;
2133 case relocInfo::oop_type: {
2134 assert(code_blob->is_nmethod(), "sanity check");
2135 oop_Relocation* r = (oop_Relocation*)iter.reloc();
2136 if (r->oop_is_immediate()) {
2137 assert(oop_list != nullptr, "sanity check");
2138 Handle h = oop_list->at(reloc_data[j]);
2139 r->set_value(cast_from_oop<address>(h()));
2140 } else {
2141 r->fix_oop_relocation();
2142 }
2143 break;
2144 }
2145 case relocInfo::metadata_type: {
2146 assert(code_blob->is_nmethod(), "sanity check");
2147 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2148 Metadata* m;
2149 if (r->metadata_is_immediate()) {
2150 assert(metadata_list != nullptr, "sanity check");
2151 m = metadata_list->at(reloc_data[j]);
2152 } else {
2153 // Get already updated value from nmethod.
2154 int index = r->metadata_index();
2155 m = code_blob->as_nmethod()->metadata_at(index);
2156 }
2157 r->set_value((address)m);
2158 break;
2159 }
2160 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
2161 case relocInfo::opt_virtual_call_type:
2162 case relocInfo::static_call_type: {
2163 address dest = _cache->address_for_id(reloc_data[j]);
2164 if (dest != (address)-1) {
2165 ((CallRelocation*)iter.reloc())->set_destination(dest);
2166 }
2167 break;
2168 }
2169 case relocInfo::trampoline_stub_type: {
2170 address dest = _cache->address_for_id(reloc_data[j]);
2171 if (dest != (address)-1) {
2172 ((trampoline_stub_Relocation*)iter.reloc())->set_destination(dest);
2173 }
2174 break;
2175 }
2176 case relocInfo::static_stub_type:
2177 break;
2178 case relocInfo::runtime_call_type: {
2179 address dest = _cache->address_for_id(reloc_data[j]);
2180 if (dest != (address)-1) {
2181 ((CallRelocation*)iter.reloc())->set_destination(dest);
2182 }
2183 break;
2184 }
2185 case relocInfo::runtime_call_w_cp_type:
2186 // this relocation should not be in cache (see write_relocations)
2187 assert(false, "runtime_call_w_cp_type relocation is not implemented");
2188 break;
2189 case relocInfo::external_word_type: {
2190 address target = _cache->address_for_id(reloc_data[j]);
2191 // Add external address to global table
2192 int index = ExternalsRecorder::find_index(target);
2193 // Update index in relocation
2194 Relocation::add_jint(iter.data(), index);
2195 external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
2196 assert(reloc->target() == target, "sanity");
2197 reloc->set_value(target); // Patch address in the code
2198 break;
2199 }
2200 case relocInfo::internal_word_type: {
2201 internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
2202 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2203 break;
2204 }
2205 case relocInfo::section_word_type: {
2206 section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
2207 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
2208 break;
2209 }
2210 case relocInfo::poll_type:
2211 break;
2212 case relocInfo::poll_return_type:
2213 break;
2214 case relocInfo::post_call_nop_type:
2215 break;
2216 case relocInfo::entry_guard_type:
2217 break;
2218 default:
2219 assert(false,"relocation %d unimplemented", (int)iter.type());
2220 break;
2221 }
2222 if (log.is_enabled()) {
2223 iter.print_current_on(&log);
2224 }
2225 j++;
2226 }
2227 assert(j == count, "sanity");
2228 }
2229
2230 bool AOTCodeCache::write_nmethod_reloc_immediates(GrowableArray<Handle>& oop_list, GrowableArray<Metadata*>& metadata_list) {
2231 int count = oop_list.length();
2232 if (!write_bytes(&count, sizeof(int))) {
2233 return false;
2234 }
2235 for (GrowableArrayIterator<Handle> iter = oop_list.begin();
2236 iter != oop_list.end(); ++iter) {
2237 Handle h = *iter;
2238 if (!write_oop(h())) {
2239 return false;
2240 }
2241 }
2242
2243 count = metadata_list.length();
2244 if (!write_bytes(&count, sizeof(int))) {
2245 return false;
2246 }
2247 for (GrowableArrayIterator<Metadata*> iter = metadata_list.begin();
2248 iter != metadata_list.end(); ++iter) {
2249 Metadata* m = *iter;
2250 if (!write_metadata(m)) {
2251 return false;
2252 }
2253 }
2254 return true;
2255 }
2256
2257 bool AOTCodeCache::write_metadata(nmethod* nm) {
2258 int count = nm->metadata_count()-1;
2259 if (!write_bytes(&count, sizeof(int))) {
2260 return false;
2261 }
2262 for (Metadata** p = nm->metadata_begin(); p < nm->metadata_end(); p++) {
2263 if (!write_metadata(*p)) {
2264 return false;
2265 }
2266 }
2267 return true;
2268 }
2269
2270 bool AOTCodeCache::write_metadata(Metadata* m) {
2271 uint n = 0;
2272 if (m == nullptr) {
2273 DataKind kind = DataKind::Null;
2274 n = write_bytes(&kind, sizeof(int));
2275 if (n != sizeof(int)) {
2276 return false;
2277 }
2278 } else if (m == (Metadata*)Universe::non_oop_word()) {
2279 DataKind kind = DataKind::No_Data;
2280 n = write_bytes(&kind, sizeof(int));
2281 if (n != sizeof(int)) {
2282 return false;
2283 }
2284 } else if (m->is_klass()) {
2285 if (!write_klass((Klass*)m)) {
2286 return false;
2287 }
2288 } else if (m->is_method()) {
2289 if (!write_method((Method*)m)) {
2290 return false;
2291 }
2292 } else if (m->is_methodCounters()) {
2293 DataKind kind = DataKind::MethodCnts;
2294 n = write_bytes(&kind, sizeof(int));
2295 if (n != sizeof(int)) {
2296 return false;
2297 }
2298 if (!write_method(((MethodCounters*)m)->method())) {
2299 return false;
2300 }
2301 log_debug(aot, codecache, metadata)("%d (L%d): Write MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2302 } else { // Not supported
2303 fatal("metadata : " INTPTR_FORMAT " unimplemented", p2i(m));
2304 return false;
2305 }
2306 return true;
2307 }
2308
2309 Metadata* AOTCodeReader::read_metadata(const methodHandle& comp_method) {
2310 uint code_offset = read_position();
2311 Metadata* m = nullptr;
2312 DataKind kind = *(DataKind*)addr(code_offset);
2313 code_offset += sizeof(DataKind);
2314 set_read_position(code_offset);
2315 if (kind == DataKind::Null) {
2316 m = (Metadata*)nullptr;
2317 } else if (kind == DataKind::No_Data) {
2318 m = (Metadata*)Universe::non_oop_word();
2319 } else if (kind == DataKind::Klass) {
2320 m = (Metadata*)read_klass(comp_method);
2321 } else if (kind == DataKind::Method) {
2322 m = (Metadata*)read_method(comp_method);
2323 } else if (kind == DataKind::MethodCnts) {
2324 kind = *(DataKind*)addr(code_offset);
2325 code_offset += sizeof(DataKind);
2326 set_read_position(code_offset);
2327 m = (Metadata*)read_method(comp_method);
2328 if (m != nullptr) {
2329 Method* method = (Method*)m;
2330 m = method->get_method_counters(Thread::current());
2331 if (m == nullptr) {
2332 set_lookup_failed();
2333 log_debug(aot, codecache, metadata)("%d (L%d): Failed to get MethodCounters", compile_id(), comp_level());
2334 } else {
2335 log_debug(aot, codecache, metadata)("%d (L%d): Read MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2336 }
2337 }
2338 } else {
2339 set_lookup_failed();
2340 log_debug(aot, codecache, metadata)("%d (L%d): Unknown metadata's kind: %d", compile_id(), comp_level(), (int)kind);
2341 }
2342 return m;
2343 }
2344
2345 bool AOTCodeCache::write_method(Method* method) {
2346 ResourceMark rm; // To method's name printing
2347 if (AOTCacheAccess::can_generate_aot_code(method)) {
2348 DataKind kind = DataKind::Method;
2349 uint n = write_bytes(&kind, sizeof(int));
2350 if (n != sizeof(int)) {
2351 return false;
2352 }
2353 uint method_offset = AOTCacheAccess::delta_from_base_address((address)method);
2354 n = write_bytes(&method_offset, sizeof(uint));
2355 if (n != sizeof(uint)) {
2356 return false;
2357 }
2358 log_debug(aot, codecache, metadata)("%d (L%d): Wrote method: %s @ 0x%08x",
2359 compile_id(), comp_level(), method->name_and_sig_as_C_string(), method_offset);
2360 return true;
2361 }
2362 log_debug(aot, codecache, metadata)("%d (L%d): Method is not archived: %s",
2363 compile_id(), comp_level(), method->name_and_sig_as_C_string());
2364 set_lookup_failed();
2365 return false;
2366 }
2367
2368 Method* AOTCodeReader::read_method(const methodHandle& comp_method) {
2369 uint code_offset = read_position();
2370 uint method_offset = *(uint*)addr(code_offset);
2371 code_offset += sizeof(uint);
2372 set_read_position(code_offset);
2373 Method* m = AOTCacheAccess::convert_offset_to_method(method_offset);
2374 if (!AOTMetaspace::in_aot_cache((address)m)) {
2375 // Something changed in CDS
2376 set_lookup_failed();
2377 log_debug(aot, codecache, metadata)("Lookup failed for shared method: " INTPTR_FORMAT " is not in CDS ", p2i((address)m));
2378 return nullptr;
2379 }
2380 assert(m->is_method(), "sanity");
2381 ResourceMark rm;
2382 Klass* k = m->method_holder();
2383 if (!k->is_instance_klass()) {
2384 set_lookup_failed();
2385 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not instance klass",
2386 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2387 return nullptr;
2388 } else if (!AOTMetaspace::in_aot_cache((address)k)) {
2389 set_lookup_failed();
2390 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not in CDS",
2391 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2392 return nullptr;
2393 } else if (!InstanceKlass::cast(k)->is_loaded()) {
2394 set_lookup_failed();
2395 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not loaded",
2396 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2397 return nullptr;
2398 } else if (!InstanceKlass::cast(k)->is_linked()) {
2399 set_lookup_failed();
2400 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for holder %s: not linked%s",
2401 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name(), (_preload ? " for code preload" : ""));
2402 return nullptr;
2403 }
2404 log_debug(aot, codecache, metadata)("%d (L%d): Shared method lookup: %s",
2405 compile_id(), comp_level(), m->name_and_sig_as_C_string());
2406 return m;
2407 }
2408
2409 bool AOTCodeCache::write_klass(Klass* klass) {
2410 uint array_dim = 0;
2411 if (klass->is_objArray_klass()) {
2412 array_dim = ObjArrayKlass::cast(klass)->dimension();
2413 klass = ObjArrayKlass::cast(klass)->bottom_klass(); // overwrites klass
2414 }
2415 uint init_state = 0;
2416 bool can_write = true;
2417 if (klass->is_instance_klass()) {
2418 InstanceKlass* ik = InstanceKlass::cast(klass);
2419 init_state = (ik->is_initialized() ? 1 : 0);
2420 can_write = AOTCacheAccess::can_generate_aot_code_for(ik);
2421 } else {
2422 can_write = AOTCacheAccess::can_generate_aot_code(klass);
2423 }
2424 ResourceMark rm;
2425 uint state = (array_dim << 1) | (init_state & 1);
2426 if (can_write) {
2427 DataKind kind = DataKind::Klass;
2428 uint n = write_bytes(&kind, sizeof(int));
2429 if (n != sizeof(int)) {
2430 return false;
2431 }
2432 // Record state of instance klass initialization and array dimentions.
2433 n = write_bytes(&state, sizeof(int));
2434 if (n != sizeof(int)) {
2435 return false;
2436 }
2437 uint klass_offset = AOTCacheAccess::delta_from_base_address((address)klass);
2438 n = write_bytes(&klass_offset, sizeof(uint));
2439 if (n != sizeof(uint)) {
2440 return false;
2441 }
2442 log_debug(aot, codecache, metadata)("%d (L%d): Registered klass: %s%s%s @ 0x%08x",
2443 compile_id(), comp_level(), klass->external_name(),
2444 (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2445 (array_dim > 0 ? " (object array)" : ""), klass_offset);
2446 return true;
2447 }
2448 log_debug(aot, codecache, metadata)("%d (L%d): Klassis not archived: %s%s%s",
2449 compile_id(), comp_level(), klass->external_name(),
2450 (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
2451 (array_dim > 0 ? " (object array)" : ""));
2452 set_lookup_failed();
2453 return false;
2454 }
2455
2456 Klass* AOTCodeReader::read_klass(const methodHandle& comp_method) {
2457 uint code_offset = read_position();
2458 uint state = *(uint*)addr(code_offset);
2459 uint init_state = (state & 1);
2460 uint array_dim = (state >> 1);
2461 code_offset += sizeof(int);
2462 uint klass_offset = *(uint*)addr(code_offset);
2463 code_offset += sizeof(uint);
2464 set_read_position(code_offset);
2465 Klass* k = AOTCacheAccess::convert_offset_to_klass(klass_offset);
2466 if (!AOTMetaspace::in_aot_cache((address)k)) {
2467 // Something changed in CDS
2468 set_lookup_failed();
2469 log_debug(aot, codecache, metadata)("Lookup failed for shared klass: " INTPTR_FORMAT " is not in CDS ", p2i((address)k));
2470 return nullptr;
2471 }
2472 assert(k->is_klass(), "sanity");
2473 ResourceMark rm;
2474 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_loaded()) {
2475 set_lookup_failed();
2476 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not loaded",
2477 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2478 return nullptr;
2479 } else
2480 // Allow not initialized klass which was uninitialized during code caching or for preload
2481 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_initialized() && (init_state == 1) && !_preload) {
2482 set_lookup_failed();
2483 log_debug(aot, codecache, metadata)("%d '%s' (L%d): Lookup failed for klass %s: not initialized",
2484 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
2485 return nullptr;
2486 }
2487 if (array_dim > 0) {
2488 assert(k->is_instance_klass() || k->is_typeArray_klass(), "sanity check");
2489 Klass* ak = k->array_klass_or_null(array_dim);
2490 // FIXME: what would it take to create an array class on the fly?
2491 // Klass* ak = k->array_klass(dim, JavaThread::current());
2492 // guarantee(JavaThread::current()->pending_exception() == nullptr, "");
2493 if (ak == nullptr) {
2494 set_lookup_failed();
2495 log_debug(aot, codecache, metadata)("%d (L%d): %d-dimension array klass lookup failed: %s",
2496 compile_id(), comp_level(), array_dim, k->external_name());
2497 }
2498 log_debug(aot, codecache, metadata)("%d (L%d): Klass lookup: %s (object array)", compile_id(), comp_level(), k->external_name());
2499 return ak;
2500 } else {
2501 log_debug(aot, codecache, metadata)("%d (L%d): Shared klass lookup: %s",
2502 compile_id(), comp_level(), k->external_name());
2503 return k;
2504 }
2505 }
2506
2507 bool AOTCodeCache::write_oop(jobject& jo) {
2508 oop obj = JNIHandles::resolve(jo);
2509 return write_oop(obj);
2510 }
2511
2512 bool AOTCodeCache::write_oop(oop obj) {
2513 DataKind kind;
2514 uint n = 0;
2515 if (obj == nullptr) {
2516 kind = DataKind::Null;
2517 n = write_bytes(&kind, sizeof(int));
2518 if (n != sizeof(int)) {
2519 return false;
2520 }
2521 } else if (cast_from_oop<void *>(obj) == Universe::non_oop_word()) {
2522 kind = DataKind::No_Data;
2523 n = write_bytes(&kind, sizeof(int));
2524 if (n != sizeof(int)) {
2525 return false;
2526 }
2527 } else if (java_lang_Class::is_instance(obj)) {
2528 if (java_lang_Class::is_primitive(obj)) {
2529 int bt = (int)java_lang_Class::primitive_type(obj);
2530 kind = DataKind::Primitive;
2531 n = write_bytes(&kind, sizeof(int));
2532 if (n != sizeof(int)) {
2533 return false;
2534 }
2535 n = write_bytes(&bt, sizeof(int));
2536 if (n != sizeof(int)) {
2537 return false;
2538 }
2539 log_debug(aot, codecache, oops)("%d (L%d): Write primitive type klass: %s", compile_id(), comp_level(), type2name((BasicType)bt));
2540 } else {
2541 Klass* klass = java_lang_Class::as_Klass(obj);
2542 if (!write_klass(klass)) {
2543 return false;
2544 }
2545 }
2546 } else if (java_lang_String::is_instance(obj)) { // herere
2547 int k = AOTCacheAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2548 ResourceMark rm;
2549 size_t length_sz = 0;
2550 const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2551 if (k >= 0) {
2552 kind = DataKind::String;
2553 n = write_bytes(&kind, sizeof(int));
2554 if (n != sizeof(int)) {
2555 return false;
2556 }
2557 n = write_bytes(&k, sizeof(int));
2558 if (n != sizeof(int)) {
2559 return false;
2560 }
2561 log_debug(aot, codecache, oops)("%d (L%d): Write String object: " PTR_FORMAT " : %s", compile_id(), comp_level(), p2i(obj), string);
2562 return true;
2563 }
2564 // Not archived String object - bailout
2565 set_lookup_failed();
2566 log_debug(aot, codecache, oops)("%d (L%d): Not archived String object: " PTR_FORMAT " : %s",
2567 compile_id(), comp_level(), p2i(obj), string);
2568 return false;
2569 } else if (java_lang_Module::is_instance(obj)) {
2570 fatal("Module object unimplemented");
2571 } else if (java_lang_ClassLoader::is_instance(obj)) {
2572 if (obj == SystemDictionary::java_system_loader()) {
2573 kind = DataKind::SysLoader;
2574 log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_system_loader", compile_id(), comp_level());
2575 } else if (obj == SystemDictionary::java_platform_loader()) {
2576 kind = DataKind::PlaLoader;
2577 log_debug(aot, codecache, oops)("%d (L%d): Write ClassLoader: java_platform_loader", compile_id(), comp_level());
2578 } else {
2579 ResourceMark rm;
2580 set_lookup_failed();
2581 log_debug(aot, codecache, oops)("%d (L%d): Not supported Class Loader: " PTR_FORMAT " : %s",
2582 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2583 return false;
2584 }
2585 n = write_bytes(&kind, sizeof(int));
2586 if (n != sizeof(int)) {
2587 return false;
2588 }
2589 } else { // herere
2590 ResourceMark rm;
2591 int k = AOTCacheAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2592 if (k >= 0) {
2593 kind = DataKind::MH_Oop;
2594 n = write_bytes(&kind, sizeof(int));
2595 if (n != sizeof(int)) {
2596 return false;
2597 }
2598 n = write_bytes(&k, sizeof(int));
2599 if (n != sizeof(int)) {
2600 return false;
2601 }
2602 log_debug(aot, codecache, oops)("%d (L%d): Write MH object: " PTR_FORMAT " : %s",
2603 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2604 return true;
2605 }
2606 // Not archived Java object - bailout
2607 set_lookup_failed();
2608 log_debug(aot, codecache, oops)("%d (L%d): Not archived Java object: " PTR_FORMAT " : %s",
2609 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2610 return false;
2611 }
2612 return true;
2613 }
2614
2615 oop AOTCodeReader::read_oop(JavaThread* thread, const methodHandle& comp_method) {
2616 uint code_offset = read_position();
2617 oop obj = nullptr;
2618 DataKind kind = *(DataKind*)addr(code_offset);
2619 code_offset += sizeof(DataKind);
2620 set_read_position(code_offset);
2621 if (kind == DataKind::Null) {
2622 return nullptr;
2623 } else if (kind == DataKind::No_Data) {
2624 return cast_to_oop(Universe::non_oop_word());
2625 } else if (kind == DataKind::Klass) {
2626 Klass* k = read_klass(comp_method);
2627 if (k == nullptr) {
2628 return nullptr;
2629 }
2630 obj = k->java_mirror();
2631 if (obj == nullptr) {
2632 set_lookup_failed();
2633 log_debug(aot, codecache, oops)("Lookup failed for java_mirror of klass %s", k->external_name());
2634 return nullptr;
2635 }
2636 } else if (kind == DataKind::Primitive) {
2637 code_offset = read_position();
2638 int t = *(int*)addr(code_offset);
2639 code_offset += sizeof(int);
2640 set_read_position(code_offset);
2641 BasicType bt = (BasicType)t;
2642 obj = java_lang_Class::primitive_mirror(bt);
2643 log_debug(aot, codecache, oops)("%d (L%d): Read primitive type klass: %s", compile_id(), comp_level(), type2name(bt));
2644 } else if (kind == DataKind::String) {
2645 code_offset = read_position();
2646 int k = *(int*)addr(code_offset);
2647 code_offset += sizeof(int);
2648 set_read_position(code_offset);
2649 obj = AOTCacheAccess::get_archived_object(k);
2650 if (obj == nullptr) {
2651 set_lookup_failed();
2652 log_debug(aot, codecache, oops)("Lookup failed for String object");
2653 return nullptr;
2654 }
2655 assert(java_lang_String::is_instance(obj), "must be string");
2656
2657 ResourceMark rm;
2658 size_t length_sz = 0;
2659 const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2660 log_debug(aot, codecache, oops)("%d (L%d): Read String object: %s", compile_id(), comp_level(), string);
2661 } else if (kind == DataKind::SysLoader) {
2662 obj = SystemDictionary::java_system_loader();
2663 log_debug(aot, codecache, oops)("%d (L%d): Read java_system_loader", compile_id(), comp_level());
2664 } else if (kind == DataKind::PlaLoader) {
2665 obj = SystemDictionary::java_platform_loader();
2666 log_debug(aot, codecache, oops)("%d (L%d): Read java_platform_loader", compile_id(), comp_level());
2667 } else if (kind == DataKind::MH_Oop) {
2668 code_offset = read_position();
2669 int k = *(int*)addr(code_offset);
2670 code_offset += sizeof(int);
2671 set_read_position(code_offset);
2672 obj = AOTCacheAccess::get_archived_object(k);
2673 if (obj == nullptr) {
2674 set_lookup_failed();
2675 log_debug(aot, codecache, oops)("Lookup failed for MH object");
2676 return nullptr;
2677 }
2678 ResourceMark rm;
2679 log_debug(aot, codecache, oops)("%d (L%d): Read MH object: " PTR_FORMAT " : %s",
2680 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2681 } else {
2682 set_lookup_failed();
2683 log_debug(aot, codecache, oops)("%d (L%d): Unknown oop's kind: %d",
2684 compile_id(), comp_level(), (int)kind);
2685 return nullptr;
2686 }
2687 return obj;
2688 }
2689
2690 bool AOTCodeReader::read_oop_metadata_list(JavaThread* thread, ciMethod* target, GrowableArray<Handle> &oop_list, GrowableArray<Metadata*> &metadata_list, OopRecorder* oop_recorder) {
2691 methodHandle comp_method(JavaThread::current(), target->get_Method());
2692 JavaThread* current = JavaThread::current();
2693 uint offset = read_position();
2694 int count = *(int *)addr(offset);
2695 offset += sizeof(int);
2696 set_read_position(offset);
2697 for (int i = 0; i < count; i++) {
2698 oop obj = read_oop(current, comp_method);
2699 if (lookup_failed()) {
2700 return false;
2701 }
2702 Handle h(thread, obj);
2703 oop_list.append(h);
2704 if (oop_recorder != nullptr) {
2705 jobject jo = JNIHandles::make_local(thread, obj);
2706 if (oop_recorder->is_real(jo)) {
2707 oop_recorder->find_index(jo);
2708 } else {
2709 oop_recorder->allocate_oop_index(jo);
2710 }
2711 }
2712 LogStreamHandle(Debug, aot, codecache, oops) log;
2713 if (log.is_enabled()) {
2714 log.print("%d: " INTPTR_FORMAT " ", i, p2i(obj));
2715 if (obj == Universe::non_oop_word()) {
2716 log.print("non-oop word");
2717 } else if (obj == nullptr) {
2718 log.print("nullptr-oop");
2719 } else {
2720 obj->print_value_on(&log);
2721 }
2722 log.cr();
2723 }
2724 }
2725
2726 offset = read_position();
2727 count = *(int *)addr(offset);
2728 offset += sizeof(int);
2729 set_read_position(offset);
2730 for (int i = 0; i < count; i++) {
2731 Metadata* m = read_metadata(comp_method);
2732 if (lookup_failed()) {
2733 return false;
2734 }
2735 metadata_list.append(m);
2736 if (oop_recorder != nullptr) {
2737 if (oop_recorder->is_real(m)) {
2738 oop_recorder->find_index(m);
2739 } else {
2740 oop_recorder->allocate_metadata_index(m);
2741 }
2742 }
2743 LogTarget(Debug, aot, codecache, metadata) log;
2744 if (log.is_enabled()) {
2745 LogStream ls(log);
2746 ls.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
2747 if (m == (Metadata*)Universe::non_oop_word()) {
2748 ls.print("non-metadata word");
2749 } else if (m == nullptr) {
2750 ls.print("nullptr-oop");
2751 } else {
2752 Metadata::print_value_on_maybe_null(&ls, m);
2753 }
2754 ls.cr();
2755 }
2756 }
2757 return true;
2758 }
2759
2760 bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
2761 ImmutableOopMapSet* oopmaps = cb.oop_maps();
2762 int oopmaps_size = oopmaps->nr_of_bytes();
2763 if (!write_bytes(&oopmaps_size, sizeof(int))) {
2764 return false;
2765 }
2766 uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
2767 if (n != (uint)oopmaps->nr_of_bytes()) {
2768 return false;
2769 }
2770 return true;
2771 }
2772
2773 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
2774 uint offset = read_position();
2775 int size = *(int *)addr(offset);
2776 offset += sizeof(int);
2777 ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
2778 offset += size;
2779 set_read_position(offset);
2780 return oopmaps;
2781 }
2782
2783 bool AOTCodeCache::write_oops(nmethod* nm) {
2784 int count = nm->oops_count()-1;
2785 if (!write_bytes(&count, sizeof(int))) {
2786 return false;
2787 }
2788 for (oop* p = nm->oops_begin(); p < nm->oops_end(); p++) {
2789 if (!write_oop(*p)) {
2790 return false;
2791 }
2792 }
2793 return true;
2794 }
2795
2796 #ifndef PRODUCT
2797 bool AOTCodeCache::write_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2798 // Write asm remarks
2799 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2800 if (count_ptr == nullptr) {
2801 return false;
2802 }
2803 uint count = 0;
2804 bool result = asm_remarks.iterate([&] (uint offset, const char* str) -> bool {
2805 log_trace(aot, codecache, stubs)("asm remark offset=%d, str='%s'", offset, str);
2806 uint n = write_bytes(&offset, sizeof(uint));
2807 if (n != sizeof(uint)) {
2808 return false;
2809 }
2810 if (use_string_table) {
2811 const char* cstr = add_C_string(str);
2812 int id = _table->id_for_C_string((address)cstr);
2813 assert(id != -1, "asm remark string '%s' not found in AOTCodeAddressTable", str);
2814 n = write_bytes(&id, sizeof(int));
2815 if (n != sizeof(int)) {
2816 return false;
2817 }
2818 } else {
2819 n = write_bytes(str, (uint)strlen(str) + 1);
2820 if (n != strlen(str) + 1) {
2821 return false;
2822 }
2823 }
2824 count += 1;
2825 return true;
2826 });
2827 *count_ptr = count;
2828 return result;
2829 }
2830
2831 void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks, bool use_string_table) {
2832 // Read asm remarks
2833 uint offset = read_position();
2834 uint count = *(uint *)addr(offset);
2835 offset += sizeof(uint);
2836 for (uint i = 0; i < count; i++) {
2837 uint remark_offset = *(uint *)addr(offset);
2838 offset += sizeof(uint);
2839 const char* remark = nullptr;
2840 if (use_string_table) {
2841 int remark_string_id = *(uint *)addr(offset);
2842 offset += sizeof(int);
2843 remark = (const char*)_cache->address_for_C_string(remark_string_id);
2844 } else {
2845 remark = (const char*)addr(offset);
2846 offset += (uint)strlen(remark)+1;
2847 }
2848 asm_remarks.insert(remark_offset, remark);
2849 }
2850 set_read_position(offset);
2851 }
2852
2853 bool AOTCodeCache::write_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2854 // Write dbg strings
2855 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
2856 if (count_ptr == nullptr) {
2857 return false;
2858 }
2859 uint count = 0;
2860 bool result = dbg_strings.iterate([&] (const char* str) -> bool {
2861 log_trace(aot, codecache, stubs)("dbg string=%s", str);
2862 if (use_string_table) {
2863 const char* cstr = add_C_string(str);
2864 int id = _table->id_for_C_string((address)cstr);
2865 assert(id != -1, "db string '%s' not found in AOTCodeAddressTable", str);
2866 uint n = write_bytes(&id, sizeof(int));
2867 if (n != sizeof(int)) {
2868 return false;
2869 }
2870 } else {
2871 uint n = write_bytes(str, (uint)strlen(str) + 1);
2872 if (n != strlen(str) + 1) {
2873 return false;
2874 }
2875 }
2876 count += 1;
2877 return true;
2878 });
2879 *count_ptr = count;
2880 return result;
2881 }
2882
2883 void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings, bool use_string_table) {
2884 // Read dbg strings
2885 uint offset = read_position();
2886 uint count = *(uint *)addr(offset);
2887 offset += sizeof(uint);
2888 for (uint i = 0; i < count; i++) {
2889 const char* str = nullptr;
2890 if (use_string_table) {
2891 int string_id = *(uint *)addr(offset);
2892 offset += sizeof(int);
2893 str = (const char*)_cache->address_for_C_string(string_id);
2894 } else {
2895 str = (const char*)addr(offset);
2896 offset += (uint)strlen(str)+1;
2897 }
2898 dbg_strings.insert(str);
2899 }
2900 set_read_position(offset);
2901 }
2902 #endif // PRODUCT
2903
2904 //======================= AOTCodeAddressTable ===============
2905
2906 // address table ids for generated routines, external addresses and C
2907 // string addresses are partitioned into positive integer ranges
2908 // defined by the following positive base and max values
2909 // i.e. [_extrs_base, _extrs_base + _extrs_max -1],
2910 // [_stubs_base, _stubs_base + _stubs_max -1],
2911 // ...
2912 // [_c_str_base, _c_str_base + _c_str_max -1],
2913 #define _extrs_max 140
2914 #define _stubs_max 210
2915 #define _shared_blobs_max 25
2916 #define _C1_blobs_max 50
2917 #define _C2_blobs_max 25
2918 #define _blobs_max (_shared_blobs_max+_C1_blobs_max+_C2_blobs_max)
2919 #define _all_max (_extrs_max+_stubs_max+_blobs_max)
2920
2921 #define _extrs_base 0
2922 #define _stubs_base (_extrs_base + _extrs_max)
2923 #define _shared_blobs_base (_stubs_base + _stubs_max)
2924 #define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max)
2925 #define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max)
2926 #define _blobs_end (_shared_blobs_base + _blobs_max)
2927 #if (_C2_blobs_base >= _all_max)
2928 #error AOTCodeAddressTable ranges need adjusting
2929 #endif
2930
2931 #define SET_ADDRESS(type, addr) \
2932 { \
2933 type##_addr[type##_length++] = (address) (addr); \
2934 assert(type##_length <= type##_max, "increase size"); \
2935 }
2936
2937 static bool initializing_extrs = false;
2938
2939 void AOTCodeAddressTable::init_extrs() {
2940 if (_extrs_complete || initializing_extrs) return; // Done already
2941
2942 assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting");
2943
2944 initializing_extrs = true;
2945 _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
2946
2947 _extrs_length = 0;
2948
2949 // Record addresses of VM runtime methods
2950 SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite);
2951 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method);
2952 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract);
2953 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss);
2954 {
2955 // Required by Shared blobs
2956 SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info);
2957 SET_ADDRESS(_extrs, Deoptimization::unpack_frames);
2958 SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception);
2959 SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C);
2960 SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C);
2961 SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C);
2962 SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2963 SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError);
2964 SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError);
2965 SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError);
2966 SET_ADDRESS(_extrs, SharedRuntime::throw_NullPointerException_at_call);
2967 SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError);
2968 SET_ADDRESS(_extrs, CompressedOops::base_addr());
2969 SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr());
2970 }
2971 {
2972 // Required by initial stubs
2973 SET_ADDRESS(_extrs, StubRoutines::crc_table_addr());
2974 #if defined(AMD64)
2975 SET_ADDRESS(_extrs, StubRoutines::crc32c_table_addr());
2976 #endif
2977 }
2978
2979 #ifdef COMPILER1
2980 {
2981 // Required by C1 blobs
2982 SET_ADDRESS(_extrs, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc));
2983 SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address);
2984 SET_ADDRESS(_extrs, SharedRuntime::register_finalizer);
2985 SET_ADDRESS(_extrs, Runtime1::is_instance_of);
2986 SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc);
2987 SET_ADDRESS(_extrs, Runtime1::check_abort_on_vm_exception);
2988 SET_ADDRESS(_extrs, Runtime1::new_instance);
2989 SET_ADDRESS(_extrs, Runtime1::counter_overflow);
2990 SET_ADDRESS(_extrs, Runtime1::new_type_array);
2991 SET_ADDRESS(_extrs, Runtime1::new_object_array);
2992 SET_ADDRESS(_extrs, Runtime1::new_multi_array);
2993 SET_ADDRESS(_extrs, Runtime1::throw_range_check_exception);
2994 SET_ADDRESS(_extrs, Runtime1::throw_index_exception);
2995 SET_ADDRESS(_extrs, Runtime1::throw_div0_exception);
2996 SET_ADDRESS(_extrs, Runtime1::throw_null_pointer_exception);
2997 SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception);
2998 SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception);
2999 SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error);
3000 SET_ADDRESS(_extrs, Runtime1::monitorenter);
3001 SET_ADDRESS(_extrs, Runtime1::monitorexit);
3002 SET_ADDRESS(_extrs, Runtime1::deoptimize);
3003 SET_ADDRESS(_extrs, Runtime1::access_field_patching);
3004 SET_ADDRESS(_extrs, Runtime1::move_klass_patching);
3005 SET_ADDRESS(_extrs, Runtime1::move_mirror_patching);
3006 SET_ADDRESS(_extrs, Runtime1::move_appendix_patching);
3007 SET_ADDRESS(_extrs, Runtime1::predicate_failed_trap);
3008 SET_ADDRESS(_extrs, Runtime1::unimplemented_entry);
3009 SET_ADDRESS(_extrs, Runtime1::trace_block_entry);
3010 #ifdef X86
3011 SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool);
3012 SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool);
3013 SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool);
3014 SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool);
3015 #endif
3016 #ifndef PRODUCT
3017 SET_ADDRESS(_extrs, os::breakpoint);
3018 #endif
3019 }
3020 #endif // COMPILER1
3021
3022 #ifdef COMPILER2
3023 {
3024 // Required by C2 blobs
3025 SET_ADDRESS(_extrs, Deoptimization::uncommon_trap);
3026 SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C);
3027 SET_ADDRESS(_extrs, OptoRuntime::new_instance_C);
3028 SET_ADDRESS(_extrs, OptoRuntime::new_array_C);
3029 SET_ADDRESS(_extrs, OptoRuntime::new_array_nozero_C);
3030 SET_ADDRESS(_extrs, OptoRuntime::multianewarray2_C);
3031 SET_ADDRESS(_extrs, OptoRuntime::multianewarray3_C);
3032 SET_ADDRESS(_extrs, OptoRuntime::multianewarray4_C);
3033 SET_ADDRESS(_extrs, OptoRuntime::multianewarray5_C);
3034 SET_ADDRESS(_extrs, OptoRuntime::multianewarrayN_C);
3035 #if INCLUDE_JVMTI
3036 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_start);
3037 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_end);
3038 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_mount);
3039 SET_ADDRESS(_extrs, SharedRuntime::notify_jvmti_vthread_unmount);
3040 #endif
3041 SET_ADDRESS(_extrs, OptoRuntime::complete_monitor_locking_C);
3042 SET_ADDRESS(_extrs, OptoRuntime::monitor_notify_C);
3043 SET_ADDRESS(_extrs, OptoRuntime::monitor_notifyAll_C);
3044 SET_ADDRESS(_extrs, OptoRuntime::rethrow_C);
3045 SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C);
3046 SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C);
3047 SET_ADDRESS(_extrs, OptoRuntime::class_init_barrier_C);
3048 #if defined(AMD64)
3049 // Use by C2 intinsic
3050 SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31());
3051 #endif
3052 }
3053 #endif // COMPILER2
3054
3055 // Record addresses of VM runtime methods and data structs
3056 BarrierSet* bs = BarrierSet::barrier_set();
3057 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
3058 SET_ADDRESS(_extrs, ci_card_table_address_as<address>());
3059 }
3060
3061 #if INCLUDE_G1GC
3062 SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_post_entry);
3063 SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry);
3064 #endif
3065
3066 #if INCLUDE_SHENANDOAHGC
3067 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_oop);
3068 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
3069 SET_ADDRESS(_extrs, ShenandoahRuntime::clone_barrier);
3070 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong);
3071 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong_narrow);
3072 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak);
3073 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak_narrow);
3074 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom);
3075 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
3076 SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre);
3077 #endif
3078
3079 #if INCLUDE_ZGC
3080 SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
3081 #if defined(AMD64)
3082 SET_ADDRESS(_extrs, &ZPointerLoadShift);
3083 #endif
3084 #endif // INCLUDE_ZGC
3085
3086 SET_ADDRESS(_extrs, SharedRuntime::log_jni_monitor_still_held);
3087 SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry);
3088 SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages);
3089 SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit);
3090
3091 SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C);
3092 SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone);
3093 #if defined(AMD64) && !defined(ZERO)
3094 SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply);
3095 SET_ADDRESS(_extrs, SharedRuntime::montgomery_square);
3096 #endif // AMD64
3097 SET_ADDRESS(_extrs, SharedRuntime::d2f);
3098 SET_ADDRESS(_extrs, SharedRuntime::d2i);
3099 SET_ADDRESS(_extrs, SharedRuntime::d2l);
3100 SET_ADDRESS(_extrs, SharedRuntime::dcos);
3101 SET_ADDRESS(_extrs, SharedRuntime::dexp);
3102 SET_ADDRESS(_extrs, SharedRuntime::dlog);
3103 SET_ADDRESS(_extrs, SharedRuntime::dlog10);
3104 SET_ADDRESS(_extrs, SharedRuntime::dpow);
3105 SET_ADDRESS(_extrs, SharedRuntime::dsin);
3106 SET_ADDRESS(_extrs, SharedRuntime::dtan);
3107 SET_ADDRESS(_extrs, SharedRuntime::f2i);
3108 SET_ADDRESS(_extrs, SharedRuntime::f2l);
3109 #ifndef ZERO
3110 SET_ADDRESS(_extrs, SharedRuntime::drem);
3111 SET_ADDRESS(_extrs, SharedRuntime::frem);
3112 #endif
3113 SET_ADDRESS(_extrs, SharedRuntime::l2d);
3114 SET_ADDRESS(_extrs, SharedRuntime::l2f);
3115 SET_ADDRESS(_extrs, SharedRuntime::ldiv);
3116 SET_ADDRESS(_extrs, SharedRuntime::lmul);
3117 SET_ADDRESS(_extrs, SharedRuntime::lrem);
3118
3119 SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset());
3120 SET_ADDRESS(_extrs, Thread::current);
3121
3122 SET_ADDRESS(_extrs, os::javaTimeMillis);
3123 SET_ADDRESS(_extrs, os::javaTimeNanos);
3124 // For JFR
3125 SET_ADDRESS(_extrs, os::elapsed_counter);
3126 #if defined(X86) && !defined(ZERO)
3127 SET_ADDRESS(_extrs, Rdtsc::elapsed_counter);
3128 #endif
3129
3130 #if INCLUDE_JVMTI
3131 SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc);
3132 SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events);
3133 #endif /* INCLUDE_JVMTI */
3134
3135 #ifndef PRODUCT
3136 SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr);
3137 SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure);
3138 #endif
3139
3140 #ifndef ZERO
3141 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
3142 SET_ADDRESS(_extrs, MacroAssembler::debug64);
3143 #endif
3144 #if defined(AARCH64)
3145 SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper);
3146 #endif
3147 #endif // ZERO
3148
3149 // addresses of fields in AOT runtime constants area
3150 address* p = AOTRuntimeConstants::field_addresses_list();
3151 while (*p != nullptr) {
3152 SET_ADDRESS(_extrs, *p++);
3153 }
3154
3155 _extrs_complete = true;
3156 log_info(aot, codecache, init)("External addresses recorded");
3157 }
3158
3159 static bool initializing_early_stubs = false;
3160
3161 void AOTCodeAddressTable::init_early_stubs() {
3162 if (_complete || initializing_early_stubs) return; // Done already
3163 initializing_early_stubs = true;
3164 _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
3165 _stubs_length = 0;
3166 SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry());
3167
3168 {
3169 // Required by C1 blobs
3170 #if defined(AMD64) && !defined(ZERO)
3171 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip());
3172 SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup());
3173 #endif // AMD64
3174 }
3175
3176 _early_stubs_complete = true;
3177 log_info(aot, codecache, init)("Early stubs recorded");
3178 }
3179
3180 static bool initializing_shared_blobs = false;
3181
3182 void AOTCodeAddressTable::init_shared_blobs() {
3183 if (_complete || initializing_shared_blobs) return; // Done already
3184 initializing_shared_blobs = true;
3185 address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode);
3186
3187 // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel
3188 _shared_blobs_addr = blobs_addr;
3189 _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max;// C1 blobs addresses stored after shared blobs
3190 _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs
3191
3192 _shared_blobs_length = 0;
3193 _C1_blobs_length = 0;
3194 _C2_blobs_length = 0;
3195
3196 // clear the address table
3197 memset(blobs_addr, 0, sizeof(address)* _blobs_max);
3198
3199 // Record addresses of generated code blobs
3200 SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub());
3201 SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub());
3202 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack());
3203 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception());
3204 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution());
3205 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls());
3206 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub());
3207 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_virtual_call_stub());
3208 SET_ADDRESS(_shared_blobs, SharedRuntime::get_resolve_static_call_stub());
3209 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->entry_point());
3210 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point());
3211 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point());
3212 #ifdef COMPILER2
3213 // polling_page_vectors_safepoint_handler_blob can be nullptr if AVX feature is not present or is disabled
3214 if (SharedRuntime::polling_page_vectors_safepoint_handler_blob() != nullptr) {
3215 SET_ADDRESS(_shared_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point());
3216 }
3217 #endif
3218 #if INCLUDE_JVMCI
3219 if (EnableJVMCI) {
3220 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap());
3221 SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap());
3222 }
3223 #endif
3224 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_AbstractMethodError_entry());
3225 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry());
3226 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_NullPointerException_at_call_entry());
3227 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_StackOverflowError_entry());
3228 SET_ADDRESS(_shared_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry());
3229
3230 assert(_shared_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _shared_blobs_length);
3231 _shared_blobs_complete = true;
3232 log_info(aot, codecache, init)("All shared blobs recorded");
3233 }
3234
3235 static bool initializing_stubs = false;
3236 void AOTCodeAddressTable::init_stubs() {
3237 if (_complete || initializing_stubs) return; // Done already
3238 assert(_early_stubs_complete, "early stubs whould be initialized");
3239 initializing_stubs = true;
3240
3241 // Stubs
3242 SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier());
3243 SET_ADDRESS(_stubs, StubRoutines::atomic_xchg_entry());
3244 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_entry());
3245 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_long_entry());
3246 SET_ADDRESS(_stubs, StubRoutines::atomic_add_entry());
3247 SET_ADDRESS(_stubs, StubRoutines::fence_entry());
3248
3249 SET_ADDRESS(_stubs, StubRoutines::cont_thaw());
3250 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrier());
3251 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrierExc());
3252
3253 JFR_ONLY(SET_ADDRESS(_stubs, SharedRuntime::jfr_write_checkpoint());)
3254
3255 SET_ADDRESS(_stubs, StubRoutines::jbyte_arraycopy());
3256 SET_ADDRESS(_stubs, StubRoutines::jshort_arraycopy());
3257 SET_ADDRESS(_stubs, StubRoutines::jint_arraycopy());
3258 SET_ADDRESS(_stubs, StubRoutines::jlong_arraycopy());
3259 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy);
3260 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy_uninit);
3261
3262 SET_ADDRESS(_stubs, StubRoutines::jbyte_disjoint_arraycopy());
3263 SET_ADDRESS(_stubs, StubRoutines::jshort_disjoint_arraycopy());
3264 SET_ADDRESS(_stubs, StubRoutines::jint_disjoint_arraycopy());
3265 SET_ADDRESS(_stubs, StubRoutines::jlong_disjoint_arraycopy());
3266 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy);
3267 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy_uninit);
3268
3269 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_arraycopy());
3270 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_arraycopy());
3271 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_arraycopy());
3272 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_arraycopy());
3273 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy);
3274 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy_uninit);
3275
3276 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_disjoint_arraycopy());
3277 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_disjoint_arraycopy());
3278 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_disjoint_arraycopy());
3279 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_disjoint_arraycopy());
3280 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy);
3281 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit);
3282
3283 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy);
3284 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy_uninit);
3285
3286 SET_ADDRESS(_stubs, StubRoutines::unsafe_arraycopy());
3287 SET_ADDRESS(_stubs, StubRoutines::generic_arraycopy());
3288
3289 SET_ADDRESS(_stubs, StubRoutines::jbyte_fill());
3290 SET_ADDRESS(_stubs, StubRoutines::jshort_fill());
3291 SET_ADDRESS(_stubs, StubRoutines::jint_fill());
3292 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_fill());
3293 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_fill());
3294 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_fill());
3295
3296 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback());
3297 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback_sync());
3298
3299 SET_ADDRESS(_stubs, StubRoutines::aescrypt_encryptBlock());
3300 SET_ADDRESS(_stubs, StubRoutines::aescrypt_decryptBlock());
3301 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_encryptAESCrypt());
3302 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_decryptAESCrypt());
3303 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_encryptAESCrypt());
3304 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_decryptAESCrypt());
3305 SET_ADDRESS(_stubs, StubRoutines::poly1305_processBlocks());
3306 SET_ADDRESS(_stubs, StubRoutines::counterMode_AESCrypt());
3307 SET_ADDRESS(_stubs, StubRoutines::ghash_processBlocks());
3308 SET_ADDRESS(_stubs, StubRoutines::chacha20Block());
3309 SET_ADDRESS(_stubs, StubRoutines::base64_encodeBlock());
3310 SET_ADDRESS(_stubs, StubRoutines::base64_decodeBlock());
3311 SET_ADDRESS(_stubs, StubRoutines::md5_implCompress());
3312 SET_ADDRESS(_stubs, StubRoutines::md5_implCompressMB());
3313 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompress());
3314 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompressMB());
3315 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompress());
3316 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompressMB());
3317 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompress());
3318 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompressMB());
3319 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompress());
3320 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompressMB());
3321 SET_ADDRESS(_stubs, StubRoutines::double_keccak());
3322 SET_ADDRESS(_stubs, StubRoutines::intpoly_assign());
3323 SET_ADDRESS(_stubs, StubRoutines::intpoly_montgomeryMult_P256());
3324 SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostNtt());
3325 SET_ADDRESS(_stubs, StubRoutines::dilithiumAlmostInverseNtt());
3326 SET_ADDRESS(_stubs, StubRoutines::dilithiumNttMult());
3327 SET_ADDRESS(_stubs, StubRoutines::dilithiumMontMulByConstant());
3328 SET_ADDRESS(_stubs, StubRoutines::dilithiumDecomposePoly());
3329
3330 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32());
3331 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32C());
3332 SET_ADDRESS(_stubs, StubRoutines::updateBytesAdler32());
3333
3334 SET_ADDRESS(_stubs, StubRoutines::multiplyToLen());
3335 SET_ADDRESS(_stubs, StubRoutines::squareToLen());
3336 SET_ADDRESS(_stubs, StubRoutines::mulAdd());
3337 SET_ADDRESS(_stubs, StubRoutines::montgomeryMultiply());
3338 SET_ADDRESS(_stubs, StubRoutines::montgomerySquare());
3339 SET_ADDRESS(_stubs, StubRoutines::bigIntegerRightShift());
3340 SET_ADDRESS(_stubs, StubRoutines::bigIntegerLeftShift());
3341 SET_ADDRESS(_stubs, StubRoutines::galoisCounterMode_AESCrypt());
3342
3343 SET_ADDRESS(_stubs, StubRoutines::vectorizedMismatch());
3344
3345 SET_ADDRESS(_stubs, StubRoutines::unsafe_setmemory());
3346
3347 SET_ADDRESS(_stubs, StubRoutines::dexp());
3348 SET_ADDRESS(_stubs, StubRoutines::dlog());
3349 SET_ADDRESS(_stubs, StubRoutines::dlog10());
3350 SET_ADDRESS(_stubs, StubRoutines::dpow());
3351 SET_ADDRESS(_stubs, StubRoutines::dsin());
3352 SET_ADDRESS(_stubs, StubRoutines::dcos());
3353 SET_ADDRESS(_stubs, StubRoutines::dlibm_reduce_pi04l());
3354 SET_ADDRESS(_stubs, StubRoutines::dlibm_sin_cos_huge());
3355 SET_ADDRESS(_stubs, StubRoutines::dlibm_tan_cot_huge());
3356 SET_ADDRESS(_stubs, StubRoutines::dtan());
3357
3358 SET_ADDRESS(_stubs, StubRoutines::f2hf_adr());
3359 SET_ADDRESS(_stubs, StubRoutines::hf2f_adr());
3360
3361 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
3362 SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_stub(slot));
3363 }
3364 SET_ADDRESS(_stubs, StubRoutines::lookup_secondary_supers_table_slow_path_stub());
3365
3366 #if defined(AMD64) && !defined(ZERO)
3367 SET_ADDRESS(_stubs, StubRoutines::x86::d2i_fixup());
3368 SET_ADDRESS(_stubs, StubRoutines::x86::f2i_fixup());
3369 SET_ADDRESS(_stubs, StubRoutines::x86::f2l_fixup());
3370 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_mask());
3371 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_flip());
3372 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_mask());
3373 SET_ADDRESS(_stubs, StubRoutines::x86::vector_popcount_lut());
3374 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_mask());
3375 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_flip());
3376 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_mask());
3377 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_flip());
3378 SET_ADDRESS(_stubs, StubRoutines::x86::vector_int_shuffle_mask());
3379 SET_ADDRESS(_stubs, StubRoutines::x86::vector_byte_shuffle_mask());
3380 SET_ADDRESS(_stubs, StubRoutines::x86::vector_short_shuffle_mask());
3381 SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_shuffle_mask());
3382 SET_ADDRESS(_stubs, StubRoutines::x86::vector_long_sign_mask());
3383 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_int());
3384 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_short());
3385 SET_ADDRESS(_stubs, StubRoutines::x86::vector_reverse_byte_perm_mask_long());
3386 // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
3387 // See C2_MacroAssembler::load_iota_indices().
3388 for (int i = 0; i < 6; i++) {
3389 SET_ADDRESS(_stubs, StubRoutines::x86::vector_iota_indices() + i * 64);
3390 }
3391 #endif
3392 #if defined(AARCH64) && !defined(ZERO)
3393 SET_ADDRESS(_stubs, StubRoutines::aarch64::zero_blocks());
3394 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives());
3395 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives_long());
3396 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_array_equals());
3397 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LL());
3398 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UU());
3399 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LU());
3400 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UL());
3401 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ul());
3402 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ll());
3403 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_uu());
3404 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_byte_array_inflate());
3405 SET_ADDRESS(_stubs, StubRoutines::aarch64::spin_wait());
3406
3407 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BOOLEAN));
3408 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BYTE));
3409 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_SHORT));
3410 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_CHAR));
3411 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_INT));
3412 #endif
3413
3414 _complete = true;
3415 log_info(aot, codecache, init)("Stubs recorded");
3416 }
3417
3418 void AOTCodeAddressTable::init_early_c1() {
3419 #ifdef COMPILER1
3420 // Runtime1 Blobs
3421 StubId id = StubInfo::stub_base(StubGroup::C1);
3422 // include forward_exception in range we publish
3423 StubId limit = StubInfo::next(StubId::c1_forward_exception_id);
3424 for (; id != limit; id = StubInfo::next(id)) {
3425 if (Runtime1::blob_for(id) == nullptr) {
3426 log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3427 continue;
3428 }
3429 if (Runtime1::entry_for(id) == nullptr) {
3430 log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3431 continue;
3432 }
3433 address entry = Runtime1::entry_for(id);
3434 SET_ADDRESS(_C1_blobs, entry);
3435 }
3436 #endif // COMPILER1
3437 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3438 _early_c1_complete = true;
3439 }
3440
3441 void AOTCodeAddressTable::init_c1() {
3442 #ifdef COMPILER1
3443 // Runtime1 Blobs
3444 assert(_early_c1_complete, "early C1 blobs should be initialized");
3445 StubId id = StubInfo::next(StubId::c1_forward_exception_id);
3446 StubId limit = StubInfo::next(StubInfo::stub_max(StubGroup::C1));
3447 for (; id != limit; id = StubInfo::next(id)) {
3448 if (Runtime1::blob_for(id) == nullptr) {
3449 log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
3450 continue;
3451 }
3452 if (Runtime1::entry_for(id) == nullptr) {
3453 log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
3454 continue;
3455 }
3456 address entry = Runtime1::entry_for(id);
3457 SET_ADDRESS(_C1_blobs, entry);
3458 }
3459 #if INCLUDE_G1GC
3460 if (UseG1GC) {
3461 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3462 address entry = bs->pre_barrier_c1_runtime_code_blob()->code_begin();
3463 SET_ADDRESS(_C1_blobs, entry);
3464 entry = bs->post_barrier_c1_runtime_code_blob()->code_begin();
3465 SET_ADDRESS(_C1_blobs, entry);
3466 }
3467 #endif // INCLUDE_G1GC
3468 #if INCLUDE_ZGC
3469 if (UseZGC) {
3470 ZBarrierSetC1* bs = (ZBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3471 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_oop_field_preloaded_runtime_stub);
3472 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_weak_oop_field_preloaded_runtime_stub);
3473 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_with_healing);
3474 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_without_healing);
3475 }
3476 #endif // INCLUDE_ZGC
3477 #if INCLUDE_SHENANDOAHGC
3478 if (UseShenandoahGC) {
3479 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
3480 SET_ADDRESS(_C1_blobs, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
3481 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_rt_code_blob()->code_begin());
3482 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin());
3483 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_weak_rt_code_blob()->code_begin());
3484 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_phantom_rt_code_blob()->code_begin());
3485 }
3486 #endif // INCLUDE_SHENANDOAHGC
3487 #endif // COMPILER1
3488
3489 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
3490 _c1_complete = true;
3491 log_info(aot, codecache, init)("Runtime1 Blobs recorded");
3492 }
3493
3494 void AOTCodeAddressTable::init_c2() {
3495 #ifdef COMPILER2
3496 // OptoRuntime Blobs
3497 SET_ADDRESS(_C2_blobs, OptoRuntime::uncommon_trap_blob()->entry_point());
3498 SET_ADDRESS(_C2_blobs, OptoRuntime::exception_blob()->entry_point());
3499 SET_ADDRESS(_C2_blobs, OptoRuntime::new_instance_Java());
3500 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_Java());
3501 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_nozero_Java());
3502 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray2_Java());
3503 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray3_Java());
3504 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray4_Java());
3505 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray5_Java());
3506 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarrayN_Java());
3507 SET_ADDRESS(_C2_blobs, OptoRuntime::vtable_must_compile_stub());
3508 SET_ADDRESS(_C2_blobs, OptoRuntime::complete_monitor_locking_Java());
3509 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notify_Java());
3510 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notifyAll_Java());
3511 SET_ADDRESS(_C2_blobs, OptoRuntime::rethrow_stub());
3512 SET_ADDRESS(_C2_blobs, OptoRuntime::slow_arraycopy_Java());
3513 SET_ADDRESS(_C2_blobs, OptoRuntime::register_finalizer_Java());
3514 SET_ADDRESS(_C2_blobs, OptoRuntime::class_init_barrier_Java());
3515 #if INCLUDE_JVMTI
3516 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_start());
3517 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_end());
3518 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_mount());
3519 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_unmount());
3520 #endif /* INCLUDE_JVMTI */
3521 #endif
3522
3523 assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length);
3524 _c2_complete = true;
3525 log_info(aot, codecache, init)("OptoRuntime Blobs recorded");
3526 }
3527 #undef SET_ADDRESS
3528
3529 AOTCodeAddressTable::~AOTCodeAddressTable() {
3530 if (_extrs_addr != nullptr) {
3531 FREE_C_HEAP_ARRAY(address, _extrs_addr);
3532 }
3533 if (_stubs_addr != nullptr) {
3534 FREE_C_HEAP_ARRAY(address, _stubs_addr);
3535 }
3536 if (_shared_blobs_addr != nullptr) {
3537 FREE_C_HEAP_ARRAY(address, _shared_blobs_addr);
3538 }
3539 }
3540
3541 #ifdef PRODUCT
3542 #define MAX_STR_COUNT 200
3543 #else
3544 #define MAX_STR_COUNT 500
3545 #endif
3546 #define _c_str_max MAX_STR_COUNT
3547 static const int _c_str_base = _all_max;
3548
3549 static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings
3550 static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; // Our duplicates
3551 static int _C_strings_count = 0;
3552 static int _C_strings_s[MAX_STR_COUNT] = {0};
3553 static int _C_strings_id[MAX_STR_COUNT] = {0};
3554 static int _C_strings_used = 0;
3555
3556 void AOTCodeCache::load_strings() {
3557 uint strings_count = _load_header->strings_count();
3558 if (strings_count == 0) {
3559 return;
3560 }
3561 uint strings_offset = _load_header->strings_offset();
3562 uint* string_lengths = (uint*)addr(strings_offset);
3563 strings_offset += (strings_count * sizeof(uint));
3564 uint strings_size = _load_header->entries_offset() - strings_offset;
3565 // We have to keep cached strings longer than _cache buffer
3566 // because they are refernced from compiled code which may
3567 // still be executed on VM exit after _cache is freed.
3568 char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
3569 memcpy(p, addr(strings_offset), strings_size);
3570 _C_strings_buf = p;
3571 assert(strings_count <= MAX_STR_COUNT, "sanity");
3572 for (uint i = 0; i < strings_count; i++) {
3573 _C_strings[i] = p;
3574 uint len = string_lengths[i];
3575 _C_strings_s[i] = i;
3576 _C_strings_id[i] = i;
3577 p += len;
3578 }
3579 assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
3580 _C_strings_count = strings_count;
3581 _C_strings_used = strings_count;
3582 log_debug(aot, codecache, init)(" Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset);
3583 }
3584
3585 int AOTCodeCache::store_strings() {
3586 if (_C_strings_used > 0) {
3587 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3588 uint offset = _write_position;
3589 uint length = 0;
3590 uint* lengths = (uint *)reserve_bytes(sizeof(uint) * _C_strings_used);
3591 if (lengths == nullptr) {
3592 return -1;
3593 }
3594 for (int i = 0; i < _C_strings_used; i++) {
3595 const char* str = _C_strings[_C_strings_s[i]];
3596 uint len = (uint)strlen(str) + 1;
3597 length += len;
3598 assert(len < 1000, "big string: %s", str);
3599 lengths[i] = len;
3600 uint n = write_bytes(str, len);
3601 if (n != len) {
3602 return -1;
3603 }
3604 }
3605 log_debug(aot, codecache, exit)(" Wrote %d C strings of total length %d at offset %d to AOT Code Cache",
3606 _C_strings_used, length, offset);
3607 }
3608 return _C_strings_used;
3609 }
3610
3611 const char* AOTCodeCache::add_C_string(const char* str) {
3612 if (is_on_for_dump() && str != nullptr) {
3613 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3614 AOTCodeAddressTable* table = addr_table();
3615 if (table != nullptr) {
3616 return table->add_C_string(str);
3617 }
3618 }
3619 return str;
3620 }
3621
3622 const char* AOTCodeAddressTable::add_C_string(const char* str) {
3623 if (_extrs_complete) {
3624 // Check previous strings address
3625 for (int i = 0; i < _C_strings_count; i++) {
3626 if (_C_strings_in[i] == str) {
3627 return _C_strings[i]; // Found previous one - return our duplicate
3628 } else if (strcmp(_C_strings[i], str) == 0) {
3629 return _C_strings[i];
3630 }
3631 }
3632 // Add new one
3633 if (_C_strings_count < MAX_STR_COUNT) {
3634 // Passed in string can be freed and used space become inaccessible.
3635 // Keep original address but duplicate string for future compare.
3636 _C_strings_id[_C_strings_count] = -1; // Init
3637 _C_strings_in[_C_strings_count] = str;
3638 const char* dup = os::strdup(str);
3639 _C_strings[_C_strings_count++] = dup;
3640 log_trace(aot, codecache, stringtable)("add_C_string: [%d] " INTPTR_FORMAT " '%s'", _C_strings_count, p2i(dup), dup);
3641 return dup;
3642 } else {
3643 assert(false, "Number of C strings >= MAX_STR_COUNT");
3644 }
3645 }
3646 return str;
3647 }
3648
3649 int AOTCodeAddressTable::id_for_C_string(address str) {
3650 if (str == nullptr) {
3651 return -1;
3652 }
3653 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
3654 for (int i = 0; i < _C_strings_count; i++) {
3655 if (_C_strings[i] == (const char*)str) { // found
3656 int id = _C_strings_id[i];
3657 if (id >= 0) {
3658 assert(id < _C_strings_used, "%d >= %d", id , _C_strings_used);
3659 return id; // Found recorded
3660 }
3661 // Not found in recorded, add new
3662 id = _C_strings_used++;
3663 _C_strings_s[id] = i;
3664 _C_strings_id[i] = id;
3665 return id;
3666 }
3667 }
3668 return -1;
3669 }
3670
3671 address AOTCodeAddressTable::address_for_C_string(int idx) {
3672 assert(idx < _C_strings_count, "sanity");
3673 return (address)_C_strings[idx];
3674 }
3675
3676 static int search_address(address addr, address* table, uint length) {
3677 for (int i = 0; i < (int)length; i++) {
3678 if (table[i] == addr) {
3679 return i;
3680 }
3681 }
3682 return BAD_ADDRESS_ID;
3683 }
3684
3685 address AOTCodeAddressTable::address_for_id(int idx) {
3686 assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3687 if (idx == -1) {
3688 return (address)-1;
3689 }
3690 uint id = (uint)idx;
3691 // special case for symbols based relative to os::init
3692 if (id > (_c_str_base + _c_str_max)) {
3693 return (address)os::init + idx;
3694 }
3695 if (idx < 0) {
3696 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3697 return nullptr;
3698 }
3699 // no need to compare unsigned id against 0
3700 if (/* id >= _extrs_base && */ id < _extrs_length) {
3701 return _extrs_addr[id - _extrs_base];
3702 }
3703 if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3704 return _stubs_addr[id - _stubs_base];
3705 }
3706 if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
3707 return _stubs_addr[id - _stubs_base];
3708 }
3709 if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) {
3710 return _shared_blobs_addr[id - _shared_blobs_base];
3711 }
3712 if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3713 return _C1_blobs_addr[id - _C1_blobs_base];
3714 }
3715 if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
3716 return _C1_blobs_addr[id - _C1_blobs_base];
3717 }
3718 if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) {
3719 return _C2_blobs_addr[id - _C2_blobs_base];
3720 }
3721 if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
3722 return address_for_C_string(id - _c_str_base);
3723 }
3724 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
3725 return nullptr;
3726 }
3727
3728 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* blob) {
3729 assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete");
3730 int id = -1;
3731 if (addr == (address)-1) { // Static call stub has jump to itself
3732 return id;
3733 }
3734 // Check card_table_base address first since it can point to any address
3735 BarrierSet* bs = BarrierSet::barrier_set();
3736 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
3737 if (addr == ci_card_table_address_as<address>()) {
3738 id = search_address(addr, _extrs_addr, _extrs_length);
3739 assert(id > 0 && _extrs_addr[id - _extrs_base] == addr, "sanity");
3740 return id;
3741 }
3742 }
3743
3744 // Seach for C string
3745 id = id_for_C_string(addr);
3746 if (id >= 0) {
3747 return id + _c_str_base;
3748 }
3749 if (StubRoutines::contains(addr)) {
3750 // Search in stubs
3751 id = search_address(addr, _stubs_addr, _stubs_length);
3752 if (id == BAD_ADDRESS_ID) {
3753 StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
3754 if (desc == nullptr) {
3755 desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset);
3756 }
3757 const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
3758 assert(false, "Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
3759 } else {
3760 return _stubs_base + id;
3761 }
3762 } else {
3763 CodeBlob* cb = CodeCache::find_blob(addr);
3764 if (cb != nullptr) {
3765 int id_base = _shared_blobs_base;
3766 // Search in code blobs
3767 id = search_address(addr, _shared_blobs_addr, _shared_blobs_length);
3768 if (id == BAD_ADDRESS_ID) {
3769 id_base = _C1_blobs_base;
3770 // search C1 blobs
3771 id = search_address(addr, _C1_blobs_addr, _C1_blobs_length);
3772 }
3773 if (id == BAD_ADDRESS_ID) {
3774 id_base = _C2_blobs_base;
3775 // search C2 blobs
3776 id = search_address(addr, _C2_blobs_addr, _C2_blobs_length);
3777 }
3778 if (id == BAD_ADDRESS_ID) {
3779 assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name());
3780 } else {
3781 return id_base + id;
3782 }
3783 } else {
3784 // Search in runtime functions
3785 id = search_address(addr, _extrs_addr, _extrs_length);
3786 if (id == BAD_ADDRESS_ID) {
3787 ResourceMark rm;
3788 const int buflen = 1024;
3789 char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
3790 int offset = 0;
3791 if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
3792 if (offset > 0) {
3793 // Could be address of C string
3794 uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
3795 CompileTask* task = ciEnv::current()->task();
3796 uint compile_id = 0;
3797 uint comp_level =0;
3798 if (task != nullptr) { // this could be called from compiler runtime initialization (compiler blobs)
3799 compile_id = task->compile_id();
3800 comp_level = task->comp_level();
3801 }
3802 log_debug(aot, codecache)("%d (L%d): Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
3803 compile_id, comp_level, p2i(addr), dist, (const char*)addr);
3804 assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
3805 return dist;
3806 }
3807 reloc.print_current_on(tty);
3808 blob->print_on(tty);
3809 blob->print_code_on(tty);
3810 assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);
3811 } else {
3812 reloc.print_current_on(tty);
3813 blob->print_on(tty);
3814 blob->print_code_on(tty);
3815 os::find(addr, tty);
3816 assert(false, "Address " INTPTR_FORMAT " for <unknown>/('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr);
3817 }
3818 } else {
3819 return _extrs_base + id;
3820 }
3821 }
3822 }
3823 return id;
3824 }
3825
3826 #undef _extrs_max
3827 #undef _stubs_max
3828 #undef _shared_blobs_max
3829 #undef _C1_blobs_max
3830 #undef _C2_blobs_max
3831 #undef _blobs_max
3832 #undef _extrs_base
3833 #undef _stubs_base
3834 #undef _shared_blobs_base
3835 #undef _C1_blobs_base
3836 #undef _C2_blobs_base
3837 #undef _blobs_end
3838
3839 void AOTRuntimeConstants::initialize_from_runtime() {
3840 BarrierSet* bs = BarrierSet::barrier_set();
3841 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
3842 CardTableBarrierSet* ctbs = ((CardTableBarrierSet*)bs);
3843 _aot_runtime_constants._grain_shift = ctbs->grain_shift();
3844 _aot_runtime_constants._card_shift = ctbs->card_shift();
3845 }
3846 }
3847
3848 AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
3849
3850 address AOTRuntimeConstants::_field_addresses_list[] = {
3851 grain_shift_address(),
3852 card_shift_address(),
3853 nullptr
3854 };
3855
3856
3857 void AOTCodeCache::wait_for_no_nmethod_readers() {
3858 while (true) {
3859 int cur = Atomic::load(&_nmethod_readers);
3860 int upd = -(cur + 1);
3861 if (cur >= 0 && Atomic::cmpxchg(&_nmethod_readers, cur, upd) == cur) {
3862 // Success, no new readers should appear.
3863 break;
3864 }
3865 }
3866
3867 // Now wait for all readers to leave.
3868 SpinYield w;
3869 while (Atomic::load(&_nmethod_readers) != -1) {
3870 w.wait();
3871 }
3872 }
3873
3874 AOTCodeCache::ReadingMark::ReadingMark() {
3875 while (true) {
3876 int cur = Atomic::load(&_nmethod_readers);
3877 if (cur < 0) {
3878 // Cache is already closed, cannot proceed.
3879 _failed = true;
3880 return;
3881 }
3882 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3883 // Successfully recorded ourselves as entered.
3884 _failed = false;
3885 return;
3886 }
3887 }
3888 }
3889
3890 AOTCodeCache::ReadingMark::~ReadingMark() {
3891 if (_failed) {
3892 return;
3893 }
3894 while (true) {
3895 int cur = Atomic::load(&_nmethod_readers);
3896 if (cur > 0) {
3897 // Cache is open, we are counting down towards 0.
3898 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur - 1) == cur) {
3899 return;
3900 }
3901 } else {
3902 // Cache is closed, we are counting up towards -1.
3903 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
3904 return;
3905 }
3906 }
3907 }
3908 }
3909
3910 void AOTCodeCache::print_timers_on(outputStream* st) {
3911 if (is_using_code()) {
3912 st->print_cr (" AOT Code Load Time: %7.3f s", _t_totalLoad.seconds());
3913 st->print_cr (" nmethod register: %7.3f s", _t_totalRegister.seconds());
3914 st->print_cr (" find AOT code entry: %7.3f s", _t_totalFind.seconds());
3915 }
3916 if (is_dumping_code()) {
3917 st->print_cr (" AOT Code Store Time: %7.3f s", _t_totalStore.seconds());
3918 }
3919 }
3920
3921 AOTCodeStats AOTCodeStats::add_aot_code_stats(AOTCodeStats stats1, AOTCodeStats stats2) {
3922 AOTCodeStats result;
3923 for (int kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3924 result.ccstats._kind_cnt[kind] = stats1.entry_count(kind) + stats2.entry_count(kind);
3925 }
3926
3927 for (int lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3928 result.ccstats._nmethod_cnt[lvl] = stats1.nmethod_count(lvl) + stats2.nmethod_count(lvl);
3929 }
3930 result.ccstats._clinit_barriers_cnt = stats1.clinit_barriers_count() + stats2.clinit_barriers_count();
3931 return result;
3932 }
3933
3934 void AOTCodeCache::log_stats_on_exit() {
3935 LogStreamHandle(Debug, aot, codecache, exit) log;
3936 if (log.is_enabled()) {
3937 AOTCodeStats prev_stats;
3938 AOTCodeStats current_stats;
3939 AOTCodeStats total_stats;
3940 uint max_size = 0;
3941
3942 uint load_count = (_load_header != nullptr) ? _load_header->entries_count() : 0;
3943
3944 for (uint i = 0; i < load_count; i++) {
3945 prev_stats.collect_entry_stats(&_load_entries[i]);
3946 if (max_size < _load_entries[i].size()) {
3947 max_size = _load_entries[i].size();
3948 }
3949 }
3950 for (uint i = 0; i < _store_entries_cnt; i++) {
3951 current_stats.collect_entry_stats(&_store_entries[i]);
3952 if (max_size < _store_entries[i].size()) {
3953 max_size = _store_entries[i].size();
3954 }
3955 }
3956 total_stats = AOTCodeStats::add_aot_code_stats(prev_stats, current_stats);
3957
3958 log.print_cr("Wrote %d AOTCodeEntry entries(%u max size) to AOT Code Cache",
3959 total_stats.total_count(), max_size);
3960 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3961 if (total_stats.entry_count(kind) > 0) {
3962 log.print_cr(" %s: total=%u(old=%u+new=%u)",
3963 aot_code_entry_kind_name[kind], total_stats.entry_count(kind), prev_stats.entry_count(kind), current_stats.entry_count(kind));
3964 if (kind == AOTCodeEntry::Code) {
3965 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3966 if (total_stats.nmethod_count(lvl) > 0) {
3967 log.print_cr(" Tier %d: total=%u(old=%u+new=%u)",
3968 lvl, total_stats.nmethod_count(lvl), prev_stats.nmethod_count(lvl), current_stats.nmethod_count(lvl));
3969 }
3970 }
3971 }
3972 }
3973 }
3974 log.print_cr("Total=%u(old=%u+new=%u)", total_stats.total_count(), prev_stats.total_count(), current_stats.total_count());
3975 }
3976 }
3977
3978 static void print_helper1(outputStream* st, const char* name, int count) {
3979 if (count > 0) {
3980 st->print(" %s=%d", name, count);
3981 }
3982 }
3983
3984 void AOTCodeCache::print_statistics_on(outputStream* st) {
3985 AOTCodeCache* cache = open_for_use();
3986 if (cache != nullptr) {
3987 ReadingMark rdmk;
3988 if (rdmk.failed()) {
3989 // Cache is closed, cannot touch anything.
3990 return;
3991 }
3992
3993 uint count = cache->_load_header->entries_count();
3994 AOTCodeEntry* load_entries = (AOTCodeEntry*)cache->addr(cache->_load_header->entries_offset());
3995
3996 AOTCodeStats stats;
3997 for (uint i = 0; i < count; i++) {
3998 stats.collect_all_stats(&load_entries[i]);
3999 }
4000
4001 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
4002 if (stats.entry_count(kind) > 0) {
4003 st->print(" %s:", aot_code_entry_kind_name[kind]);
4004 print_helper1(st, "total", stats.entry_count(kind));
4005 print_helper1(st, "loaded", stats.entry_loaded_count(kind));
4006 print_helper1(st, "invalidated", stats.entry_invalidated_count(kind));
4007 print_helper1(st, "failed", stats.entry_load_failed_count(kind));
4008 st->cr();
4009 }
4010 if (kind == AOTCodeEntry::Code) {
4011 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
4012 if (stats.nmethod_count(lvl) > 0) {
4013 st->print(" AOT Code T%d", lvl);
4014 print_helper1(st, "total", stats.nmethod_count(lvl));
4015 print_helper1(st, "loaded", stats.nmethod_loaded_count(lvl));
4016 print_helper1(st, "invalidated", stats.nmethod_invalidated_count(lvl));
4017 print_helper1(st, "failed", stats.nmethod_load_failed_count(lvl));
4018 if (lvl == AOTCompLevel_count-1) {
4019 print_helper1(st, "has_clinit_barriers", stats.clinit_barriers_count());
4020 }
4021 st->cr();
4022 }
4023 }
4024 }
4025 }
4026 LogStreamHandle(Debug, aot, codecache, init) log;
4027 if (log.is_enabled()) {
4028 AOTCodeCache::print_unused_entries_on(&log);
4029 }
4030 LogStreamHandle(Trace, aot, codecache) aot_info;
4031 // need a lock to traverse the code cache
4032 if (aot_info.is_enabled()) {
4033 MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag);
4034 NMethodIterator iter(NMethodIterator::all);
4035 while (iter.next()) {
4036 nmethod* nm = iter.method();
4037 if (nm->is_in_use() && !nm->is_native_method() && !nm->is_osr_method()) {
4038 aot_info.print("%5d:%c%c%c%d:", nm->compile_id(),
4039 (nm->method()->in_aot_cache() ? 'S' : ' '),
4040 (nm->is_aot() ? 'A' : ' '),
4041 (nm->preloaded() ? 'P' : ' '),
4042 nm->comp_level());
4043 print_helper(nm, &aot_info);
4044 aot_info.print(": ");
4045 CompileTask::print(&aot_info, nm, nullptr, true /*short_form*/);
4046 LogStreamHandle(Trace, aot, codecache) aot_debug;
4047 if (aot_debug.is_enabled()) {
4048 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), nm->method()));
4049 if (mtd != nullptr) {
4050 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
4051 aot_debug.print(" CTD: "); ctd->print_on(&aot_debug); aot_debug.cr();
4052 });
4053 }
4054 }
4055 }
4056 }
4057 }
4058 }
4059 }
4060
4061 void AOTCodeEntry::print(outputStream* st) const {
4062 st->print_cr(" AOT Code Cache entry " INTPTR_FORMAT " [kind: %d, id: " UINT32_FORMAT_X_0 ", offset: %d, size: %d, comp_level: %d, comp_id: %d, %s%s%s%s]",
4063 p2i(this), (int)_kind, _id, _offset, _size, _comp_level, _comp_id,
4064 (_not_entrant? "not_entrant" : "entrant"),
4065 (_loaded ? ", loaded" : ""),
4066 (_has_clinit_barriers ? ", has_clinit_barriers" : ""),
4067 (_for_preload ? ", for_preload" : ""));
4068 }
4069
4070 // This is called after initialize() but before init2()
4071 // and _cache is not set yet.
4072 void AOTCodeCache::print_on(outputStream* st) {
4073 if (opened_cache != nullptr && opened_cache->for_use()) {
4074 ReadingMark rdmk;
4075 if (rdmk.failed()) {
4076 // Cache is closed, cannot touch anything.
4077 return;
4078 }
4079
4080 st->print_cr("\nAOT Code Cache");
4081 uint count = opened_cache->_load_header->entries_count();
4082 uint* search_entries = (uint*)opened_cache->addr(opened_cache->_load_header->search_table_offset()); // [id, index]
4083 AOTCodeEntry* load_entries = (AOTCodeEntry*)opened_cache->addr(opened_cache->_load_header->entries_offset());
4084
4085 for (uint i = 0; i < count; i++) {
4086 int index = search_entries[2*i + 1];
4087 AOTCodeEntry* entry = &(load_entries[index]);
4088
4089 uint entry_position = entry->offset();
4090 uint name_offset = entry->name_offset() + entry_position;
4091 const char* saved_name = opened_cache->addr(name_offset);
4092
4093 st->print_cr("%4u: %10s idx:%4u Id:%u L%u size=%u '%s' %s%s%s%s",
4094 i, aot_code_entry_kind_name[entry->kind()], index, entry->id(), entry->comp_level(),
4095 entry->size(), saved_name,
4096 entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
4097 entry->for_preload() ? " for_preload" : "",
4098 entry->is_loaded() ? " loaded" : "",
4099 entry->not_entrant() ? " not_entrant" : "");
4100
4101 st->print_raw(" ");
4102 AOTCodeReader reader(opened_cache, entry, nullptr);
4103 reader.print_on(st);
4104 }
4105 }
4106 }
4107
4108 void AOTCodeCache::print_unused_entries_on(outputStream* st) {
4109 LogStreamHandle(Info, aot, codecache, init) info;
4110 if (info.is_enabled()) {
4111 AOTCodeCache::iterate([&](AOTCodeEntry* entry) {
4112 if (entry->is_code() && !entry->is_loaded()) {
4113 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), entry->method()));
4114 if (mtd != nullptr) {
4115 if (mtd->has_holder()) {
4116 if (mtd->holder()->method_holder()->is_initialized()) {
4117 ResourceMark rm;
4118 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
4119 if ((uint)ctd->level() == entry->comp_level()) {
4120 if (ctd->init_deps_left_acquire() == 0) {
4121 nmethod* nm = mtd->holder()->code();
4122 if (nm == nullptr) {
4123 if (mtd->holder()->queued_for_compilation()) {
4124 return; // scheduled for compilation
4125 }
4126 } else if ((uint)nm->comp_level() >= entry->comp_level()) {
4127 return; // already online compiled and superseded by a more optimal method
4128 }
4129 info.print("AOT Code Cache entry not loaded: ");
4130 ctd->print_on(&info);
4131 info.cr();
4132 }
4133 }
4134 });
4135 } else {
4136 // not yet initialized
4137 }
4138 } else {
4139 info.print("AOT Code Cache entry doesn't have a holder: ");
4140 mtd->print_on(&info);
4141 info.cr();
4142 }
4143 }
4144 }
4145 });
4146 }
4147 }
4148
4149 void AOTCodeReader::print_on(outputStream* st) {
4150 uint entry_position = _entry->offset();
4151 set_read_position(entry_position);
4152
4153 // Read name
4154 uint name_offset = entry_position + _entry->name_offset();
4155 uint name_size = _entry->name_size(); // Includes '/0'
4156 const char* name = addr(name_offset);
4157
4158 st->print_cr(" name: %s", name);
4159 }
4160