1 /*
2 * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25
26 #include "asm/macroAssembler.hpp"
27 #include "cds/aotCacheAccess.hpp"
28 #include "cds/aotMetaspace.hpp"
29 #include "cds/cds_globals.hpp"
30 #include "cds/cdsConfig.hpp"
31 #include "cds/heapShared.hpp"
32 #include "ci/ciUtilities.hpp"
33 #include "classfile/javaAssertions.hpp"
34 #include "code/aotCodeCache.hpp"
35 #include "code/codeCache.hpp"
36 #include "gc/shared/barrierSetAssembler.hpp"
37 #include "gc/shared/barrierSetNMethod.hpp"
38 #include "gc/shared/cardTableBarrierSet.hpp"
39 #include "gc/shared/gcConfig.hpp"
40 #include "logging/logStream.hpp"
41 #include "memory/memoryReserver.hpp"
42 #include "prims/jvmtiThreadState.hpp"
43 #include "prims/upcallLinker.hpp"
44 #include "runtime/deoptimization.hpp"
45 #include "runtime/flags/flagSetting.hpp"
46 #include "runtime/globals_extension.hpp"
47 #include "runtime/icache.hpp"
48 #include "runtime/java.hpp"
49 #include "runtime/mutexLocker.hpp"
50 #include "runtime/os.inline.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubInfo.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "utilities/copy.hpp"
55 #ifdef COMPILER1
56 #include "c1/c1_Runtime1.hpp"
57 #endif
58 #ifdef COMPILER2
59 #include "opto/runtime.hpp"
60 #endif
61 #if INCLUDE_G1GC
62 #include "gc/g1/g1BarrierSetRuntime.hpp"
63 #include "gc/g1/g1HeapRegion.hpp"
64 #endif
65 #if INCLUDE_SHENANDOAHGC
66 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
67 #include "gc/shenandoah/shenandoahRuntime.hpp"
68 #endif
69 #if INCLUDE_ZGC
70 #include "gc/z/zBarrierSetRuntime.hpp"
71 #endif
72
73 #include <errno.h>
74 #include <sys/stat.h>
75
76 const char* aot_code_entry_kind_name[] = {
77 #define DECL_KIND_STRING(kind) XSTR(kind),
78 DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
79 #undef DECL_KIND_STRING
80 };
81
82 // Stream to printing AOTCodeCache loading failure.
83 // Print to error channel when -XX:AOTMode is set to "on"
84 static LogStream& load_failure_log() {
85 static LogStream err_stream(LogLevel::Error, LogTagSetMapping<LOG_TAGS(aot, codecache, init)>::tagset());
86 static LogStream inf_stream(LogLevel::Info, LogTagSetMapping<LOG_TAGS(aot, codecache, init)>::tagset());
87 if (RequireSharedSpaces || AbortVMOnAOTCodeFailure) {
88 return err_stream;
89 } else {
90 static LogStream aot_stream(LogLevel::Info, LogTagSetMapping<LOG_TAGS(aot)>::tagset());
91 return inf_stream.is_enabled() ? inf_stream : aot_stream;
92 }
93 }
94
95 // Report AOT code cache failure and exit VM
96 // if (AOTMode is `on` and AbortVMOnAOTCodeFailure is default)
97 // or AbortVMOnAOTCodeFailure is `true`.
98 //
99 // Note, specifying -XX:-AbortVMOnAOTCodeFailure on command line
100 // will prevent aborting VM when AOTMode is `on`. It is used for testing.
101
102 static void report_load_failure() {
103 bool abort_vm = AbortVMOnAOTCodeFailure ||
104 (FLAG_IS_DEFAULT(AbortVMOnAOTCodeFailure) && RequireSharedSpaces);
105 if (abort_vm) {
106 vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
107 }
108 load_failure_log().print_cr("Unable to use AOT Code Cache.");
109 AOTCodeCache::disable_caching();
110 }
111
112 static void report_store_failure() {
113 if (AbortVMOnAOTCodeFailure) {
114 tty->print_cr("Unable to create AOT Code Cache.");
115 vm_abort(false);
116 }
117 log_error(aot, codecache, exit)("Unable to create AOT Code Cache.");
118 AOTCodeCache::disable_caching();
119 }
120
121 // The sequence of AOT code caching flags and parametters settings.
122 //
123 // 1. The initial AOT code caching flags setting is done
124 // during call to CDSConfig::check_vm_args_consistency().
125 //
126 // 2. The earliest AOT code state check done in compilationPolicy_init()
127 // where we set number of compiler threads for AOT assembly phase.
128 //
129 // 3. We determine presence of AOT code in AOT Cache in
130 // AOTMetaspace::open_static_archive() which is calles
131 // after compilationPolicy_init() but before codeCache_init().
132 //
133 // 4. AOTCodeCache::initialize() is called during universe_init()
134 // and does final AOT state and flags settings.
135 //
136 // 5. Finally AOTCodeCache::init2() is called after universe_init()
137 // when all GC settings are finalized.
138
139 // Next methods determine which action we do with AOT code depending
140 // on phase of AOT process: assembly or production.
141
142 bool AOTCodeCache::is_dumping_adapter() {
143 return AOTAdapterCaching && is_on_for_dump();
144 }
145
146 bool AOTCodeCache::is_using_adapter() {
147 return AOTAdapterCaching && is_on_for_use();
148 }
149
150 bool AOTCodeCache::is_dumping_stub() {
151 return AOTStubCaching && is_on_for_dump();
152 }
153
154 bool AOTCodeCache::is_using_stub() {
155 return AOTStubCaching && is_on_for_use();
156 }
157
158 // Next methods could be called regardless AOT code cache status.
159 // Initially they are called during flags parsing and finilized
160 // in AOTCodeCache::initialize().
161 void AOTCodeCache::enable_caching() {
162 FLAG_SET_ERGO_IF_DEFAULT(AOTStubCaching, true);
163 FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true);
164 }
165
166 void AOTCodeCache::disable_caching() {
167 FLAG_SET_ERGO(AOTStubCaching, false);
168 FLAG_SET_ERGO(AOTAdapterCaching, false);
169 }
170
171 bool AOTCodeCache::is_caching_enabled() {
172 return AOTStubCaching || AOTAdapterCaching;
173 }
174
175 static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
176 assert(AOTCodeEntry::is_valid_entry_kind(kind), "invalid AOTCodeEntry kind %d", (int)kind);
177 // There can be a conflict of id between an Adapter and *Blob, but that should not cause any functional issue
178 // becasue both id and kind are used to find an entry, and that combination should be unique
179 if (kind == AOTCodeEntry::Adapter) {
180 return id;
181 } else if (kind == AOTCodeEntry::SharedBlob) {
182 assert(StubInfo::is_shared(static_cast<BlobId>(id)), "not a shared blob id %d", id);
183 return id;
184 } else if (kind == AOTCodeEntry::C1Blob) {
185 assert(StubInfo::is_c1(static_cast<BlobId>(id)), "not a c1 blob id %d", id);
186 return id;
187 } else if (kind == AOTCodeEntry::C2Blob) {
188 assert(StubInfo::is_c2(static_cast<BlobId>(id)), "not a c2 blob id %d", id);
189 return id;
190 } else {
191 // kind must be AOTCodeEntry::StubGenBlob
192 assert(StubInfo::is_stubgen(static_cast<BlobId>(id)), "not a stubgen blob id %d", id);
193 return id;
194 }
195 }
196
197 static uint _max_aot_code_size = 0;
198 uint AOTCodeCache::max_aot_code_size() {
199 return _max_aot_code_size;
200 }
201
202 // It is called from AOTMetaspace::initialize_shared_spaces()
203 // which is called from universe_init().
204 // At this point all AOT class linking seetings are finilized
205 // and AOT cache is open so we can map AOT code region.
206 void AOTCodeCache::initialize() {
207 #if defined(ZERO) || !(defined(AMD64) || defined(AARCH64))
208 log_info(aot, codecache, init)("AOT Code Cache is not supported on this platform.");
209 disable_caching();
210 return;
211 #else
212 if (FLAG_IS_DEFAULT(AOTCache)) {
213 log_info(aot, codecache, init)("AOT Code Cache is not used: AOTCache is not specified.");
214 disable_caching();
215 return; // AOTCache must be specified to dump and use AOT code
216 }
217
218 if (VerifyOops) {
219 // Disable AOT stub caching when VerifyOops flag is on.
220 // Verify oops code generated a lot of C strings which overflow
221 // AOT C string table (which has fixed size).
222 // AOT C string table will be reworked later to handle such cases.
223 log_info(aot, codecache, init)("AOT Stub Caching is not supported with VerifyOops.");
224 FLAG_SET_ERGO(AOTStubCaching, false);
225 if (InlineTypePassFieldsAsArgs) {
226 log_info(aot, codecache, init)("AOT Adapter Caching is not supported with VerifyOops + InlineTypePassFieldsAsArgs.");
227 FLAG_SET_ERGO(AOTAdapterCaching, false);
228 }
229 }
230
231 bool is_dumping = false;
232 bool is_using = false;
233 if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) {
234 is_dumping = true;
235 enable_caching();
236 is_dumping = is_caching_enabled();
237 } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) {
238 enable_caching();
239 is_using = is_caching_enabled();
240 } else {
241 log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used.");
242 disable_caching();
243 return; // nothing to do
244 }
245 if (!(is_dumping || is_using)) {
246 disable_caching();
247 return; // AOT code caching disabled on command line
248 }
249 _max_aot_code_size = AOTCodeMaxSize;
250 if (!FLAG_IS_DEFAULT(AOTCodeMaxSize)) {
251 if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) {
252 _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity());
253 log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K));
254 }
255 }
256 size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0;
257 if (is_using && aot_code_size == 0) {
258 log_info(aot, codecache, init)("AOT Code Cache is empty");
259 disable_caching();
260 return;
261 }
262 if (!open_cache(is_dumping, is_using)) {
263 if (is_using) {
264 report_load_failure();
265 } else {
266 report_store_failure();
267 }
268 return;
269 }
270 if (is_dumping) {
271 FLAG_SET_DEFAULT(ForceUnreachable, true);
272 }
273 FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
274 #endif // defined(AMD64) || defined(AARCH64)
275 }
276
277 static AOTCodeCache* opened_cache = nullptr; // Use this until we verify the cache
278 AOTCodeCache* AOTCodeCache::_cache = nullptr;
279 DEBUG_ONLY( bool AOTCodeCache::_passed_init2 = false; )
280
281 // It is called after universe_init() when all GC settings are finalized.
282 void AOTCodeCache::init2() {
283 DEBUG_ONLY( _passed_init2 = true; )
284 if (opened_cache == nullptr) {
285 return;
286 }
287 if (!opened_cache->verify_config()) {
288 delete opened_cache;
289 opened_cache = nullptr;
290 report_load_failure();
291 return;
292 }
293
294 // Report contents of AOT code cache after verification passed
295 Header* header = opened_cache->_load_header;
296 if (header != nullptr) { // Loading AOT code
297 log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", header->entries_count());
298 log_debug(aot, codecache, init)(" Adapters: total=%u", header->adapters_count());
299 log_debug(aot, codecache, init)(" Shared Blobs: total=%u", header->shared_blobs_count());
300 log_debug(aot, codecache, init)(" StubGen Blobs: total=%d", header->stubgen_blobs_count());
301 log_debug(aot, codecache, init)(" C1 Blobs: total=%u", header->C1_blobs_count());
302 log_debug(aot, codecache, init)(" C2 Blobs: total=%u", header->C2_blobs_count());
303 log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", header->cache_size());
304
305 // Read strings
306 opened_cache->load_strings();
307 }
308 // initialize aot runtime constants as appropriate to this runtime
309 AOTRuntimeConstants::initialize_from_runtime();
310
311 // initialize the table of external routines so we can save
312 // generated code blobs that reference them
313 AOTCodeAddressTable* table = opened_cache->_table;
314 assert(table != nullptr, "should be initialized already");
315 table->init_extrs();
316
317 // Now cache and address table are ready for AOT code generation
318 _cache = opened_cache;
319 }
320
321 bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
322 opened_cache = new AOTCodeCache(is_dumping, is_using);
323 if (opened_cache->failed()) {
324 delete opened_cache;
325 opened_cache = nullptr;
326 return false;
327 }
328 return true;
329 }
330
331 // Called after continuations_init() when continuation stub callouts
332 // have been initialized
333 void AOTCodeCache::init3() {
334 if (opened_cache == nullptr) {
335 return;
336 }
337 // initialize external routines for continuations so we can save
338 // generated continuation blob that references them
339 AOTCodeAddressTable* table = opened_cache->_table;
340 assert(table != nullptr, "should be initialized already");
341 table->init_extrs2();
342 }
343
344 void AOTCodeCache::dump() {
345 if (is_on()) {
346 assert(is_on_for_dump(), "should be called only when dumping AOT code");
347 MutexLocker ml(Compile_lock);
348 _cache->finish_write();
349 }
350 }
351
352 #define DATA_ALIGNMENT HeapWordSize
353
354 AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
355 _load_header(nullptr),
356 _load_buffer(nullptr),
357 _store_buffer(nullptr),
358 _C_store_buffer(nullptr),
359 _write_position(0),
360 _load_size(0),
361 _store_size(0),
362 _for_use(is_using),
363 _for_dump(is_dumping),
364 _failed(false),
365 _lookup_failed(false),
366 _table(nullptr),
367 _load_entries(nullptr),
368 _search_entries(nullptr),
369 _store_entries(nullptr),
370 _C_strings_buf(nullptr),
371 _store_entries_cnt(0)
372 {
373 // Read header at the begining of cache
374 if (_for_use) {
375 // Read cache
376 size_t load_size = AOTCacheAccess::get_aot_code_region_size();
377 ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode);
378 if (!rs.is_reserved()) {
379 log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size);
380 set_failed();
381 return;
382 }
383 if (!AOTCacheAccess::map_aot_code_region(rs)) {
384 log_warning(aot, codecache, init)("Failed to read/mmap cached code region into AOT Code Cache");
385 set_failed();
386 return;
387 }
388
389 _load_size = (uint)load_size;
390 _load_buffer = (char*)rs.base();
391 assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
392 log_debug(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " at AOT Code Cache", _load_size, p2i(_load_buffer));
393
394 _load_header = (Header*)addr(0);
395 if (!_load_header->verify(_load_size)) {
396 set_failed();
397 return;
398 }
399 }
400 if (_for_dump) {
401 _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
402 _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
403 // Entries allocated at the end of buffer in reverse (as on stack).
404 _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
405 log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size());
406 }
407 _table = new AOTCodeAddressTable();
408 }
409
410 void AOTCodeCache::add_stub_entries(StubId stub_id, address start, GrowableArray<address> *entries, int begin_idx) {
411 EntryId entry_id = StubInfo::entry_base(stub_id);
412 add_stub_entry(entry_id, start);
413 // skip past first entry
414 entry_id = StubInfo::next_in_stub(stub_id, entry_id);
415 // now check for any more entries
416 int count = StubInfo::entry_count(stub_id) - 1;
417 assert(start != nullptr, "invalid start address for stub %s", StubInfo::name(stub_id));
418 assert(entries == nullptr || begin_idx + count <= entries->length(), "sanity");
419 // write any extra entries
420 for (int i = 0; i < count; i++) {
421 assert(entry_id != EntryId::NO_ENTRYID, "not enough entries for stub %s", StubInfo::name(stub_id));
422 address a = entries->at(begin_idx + i);
423 add_stub_entry(entry_id, a);
424 entry_id = StubInfo::next_in_stub(stub_id, entry_id);
425 }
426 assert(entry_id == EntryId::NO_ENTRYID, "too many entries for stub %s", StubInfo::name(stub_id));
427 }
428
429 void AOTCodeCache::add_stub_entry(EntryId entry_id, address a) {
430 if (a != nullptr) {
431 if (_table != nullptr) {
432 log_trace(aot, codecache, stubs)("Publishing stub entry %s at address " INTPTR_FORMAT, StubInfo::name(entry_id), p2i(a));
433 return _table->add_stub_entry(entry_id, a);
434 }
435 }
436 }
437
438 void AOTCodeCache::set_shared_stubs_complete() {
439 AOTCodeAddressTable* table = addr_table();
440 if (table != nullptr) {
441 table->set_shared_stubs_complete();
442 }
443 }
444
445 void AOTCodeCache::set_c1_stubs_complete() {
446 AOTCodeAddressTable* table = addr_table();
447 if (table != nullptr) {
448 table->set_c1_stubs_complete();
449 }
450 }
451
452 void AOTCodeCache::set_c2_stubs_complete() {
453 AOTCodeAddressTable* table = addr_table();
454 if (table != nullptr) {
455 table->set_c2_stubs_complete();
456 }
457 }
458
459 void AOTCodeCache::set_stubgen_stubs_complete() {
460 AOTCodeAddressTable* table = addr_table();
461 if (table != nullptr) {
462 table->set_stubgen_stubs_complete();
463 }
464 }
465
466 void AOTCodeCache::Config::record(uint cpu_features_offset) {
467
468 #define AOTCODECACHE_SAVE_VAR(type, name) _saved_ ## name = name;
469 #define AOTCODECACHE_SAVE_FUN(type, name, fun) _saved_ ## name = fun;
470
471 AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_SAVE_VAR, AOTCODECACHE_SAVE_FUN);
472
473 // Special configs that cannot be checked with macros
474 _compressedOopBase = CompressedOops::base();
475 _compressedOopShift = CompressedOops::shift();
476
477 #if defined(X86) && !defined(ZERO)
478 _useUnalignedLoadStores = UseUnalignedLoadStores;
479 #endif
480
481 #if defined(AARCH64) && !defined(ZERO)
482 _avoidUnalignedAccesses = AvoidUnalignedAccesses;
483 #endif
484
485 _cpu_features_offset = cpu_features_offset;
486 }
487
488 bool AOTCodeCache::Config::verify_cpu_features(AOTCodeCache* cache) const {
489 LogStreamHandle(Debug, aot, codecache, init) log;
490 uint offset = _cpu_features_offset;
491 uint cpu_features_size = *(uint *)cache->addr(offset);
492 assert(cpu_features_size == (uint)VM_Version::cpu_features_size(), "must be");
493 offset += sizeof(uint);
494
495 void* cached_cpu_features_buffer = (void *)cache->addr(offset);
496 if (log.is_enabled()) {
497 ResourceMark rm; // required for stringStream::as_string()
498 stringStream ss;
499 VM_Version::get_cpu_features_name(cached_cpu_features_buffer, ss);
500 log.print_cr("CPU features recorded in AOTCodeCache: %s", ss.as_string());
501 }
502
503 if (!VM_Version::verify_aot_code_cache_features(cached_cpu_features_buffer)) {
504 if (load_failure_log().is_enabled()) {
505 ResourceMark rm; // required for stringStream::as_string()
506 load_failure_log().print_cr("AOT Code Cache disabled: cpu features are incompatible");
507 char* runtime_cpu_features = NEW_RESOURCE_ARRAY(char, VM_Version::cpu_features_size());
508 VM_Version::store_cpu_features(runtime_cpu_features);
509
510 stringStream missing_features;
511 VM_Version::get_missing_features_name(cached_cpu_features_buffer, runtime_cpu_features, missing_features);
512 if (!missing_features.is_empty()) {
513 load_failure_log().print_cr("cpu features that are required: \"%s\"", missing_features.as_string());
514 }
515
516 stringStream additional_features;
517 VM_Version::get_missing_features_name(runtime_cpu_features, cached_cpu_features_buffer, additional_features);
518 if (!additional_features.is_empty()) {
519 load_failure_log().print("cpu features that are additional: \"%s\"", additional_features.as_string());
520 }
521 load_failure_log().print_cr("");
522 }
523 return false;
524 }
525 return true;
526 }
527
528 #define AOTCODECACHE_DISABLED_MSG "AOT Code Cache disabled: it was created with %s = "
529
530 // Special case, print "GC = ..." to be more understandable.
531 inline void log_config_mismatch(CollectedHeap::Name saved, CollectedHeap::Name current, const char* name/*unused*/) {
532 load_failure_log().print_cr("AOT Code Cache disabled: it was created with GC = \"%s\" vs current \"%s\"",
533 GCConfig::hs_err_name(saved), GCConfig::hs_err_name(current));
534 }
535
536 inline void log_config_mismatch(bool saved, bool current, const char* name) {
537 load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%s vs current %s", name,
538 saved ? "true" : "false", current ? "true" : "false");
539 }
540
541 inline void log_config_mismatch(int saved, int current, const char* name) {
542 load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%d vs current %d", name, saved, current);
543 }
544
545 inline void log_config_mismatch(uint saved, uint current, const char* name) {
546 load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%u vs current %u", name, saved, current);
547 }
548
549 #ifdef _LP64
550 inline void log_config_mismatch(intx saved, intx current, const char* name) {
551 load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zd vs current %zd", name, saved, current);
552 }
553
554 inline void log_config_mismatch(uintx saved, uintx current, const char* name) {
555 load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zu vs current %zu", name, saved, current);
556 }
557 #endif
558
559 template <typename T>
560 bool check_config(T saved, T current, const char* name) {
561 if (saved != current) {
562 log_config_mismatch(saved, current, name);
563 return false;
564 } else {
565 return true;
566 }
567 }
568
569 bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
570 // check CPU features before checking flags that may be
571 // auto-configured in response to them
572 if (!verify_cpu_features(cache)) {
573 return false;
574 }
575
576 // Tests for config options which might affect validity of adapters,
577 // stubs or nmethods. Currently we take a pessemistic stand and
578 // drop the whole cache if any of these are changed.
579
580 #define AOTCODECACHE_CHECK_VAR(type, name) \
581 if (!check_config(_saved_ ## name, name, #name)) { return false; }
582 #define AOTCODECACHE_CHECK_FUN(type, name, fun) \
583 if (!check_config(_saved_ ## name, fun, #fun)) { return false; }
584
585 AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_CHECK_VAR, AOTCODECACHE_CHECK_FUN);
586
587 // Special configs that cannot be checked with macros
588 #define COMPRESSED_OOPS_HINT "Consider adding -XX:+AOTCompatibleOopCompression when creating the AOT cache"
589
590 if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
591 load_failure_log().print_cr("AOT Code Cache disabled: incompatible CompressedOops::base(): %p vs current %p",
592 _compressedOopBase, CompressedOops::base());
593 load_failure_log().print_cr(COMPRESSED_OOPS_HINT);
594 return false;
595 }
596
597 if (!check_config(_compressedOopShift, CompressedOops::shift(), "CompressedOops::shift()")) {
598 load_failure_log().print_cr(COMPRESSED_OOPS_HINT);
599 return false;
600 }
601
602 #if defined(X86) && !defined(ZERO)
603 // switching off UseUnalignedLoadStores can affect validity of fill
604 // stubs
605 if (_useUnalignedLoadStores && !UseUnalignedLoadStores) {
606 log_config_mismatch(_useUnalignedLoadStores, UseUnalignedLoadStores, "UseUnalignedLoadStores");
607 return false;
608 }
609 #endif // defined(X86) && !defined(ZERO)
610
611 #if defined(AARCH64) && !defined(ZERO)
612 // switching on AvoidUnalignedAccesses may affect validity of array
613 // copy stubs and nmethods
614 if (!_avoidUnalignedAccesses && AvoidUnalignedAccesses) {
615 log_config_mismatch(_avoidUnalignedAccesses, AvoidUnalignedAccesses, "AvoidUnalignedAccesses");
616 return false;
617 }
618 #endif // defined(AARCH64) && !defined(ZERO)
619
620 return true;
621 }
622
623 bool AOTCodeCache::Header::verify(uint load_size) const {
624 if (_version != AOT_CODE_VERSION) {
625 load_failure_log().print_cr("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
626 return false;
627 }
628 if (load_size < _cache_size) {
629 load_failure_log().print_cr("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
630 return false;
631 }
632 return true;
633 }
634
635 AOTCodeCache* AOTCodeCache::open_for_use() {
636 if (AOTCodeCache::is_on_for_use()) {
637 return AOTCodeCache::cache();
638 }
639 return nullptr;
640 }
641
642 AOTCodeCache* AOTCodeCache::open_for_dump() {
643 if (AOTCodeCache::is_on_for_dump()) {
644 AOTCodeCache* cache = AOTCodeCache::cache();
645 cache->clear_lookup_failed(); // Reset bit
646 return cache;
647 }
648 return nullptr;
649 }
650
651 void copy_bytes(const char* from, address to, uint size) {
652 assert((int)size > 0, "sanity");
653 memcpy(to, from, size);
654 log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
655 }
656
657 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) {
658 _cache = cache;
659 _entry = entry;
660 _load_buffer = cache->cache_buffer();
661 _read_position = 0;
662 _lookup_failed = false;
663 _name = nullptr;
664 _reloc_data = nullptr;
665 _reloc_count = 0;
666 _oop_maps = nullptr;
667 _entry_kind = AOTCodeEntry::None;
668 _stub_data = nullptr;
669 _id = -1;
670 }
671
672 void AOTCodeReader::set_read_position(uint pos) {
673 if (pos == _read_position) {
674 return;
675 }
676 assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
677 _read_position = pos;
678 }
679
680 uint AOTCodeReader::align_read_int() {
681 return align_up(_read_position, sizeof(int));
682 }
683
684 bool AOTCodeCache::set_write_position(uint pos) {
685 if (pos == _write_position) {
686 return true;
687 }
688 if (_store_size < _write_position) {
689 _store_size = _write_position; // Adjust during write
690 }
691 assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
692 _write_position = pos;
693 return true;
694 }
695
696 static char align_buffer[256] = { 0 };
697
698 bool AOTCodeCache::align_write_bytes(uint alignment) {
699 uint padding = alignment - (_write_position & (alignment - 1));
700 if (padding == alignment) {
701 return true;
702 }
703 uint n = write_bytes((const void*)&align_buffer, padding);
704 if (n != padding) {
705 return false;
706 }
707 log_trace(aot, codecache)("Adjust write alignment to %d bytes in AOT Code Cache", alignment);
708 return true;
709 }
710
711 bool AOTCodeCache::align_write() {
712 // We are not executing code from cache - we copy it by bytes first.
713 // No need for big alignment (or at all).
714 return align_write_bytes(DATA_ALIGNMENT);
715 }
716
717 bool AOTCodeCache::align_write_int() {
718 return align_write_bytes(sizeof(int));
719 }
720
721 // Check to see if AOT code cache has required space to store "nbytes" of data
722 address AOTCodeCache::reserve_bytes(uint nbytes) {
723 assert(for_dump(), "Code Cache file is not created");
724 uint new_position = _write_position + nbytes;
725 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
726 log_warning(aot,codecache)("Failed to ensure %d bytes at offset %d in AOT Code Cache. Increase AOTCodeMaxSize.",
727 nbytes, _write_position);
728 set_failed();
729 report_store_failure();
730 return nullptr;
731 }
732 address buffer = (address)(_store_buffer + _write_position);
733 log_trace(aot, codecache)("Reserved %d bytes at offset %d in AOT Code Cache", nbytes, _write_position);
734 _write_position += nbytes;
735 if (_store_size < _write_position) {
736 _store_size = _write_position;
737 }
738 return buffer;
739 }
740
741 uint AOTCodeCache::write_bytes(const void* buffer, uint nbytes) {
742 assert(for_dump(), "Code Cache file is not created");
743 if (nbytes == 0) {
744 return 0;
745 }
746 uint new_position = _write_position + nbytes;
747 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
748 log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.",
749 nbytes, _write_position);
750 set_failed();
751 report_store_failure();
752 return 0;
753 }
754 copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
755 log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
756 _write_position += nbytes;
757 if (_store_size < _write_position) {
758 _store_size = _write_position;
759 }
760 return nbytes;
761 }
762
763 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
764 return (void*)(cache->add_entry());
765 }
766
767 static bool check_entry(AOTCodeEntry::Kind kind, uint id, AOTCodeEntry* entry) {
768 if (entry->kind() == kind) {
769 assert(entry->id() == id, "sanity");
770 return true; // Found
771 }
772 return false;
773 }
774
775 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id) {
776 assert(_for_use, "sanity");
777 uint count = _load_header->entries_count();
778 if (_load_entries == nullptr) {
779 // Read it
780 _search_entries = (uint*)addr(_load_header->entries_offset()); // [id, index]
781 _load_entries = (AOTCodeEntry*)(_search_entries + 2 * count);
782 log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
783 }
784 // Binary search
785 int l = 0;
786 int h = count - 1;
787 while (l <= h) {
788 int mid = (l + h) >> 1;
789 int ix = mid * 2;
790 uint is = _search_entries[ix];
791 if (is == id) {
792 int index = _search_entries[ix + 1];
793 AOTCodeEntry* entry = &(_load_entries[index]);
794 if (check_entry(kind, id, entry)) {
795 return entry; // Found
796 }
797 // Linear search around to handle id collission
798 for (int i = mid - 1; i >= l; i--) { // search back
799 ix = i * 2;
800 is = _search_entries[ix];
801 if (is != id) {
802 break;
803 }
804 index = _search_entries[ix + 1];
805 AOTCodeEntry* entry = &(_load_entries[index]);
806 if (check_entry(kind, id, entry)) {
807 return entry; // Found
808 }
809 }
810 for (int i = mid + 1; i <= h; i++) { // search forward
811 ix = i * 2;
812 is = _search_entries[ix];
813 if (is != id) {
814 break;
815 }
816 index = _search_entries[ix + 1];
817 AOTCodeEntry* entry = &(_load_entries[index]);
818 if (check_entry(kind, id, entry)) {
819 return entry; // Found
820 }
821 }
822 break; // Not found match
823 } else if (is < id) {
824 l = mid + 1;
825 } else {
826 h = mid - 1;
827 }
828 }
829 return nullptr;
830 }
831
832 extern "C" {
833 static int uint_cmp(const void *i, const void *j) {
834 uint a = *(uint *)i;
835 uint b = *(uint *)j;
836 return a > b ? 1 : a < b ? -1 : 0;
837 }
838 }
839
840 void AOTCodeCache::store_cpu_features(char*& buffer, uint buffer_size) {
841 uint* size_ptr = (uint *)buffer;
842 *size_ptr = buffer_size;
843 buffer += sizeof(uint);
844
845 VM_Version::store_cpu_features(buffer);
846 log_debug(aot, codecache, exit)("CPU features recorded in AOTCodeCache: %s", VM_Version::features_string());
847 buffer += buffer_size;
848 buffer = align_up(buffer, DATA_ALIGNMENT);
849 }
850
851 bool AOTCodeCache::finish_write() {
852 if (!align_write()) {
853 return false;
854 }
855 uint strings_offset = _write_position;
856 int strings_count = store_strings();
857 if (strings_count < 0) {
858 return false;
859 }
860 if (!align_write()) {
861 return false;
862 }
863 uint strings_size = _write_position - strings_offset;
864
865 uint entries_count = 0; // Number of entrant (useful) code entries
866 uint entries_offset = _write_position;
867
868 uint store_count = _store_entries_cnt;
869 if (store_count > 0) {
870 uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT);
871 uint code_count = store_count;
872 uint search_count = code_count * 2;
873 uint search_size = search_count * sizeof(uint);
874 uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
875 // _write_position includes size of code and strings
876 uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
877 uint cpu_features_size = VM_Version::cpu_features_size();
878 uint total_cpu_features_size = sizeof(uint) + cpu_features_size; // sizeof(uint) to store cpu_features_size
879 uint total_size = header_size + _write_position + code_alignment + search_size + entries_size +
880 align_up(total_cpu_features_size, DATA_ALIGNMENT);
881 assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size());
882
883 // Allocate in AOT Cache buffer
884 char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT);
885 char* start = align_up(buffer, DATA_ALIGNMENT);
886 char* current = start + header_size; // Skip header
887
888 uint cpu_features_offset = current - start;
889 store_cpu_features(current, cpu_features_size);
890 assert(is_aligned(current, DATA_ALIGNMENT), "sanity check");
891 assert(current < start + total_size, "sanity check");
892
893 // Create ordered search table for entries [id, index];
894 uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
895
896 AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
897 uint adapters_count = 0;
898 uint shared_blobs_count = 0;
899 uint stubgen_blobs_count = 0;
900 uint C1_blobs_count = 0;
901 uint C2_blobs_count = 0;
902 uint max_size = 0;
903 // AOTCodeEntry entries were allocated in reverse in store buffer.
904 // Process them in reverse order to cache first code first.
905 for (int i = store_count - 1; i >= 0; i--) {
906 entries_address[i].set_next(nullptr); // clear pointers before storing data
907 uint size = align_up(entries_address[i].size(), DATA_ALIGNMENT);
908 if (size > max_size) {
909 max_size = size;
910 }
911 copy_bytes((_store_buffer + entries_address[i].offset()), (address)current, size);
912 entries_address[i].set_offset(current - start); // New offset
913 current += size;
914 uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry));
915 if (n != sizeof(AOTCodeEntry)) {
916 FREE_C_HEAP_ARRAY(search);
917 return false;
918 }
919 search[entries_count*2 + 0] = entries_address[i].id();
920 search[entries_count*2 + 1] = entries_count;
921 entries_count++;
922 AOTCodeEntry::Kind kind = entries_address[i].kind();
923 if (kind == AOTCodeEntry::Adapter) {
924 adapters_count++;
925 } else if (kind == AOTCodeEntry::SharedBlob) {
926 shared_blobs_count++;
927 } else if (kind == AOTCodeEntry::StubGenBlob) {
928 stubgen_blobs_count++;
929 } else if (kind == AOTCodeEntry::C1Blob) {
930 C1_blobs_count++;
931 } else if (kind == AOTCodeEntry::C2Blob) {
932 C2_blobs_count++;
933 }
934 }
935 if (entries_count == 0) {
936 log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires");
937 FREE_C_HEAP_ARRAY(search);
938 return true; // Nothing to write
939 }
940 assert(entries_count <= store_count, "%d > %d", entries_count, store_count);
941 // Write strings
942 if (strings_count > 0) {
943 copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
944 strings_offset = (current - start); // New offset
945 current += strings_size;
946 }
947
948 uint new_entries_offset = (current - start); // New offset
949 // Sort and store search table
950 qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
951 search_size = 2 * entries_count * sizeof(uint);
952 copy_bytes((const char*)search, (address)current, search_size);
953 FREE_C_HEAP_ARRAY(search);
954 current += search_size;
955
956 // Write entries
957 entries_size = entries_count * sizeof(AOTCodeEntry); // New size
958 copy_bytes((_store_buffer + entries_offset), (address)current, entries_size);
959 current += entries_size;
960 uint size = (current - start);
961 assert(size <= total_size, "%d > %d", size , total_size);
962
963 log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count);
964 log_debug(aot, codecache, exit)(" Shared Blobs: total=%d", shared_blobs_count);
965 log_debug(aot, codecache, exit)(" StubGen Blobs: total=%d", stubgen_blobs_count);
966 log_debug(aot, codecache, exit)(" C1 Blobs: total=%d", C1_blobs_count);
967 log_debug(aot, codecache, exit)(" C2 Blobs: total=%d", C2_blobs_count);
968 log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
969
970 // Finalize header
971 AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
972 header->init(size, (uint)strings_count, strings_offset,
973 entries_count, new_entries_offset,
974 adapters_count, shared_blobs_count,
975 stubgen_blobs_count, C1_blobs_count,
976 C2_blobs_count, cpu_features_offset);
977
978 log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count);
979 }
980 return true;
981 }
982
983 //------------------Store/Load AOT code ----------------------
984
985 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data, CodeBuffer* code_buffer) {
986 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
987
988 // we only expect stub data and a code buffer for a multi stub blob
989 assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr),
990 "entry_kind %d does not match stub_data pointer %p",
991 entry_kind, stub_data);
992
993 assert((stub_data == nullptr) == (code_buffer == nullptr),
994 "stub data and code buffer must both be null or both non null");
995
996 // If this is a stub and the cache is on for either load or dump we
997 // need to insert the stub entries into the AOTCacheAddressTable so
998 // that relocs which refer to entries defined by this blob get
999 // translated correctly.
1000 //
1001 // Entry insertion needs to be be done up front before writing the
1002 // blob because some blobs rely on internal daisy-chain references
1003 // from one entry to another.
1004 //
1005 // Entry insertion also needs to be done even if the cache is open
1006 // for use but not for dump. This may be needed when an archived
1007 // blob omits some entries -- either because of a config change or a
1008 // load failure -- with the result that the entries end up being
1009 // generated. These generated entry addresses may be needed to
1010 // resolve references from subsequently loaded blobs (for either
1011 // stubs or nmethods).
1012
1013 if (is_on() && AOTCodeEntry::is_blob(entry_kind)) {
1014 publish_stub_addresses(blob, (BlobId)id, stub_data);
1015 }
1016
1017 AOTCodeCache* cache = open_for_dump();
1018 if (cache == nullptr) {
1019 return false;
1020 }
1021 if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
1022 return false;
1023 }
1024 if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
1025 return false;
1026 }
1027 log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1028
1029 #ifdef ASSERT
1030 LogStreamHandle(Trace, aot, codecache, stubs) log;
1031 if (log.is_enabled()) {
1032 FlagSetting fs(PrintRelocations, true);
1033 blob.print_on(&log);
1034 }
1035 #endif
1036 // we need to take a lock to prevent race between compiler threads generating AOT code
1037 // and the main thread generating adapter
1038 MutexLocker ml(Compile_lock);
1039 if (!is_on()) {
1040 return false; // AOT code cache was already dumped and closed.
1041 }
1042 if (!cache->align_write()) {
1043 return false;
1044 }
1045 uint entry_position = cache->_write_position;
1046
1047 uint blob_offset = cache->_write_position - entry_position;
1048 // Code blob's size is aligned to oopSize
1049 address archive_buffer = cache->reserve_bytes(blob.size());
1050 if (archive_buffer == nullptr) {
1051 return false;
1052 }
1053 CodeBlob::archive_blob(&blob, archive_buffer);
1054
1055 // For a relocatable code blob its relocations are linked from the
1056 // blob. However, for a non-relocatable (stubgen) blob we only have
1057 // transient relocations attached to the code buffer that are added
1058 // in order to support AOT-load time patching. in either case, we
1059 // need to explicitly save these relocs when storing the blob to the
1060 // archive so we can then reload them and reattach them to either
1061 // the blob or to a code buffer when we reload the blob into a
1062 // production JVM.
1063 //
1064 // Either way we are then in a position to iterate over the relocs
1065 // and AOT patch the ones that refer to code that may move between
1066 // assembly and production time. We also need to save and restore
1067 // AOT address table indexes for the target addresses of affected
1068 // relocs. That happens below.
1069
1070 int reloc_count;
1071 address reloc_data;
1072 if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
1073 CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS);
1074 reloc_count = (cs->has_locs() ? cs->locs_count() : 0);
1075 reloc_data = (reloc_count > 0 ? (address)cs->locs_start() : nullptr);
1076 } else {
1077 reloc_count = blob.relocation_size() / sizeof(relocInfo);
1078 reloc_data = (address)blob.relocation_begin();
1079 }
1080 uint n = cache->write_bytes(&reloc_count, sizeof(int));
1081 if (n != sizeof(int)) {
1082 return false;
1083 }
1084 if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
1085 // align to heap word size before writing the relocs so we can
1086 // install them into a code buffer when they get restored
1087 if (!cache->align_write()) {
1088 return false;
1089 }
1090 }
1091 uint reloc_data_size = (uint)(reloc_count * sizeof(relocInfo));
1092 n = cache->write_bytes(reloc_data, reloc_data_size);
1093 if (n != reloc_data_size) {
1094 return false;
1095 }
1096
1097 bool has_oop_maps = false;
1098 if (blob.oop_maps() != nullptr) {
1099 if (!cache->write_oop_map_set(blob)) {
1100 return false;
1101 }
1102 has_oop_maps = true;
1103 }
1104
1105 // In the case of a multi-stub blob we need to write start, end,
1106 // secondary entries and extras. For any other blob entry addresses
1107 // beyond the blob start will be stored in the blob as offsets.
1108 if (stub_data != nullptr) {
1109 if (!cache->write_stub_data(blob, stub_data)) {
1110 return false;
1111 }
1112 }
1113
1114 // now we have added all the other data we can write details of any
1115 // extra the AOT relocations
1116
1117 bool write_ok = true;
1118 if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
1119 if (reloc_count > 0) {
1120 CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS);
1121 RelocIterator iter(cs);
1122 write_ok = cache->write_relocations(blob, iter);
1123 }
1124 } else {
1125 RelocIterator iter(&blob);
1126 write_ok = cache->write_relocations(blob, iter);
1127 }
1128
1129 if (!write_ok) {
1130 if (!cache->failed()) {
1131 // We may miss an address in AOT table - skip this code blob.
1132 cache->set_write_position(entry_position);
1133 }
1134 return false;
1135 }
1136
1137 #ifndef PRODUCT
1138 // Write asm remarks after relocation info
1139 if (!cache->write_asm_remarks(blob)) {
1140 return false;
1141 }
1142 if (!cache->write_dbg_strings(blob)) {
1143 return false;
1144 }
1145 #endif /* PRODUCT */
1146
1147 // Write name after code comments
1148 uint name_offset = cache->_write_position - entry_position;
1149 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1150 n = cache->write_bytes(name, name_size);
1151 if (n != name_size) {
1152 return false;
1153 }
1154
1155 uint entry_size = cache->_write_position - entry_position;
1156
1157 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
1158 entry_position, entry_size, name_offset, name_size,
1159 blob_offset, has_oop_maps, blob.content_begin());
1160 log_debug(aot, codecache, stubs)("Wrote code blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1161 return true;
1162 }
1163
1164 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1165 assert(!AOTCodeEntry::is_blob(entry_kind),
1166 "wrong entry kind for numeric id %d", id);
1167 return store_code_blob(blob, entry_kind, (uint)id, name, nullptr, nullptr);
1168 }
1169
1170 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) {
1171 assert(AOTCodeEntry::is_single_stub_blob(entry_kind),
1172 "wrong entry kind for blob id %s", StubInfo::name(id));
1173 return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), nullptr, nullptr);
1174 }
1175
1176 bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data, CodeBuffer* code_buffer) {
1177 assert(AOTCodeEntry::is_multi_stub_blob(entry_kind),
1178 "wrong entry kind for multi stub blob id %s", StubInfo::name(id));
1179 return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), stub_data, code_buffer);
1180 }
1181
1182 bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) {
1183 if (!align_write_int()) {
1184 return false;
1185 }
1186 BlobId blob_id = stub_data->blob_id();
1187 StubId stub_id = StubInfo::stub_base(blob_id);
1188 address blob_base = blob.code_begin();
1189 int stub_cnt = StubInfo::stub_count(blob_id);
1190 int n;
1191
1192 LogStreamHandle(Trace, aot, codecache, stubs) log;
1193
1194 if (log.is_enabled()) {
1195 log.print_cr("======== Stub data starts at offset %d", _write_position);
1196 }
1197
1198 for (int i = 0; i < stub_cnt; i++, stub_id = StubInfo::next_in_blob(blob_id, stub_id)) {
1199 // for each stub we find in the ranges list we write an int
1200 // sequence <stubid,start,end,N,offset1, ... offsetN> where
1201 //
1202 // - start_pos is the stub start address encoded as a code section offset
1203 //
1204 // - end is the stub end address encoded as an offset from start
1205 //
1206 // - N counts the number of stub-local entries/extras
1207 //
1208 // - offseti is a stub-local entry/extra address encoded as len for
1209 // a null address otherwise as an offset in range [1,len-1]
1210
1211 StubAddrRange& range = stub_data->get_range(i);
1212 GrowableArray<address>& addresses = stub_data->address_array();
1213 int base = range.start_index();
1214 if (base >= 0) {
1215 n = write_bytes(&stub_id, sizeof(StubId));
1216 if (n != sizeof(StubId)) {
1217 return false;
1218 }
1219 address start = addresses.at(base);
1220 assert (blob_base <= start, "sanity");
1221 uint offset = (uint)(start - blob_base);
1222 n = write_bytes(&offset, sizeof(uint));
1223 if (n != sizeof(int)) {
1224 return false;
1225 }
1226 address end = addresses.at(base + 1);
1227 assert (start < end, "sanity");
1228 offset = (uint)(end - start);
1229 n = write_bytes(&offset, sizeof(uint));
1230 if (n != sizeof(int)) {
1231 return false;
1232 }
1233 // write number of secondary and extra entries
1234 int count = range.count() - 2;
1235 n = write_bytes(&count, sizeof(int));
1236 if (n != sizeof(int)) {
1237 return false;
1238 }
1239 for (int j = 0; j < count; j++) {
1240 address next = addresses.at(base + 2 + j);
1241 if (next != nullptr) {
1242 // n.b. This maps next == end to the stub length which
1243 // means we will reconstitute the address as nullptr. That
1244 // happens when we have a handler range covers the end of
1245 // a stub and needs to be handled specially by the client
1246 // that restores the extras.
1247 assert(start <= next && next <= end, "sanity");
1248 offset = (uint)(next - start);
1249 } else {
1250 // this can happen when a stub is not generated or an
1251 // extra is the common handler target
1252 offset = NULL_ADDRESS_MARKER;
1253 }
1254 n = write_bytes(&offset, sizeof(uint));
1255 if (n != sizeof(int)) {
1256 return false;
1257 }
1258 }
1259 if (log.is_enabled()) {
1260 log.print_cr("======== wrote stub %s and %d addresses up to offset %d",
1261 StubInfo::name(stub_id), range.count(), _write_position);
1262 }
1263 }
1264 }
1265 // we should have exhausted all stub ids in the blob
1266 assert(stub_id == StubId::NO_STUBID, "sanity");
1267 // write NO_STUBID as an end marker
1268 n = write_bytes(&stub_id, sizeof(StubId));
1269 if (n != sizeof(StubId)) {
1270 return false;
1271 }
1272
1273 if (log.is_enabled()) {
1274 log.print_cr("======== Stub data ends at offset %d", _write_position);
1275 }
1276
1277 return true;
1278 }
1279
1280 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data) {
1281 AOTCodeCache* cache = open_for_use();
1282 if (cache == nullptr) {
1283 return nullptr;
1284 }
1285 assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
1286
1287 assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr),
1288 "entry_kind %d does not match stub_data pointer %p",
1289 entry_kind, stub_data);
1290
1291 if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
1292 return nullptr;
1293 }
1294 if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
1295 return nullptr;
1296 }
1297 log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
1298
1299 AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
1300 if (entry == nullptr) {
1301 return nullptr;
1302 }
1303 AOTCodeReader reader(cache, entry);
1304 CodeBlob* blob = reader.compile_code_blob(name, entry_kind, id, stub_data);
1305
1306 log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
1307 (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
1308 return blob;
1309 }
1310
1311 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
1312 assert(!AOTCodeEntry::is_blob(entry_kind),
1313 "wrong entry kind for numeric id %d", id);
1314 return load_code_blob(entry_kind, (uint)id, name, nullptr);
1315 }
1316
1317 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
1318 assert(AOTCodeEntry::is_single_stub_blob(entry_kind),
1319 "wrong entry kind for blob id %s", StubInfo::name(id));
1320 return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), nullptr);
1321 }
1322
1323 CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data) {
1324 assert(AOTCodeEntry::is_multi_stub_blob(entry_kind),
1325 "wrong entry kind for blob id %s", StubInfo::name(id));
1326 return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), stub_data);
1327 }
1328
1329 CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data) {
1330 uint entry_position = _entry->offset();
1331
1332 // Read name
1333 uint name_offset = entry_position + _entry->name_offset();
1334 uint name_size = _entry->name_size(); // Includes '/0'
1335 const char* stored_name = addr(name_offset);
1336
1337 if (strncmp(stored_name, name, (name_size - 1)) != 0) {
1338 log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'",
1339 stored_name, name);
1340 set_lookup_failed(); // Skip this blob
1341 return nullptr;
1342 }
1343 _name = stored_name;
1344
1345 // Read archived code blob and related info
1346 uint offset = entry_position + _entry->blob_offset();
1347 CodeBlob* archived_blob = (CodeBlob*)addr(offset);
1348 offset += archived_blob->size();
1349
1350 _reloc_count = *(int*)addr(offset);
1351 offset += sizeof(int);
1352 if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
1353 // position of relocs will have been aligned to heap word size so
1354 // we can install them into a code buffer
1355 offset = align_up(offset, DATA_ALIGNMENT);
1356 }
1357 _reloc_data = (address)addr(offset);
1358 offset += _reloc_count * sizeof(relocInfo);
1359 set_read_position(offset);
1360
1361 if (_entry->has_oop_maps()) {
1362 _oop_maps = read_oop_map_set();
1363 }
1364
1365 // record current context for use by that callback
1366 _stub_data = stub_data;
1367 _entry_kind = entry_kind;
1368 _id = id;
1369
1370 // CodeBlob::restore() calls AOTCodeReader::restore()
1371
1372 CodeBlob* code_blob = CodeBlob::create(archived_blob, this);
1373
1374 if (code_blob == nullptr) { // no space left in CodeCache
1375 return nullptr;
1376 }
1377
1378 #ifdef ASSERT
1379 LogStreamHandle(Trace, aot, codecache, stubs) log;
1380 if (log.is_enabled()) {
1381 FlagSetting fs(PrintRelocations, true);
1382 code_blob->print_on(&log);
1383 }
1384 #endif
1385 return code_blob;
1386 }
1387
1388 void AOTCodeReader::restore(CodeBlob* code_blob) {
1389 precond(AOTCodeCache::is_on_for_use());
1390 precond(_name != nullptr);
1391 precond(_reloc_data != nullptr);
1392
1393 code_blob->set_name(_name);
1394 // Saved relocations need restoring except for the case of a
1395 // multi-stub blob which has no runtime relocations. However, we may
1396 // still have saved some (re-)load time relocs that were attached to
1397 // the generator's code buffer. We don't attach them to the blob but
1398 // they get processed below by fix_relocations.
1399 if (!AOTCodeEntry::is_multi_stub_blob(_entry_kind)) {
1400 code_blob->restore_mutable_data(_reloc_data);
1401 }
1402 code_blob->set_oop_maps(_oop_maps);
1403
1404 // if this is a multi stub blob load its entries
1405 if (AOTCodeEntry::is_blob(_entry_kind)) {
1406 BlobId blob_id = static_cast<BlobId>(_id);
1407 if (StubInfo::is_stubgen(blob_id)) {
1408 assert(_stub_data != nullptr, "sanity");
1409 read_stub_data(code_blob, _stub_data);
1410 }
1411 // publish entries found either in stub_data or as offsets in blob
1412 AOTCodeCache::publish_stub_addresses(*code_blob, blob_id, _stub_data);
1413 }
1414
1415 // Now that all the entry points are in the address table we can
1416 // read all the extra reloc info and fix up any addresses that need
1417 // patching to adjust for a new location in a new JVM. We can be
1418 // sure to correctly update all runtime references, including
1419 // cross-linked stubs that are internally daisy-chained. If
1420 // relocation fails and we have to re-generate any of the stubs then
1421 // the entry points for newly generated stubs will get updated,
1422 // ensuring that any other stubs or nmethods we need to relocate
1423 // will use the correct address.
1424
1425 // if we have a relocatable code blob then the relocs are already
1426 // attached to the blob and we can iterate over it to find the ones
1427 // we need to patch. With a non-relocatable code blob we need to
1428 // wrap it with a CodeBuffer and then reattach the relocs to the
1429 // code buffer.
1430
1431 if (AOTCodeEntry::is_multi_stub_blob(_entry_kind)) {
1432 // the blob doesn't have any proper runtime relocs but we can
1433 // reinstate the AOT-load time relocs we saved from the code
1434 // buffer that generated this blob in a new code buffer and use
1435 // the latter to iterate over them
1436 if (_reloc_count > 0) {
1437 CodeBuffer code_buffer(code_blob);
1438 relocInfo* locs = (relocInfo*)_reloc_data;
1439 code_buffer.insts()->initialize_shared_locs(locs, _reloc_count);
1440 code_buffer.insts()->set_locs_end(locs + _reloc_count);
1441 CodeSection *cs = code_buffer.code_section(CodeBuffer::SECT_INSTS);
1442 RelocIterator reloc_iter(cs);
1443 fix_relocations(code_blob, reloc_iter);
1444 }
1445 } else {
1446 // the AOT-load time relocs will be in the blob's restored relocs
1447 RelocIterator reloc_iter(code_blob);
1448 fix_relocations(code_blob, reloc_iter);
1449 }
1450
1451 #ifndef PRODUCT
1452 code_blob->asm_remarks().init();
1453 read_asm_remarks(code_blob->asm_remarks());
1454 code_blob->dbg_strings().init();
1455 read_dbg_strings(code_blob->dbg_strings());
1456 #endif // PRODUCT
1457 }
1458
1459 void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) {
1460 GrowableArray<address>& addresses = stub_data->address_array();
1461 // Read the list of stub ids and associated start, end, secondary
1462 // and extra addresses and install them in the stub data.
1463 //
1464 // Also insert all start and secondary addresses into the AOTCache
1465 // address table so we correctly relocate this blob and any followng
1466 // blobs/nmethods.
1467 //
1468 // n.b. if an error occurs and we need to regenerate any of these
1469 // stubs the address table will be updated as a side-effect of
1470 // regeneration.
1471
1472 address blob_base = code_blob->code_begin();
1473 uint blob_size = (uint)(code_blob->code_end() - blob_base);
1474 uint offset = align_read_int();
1475 LogStreamHandle(Trace, aot, codecache, stubs) log;
1476 if (log.is_enabled()) {
1477 log.print_cr("======== Stub data starts at offset %d", offset);
1478 }
1479 // read stub and entries until we see NO_STUBID
1480 StubId stub_id = *(StubId*)addr(offset); offset += sizeof(StubId);
1481 // we ought to have at least one saved stub in the blob
1482 assert(stub_id != StubId::NO_STUBID, "blob %s contains no stubs!", StubInfo::name(stub_data->blob_id()));
1483 while (stub_id != StubId::NO_STUBID) {
1484 assert(StubInfo::blob(stub_id) == stub_data->blob_id(), "sanity");
1485 int idx = StubInfo::stubgen_offset_in_blob(stub_data->blob_id(), stub_id);
1486 StubAddrRange& range = stub_data->get_range(idx);
1487 // we should only see a stub once
1488 assert(range.start_index() < 0, "repeated entry for stub %s", StubInfo::name(stub_id));
1489 int address_base = addresses.length();
1490 // start is an offset from the blob base
1491 uint start = *(uint*)addr(offset); offset += sizeof(uint);
1492 assert(start < blob_size, "stub %s start offset %d exceeds buffer length %d", StubInfo::name(stub_id), start, blob_size);
1493 address stub_start = blob_base + start;
1494 addresses.append(stub_start);
1495 // end is an offset from the stub start
1496 uint end = *(uint*)addr(offset); offset += sizeof(uint);
1497 assert(start + end <= blob_size, "stub %s end offset %d exceeds remaining buffer length %d", StubInfo::name(stub_id), end, blob_size - start);
1498 addresses.append(stub_start + end);
1499 // read count of secondary entries plus extras
1500 int entries_count = *(int*)addr(offset); offset += sizeof(int);
1501 assert(entries_count >= (StubInfo::entry_count(stub_id) - 1), "not enough entries for %s", StubInfo::name(stub_id));
1502 for (int i = 0; i < entries_count; i++) {
1503 // entry offset is an offset from the stub start less than or
1504 // equal to end
1505 uint entry = *(uint*)addr(offset); offset += sizeof(uint);
1506 if (entry <= end) {
1507 // entry addresses may not address end but extras can
1508 assert(entry < end || i >= StubInfo::entry_count(stub_id),
1509 "entry offset 0x%x exceeds stub length 0x%x for stub %s",
1510 entry, end, StubInfo::name(stub_id));
1511 addresses.append(stub_start + entry);
1512 } else {
1513 // special case: entry encodes a nullptr
1514 assert(entry == AOTCodeCache::NULL_ADDRESS_MARKER, "stub %s entry offset %d lies beyond stub end %d and does not equal NULL_ADDRESS_MARKER", StubInfo::name(stub_id), entry, end);
1515 addresses.append(nullptr);
1516 }
1517 }
1518 if (log.is_enabled()) {
1519 log.print_cr("======== read stub %s and %d addresses up to offset %d",
1520 StubInfo::name(stub_id), 2 + entries_count, offset);
1521 }
1522 range.init_entry(address_base, 2 + entries_count);
1523 // move on to next stub or NO_STUBID
1524 stub_id = *(StubId*)addr(offset); offset += sizeof(StubId);
1525 }
1526 if (log.is_enabled()) {
1527 log.print_cr("======== Stub data ends at offset %d", offset);
1528 }
1529
1530 set_read_position(offset);
1531 }
1532
1533 void AOTCodeCache::publish_external_addresses(GrowableArray<address>& addresses) {
1534 DEBUG_ONLY( _passed_init2 = true; )
1535 if (opened_cache == nullptr) {
1536 return;
1537 }
1538
1539 cache()->_table->add_external_addresses(addresses);
1540 }
1541
1542 void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, AOTStubData *stub_data) {
1543 if (stub_data != nullptr) {
1544 // register all entries in stub
1545 assert(StubInfo::stub_count(blob_id) > 1,
1546 "multiple stub data provided for single stub blob %s",
1547 StubInfo::name(blob_id));
1548 assert(blob_id == stub_data->blob_id(),
1549 "blob id %s does not match id in stub data %s",
1550 StubInfo::name(blob_id),
1551 StubInfo::name(stub_data->blob_id()));
1552 // iterate over all stubs in the blob
1553 StubId stub_id = StubInfo::stub_base(blob_id);
1554 int stub_cnt = StubInfo::stub_count(blob_id);
1555 GrowableArray<address>& addresses = stub_data->address_array();
1556 for (int i = 0; i < stub_cnt; i++) {
1557 assert(stub_id != StubId::NO_STUBID, "sanity");
1558 StubAddrRange& range = stub_data->get_range(i);
1559 int base = range.start_index();
1560 if (base >= 0) {
1561 cache()->add_stub_entries(stub_id, addresses.at(base), &addresses, base + 2);
1562 }
1563 stub_id = StubInfo::next_in_blob(blob_id, stub_id);
1564 }
1565 // we should have exhausted all stub ids in the blob
1566 assert(stub_id == StubId::NO_STUBID, "sanity");
1567 } else {
1568 // register entry or entries for a single stub blob
1569 StubId stub_id = StubInfo::stub_base(blob_id);
1570 assert(StubInfo::stub_count(blob_id) == 1,
1571 "multiple stub blob %s provided without stub data",
1572 StubInfo::name(blob_id));
1573 address start = code_blob.code_begin();
1574 if (StubInfo::entry_count(stub_id) == 1) {
1575 assert(!code_blob.is_deoptimization_stub(), "expecting multiple entries for stub %s", StubInfo::name(stub_id));
1576 // register the blob base address as the only entry
1577 cache()->add_stub_entries(stub_id, start);
1578 } else {
1579 assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id));
1580 DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob();
1581 assert(deopt_blob->unpack() == start, "unexpected offset 0x%x for deopt stub entry", (int)(deopt_blob->unpack() - start));
1582 GrowableArray<address> addresses;
1583 addresses.append(deopt_blob->unpack_with_exception());
1584 addresses.append(deopt_blob->unpack_with_reexecution());
1585 addresses.append(deopt_blob->unpack_with_exception_in_tls());
1586 cache()->add_stub_entries(stub_id, start, &addresses, 0);
1587 }
1588 }
1589 }
1590
1591 // ------------ process code and data --------------
1592
1593 // Can't use -1. It is valid value for jump to iteself destination
1594 // used by static call stub: see NativeJump::jump_destination().
1595 #define BAD_ADDRESS_ID -2
1596
1597 bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) {
1598 if (!align_write_int()) {
1599 return false;
1600 }
1601 GrowableArray<uint> reloc_data;
1602 LogStreamHandle(Trace, aot, codecache, reloc) log;
1603 while (iter.next()) {
1604 int idx = reloc_data.append(0); // default value
1605 switch (iter.type()) {
1606 case relocInfo::none:
1607 break;
1608 case relocInfo::runtime_call_type: {
1609 // Record offset of runtime destination
1610 CallRelocation* r = (CallRelocation*)iter.reloc();
1611 address dest = r->destination();
1612 if (dest == r->addr()) { // possible call via trampoline on Aarch64
1613 dest = (address)-1; // do nothing in this case when loading this relocation
1614 }
1615 int id = _table->id_for_address(dest, iter, &code_blob);
1616 if (id == BAD_ADDRESS_ID) {
1617 return false;
1618 }
1619 reloc_data.at_put(idx, id);
1620 break;
1621 }
1622 case relocInfo::runtime_call_w_cp_type:
1623 log_debug(aot, codecache, reloc)("runtime_call_w_cp_type relocation is not implemented");
1624 return false;
1625 case relocInfo::external_word_type: {
1626 // Record offset of runtime target
1627 address target = ((external_word_Relocation*)iter.reloc())->target();
1628 int id = _table->id_for_address(target, iter, &code_blob);
1629 if (id == BAD_ADDRESS_ID) {
1630 return false;
1631 }
1632 reloc_data.at_put(idx, id);
1633 break;
1634 }
1635 case relocInfo::internal_word_type:
1636 break;
1637 case relocInfo::section_word_type:
1638 break;
1639 case relocInfo::post_call_nop_type:
1640 break;
1641 default:
1642 log_debug(aot, codecache, reloc)("relocation %d unimplemented", (int)iter.type());
1643 return false;
1644 break;
1645 }
1646 if (log.is_enabled()) {
1647 iter.print_current_on(&log);
1648 }
1649 }
1650
1651 // Write additional relocation data: uint per relocation
1652 // Write the count first
1653 int count = reloc_data.length();
1654 write_bytes(&count, sizeof(int));
1655 if (log.is_enabled()) {
1656 log.print_cr("======== extra relocations count=%d", count);
1657 log.print( " {");
1658 }
1659 bool first = true;
1660 for (GrowableArrayIterator<uint> iter = reloc_data.begin();
1661 iter != reloc_data.end(); ++iter) {
1662 uint value = *iter;
1663 int n = write_bytes(&value, sizeof(uint));
1664 if (n != sizeof(uint)) {
1665 return false;
1666 }
1667 if (log.is_enabled()) {
1668 if (first) {
1669 first = false;
1670 log.print("%d", value);
1671 } else {
1672 log.print(", %d", value);
1673 }
1674 }
1675 }
1676 if (log.is_enabled()) {
1677 log.print_cr("}");
1678 }
1679 return true;
1680 }
1681
1682 void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) {
1683 uint offset = align_read_int();
1684 int reloc_count = *(int*)addr(offset);
1685 offset += sizeof(int);
1686 uint* reloc_data = (uint*)addr(offset);
1687 offset += (reloc_count * sizeof(uint));
1688 set_read_position(offset);
1689
1690 LogStreamHandle(Trace, aot, codecache, reloc) log;
1691 if (log.is_enabled()) {
1692 log.print_cr("======== extra relocations count=%d", reloc_count);
1693 log.print(" {");
1694 for(int i = 0; i < reloc_count; i++) {
1695 if (i == 0) {
1696 log.print("%d", reloc_data[i]);
1697 } else {
1698 log.print(", %d", reloc_data[i]);
1699 }
1700 }
1701 log.print_cr("}");
1702 }
1703
1704 int j = 0;
1705 while (iter.next()) {
1706 switch (iter.type()) {
1707 case relocInfo::none:
1708 break;
1709 case relocInfo::runtime_call_type: {
1710 address dest = _cache->address_for_id(reloc_data[j]);
1711 if (dest != (address)-1) {
1712 ((CallRelocation*)iter.reloc())->set_destination(dest);
1713 }
1714 break;
1715 }
1716 case relocInfo::runtime_call_w_cp_type:
1717 // this relocation should not be in cache (see write_relocations)
1718 assert(false, "runtime_call_w_cp_type relocation is not implemented");
1719 break;
1720 case relocInfo::external_word_type: {
1721 address target = _cache->address_for_id(reloc_data[j]);
1722 // Add external address to global table
1723 int index = ExternalsRecorder::find_index(target);
1724 // Update index in relocation
1725 Relocation::add_jint(iter.data(), index);
1726 external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
1727 assert(reloc->target() == target, "sanity");
1728 reloc->set_value(target); // Patch address in the code
1729 break;
1730 }
1731 case relocInfo::internal_word_type: {
1732 internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
1733 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
1734 break;
1735 }
1736 case relocInfo::section_word_type: {
1737 section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
1738 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin());
1739 break;
1740 }
1741 case relocInfo::post_call_nop_type:
1742 break;
1743 default:
1744 assert(false,"relocation %d unimplemented", (int)iter.type());
1745 break;
1746 }
1747 if (log.is_enabled()) {
1748 iter.print_current_on(&log);
1749 }
1750 j++;
1751 }
1752 assert(j == reloc_count, "sanity");
1753 }
1754
1755 bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
1756 if (!align_write_int()) {
1757 return false;
1758 }
1759 ImmutableOopMapSet* oopmaps = cb.oop_maps();
1760 int oopmaps_size = oopmaps->nr_of_bytes();
1761 if (!write_bytes(&oopmaps_size, sizeof(int))) {
1762 return false;
1763 }
1764 uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
1765 if (n != (uint)oopmaps->nr_of_bytes()) {
1766 return false;
1767 }
1768 return true;
1769 }
1770
1771 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
1772 uint offset = align_read_int();
1773 int size = *(int *)addr(offset);
1774 offset += sizeof(int);
1775 ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
1776 offset += size;
1777 set_read_position(offset);
1778 return oopmaps;
1779 }
1780
1781 #ifndef PRODUCT
1782 bool AOTCodeCache::write_asm_remarks(CodeBlob& cb) {
1783 if (!align_write_int()) {
1784 return false;
1785 }
1786 // Write asm remarks
1787 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
1788 if (count_ptr == nullptr) {
1789 return false;
1790 }
1791 uint count = 0;
1792 bool result = cb.asm_remarks().iterate([&] (uint offset, const char* str) -> bool {
1793 log_trace(aot, codecache, stubs)("asm remark offset=%d, str='%s'", offset, str);
1794 uint n = write_bytes(&offset, sizeof(uint));
1795 if (n != sizeof(uint)) {
1796 return false;
1797 }
1798 const char* cstr = add_C_string(str);
1799 int id = _table->id_for_C_string((address)cstr);
1800 assert(id != BAD_ADDRESS_ID, "asm remark string '%s' not found in AOTCodeAddressTable", str);
1801 n = write_bytes(&id, sizeof(int));
1802 if (n != sizeof(int)) {
1803 return false;
1804 }
1805 count += 1;
1806 return true;
1807 });
1808 *count_ptr = count;
1809 return result;
1810 }
1811
1812 void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks) {
1813 // Read asm remarks
1814 uint offset = align_read_int();
1815 uint count = *(uint *)addr(offset);
1816 offset += sizeof(uint);
1817 for (uint i = 0; i < count; i++) {
1818 uint remark_offset = *(uint *)addr(offset);
1819 offset += sizeof(uint);
1820 int remark_string_id = *(uint *)addr(offset);
1821 offset += sizeof(int);
1822 const char* remark = (const char*)_cache->address_for_C_string(remark_string_id);
1823 asm_remarks.insert(remark_offset, remark);
1824 }
1825 set_read_position(offset);
1826 }
1827
1828 bool AOTCodeCache::write_dbg_strings(CodeBlob& cb) {
1829 if (!align_write_int()) {
1830 return false;
1831 }
1832 // Write dbg strings
1833 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
1834 if (count_ptr == nullptr) {
1835 return false;
1836 }
1837 uint count = 0;
1838 bool result = cb.dbg_strings().iterate([&] (const char* str) -> bool {
1839 log_trace(aot, codecache, stubs)("dbg string=%s", str);
1840 const char* cstr = add_C_string(str);
1841 int id = _table->id_for_C_string((address)cstr);
1842 assert(id != BAD_ADDRESS_ID, "db string '%s' not found in AOTCodeAddressTable", str);
1843 uint n = write_bytes(&id, sizeof(int));
1844 if (n != sizeof(int)) {
1845 return false;
1846 }
1847 count += 1;
1848 return true;
1849 });
1850 *count_ptr = count;
1851 return result;
1852 }
1853
1854 void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) {
1855 // Read dbg strings
1856 uint offset = align_read_int();
1857 uint count = *(uint *)addr(offset);
1858 offset += sizeof(uint);
1859 for (uint i = 0; i < count; i++) {
1860 int string_id = *(uint *)addr(offset);
1861 offset += sizeof(int);
1862 const char* str = (const char*)_cache->address_for_C_string(string_id);
1863 dbg_strings.insert(str);
1864 }
1865 set_read_position(offset);
1866 }
1867 #endif // PRODUCT
1868
1869 //======================= AOTCodeAddressTable ===============
1870
1871 // address table ids for generated routine entry adresses, external
1872 // addresses and C string addresses are partitioned into positive
1873 // integer ranges defined by the following positive base and max
1874 // values i.e. [_extrs_base, _extrs_base + _extrs_max -1],
1875 // [_stubs_base, _stubs_base + _stubs_max -1], [_c_str_base,
1876 // _c_str_base + _c_str_max -1],
1877
1878 #define _extrs_max 500
1879 #define _stubs_max static_cast<int>(EntryId::NUM_ENTRYIDS)
1880
1881 #define _extrs_base 0
1882 #define _stubs_base (_extrs_base + _extrs_max)
1883 #define _all_max (_stubs_base + _stubs_max)
1884
1885 // setter for external addresses and string addresses inserts new
1886 // addresses in the order they are encountered them which must remain
1887 // the same across an assembly run and subsequent production run
1888
1889 #define ADD_EXTERNAL_ADDRESS(addr) \
1890 { \
1891 hash_address((address) addr, _extrs_base + _extrs_length); \
1892 _extrs_addr[_extrs_length++] = (address) (addr); \
1893 assert(_extrs_length <= _extrs_max, "increase size"); \
1894 }
1895
1896 // insert into to the address hash table the index of an external
1897 // address or a stub address in the list of external or stub
1898 // addresses, respectively, keyed by the relevant address
1899
1900 void AOTCodeAddressTable::hash_address(address addr, int idx) {
1901 // only do this if we have a non-null address to record and the
1902 // cache is open for dumping
1903 if (addr == nullptr) {
1904 return;
1905 }
1906 // check opened_cache because this can be called before the cache is
1907 // properly initialized and only continue when dumping is enabled
1908 if (opened_cache != nullptr && opened_cache->for_dump()) {
1909 if (_hash_table == nullptr) {
1910 _hash_table = new (mtCode) AOTCodeAddressHashTable();
1911 }
1912 assert(_hash_table->get(addr) == nullptr, "repeated insert of address " INTPTR_FORMAT, p2i(addr));
1913 _hash_table->put(addr, idx);
1914 log_trace(aot, codecache)("Address " INTPTR_FORMAT " inserted into AOT Code Cache address hash table with index '%d'",
1915 p2i(addr), idx);
1916 }
1917 }
1918
1919 static bool initializing_extrs = false;
1920
1921 void AOTCodeAddressTable::init_extrs() {
1922 if (_extrs_complete || initializing_extrs) return; // Done already
1923
1924 initializing_extrs = true;
1925 _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
1926
1927 _extrs_length = 0;
1928
1929 {
1930 // Required by initial stubs
1931 ADD_EXTERNAL_ADDRESS(SharedRuntime::exception_handler_for_return_address); // used by forward_exception
1932 ADD_EXTERNAL_ADDRESS(CompressedOops::base_addr()); // used by call_stub
1933 ADD_EXTERNAL_ADDRESS(Thread::current); // used by call_stub
1934 ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_StackOverflowError);
1935 ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_delayed_StackOverflowError);
1936 }
1937
1938 // Record addresses of VM runtime methods
1939 ADD_EXTERNAL_ADDRESS(SharedRuntime::fixup_callers_callsite);
1940 ADD_EXTERNAL_ADDRESS(SharedRuntime::handle_wrong_method);
1941 ADD_EXTERNAL_ADDRESS(SharedRuntime::handle_wrong_method_abstract);
1942 ADD_EXTERNAL_ADDRESS(SharedRuntime::handle_wrong_method_ic_miss);
1943 ADD_EXTERNAL_ADDRESS(SharedRuntime::allocate_inline_types);
1944 #if defined(AARCH64) && !defined(ZERO)
1945 ADD_EXTERNAL_ADDRESS(JavaThread::aarch64_get_thread_helper);
1946 ADD_EXTERNAL_ADDRESS(BarrierSetAssembler::patching_epoch_addr());
1947 #endif
1948
1949 #ifndef PRODUCT
1950 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_jbyte_array_copy_ctr); // used by arraycopy stub on arm32 and x86_64
1951 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_jshort_array_copy_ctr); // used by arraycopy stub
1952 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_jint_array_copy_ctr); // used by arraycopy stub
1953 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_jlong_array_copy_ctr); // used by arraycopy stub
1954 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_oop_array_copy_ctr); // used by arraycopy stub
1955 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_checkcast_array_copy_ctr); // used by arraycopy stub
1956 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_unsafe_array_copy_ctr); // used by arraycopy stub
1957 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_generic_array_copy_ctr); // used by arraycopy stub
1958 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_unsafe_set_memory_ctr); // used by arraycopy stub
1959 #endif /* PRODUCT */
1960
1961 ADD_EXTERNAL_ADDRESS(SharedRuntime::enable_stack_reserved_zone);
1962
1963 #if defined(AMD64) && !defined(ZERO)
1964 ADD_EXTERNAL_ADDRESS(SharedRuntime::montgomery_multiply);
1965 ADD_EXTERNAL_ADDRESS(SharedRuntime::montgomery_square);
1966 #endif // defined(AMD64) && !defined(ZERO)
1967
1968 ADD_EXTERNAL_ADDRESS(SharedRuntime::d2f);
1969 ADD_EXTERNAL_ADDRESS(SharedRuntime::d2i);
1970 ADD_EXTERNAL_ADDRESS(SharedRuntime::d2l);
1971 ADD_EXTERNAL_ADDRESS(SharedRuntime::dcos);
1972 ADD_EXTERNAL_ADDRESS(SharedRuntime::dexp);
1973 ADD_EXTERNAL_ADDRESS(SharedRuntime::dlog);
1974 ADD_EXTERNAL_ADDRESS(SharedRuntime::dlog10);
1975 ADD_EXTERNAL_ADDRESS(SharedRuntime::dpow);
1976 #ifndef ZERO
1977 ADD_EXTERNAL_ADDRESS(SharedRuntime::drem);
1978 #endif
1979 ADD_EXTERNAL_ADDRESS(SharedRuntime::dsin);
1980 ADD_EXTERNAL_ADDRESS(SharedRuntime::dtan);
1981 ADD_EXTERNAL_ADDRESS(SharedRuntime::f2i);
1982 ADD_EXTERNAL_ADDRESS(SharedRuntime::f2l);
1983 #ifndef ZERO
1984 ADD_EXTERNAL_ADDRESS(SharedRuntime::frem);
1985 #endif
1986 ADD_EXTERNAL_ADDRESS(SharedRuntime::l2d);
1987 ADD_EXTERNAL_ADDRESS(SharedRuntime::l2f);
1988 ADD_EXTERNAL_ADDRESS(SharedRuntime::ldiv);
1989 ADD_EXTERNAL_ADDRESS(SharedRuntime::lmul);
1990 ADD_EXTERNAL_ADDRESS(SharedRuntime::lrem);
1991
1992 #if INCLUDE_JVMTI
1993 ADD_EXTERNAL_ADDRESS(&JvmtiExport::_should_notify_object_alloc);
1994 #endif /* INCLUDE_JVMTI */
1995
1996 ADD_EXTERNAL_ADDRESS(ThreadIdentifier::unsafe_offset());
1997 // already added
1998 // ADD_EXTERNAL_ADDRESS(Thread::current);
1999
2000 ADD_EXTERNAL_ADDRESS(os::javaTimeMillis);
2001 ADD_EXTERNAL_ADDRESS(os::javaTimeNanos);
2002 #ifndef PRODUCT
2003 ADD_EXTERNAL_ADDRESS(os::breakpoint);
2004 #endif
2005
2006 ADD_EXTERNAL_ADDRESS(StubRoutines::crc_table_addr());
2007 #ifndef PRODUCT
2008 ADD_EXTERNAL_ADDRESS(&SharedRuntime::_partial_subtype_ctr);
2009 #endif
2010
2011 #if INCLUDE_JFR
2012 ADD_EXTERNAL_ADDRESS(JfrIntrinsicSupport::write_checkpoint);
2013 ADD_EXTERNAL_ADDRESS(JfrIntrinsicSupport::return_lease);
2014 #endif
2015
2016 ADD_EXTERNAL_ADDRESS(UpcallLinker::handle_uncaught_exception); // used by upcall_stub_exception_handler
2017
2018 {
2019 // Required by Shared blobs
2020 ADD_EXTERNAL_ADDRESS(Deoptimization::fetch_unroll_info);
2021 ADD_EXTERNAL_ADDRESS(Deoptimization::unpack_frames);
2022 ADD_EXTERNAL_ADDRESS(SafepointSynchronize::handle_polling_page_exception);
2023 ADD_EXTERNAL_ADDRESS(SharedRuntime::resolve_opt_virtual_call_C);
2024 ADD_EXTERNAL_ADDRESS(SharedRuntime::resolve_virtual_call_C);
2025 ADD_EXTERNAL_ADDRESS(SharedRuntime::resolve_static_call_C);
2026 // already added
2027 // ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_delayed_StackOverflowError);
2028 ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_AbstractMethodError);
2029 ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_IncompatibleClassChangeError);
2030 ADD_EXTERNAL_ADDRESS(SharedRuntime::throw_NullPointerException_at_call);
2031 }
2032
2033 #ifdef COMPILER1
2034 {
2035 // Required by C1 blobs
2036 ADD_EXTERNAL_ADDRESS(static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc));
2037 ADD_EXTERNAL_ADDRESS(SharedRuntime::register_finalizer);
2038 ADD_EXTERNAL_ADDRESS(Runtime1::is_instance_of);
2039 ADD_EXTERNAL_ADDRESS(Runtime1::exception_handler_for_pc);
2040 ADD_EXTERNAL_ADDRESS(Runtime1::check_abort_on_vm_exception);
2041 ADD_EXTERNAL_ADDRESS(Runtime1::new_instance);
2042 ADD_EXTERNAL_ADDRESS(Runtime1::counter_overflow);
2043 ADD_EXTERNAL_ADDRESS(Runtime1::new_type_array);
2044 ADD_EXTERNAL_ADDRESS(Runtime1::new_object_array);
2045 ADD_EXTERNAL_ADDRESS(Runtime1::new_multi_array);
2046 ADD_EXTERNAL_ADDRESS(Runtime1::throw_range_check_exception);
2047 ADD_EXTERNAL_ADDRESS(Runtime1::throw_index_exception);
2048 ADD_EXTERNAL_ADDRESS(Runtime1::throw_div0_exception);
2049 ADD_EXTERNAL_ADDRESS(Runtime1::throw_null_pointer_exception);
2050 ADD_EXTERNAL_ADDRESS(Runtime1::throw_array_store_exception);
2051 ADD_EXTERNAL_ADDRESS(Runtime1::throw_class_cast_exception);
2052 ADD_EXTERNAL_ADDRESS(Runtime1::throw_incompatible_class_change_error);
2053 ADD_EXTERNAL_ADDRESS(Runtime1::monitorenter);
2054 ADD_EXTERNAL_ADDRESS(Runtime1::monitorexit);
2055 ADD_EXTERNAL_ADDRESS(Runtime1::deoptimize);
2056 ADD_EXTERNAL_ADDRESS(Runtime1::access_field_patching);
2057 ADD_EXTERNAL_ADDRESS(Runtime1::move_klass_patching);
2058 ADD_EXTERNAL_ADDRESS(Runtime1::move_mirror_patching);
2059 ADD_EXTERNAL_ADDRESS(Runtime1::move_appendix_patching);
2060 ADD_EXTERNAL_ADDRESS(Runtime1::predicate_failed_trap);
2061 ADD_EXTERNAL_ADDRESS(Runtime1::unimplemented_entry);
2062 ADD_EXTERNAL_ADDRESS(Runtime1::new_null_free_array);
2063 ADD_EXTERNAL_ADDRESS(Runtime1::load_flat_array);
2064 ADD_EXTERNAL_ADDRESS(Runtime1::store_flat_array);
2065 ADD_EXTERNAL_ADDRESS(Runtime1::substitutability_check);
2066 ADD_EXTERNAL_ADDRESS(Runtime1::buffer_inline_args);
2067 ADD_EXTERNAL_ADDRESS(Runtime1::buffer_inline_args_no_receiver);
2068 ADD_EXTERNAL_ADDRESS(Runtime1::throw_identity_exception);
2069 ADD_EXTERNAL_ADDRESS(Runtime1::throw_illegal_monitor_state_exception);
2070 // already added
2071 // ADD_EXTERNAL_ADDRESS(Thread::current);
2072 ADD_EXTERNAL_ADDRESS(CompressedKlassPointers::base_addr());
2073 }
2074 #endif
2075
2076 #ifdef COMPILER2
2077 {
2078 // Required by C2 blobs
2079 ADD_EXTERNAL_ADDRESS(Deoptimization::uncommon_trap);
2080 ADD_EXTERNAL_ADDRESS(OptoRuntime::handle_exception_C);
2081 ADD_EXTERNAL_ADDRESS(OptoRuntime::new_instance_C);
2082 ADD_EXTERNAL_ADDRESS(OptoRuntime::new_array_C);
2083 ADD_EXTERNAL_ADDRESS(OptoRuntime::new_array_nozero_C);
2084 ADD_EXTERNAL_ADDRESS(OptoRuntime::multianewarray2_C);
2085 ADD_EXTERNAL_ADDRESS(OptoRuntime::multianewarray3_C);
2086 ADD_EXTERNAL_ADDRESS(OptoRuntime::multianewarray4_C);
2087 ADD_EXTERNAL_ADDRESS(OptoRuntime::multianewarray5_C);
2088 ADD_EXTERNAL_ADDRESS(OptoRuntime::multianewarrayN_C);
2089 ADD_EXTERNAL_ADDRESS(OptoRuntime::complete_monitor_locking_C);
2090 ADD_EXTERNAL_ADDRESS(OptoRuntime::monitor_notify_C);
2091 ADD_EXTERNAL_ADDRESS(OptoRuntime::monitor_notifyAll_C);
2092 ADD_EXTERNAL_ADDRESS(OptoRuntime::rethrow_C);
2093 ADD_EXTERNAL_ADDRESS(OptoRuntime::slow_arraycopy_C);
2094 ADD_EXTERNAL_ADDRESS(OptoRuntime::register_finalizer_C);
2095 ADD_EXTERNAL_ADDRESS(OptoRuntime::load_unknown_inline_C);
2096 ADD_EXTERNAL_ADDRESS(OptoRuntime::store_unknown_inline_C);
2097 ADD_EXTERNAL_ADDRESS(OptoRuntime::vthread_end_first_transition_C);
2098 ADD_EXTERNAL_ADDRESS(OptoRuntime::vthread_start_final_transition_C);
2099 ADD_EXTERNAL_ADDRESS(OptoRuntime::vthread_start_transition_C);
2100 ADD_EXTERNAL_ADDRESS(OptoRuntime::vthread_end_transition_C);
2101 // already added for
2102 #if defined(AARCH64) && ! defined(PRODUCT)
2103 ADD_EXTERNAL_ADDRESS(JavaThread::verify_cross_modify_fence_failure);
2104 #endif // AARCH64 && !PRODUCT
2105 }
2106 #endif // COMPILER2
2107
2108 #if INCLUDE_G1GC
2109 ADD_EXTERNAL_ADDRESS(G1BarrierSetRuntime::write_ref_field_pre_entry);
2110 ADD_EXTERNAL_ADDRESS(G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry); // used by arraycopy stubs
2111 ADD_EXTERNAL_ADDRESS(G1BarrierSetRuntime::write_ref_array_pre_oop_entry); // used by arraycopy stubs
2112 ADD_EXTERNAL_ADDRESS(G1BarrierSetRuntime::write_ref_array_post_entry); // used by arraycopy stubs
2113 ADD_EXTERNAL_ADDRESS(BarrierSetNMethod::nmethod_stub_entry_barrier); // used by method_entry_barrier
2114
2115 #endif
2116 #if INCLUDE_SHENANDOAHGC
2117 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::write_barrier_pre);
2118 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::write_barrier_pre_narrow);
2119 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_strong);
2120 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_strong_narrow);
2121 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_strong_narrow_narrow);
2122 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_weak);
2123 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_weak_narrow);
2124 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_weak_narrow_narrow);
2125 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_phantom);
2126 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_phantom_narrow);
2127 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::load_reference_barrier_phantom_narrow_narrow);
2128 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::arraycopy_barrier_oop);
2129 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::arraycopy_barrier_narrow_oop);
2130 ADD_EXTERNAL_ADDRESS(ShenandoahRuntime::clone);
2131 #endif
2132 #if INCLUDE_ZGC
2133 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr());
2134 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_store_good_addr());
2135 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr());
2136 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
2137 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::no_keepalive_load_barrier_on_weak_oop_field_preloaded_addr());
2138 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::no_keepalive_load_barrier_on_phantom_oop_field_preloaded_addr());
2139 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr());
2140 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr());
2141 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr());
2142 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr());
2143 ADD_EXTERNAL_ADDRESS(ZBarrierSetRuntime::load_barrier_on_oop_array_addr());
2144
2145 ADD_EXTERNAL_ADDRESS(ZPointerVectorLoadBadMask);
2146 ADD_EXTERNAL_ADDRESS(ZPointerVectorStoreBadMask);
2147 ADD_EXTERNAL_ADDRESS(ZPointerVectorStoreGoodMask);
2148 #if defined(AMD64)
2149 ADD_EXTERNAL_ADDRESS(&ZPointerLoadShift);
2150 ADD_EXTERNAL_ADDRESS(&ZPointerLoadShiftTable);
2151 #endif
2152 #endif
2153 #ifndef ZERO
2154 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
2155 ADD_EXTERNAL_ADDRESS(MacroAssembler::debug64);
2156 #endif // defined(AMD64) || defined(AARCH64) || defined(RISCV64)
2157 #if defined(AMD64)
2158 ADD_EXTERNAL_ADDRESS(warning);
2159 #endif // defined(AMD64)
2160 #endif // ZERO
2161
2162 // addresses of fields in AOT runtime constants area
2163 address* p = AOTRuntimeConstants::field_addresses_list();
2164 while (*p != nullptr) {
2165 address to_add = (address)*p++;
2166 ADD_EXTERNAL_ADDRESS(to_add);
2167 }
2168
2169 log_debug(aot, codecache, init)("External addresses opened and recorded");
2170 // allocate storage for stub entries
2171 _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
2172 log_debug(aot, codecache, init)("Stub addresses opened");
2173 }
2174
2175 void AOTCodeAddressTable::init_extrs2() {
2176 assert(initializing_extrs && !_extrs_complete,
2177 "invalid sequence for init_extrs2");
2178
2179 {
2180 ADD_EXTERNAL_ADDRESS(Continuation::prepare_thaw); // used by cont_thaw
2181 ADD_EXTERNAL_ADDRESS(Continuation::thaw_entry()); // used by cont_thaw
2182 ADD_EXTERNAL_ADDRESS(ContinuationEntry::thaw_call_pc_address()); // used by cont_preempt_stub
2183 }
2184 _extrs_complete = true;
2185 initializing_extrs = false;
2186 log_debug(aot, codecache, init)("External addresses recorded and closed");
2187 }
2188
2189 void AOTCodeAddressTable::add_external_addresses(GrowableArray<address>& addresses) {
2190 assert(initializing_extrs && !_extrs_complete,
2191 "invalid sequence for add_external_addresses");
2192 for (int i = 0; i < addresses.length(); i++) {
2193 ADD_EXTERNAL_ADDRESS(addresses.at(i));
2194 }
2195 log_debug(aot, codecache, init)("Recorded %d additional external addresses",
2196 addresses.length());
2197 }
2198
2199 void AOTCodeAddressTable::add_stub_entry(EntryId entry_id, address a) {
2200 assert(_extrs_complete || initializing_extrs,
2201 "recording stub entry address before external addresses complete");
2202 assert(!(StubInfo::is_shared(StubInfo::stub(entry_id)) && _shared_stubs_complete), "too late to add shared entry");
2203 assert(!(StubInfo::is_stubgen(StubInfo::stub(entry_id)) && _stubgen_stubs_complete), "too late to add stubgen entry");
2204 assert(!(StubInfo::is_c1(StubInfo::stub(entry_id)) && _c1_stubs_complete), "too late to add c1 entry");
2205 assert(!(StubInfo::is_c2(StubInfo::stub(entry_id)) && _c2_stubs_complete), "too late to add c2 entry");
2206 log_debug(aot, stubs)("Recording address 0x%p for %s entry %s", a, StubInfo::name(StubInfo::stubgroup(entry_id)), StubInfo::name(entry_id));
2207 int idx = static_cast<int>(entry_id);
2208 hash_address(a, _stubs_base + idx);
2209 _stubs_addr[idx] = a;
2210 }
2211
2212 void AOTCodeAddressTable::set_shared_stubs_complete() {
2213 assert(!_shared_stubs_complete, "repeated close for shared stubs!");
2214 _shared_stubs_complete = true;
2215 log_debug(aot, codecache, init)("Shared stubs closed");
2216 }
2217
2218 void AOTCodeAddressTable::set_c1_stubs_complete() {
2219 assert(!_c1_stubs_complete, "repeated close for c1 stubs!");
2220 _c1_stubs_complete = true;
2221 log_debug(aot, codecache, init)("C1 stubs closed");
2222 }
2223
2224 void AOTCodeAddressTable::set_c2_stubs_complete() {
2225 assert(!_c2_stubs_complete, "repeated close for c2 stubs!");
2226 _c2_stubs_complete = true;
2227 log_debug(aot, codecache, init)("C2 stubs closed");
2228 }
2229
2230 void AOTCodeAddressTable::set_stubgen_stubs_complete() {
2231 assert(!_stubgen_stubs_complete, "repeated close for stubgen stubs!");
2232 _stubgen_stubs_complete = true;
2233 log_debug(aot, codecache, init)("StubGen stubs closed");
2234 }
2235
2236 #ifdef PRODUCT
2237 #define MAX_STR_COUNT 200
2238 #else
2239 #define MAX_STR_COUNT 2000
2240 #endif
2241 #define _c_str_max MAX_STR_COUNT
2242 static const int _c_str_base = _all_max;
2243
2244 static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings
2245 static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; // Our duplicates
2246 static int _C_strings_count = 0;
2247 static int _C_strings_s[MAX_STR_COUNT] = {0};
2248 static int _C_strings_id[MAX_STR_COUNT] = {0};
2249 static int _C_strings_used = 0;
2250
2251 void AOTCodeCache::load_strings() {
2252 uint strings_count = _load_header->strings_count();
2253 if (strings_count == 0) {
2254 return;
2255 }
2256 if (strings_count > MAX_STR_COUNT) {
2257 fatal("Invalid strings_count loaded from AOT Code Cache: %d > MAX_STR_COUNT [%d]", strings_count, MAX_STR_COUNT);
2258 return;
2259 }
2260 uint strings_offset = _load_header->strings_offset();
2261 uint* string_lengths = (uint*)addr(strings_offset);
2262 strings_offset += (strings_count * sizeof(uint));
2263 uint strings_size = _load_header->entries_offset() - strings_offset;
2264 // We have to keep cached strings longer than _cache buffer
2265 // because they are refernced from compiled code which may
2266 // still be executed on VM exit after _cache is freed.
2267 char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
2268 memcpy(p, addr(strings_offset), strings_size);
2269 _C_strings_buf = p;
2270 for (uint i = 0; i < strings_count; i++) {
2271 _C_strings[i] = p;
2272 uint len = string_lengths[i];
2273 _C_strings_s[i] = i;
2274 _C_strings_id[i] = i;
2275 log_trace(aot, codecache, stringtable)("load_strings: _C_strings[%d] " INTPTR_FORMAT " '%s'", i, p2i(p), p);
2276 p += len;
2277 }
2278 assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
2279 _C_strings_count = strings_count;
2280 _C_strings_used = strings_count;
2281 log_debug(aot, codecache, init)(" Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset);
2282 }
2283
2284 int AOTCodeCache::store_strings() {
2285 if (_C_strings_used > 0) {
2286 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
2287 uint offset = _write_position;
2288 uint length = 0;
2289 uint* lengths = (uint *)reserve_bytes(sizeof(uint) * _C_strings_used);
2290 if (lengths == nullptr) {
2291 return -1;
2292 }
2293 for (int i = 0; i < _C_strings_used; i++) {
2294 const char* str = _C_strings[_C_strings_s[i]];
2295 log_trace(aot, codecache, stringtable)("store_strings: _C_strings[%d] " INTPTR_FORMAT " '%s'", i, p2i(str), str);
2296 uint len = (uint)strlen(str) + 1;
2297 length += len;
2298 assert(len < 1000, "big string: %s", str);
2299 lengths[i] = len;
2300 uint n = write_bytes(str, len);
2301 if (n != len) {
2302 return -1;
2303 }
2304 }
2305 log_debug(aot, codecache, exit)(" Wrote %d C strings of total length %d at offset %d to AOT Code Cache",
2306 _C_strings_used, length, offset);
2307 }
2308 return _C_strings_used;
2309 }
2310
2311 const char* AOTCodeCache::add_C_string(const char* str) {
2312 if (is_on_for_dump() && str != nullptr) {
2313 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
2314 AOTCodeAddressTable* table = addr_table();
2315 if (table != nullptr) {
2316 return table->add_C_string(str);
2317 }
2318 }
2319 return str;
2320 }
2321
2322 const char* AOTCodeAddressTable::add_C_string(const char* str) {
2323 if (_extrs_complete || initializing_extrs) {
2324 // Check previous strings address
2325 for (int i = 0; i < _C_strings_count; i++) {
2326 if (_C_strings_in[i] == str) {
2327 return _C_strings[i]; // Found previous one - return our duplicate
2328 } else if (strcmp(_C_strings[i], str) == 0) {
2329 return _C_strings[i];
2330 }
2331 }
2332 // Add new one
2333 if (_C_strings_count < MAX_STR_COUNT) {
2334 // Passed in string can be freed and used space become inaccessible.
2335 // Keep original address but duplicate string for future compare.
2336 _C_strings_id[_C_strings_count] = -1; // Init
2337 _C_strings_in[_C_strings_count] = str;
2338 const char* dup = os::strdup(str);
2339 _C_strings[_C_strings_count++] = dup;
2340 log_trace(aot, codecache, stringtable)("add_C_string: [%d] " INTPTR_FORMAT " '%s'", _C_strings_count, p2i(dup), dup);
2341 return dup;
2342 } else {
2343 assert(false, "Number of C strings >= MAX_STR_COUNT");
2344 }
2345 }
2346 return str;
2347 }
2348
2349 int AOTCodeAddressTable::id_for_C_string(address str) {
2350 if (str == nullptr) {
2351 return BAD_ADDRESS_ID;
2352 }
2353 MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
2354 for (int i = 0; i < _C_strings_count; i++) {
2355 if (_C_strings[i] == (const char*)str) { // found
2356 int id = _C_strings_id[i];
2357 if (id >= 0) {
2358 assert(id < _C_strings_used, "%d >= %d", id , _C_strings_used);
2359 return id; // Found recorded
2360 }
2361 log_trace(aot, codecache, stringtable)("id_for_C_string: _C_strings[%d ==> %d] " INTPTR_FORMAT " '%s'", i, _C_strings_used, p2i(str), str);
2362 // Not found in recorded, add new
2363 id = _C_strings_used++;
2364 _C_strings_s[id] = i;
2365 _C_strings_id[i] = id;
2366 return id;
2367 }
2368 }
2369 return BAD_ADDRESS_ID;
2370 }
2371
2372 address AOTCodeAddressTable::address_for_C_string(int idx) {
2373 assert(idx < _C_strings_count, "sanity");
2374 return (address)_C_strings[idx];
2375 }
2376
2377 static int search_address(address addr, address* table, uint length) {
2378 for (int i = 0; i < (int)length; i++) {
2379 if (table[i] == addr) {
2380 return i;
2381 }
2382 }
2383 return BAD_ADDRESS_ID;
2384 }
2385
2386 address AOTCodeAddressTable::address_for_id(int idx) {
2387 assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete");
2388 if (idx == -1) {
2389 return (address)-1;
2390 }
2391 uint id = (uint)idx;
2392 // special case for symbols based relative to os::init
2393 if (id > (_c_str_base + _c_str_max)) {
2394 return (address)os::init + idx;
2395 }
2396 if (idx < 0) {
2397 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
2398 return nullptr;
2399 }
2400 // no need to compare unsigned id against 0
2401 if (/* id >= _extrs_base && */ id < _extrs_length) {
2402 return _extrs_addr[id - _extrs_base];
2403 }
2404 if (id >= _stubs_base && id < _c_str_base) {
2405 return _stubs_addr[id - _stubs_base];
2406 }
2407 if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
2408 return address_for_C_string(id - _c_str_base);
2409 }
2410 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
2411 return nullptr;
2412 }
2413
2414 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) {
2415 assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete");
2416 int id = -1;
2417 if (addr == (address)-1) { // Static call stub has jump to itself
2418 return id;
2419 }
2420 // fast path for stubs and external addresses
2421 if (_hash_table != nullptr) {
2422 int *result = _hash_table->get(addr);
2423 if (result != nullptr) {
2424 id = *result;
2425 log_trace(aot, codecache)("Address " INTPTR_FORMAT " retrieved from AOT Code Cache address hash table with index '%d'",
2426 p2i(addr), id);
2427 return id;
2428 }
2429 }
2430 // Seach for C string
2431 id = id_for_C_string(addr);
2432 if (id != BAD_ADDRESS_ID) {
2433 return id + _c_str_base;
2434 }
2435 if (StubRoutines::contains(addr) || CodeCache::find_blob(addr) != nullptr) {
2436 // Search for a matching stub entry
2437 id = search_address(addr, _stubs_addr, _stubs_max);
2438 if (id == BAD_ADDRESS_ID) {
2439 StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
2440 const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
2441 assert(false, "Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
2442 } else {
2443 return id + _stubs_base;
2444 }
2445 } else {
2446 // Search in runtime functions
2447 id = search_address(addr, _extrs_addr, _extrs_length);
2448 if (id == BAD_ADDRESS_ID) {
2449 ResourceMark rm;
2450 const int buflen = 1024;
2451 char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
2452 int offset = 0;
2453 if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
2454 if (offset > 0) {
2455 // Could be address of C string
2456 uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
2457 log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
2458 p2i(addr), dist, (const char*)addr);
2459 assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
2460 return dist;
2461 }
2462 #ifdef ASSERT
2463 reloc.print_current_on(tty);
2464 code_blob->print_on(tty);
2465 code_blob->print_code_on(tty);
2466 assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);
2467 #endif
2468 } else {
2469 #ifdef ASSERT
2470 reloc.print_current_on(tty);
2471 code_blob->print_on(tty);
2472 code_blob->print_code_on(tty);
2473 os::find(addr, tty);
2474 assert(false, "Address " INTPTR_FORMAT " for <unknown>/('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr);
2475 #endif
2476 }
2477 } else {
2478 return _extrs_base + id;
2479 }
2480 }
2481 return id;
2482 }
2483
2484 AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
2485
2486 void AOTRuntimeConstants::initialize_from_runtime() {
2487 BarrierSet* bs = BarrierSet::barrier_set();
2488 address card_table_base = nullptr;
2489 uint grain_shift = 0;
2490 address cset_base = nullptr;
2491 #if INCLUDE_G1GC
2492 if (bs->is_a(BarrierSet::G1BarrierSet)) {
2493 grain_shift = G1HeapRegion::LogOfHRGrainBytes;
2494 } else
2495 #endif
2496 #if INCLUDE_SHENANDOAHGC
2497 if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) {
2498 grain_shift = ShenandoahHeapRegion::region_size_bytes_shift_jint();
2499 cset_base = ShenandoahHeap::in_cset_fast_test_addr();
2500 } else
2501 #endif
2502 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
2503 CardTable::CardValue* base = ci_card_table_address_const();
2504 assert(base != nullptr, "unexpected byte_map_base");
2505 card_table_base = base;
2506 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
2507 grain_shift = ctbs->grain_shift();
2508 }
2509 _aot_runtime_constants._card_table_base = card_table_base;
2510 _aot_runtime_constants._grain_shift = grain_shift;
2511 _aot_runtime_constants._cset_base = cset_base;
2512 }
2513
2514 address AOTRuntimeConstants::_field_addresses_list[] = {
2515 ((address)&_aot_runtime_constants._card_table_base),
2516 ((address)&_aot_runtime_constants._grain_shift),
2517 ((address)&_aot_runtime_constants._cset_base),
2518 nullptr
2519 };
2520
2521 address AOTRuntimeConstants::card_table_base_address() {
2522 assert(UseSerialGC || UseParallelGC, "Only these GCs have constant card table base");
2523 return (address)&_aot_runtime_constants._card_table_base;
2524 }
2525
2526 // This is called after initialize() but before init2()
2527 // and _cache is not set yet.
2528 void AOTCodeCache::print_on(outputStream* st) {
2529 if (opened_cache != nullptr && opened_cache->for_use()) {
2530 st->print_cr("\nAOT Code Cache");
2531 uint count = opened_cache->_load_header->entries_count();
2532 uint* search_entries = (uint*)opened_cache->addr(opened_cache->_load_header->entries_offset()); // [id, index]
2533 AOTCodeEntry* load_entries = (AOTCodeEntry*)(search_entries + 2 * count);
2534
2535 for (uint i = 0; i < count; i++) {
2536 // Use search_entries[] to order ouput
2537 int index = search_entries[2*i + 1];
2538 AOTCodeEntry* entry = &(load_entries[index]);
2539
2540 uint entry_position = entry->offset();
2541 uint name_offset = entry->name_offset() + entry_position;
2542 const char* saved_name = opened_cache->addr(name_offset);
2543
2544 st->print_cr("%4u: %10s idx:%4u Id:%u size=%u '%s'",
2545 i, aot_code_entry_kind_name[entry->kind()], index, entry->id(), entry->size(), saved_name);
2546 }
2547 }
2548 }
2549
2550 // methods for managing entries in multi-stub blobs
2551
2552
2553 AOTStubData::AOTStubData(BlobId blob_id) :
2554 _blob_id(blob_id),
2555 _cached_blob(nullptr),
2556 _stub_cnt(0),
2557 _ranges(nullptr),
2558 _flags(0) {
2559 assert(StubInfo::is_stubgen(blob_id),
2560 "AOTStubData expects a multi-stub blob not %s",
2561 StubInfo::name(blob_id));
2562
2563 // we cannot save or restore preuniversestubs because the cache
2564 // cannot be accessed before initialising the universe
2565 if (blob_id == BlobId::stubgen_preuniverse_id) {
2566 // invalidate any attempt to use this
2567 _flags = INVALID;
2568 return;
2569 }
2570 if (AOTCodeCache::is_on()) {
2571 _flags = OPEN;
2572 // allow update of stub entry addresses
2573 if (AOTCodeCache::is_using_stub()) {
2574 // allow stub loading
2575 _flags |= USING;
2576 }
2577 if (AOTCodeCache::is_dumping_stub()) {
2578 // allow stub saving
2579 _flags |= DUMPING;
2580 }
2581 // we need to track all the blob's entries
2582 _stub_cnt = StubInfo::stub_count(_blob_id);
2583 _ranges = NEW_C_HEAP_ARRAY(StubAddrRange, _stub_cnt, mtCode);
2584 for (int i = 0; i < _stub_cnt; i++) {
2585 _ranges[i].default_init();
2586 }
2587 }
2588 }
2589
2590 bool AOTStubData::load_code_blob() {
2591 assert(is_using(), "should not call");
2592 assert(!is_invalid() && _cached_blob == nullptr, "repeated init");
2593 _cached_blob = AOTCodeCache::load_code_blob(AOTCodeEntry::StubGenBlob,
2594 _blob_id,
2595 this);
2596 if (_cached_blob == nullptr) {
2597 set_invalid();
2598 return false;
2599 } else {
2600 return true;
2601 }
2602 }
2603
2604 bool AOTStubData::store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) {
2605 assert(is_dumping(), "should not call");
2606 assert(_cached_blob == nullptr, "should not be loading and storing!");
2607 if (!AOTCodeCache::store_code_blob(new_blob,
2608 AOTCodeEntry::StubGenBlob,
2609 _blob_id, this, code_buffer)) {
2610 set_invalid();
2611 return false;
2612 } else {
2613 return true;
2614 }
2615 }
2616
2617 address AOTStubData::load_archive_data(StubId stub_id, address& end, GrowableArray<address>* entries, GrowableArray<address>* extras) {
2618 assert(StubInfo::blob(stub_id) == _blob_id, "sanity check");
2619 if (is_invalid()) {
2620 return nullptr;
2621 }
2622 int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id);
2623 assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt);
2624 // ensure we have a valid associated range
2625 StubAddrRange &range = _ranges[idx];
2626 int base = range.start_index();
2627 if (base < 0) {
2628 return nullptr;
2629 }
2630 int count = range.count();
2631 assert(base >= 0, "sanity");
2632 assert(count >= 2, "sanity");
2633 // first two saved addresses are start and end
2634 address start = _address_array.at(base);
2635 end = _address_array.at(base + 1);
2636 assert(start != nullptr, "failed to load start address of stub %s", StubInfo::name(stub_id));
2637 assert(end != nullptr, "failed to load end address of stub %s", StubInfo::name(stub_id));
2638 assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id));
2639
2640 int entry_count = StubInfo::entry_count(stub_id);
2641 // the address count must at least include the stub start, end
2642 // and secondary addresses
2643 assert(count >= entry_count + 1, "stub %s requires %d saved addresses but only has %d", StubInfo::name(stub_id), entry_count + 1, count);
2644
2645 // caller must retrieve secondary entries if and only if they exist
2646 assert((entry_count == 1) == (entries == nullptr), "trying to retrieve wrong number of entries for stub %s", StubInfo::name(stub_id));
2647 int index = 2;
2648 if (entries != nullptr) {
2649 assert(entries->length() == 0, "non-empty array when retrieving entries for stub %s!", StubInfo::name(stub_id));
2650 while (index < entry_count + 1) {
2651 address entry = _address_array.at(base + index++);
2652 assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id));
2653 entries->append(entry);
2654 }
2655 }
2656 // caller must retrieve extras if and only if they exist
2657 assert((index < count) == (extras != nullptr), "trying to retrieve wrong number of extras for stub %s", StubInfo::name(stub_id));
2658 if (extras != nullptr) {
2659 assert(extras->length() == 0, "non-empty array when retrieving extras for stub %s!", StubInfo::name(stub_id));
2660 while (index < count) {
2661 address extra = _address_array.at(base + index++);
2662 assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id));
2663 extras->append(extra);
2664 }
2665 }
2666
2667 return start;
2668 }
2669
2670 void AOTStubData::store_archive_data(StubId stub_id, address start, address end, GrowableArray<address>* entries, GrowableArray<address>* extras) {
2671 assert(StubInfo::blob(stub_id) == _blob_id, "sanity check");
2672 assert(start != nullptr, "start address cannot be null");
2673 assert(end != nullptr, "end address cannot be null");
2674 assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id));
2675 int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id);
2676 StubAddrRange& range = _ranges[idx];
2677 assert(range.start_index() == -1, "sanity");
2678 int base = _address_array.length();
2679 assert(base >= 0, "sanity");
2680 // first two saved addresses are start and end
2681 _address_array.append(start);
2682 _address_array.append(end);
2683 // caller must save secondary entries if and only if they exist
2684 assert((StubInfo::entry_count(stub_id) == 1) == (entries == nullptr), "trying to save wrong number of entries for stub %s", StubInfo::name(stub_id));
2685 if (entries != nullptr) {
2686 assert(entries->length() == StubInfo::entry_count(stub_id) - 1, "incorrect entry count %d when saving entries for stub %s!", entries->length(), StubInfo::name(stub_id));
2687 for (int i = 0; i < entries->length(); i++) {
2688 address entry = entries->at(i);
2689 assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id));
2690 _address_array.append(entry);
2691 }
2692 }
2693 // caller may wish to save extra addresses
2694 if (extras != nullptr) {
2695 for (int i = 0; i < extras->length(); i++) {
2696 address extra = extras->at(i);
2697 // handler range end may be end -- it gets restored as nullptr
2698 assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id));
2699 _address_array.append(extra);
2700 }
2701 }
2702 range.init_entry(base, _address_array.length() - base);
2703 }
2704
2705 void AOTStubData::stub_epilog(StubId stub_id) {
2706 DEBUG_ONLY(check_stored(stub_id));
2707 }
2708
2709 #ifdef ASSERT
2710 void AOTStubData::check_stored(StubId stub_id) {
2711 // Only need to check if we are dumping
2712 //
2713 // This excludes cases where the cache got closed because of error
2714 // plus the pre-universe stubs we can never store because they are
2715 // generated prior to cache opening.
2716 if (is_dumping()) {
2717 int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id);
2718 assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt);
2719 StubAddrRange& range = _ranges[idx];
2720 assert(range.start_index() != -1, "missing store_archive_data for generated stub %s", StubInfo::name(stub_id));
2721 }
2722 }
2723 #endif