1 /*
2 * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/codeBuffer.hpp"
27 #include "cds/cdsAccess.hpp"
28 #include "cds/cdsConfig.hpp"
29 #include "cds/heapShared.hpp"
30 #include "cds/metaspaceShared.hpp"
31 #include "ci/ciConstant.hpp"
32 #include "ci/ciEnv.hpp"
33 #include "ci/ciField.hpp"
34 #include "ci/ciMethod.hpp"
35 #include "ci/ciMethodData.hpp"
36 #include "ci/ciObject.hpp"
37 #include "ci/ciUtilities.inline.hpp"
38 #include "classfile/javaAssertions.hpp"
39 #include "classfile/stringTable.hpp"
40 #include "classfile/symbolTable.hpp"
41 #include "classfile/systemDictionary.hpp"
42 #include "classfile/vmClasses.hpp"
43 #include "classfile/vmIntrinsics.hpp"
44 #include "code/aotCodeCache.hpp"
45 #include "code/codeBlob.hpp"
46 #include "code/codeCache.hpp"
47 #include "code/oopRecorder.inline.hpp"
48 #include "compiler/abstractCompiler.hpp"
49 #include "compiler/compilationPolicy.hpp"
50 #include "compiler/compileBroker.hpp"
51 #include "compiler/compileTask.hpp"
52 #include "gc/g1/g1BarrierSetRuntime.hpp"
53 #include "gc/shared/gcConfig.hpp"
54 #include "logging/logStream.hpp"
55 #include "memory/memoryReserver.hpp"
56 #include "memory/universe.hpp"
57 #include "oops/klass.inline.hpp"
58 #include "oops/method.inline.hpp"
59 #include "oops/trainingData.hpp"
60 #include "prims/jvmtiThreadState.hpp"
61 #include "runtime/atomic.hpp"
62 #include "runtime/flags/flagSetting.hpp"
63 #include "runtime/globals_extension.hpp"
64 #include "runtime/handles.inline.hpp"
65 #include "runtime/java.hpp"
66 #include "runtime/jniHandles.inline.hpp"
67 #include "runtime/os.inline.hpp"
68 #include "runtime/sharedRuntime.hpp"
69 #include "runtime/stubCodeGenerator.hpp"
70 #include "runtime/stubRoutines.hpp"
71 #include "runtime/timerTrace.hpp"
72 #include "runtime/threadIdentifier.hpp"
73 #include "utilities/ostream.hpp"
74 #include "utilities/spinYield.hpp"
75 #ifdef COMPILER1
76 #include "c1/c1_Runtime1.hpp"
77 #include "c1/c1_LIRAssembler.hpp"
78 #include "gc/shared/c1/barrierSetC1.hpp"
79 #include "gc/g1/c1/g1BarrierSetC1.hpp"
80 #if INCLUDE_SHENANDOAHGC
81 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
82 #endif
83 #include "gc/z/c1/zBarrierSetC1.hpp"
84 #endif
85 #ifdef COMPILER2
86 #include "opto/runtime.hpp"
87 #endif
88 #if INCLUDE_JVMCI
89 #include "jvmci/jvmci.hpp"
90 #endif
91 #if INCLUDE_SHENANDOAHGC
92 #include "gc/shenandoah/shenandoahRuntime.hpp"
93 #endif
94
95 #include <sys/stat.h>
96 #include <errno.h>
97
98 #ifndef O_BINARY // if defined (Win32) use binary files.
99 #define O_BINARY 0 // otherwise do nothing.
100 #endif
101
102 const char* aot_code_entry_kind_name[] = {
103 #define DECL_KIND_STRING(kind) XSTR(kind),
104 DO_AOTCODEENTRY_KIND(DECL_KIND_STRING)
105 #undef DECL_KIND_STRING
106 };
107
108 static elapsedTimer _t_totalLoad;
109 static elapsedTimer _t_totalRegister;
110 static elapsedTimer _t_totalFind;
111 static elapsedTimer _t_totalStore;
112
113 AOTCodeCache* AOTCodeCache::_cache = nullptr;
114
115 static bool enable_timers() {
116 return CITime || log_is_enabled(Info, init);
117 }
118
119 static void exit_vm_on_load_failure() {
120 // Treat AOTCodeCache warnings as error when RequireSharedSpaces is on.
121 if (RequireSharedSpaces) {
122 vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
123 }
124 }
125
126 static void exit_vm_on_store_failure() {
127 // Treat AOTCodeCache warnings as error when RequireSharedSpaces is on.
128 if (RequireSharedSpaces) {
129 tty->print_cr("Unable to create startup cached code.");
130 // Failure during AOT code caching, we don't want to dump core
131 vm_abort(false);
132 }
133 }
134
135 uint AOTCodeCache::max_aot_code_size() {
136 return (uint)CachedCodeMaxSize;
137 }
138
139 void AOTCodeCache::initialize() {
140 if (LoadCachedCode && !UseSharedSpaces) {
141 return;
142 }
143 if (LoadCachedCode && CDSAccess::get_cached_code_size() == 0) {
144 LoadCachedCode = false;
145 return;
146 }
147 if (StoreCachedCode || LoadCachedCode) {
148 if (FLAG_IS_DEFAULT(ClassInitBarrierMode)) {
149 FLAG_SET_DEFAULT(ClassInitBarrierMode, 1);
150 }
151 } else if (ClassInitBarrierMode > 0) {
152 log_info(aot, codecache, init)("Set ClassInitBarrierMode to 0 because StoreCachedCode and LoadCachedCode are false.");
153 FLAG_SET_DEFAULT(ClassInitBarrierMode, 0);
154 }
155 if (LoadCachedCode || StoreCachedCode) {
156 if (!open_cache()) {
157 exit_vm_on_load_failure();
158 return;
159 }
160 if (StoreCachedCode) {
161 FLAG_SET_DEFAULT(FoldStableValues, false);
162 FLAG_SET_DEFAULT(ForceUnreachable, true);
163 }
164 FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false);
165 }
166 }
167
168 void AOTCodeCache::init2() {
169 if (!is_on()) {
170 return;
171 }
172 // After Universe initialized
173 BarrierSet* bs = BarrierSet::barrier_set();
174 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
175 address byte_map_base = ci_card_table_address_as<address>();
176 if (is_on_for_write() && !external_word_Relocation::can_be_relocated(byte_map_base)) {
177 // Bail out since we can't encode card table base address with relocation
178 log_warning(aot, codecache, init)("Can't create AOT Code Cache because card table base address is not relocatable: " INTPTR_FORMAT, p2i(byte_map_base));
179 close();
180 exit_vm_on_load_failure();
181 }
182 }
183 // initialize aot runtime constants as appropriate to this runtime
184 AOTRuntimeConstants::initialize_from_runtime();
185
186 if (!verify_vm_config()) {
187 close();
188 exit_vm_on_load_failure();
189 }
190
191 // initialize the table of external routines so we can save
192 // generated code blobs that reference them
193 init_extrs_table();
194 // initialize the table of initial stubs so we can save
195 // generated code blobs that reference them
196 init_early_stubs_table();
197 }
198
199 void AOTCodeCache::print_timers_on(outputStream* st) {
200 if (LoadCachedCode) {
201 st->print_cr (" AOT Code Load Time: %7.3f s", _t_totalLoad.seconds());
202 st->print_cr (" nmethod register: %7.3f s", _t_totalRegister.seconds());
203 st->print_cr (" find cached code: %7.3f s", _t_totalFind.seconds());
204 }
205 if (StoreCachedCode) {
206 st->print_cr (" AOT Code Store Time: %7.3f s", _t_totalStore.seconds());
207 }
208 }
209
210 bool AOTCodeCache::is_C3_on() {
211 #if INCLUDE_JVMCI
212 if (UseJVMCICompiler) {
213 return (StoreCachedCode || LoadCachedCode) && UseC2asC3;
214 }
215 #endif
216 return false;
217 }
218
219 bool AOTCodeCache::is_code_load_thread_on() {
220 return UseCodeLoadThread && LoadCachedCode;
221 }
222
223 bool AOTCodeCache::gen_preload_code(ciMethod* m, int entry_bci) {
224 VM_ENTRY_MARK;
225 return (entry_bci == InvocationEntryBci) && is_on() && _cache->gen_preload_code() &&
226 CDSAccess::can_generate_cached_code(m->get_Method());
227 }
228
229 static void print_helper(nmethod* nm, outputStream* st) {
230 AOTCodeCache::iterate([&](AOTCodeEntry* e) {
231 if (e->method() == nm->method()) {
232 ResourceMark rm;
233 stringStream ss;
234 ss.print("A%s%d", (e->for_preload() ? "P" : ""), e->comp_level());
235 if (e->decompile() > 0) {
236 ss.print("+D%d", e->decompile());
237 }
238 ss.print("[%s%s%s]",
239 (e->is_loaded() ? "L" : ""),
240 (e->load_fail() ? "F" : ""),
241 (e->not_entrant() ? "I" : ""));
242 ss.print("#%d", e->comp_id());
243
244 st->print(" %s", ss.freeze());
245 }
246 });
247 }
248
249 void AOTCodeCache::close() {
250 if (is_on()) {
251 if (AOTCodeCache::is_on_for_read()) {
252 LogStreamHandle(Info, init) log;
253 if (log.is_enabled()) {
254 log.print_cr("AOT Code Cache statistics (when closed): ");
255 AOTCodeCache::print_statistics_on(&log);
256 log.cr();
257 AOTCodeCache::print_timers_on(&log);
258
259 LogStreamHandle(Info, aot, codecache, init) log1;
260 if (log1.is_enabled()) {
261 AOTCodeCache::print_unused_entries_on(&log1);
262 }
263
264 LogStreamHandle(Info, aot, codecache) aot_info;
265 // need a lock to traverse the code cache
266 MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag);
267 if (aot_info.is_enabled()) {
268 NMethodIterator iter(NMethodIterator::all);
269 while (iter.next()) {
270 nmethod* nm = iter.method();
271 if (nm->is_in_use() && !nm->is_native_method() && !nm->is_osr_method()) {
272 aot_info.print("%5d:%c%c%c%d:", nm->compile_id(),
273 (nm->method()->is_shared() ? 'S' : ' '),
274 (nm->is_aot() ? 'A' : ' '),
275 (nm->preloaded() ? 'P' : ' '),
276 nm->comp_level());
277 print_helper(nm, &aot_info);
278 aot_info.print(": ");
279 CompileTask::print(&aot_info, nm, nullptr, true /*short_form*/);
280
281 LogStreamHandle(Debug, aot, codecache) aot_debug;
282 if (aot_debug.is_enabled()) {
283 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), nm->method()));
284 if (mtd != nullptr) {
285 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
286 aot_debug.print(" CTD: "); ctd->print_on(&aot_debug); aot_debug.cr();
287 });
288 }
289 }
290 }
291 }
292 }
293 }
294 }
295
296 delete _cache; // Free memory
297 _cache = nullptr;
298 }
299 }
300
301 void AOTCodeCache::invalidate(AOTCodeEntry* entry) {
302 // This could be concurent execution
303 if (entry != nullptr && is_on()) { // Request could come after cache is closed.
304 _cache->invalidate_entry(entry);
305 }
306 }
307
308 bool AOTCodeCache::is_loaded(AOTCodeEntry* entry) {
309 if (is_on() && _cache->cache_buffer() != nullptr) {
310 return (uint)((char*)entry - _cache->cache_buffer()) < _cache->load_size();
311 }
312 return false;
313 }
314
315 void AOTCodeCache::preload_code(JavaThread* thread) {
316 if ((ClassInitBarrierMode == 0) || !is_on_for_read()) {
317 return;
318 }
319 if ((DisableCachedCode & (1 << 3)) != 0) {
320 return; // no preloaded code (level 5);
321 }
322 _cache->preload_startup_code(thread);
323 }
324
325 AOTCodeEntry* AOTCodeCache::find_code_entry(const methodHandle& method, uint comp_level) {
326 switch (comp_level) {
327 case CompLevel_simple:
328 if ((DisableCachedCode & (1 << 0)) != 0) {
329 return nullptr;
330 }
331 break;
332 case CompLevel_limited_profile:
333 if ((DisableCachedCode & (1 << 1)) != 0) {
334 return nullptr;
335 }
336 break;
337 case CompLevel_full_optimization:
338 if ((DisableCachedCode & (1 << 2)) != 0) {
339 return nullptr;
340 }
341 break;
342
343 default: return nullptr; // Level 1, 2, and 4 only
344 }
345 TraceTime t1("Total time to find AOT code", &_t_totalFind, enable_timers(), false);
346 if (is_on() && _cache->cache_buffer() != nullptr) {
347 MethodData* md = method->method_data();
348 uint decomp = (md == nullptr) ? 0 : md->decompile_count();
349
350 ResourceMark rm;
351 const char* target_name = method->name_and_sig_as_C_string();
352 uint hash = java_lang_String::hash_code((const jbyte*)target_name, (int)strlen(target_name));
353 AOTCodeEntry* entry = _cache->find_entry(AOTCodeEntry::Code, hash, comp_level, decomp);
354 if (entry == nullptr) {
355 log_info(aot, codecache, nmethod)("Missing entry for '%s' (comp_level %d, decomp: %d, hash: " UINT32_FORMAT_X_0 ")", target_name, (uint)comp_level, decomp, hash);
356 #ifdef ASSERT
357 } else {
358 uint name_offset = entry->offset() + entry->name_offset();
359 uint name_size = entry->name_size(); // Includes '/0'
360 const char* name = _cache->cache_buffer() + name_offset;
361 if (strncmp(target_name, name, name_size) != 0) {
362 assert(false, "AOTCodeCache: saved nmethod's name '%s' is different from '%s', hash: " UINT32_FORMAT_X_0, name, target_name, hash);
363 }
364 #endif
365 }
366
367 DirectiveSet* directives = DirectivesStack::getMatchingDirective(method, nullptr);
368 if (directives->IgnorePrecompiledOption) {
369 LogStreamHandle(Info, aot, codecache, compilation) log;
370 if (log.is_enabled()) {
371 log.print("Ignore cached code entry on level %d for ", comp_level);
372 method->print_value_on(&log);
373 }
374 return nullptr;
375 }
376
377 return entry;
378 }
379 return nullptr;
380 }
381
382 void AOTCodeCache::add_C_string(const char* str) {
383 if (is_on_for_write()) {
384 _cache->add_new_C_string(str);
385 }
386 }
387
388 bool AOTCodeCache::allow_const_field(ciConstant& value) {
389 return !is_on() || !StoreCachedCode // Restrict only when we generate cache
390 // Can not trust primitive too || !is_reference_type(value.basic_type())
391 // May disable this too for now || is_reference_type(value.basic_type()) && value.as_object()->should_be_constant()
392 ;
393 }
394
395
396 bool AOTCodeCache::open_cache() {
397 AOTCodeCache* cache = new AOTCodeCache();
398 if (cache->failed()) {
399 delete cache;
400 _cache = nullptr;
401 return false;
402 }
403 _cache = cache;
404 return true;
405 }
406
407 class CachedCodeDirectory : public CachedCodeDirectoryInternal {
408 public:
409 uint _aot_code_size;
410 char* _aot_code_data;
411
412 void set_aot_code_data(uint size, char* aot_data) {
413 _aot_code_size = size;
414 CDSAccess::set_pointer(&_aot_code_data, aot_data);
415 }
416
417 static CachedCodeDirectory* create();
418 };
419
420 // Storing AOT code in the cached code region of AOT Cache:
421 //
422 // [1] Use CachedCodeDirectory to keep track of all of data related to cached code.
423 // E.g., you can build a hashtable to record what methods have been archived.
424 //
425 // [2] Memory for all data for cached code, including CachedCodeDirectory, should be
426 // allocated using CDSAccess::allocate_from_code_cache().
427 //
428 // [3] CachedCodeDirectory must be the very first allocation.
429 //
430 // [4] Two kinds of pointer can be stored:
431 // - A pointer p that points to metadata. CDSAccess::can_generate_cached_code(p) must return true.
432 // - A pointer to a buffer returned by CDSAccess::allocate_from_code_cache().
433 // (It's OK to point to an interior location within this buffer).
434 // Such pointers must be stored using CDSAccess::set_pointer()
435 //
436 // The buffers allocated by CDSAccess::allocate_from_code_cache() are in a contiguous region. At runtime, this
437 // region is mapped to the process address space. All the pointers in this buffer are relocated as necessary
438 // (e.g., to account for the runtime location of the CodeCache).
439 //
440 // This is always at the very beginning of the mmaped CDS "cc" (cached code) region
441 static CachedCodeDirectory* _cached_code_directory = nullptr;
442
443 CachedCodeDirectory* CachedCodeDirectory::create() {
444 assert(CDSAccess::is_cached_code_region_empty(), "must be");
445 CachedCodeDirectory* dir = (CachedCodeDirectory*)CDSAccess::allocate_from_code_cache(sizeof(CachedCodeDirectory));
446 dir->dumptime_init_internal();
447 return dir;
448 }
449
450 #define DATA_ALIGNMENT HeapWordSize
451
452 AOTCodeCache::AOTCodeCache() {
453 _load_header = nullptr;
454 _for_read = LoadCachedCode;
455 _for_write = StoreCachedCode;
456 _load_size = 0;
457 _store_size = 0;
458 _write_position = 0;
459 _closing = false;
460 _failed = false;
461 _lookup_failed = false;
462 _table = nullptr;
463 _load_entries = nullptr;
464 _store_entries = nullptr;
465 _C_strings_buf = nullptr;
466 _load_buffer = nullptr;
467 _store_buffer = nullptr;
468 _C_store_buffer = nullptr;
469 _store_entries_cnt = 0;
470 _gen_preload_code = false;
471 _for_preload = false; // changed while storing entry data
472 _has_clinit_barriers = false;
473
474 _compile_id = 0;
475 _comp_level = 0;
476
477 _use_meta_ptrs = UseSharedSpaces ? UseMetadataPointers : false;
478
479 if (_for_read) {
480 // Read cache
481 ReservedSpace rs = MemoryReserver::reserve(CDSAccess::get_cached_code_size(), mtCode);
482 if (!rs.is_reserved()) {
483 log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping cached code region in AOT Cache", (uint)CDSAccess::get_cached_code_size());
484 set_failed();
485 return;
486 }
487 if (!CDSAccess::map_cached_code(rs)) {
488 log_warning(aot, codecache, init)("Failed to read/mmap cached code region in AOT Cache");
489 set_failed();
490 return;
491 }
492 _cached_code_directory = (CachedCodeDirectory*)rs.base();
493 _cached_code_directory->runtime_init_internal();
494
495 _load_size = _cached_code_directory->_aot_code_size;
496 _load_buffer = _cached_code_directory->_aot_code_data;
497 assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned");
498 log_info(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " from AOT Code Cache", _load_size, p2i(_load_buffer));
499
500 _load_header = (AOTCodeCache::Header*)addr(0);
501 if (!_load_header->verify_config(_load_size)) {
502 set_failed();
503 return;
504 }
505 log_info(aot, codecache, init)("Read header from AOT Code Cache");
506 if (_load_header->has_meta_ptrs()) {
507 assert(UseSharedSpaces, "should be verified already");
508 _use_meta_ptrs = true; // Regardless UseMetadataPointers
509 UseMetadataPointers = true;
510 }
511 // Read strings
512 load_strings();
513 }
514 if (_for_write) {
515 _gen_preload_code = _use_meta_ptrs && (ClassInitBarrierMode > 0);
516
517 _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode);
518 _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT);
519 // Entries allocated at the end of buffer in reverse (as on stack).
520 _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT);
521 log_info(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size " UINT32_FORMAT " bytes", p2i(_store_buffer), max_aot_code_size());
522 }
523 _table = new AOTCodeAddressTable();
524 }
525
526 void AOTCodeCache::init_extrs_table() {
527 AOTCodeAddressTable* table = addr_table();
528 if (table != nullptr) {
529 table->init_extrs();
530 }
531 }
532 void AOTCodeCache::init_early_stubs_table() {
533 AOTCodeAddressTable* table = addr_table();
534 if (table != nullptr) {
535 table->init_early_stubs();
536 }
537 }
538 void AOTCodeCache::init_shared_blobs_table() {
539 AOTCodeAddressTable* table = addr_table();
540 if (table != nullptr) {
541 table->init_shared_blobs();
542 }
543 }
544 void AOTCodeCache::init_stubs_table() {
545 AOTCodeAddressTable* table = addr_table();
546 if (table != nullptr) {
547 table->init_stubs();
548 }
549 }
550
551 void AOTCodeCache::init_opto_table() {
552 AOTCodeAddressTable* table = addr_table();
553 if (table != nullptr) {
554 table->init_opto();
555 }
556 }
557
558 void AOTCodeCache::init_c1_table() {
559 AOTCodeAddressTable* table = addr_table();
560 if (table != nullptr) {
561 table->init_c1();
562 }
563 }
564
565 void AOTCodeCache::Config::record(bool use_meta_ptrs) {
566 _flags = 0;
567 if (use_meta_ptrs) {
568 _flags |= metadataPointers;
569 }
570 #ifdef ASSERT
571 _flags |= debugVM;
572 #endif
573 if (UseCompressedOops) {
574 _flags |= compressedOops;
575 }
576 if (UseCompressedClassPointers) {
577 _flags |= compressedClassPointers;
578 }
579 if (UseTLAB) {
580 _flags |= useTLAB;
581 }
582 if (JavaAssertions::systemClassDefault()) {
583 _flags |= systemClassAssertions;
584 }
585 if (JavaAssertions::userClassDefault()) {
586 _flags |= userClassAssertions;
587 }
588 if (EnableContended) {
589 _flags |= enableContendedPadding;
590 }
591 if (RestrictContended) {
592 _flags |= restrictContendedPadding;
593 }
594 _compressedOopShift = CompressedOops::shift();
595 _compressedKlassShift = CompressedKlassPointers::shift();
596 _contendedPaddingWidth = ContendedPaddingWidth;
597 _objectAlignment = ObjectAlignmentInBytes;
598 _gc = (uint)Universe::heap()->kind();
599 }
600
601 bool AOTCodeCache::Config::verify() const {
602 #ifdef ASSERT
603 if ((_flags & debugVM) == 0) {
604 log_warning(aot, codecache, init)("Disable AOT Code: it was created by product VM, it can't be used by debug VM");
605 return false;
606 }
607 #else
608 if ((_flags & debugVM) != 0) {
609 log_warning(aot, codecache, init)("Disable AOT Code: it was created by debug VM, it can't be used by product VM");
610 return false;
611 }
612 #endif
613
614 CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
615 if (aot_gc != Universe::heap()->kind()) {
616 log_warning(aot, codecache, init)("Disable AOT Code: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
617 return false;
618 }
619
620 if (((_flags & compressedOops) != 0) != UseCompressedOops) {
621 log_warning(aot, codecache, init)("Disable AOT Code: it was created with UseCompressedOops = %s", UseCompressedOops ? "false" : "true");
622 return false;
623 }
624 if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
625 log_warning(aot, codecache, init)("Disable AOT Code: it was created with UseCompressedClassPointers = %s", UseCompressedClassPointers ? "false" : "true");
626 return false;
627 }
628
629 if (((_flags & systemClassAssertions) != 0) != JavaAssertions::systemClassDefault()) {
630 log_warning(aot, codecache, init)("Disable AOT Code: it was created with JavaAssertions::systemClassDefault() = %s", JavaAssertions::systemClassDefault() ? "disabled" : "enabled");
631 return false;
632 }
633 if (((_flags & userClassAssertions) != 0) != JavaAssertions::userClassDefault()) {
634 log_warning(aot, codecache, init)("Disable AOT Code: it was created with JavaAssertions::userClassDefault() = %s", JavaAssertions::userClassDefault() ? "disabled" : "enabled");
635 return false;
636 }
637
638 if (((_flags & enableContendedPadding) != 0) != EnableContended) {
639 log_warning(aot, codecache, init)("Disable AOT Code: it was created with EnableContended = %s", EnableContended ? "false" : "true");
640 return false;
641 }
642 if (((_flags & restrictContendedPadding) != 0) != RestrictContended) {
643 log_warning(aot, codecache, init)("Disable AOT Code: it was created with RestrictContended = %s", RestrictContended ? "false" : "true");
644 return false;
645 }
646 if (_compressedOopShift != (uint)CompressedOops::shift()) {
647 log_warning(aot, codecache, init)("Disable AOT Code: it was created with CompressedOops::shift() = %d vs current %d", _compressedOopShift, CompressedOops::shift());
648 return false;
649 }
650 if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
651 log_warning(aot, codecache, init)("Disable AOT Code: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
652 return false;
653 }
654 if (_contendedPaddingWidth != (uint)ContendedPaddingWidth) {
655 log_warning(aot, codecache, init)("Disable AOT Code: it was created with ContendedPaddingWidth = %d vs current %d", _contendedPaddingWidth, ContendedPaddingWidth);
656 return false;
657 }
658 if (_objectAlignment != (uint)ObjectAlignmentInBytes) {
659 log_warning(aot, codecache, init)("Disable AOT Code: it was created with ObjectAlignmentInBytes = %d vs current %d", _objectAlignment, ObjectAlignmentInBytes);
660 return false;
661 }
662 return true;
663 }
664
665 bool AOTCodeCache::Header::verify_config(uint load_size) const {
666 if (_version != AOT_CODE_VERSION) {
667 log_warning(aot, codecache, init)("Disable AOT Code: different AOT Code version %d vs %d recorded in AOT Cache", AOT_CODE_VERSION, _version);
668 return false;
669 }
670 if (_cache_size != load_size) {
671 log_warning(aot, codecache, init)("Disable AOT Code: different cached code size %d vs %d recorded in AOT Cache", load_size, _cache_size);
672 return false;
673 }
674 return true;
675 }
676
677 volatile int AOTCodeCache::_nmethod_readers = 0;
678
679 AOTCodeCache::~AOTCodeCache() {
680 if (_closing) {
681 return; // Already closed
682 }
683 // Stop any further access to cache.
684 // Checked on entry to load_nmethod() and store_nmethod().
685 _closing = true;
686 if (_for_read) {
687 // Wait for all load_nmethod() finish.
688 wait_for_no_nmethod_readers();
689 }
690 // Prevent writing code into cache while we are closing it.
691 // This lock held by ciEnv::register_method() which calls store_nmethod().
692 MutexLocker ml(Compile_lock);
693 if (for_write()) { // Finalize cache
694 finish_write();
695 }
696 _load_buffer = nullptr;
697 if (_C_store_buffer != nullptr) {
698 FREE_C_HEAP_ARRAY(char, _C_store_buffer);
699 _C_store_buffer = nullptr;
700 _store_buffer = nullptr;
701 }
702 if (_table != nullptr) {
703 delete _table;
704 _table = nullptr;
705 }
706 }
707
708 AOTCodeCache* AOTCodeCache::open_for_read() {
709 if (AOTCodeCache::is_on_for_read()) {
710 return AOTCodeCache::cache();
711 }
712 return nullptr;
713 }
714
715 AOTCodeCache* AOTCodeCache::open_for_write() {
716 if (AOTCodeCache::is_on_for_write()) {
717 AOTCodeCache* cache = AOTCodeCache::cache();
718 cache->clear_lookup_failed(); // Reset bit
719 return cache;
720 }
721 return nullptr;
722 }
723
724 bool AOTCodeCache::is_address_in_aot_cache(address p) {
725 AOTCodeCache* cache = open_for_read();
726 if (cache == nullptr) {
727 return false;
728 }
729 if ((p >= (address)cache->cache_buffer()) &&
730 (p < (address)(cache->cache_buffer() + cache->load_size()))) {
731 return true;
732 }
733 return false;
734 }
735
736 static void copy_bytes(const char* from, address to, uint size) {
737 assert(size > 0, "sanity");
738 bool by_words = true;
739 if ((size > 2 * HeapWordSize) && (((intptr_t)from | (intptr_t)to) & (HeapWordSize - 1)) == 0) {
740 // Use wordwise copies if possible:
741 Copy::disjoint_words((HeapWord*)from,
742 (HeapWord*)to,
743 ((size_t)size + HeapWordSize-1) / HeapWordSize);
744 } else {
745 by_words = false;
746 Copy::conjoint_jbytes(from, to, (size_t)size);
747 }
748 log_trace(aot, codecache)("Copied %d bytes as %s from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, (by_words ? "HeapWord" : "bytes"), p2i(from), p2i(to));
749 }
750
751 void AOTCodeReader::set_read_position(uint pos) {
752 if (pos == _read_position) {
753 return;
754 }
755 assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size());
756 _read_position = pos;
757 }
758
759 bool AOTCodeCache::set_write_position(uint pos) {
760 if (pos == _write_position) {
761 return true;
762 }
763 if (_store_size < _write_position) {
764 _store_size = _write_position; // Adjust during write
765 }
766 assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size);
767 _write_position = pos;
768 return true;
769 }
770
771 static char align_buffer[256] = { 0 };
772
773 bool AOTCodeCache::align_write() {
774 // We are not executing code from cache - we copy it by bytes first.
775 // No need for big alignment (or at all).
776 uint padding = DATA_ALIGNMENT - (_write_position & (DATA_ALIGNMENT - 1));
777 if (padding == DATA_ALIGNMENT) {
778 return true;
779 }
780 uint n = write_bytes((const void*)&align_buffer, padding);
781 if (n != padding) {
782 return false;
783 }
784 log_trace(aot, codecache)("Adjust write alignment in AOT Code Cache");
785 return true;
786 }
787
788 // Check to see if AOT code cache has required space to store "nbytes" of data
789 address AOTCodeCache::reserve_bytes(uint nbytes) {
790 assert(for_write(), "Code Cache file is not created");
791 uint new_position = _write_position + nbytes;
792 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
793 log_warning(aot, codecache)("Failed to ensure %d bytes at offset %d in AOT Code Cache. Increase CachedCodeMaxSize.",
794 nbytes, _write_position);
795 set_failed();
796 exit_vm_on_store_failure();
797 return nullptr;
798 }
799 address buffer = (address)(_store_buffer + _write_position);
800 _write_position += nbytes;
801 if (_store_size < _write_position) {
802 _store_size = _write_position;
803 }
804 return buffer;
805 }
806
807 uint AOTCodeCache::write_bytes(const void* buffer, uint nbytes) {
808 assert(for_write(), "Code Cache file is not created");
809 if (nbytes == 0) {
810 return 0;
811 }
812 uint new_position = _write_position + nbytes;
813 if (new_position >= (uint)((char*)_store_entries - _store_buffer)) {
814 log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase CachedCodeMaxSize.",
815 nbytes, _write_position);
816 set_failed();
817 exit_vm_on_store_failure();
818 return 0;
819 }
820 copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes);
821 log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position);
822 _write_position += nbytes;
823 if (_store_size < _write_position) {
824 _store_size = _write_position;
825 }
826 return nbytes;
827 }
828
829 void AOTCodeEntry::update_method_for_writing() {
830 if (_method != nullptr) {
831 _method = CDSAccess::method_in_cached_code(_method);
832 }
833 }
834
835 void AOTCodeEntry::print(outputStream* st) const {
836 st->print_cr(" AOT Code Cache entry " INTPTR_FORMAT " [kind: %d, id: " UINT32_FORMAT_X_0 ", offset: %d, size: %d, comp_level: %d, comp_id: %d, decompiled: %d, %s%s%s%s%s]",
837 p2i(this), (int)_kind, _id, _offset, _size, _comp_level, _comp_id, _decompile,
838 (_not_entrant? "not_entrant" : "entrant"),
839 (_loaded ? ", loaded" : ""),
840 (_has_clinit_barriers ? ", has_clinit_barriers" : ""),
841 (_for_preload ? ", for_preload" : ""),
842 (_ignore_decompile ? ", ignore_decomp" : ""));
843 }
844
845 void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) {
846 return (void*)(cache->add_entry());
847 }
848
849 bool skip_preload(methodHandle mh) {
850 if (!mh->method_holder()->is_loaded()) {
851 return true;
852 }
853 DirectiveSet* directives = DirectivesStack::getMatchingDirective(mh, nullptr);
854 if (directives->DontPreloadOption) {
855 LogStreamHandle(Info, aot, codecache, init) log;
856 if (log.is_enabled()) {
857 log.print("Exclude preloading code for ");
858 mh->print_value_on(&log);
859 }
860 return true;
861 }
862 return false;
863 }
864
865 void AOTCodeCache::preload_startup_code(TRAPS) {
866 if (CompilationPolicy::compiler_count(CompLevel_full_optimization) == 0) {
867 // Since we reuse the CompilerBroker API to install cached code, we're required to have a JIT compiler for the
868 // level we want (that is CompLevel_full_optimization).
869 return;
870 }
871 assert(_for_read, "sanity");
872 uint count = _load_header->entries_count();
873 if (_load_entries == nullptr) {
874 // Read it
875 _search_entries = (uint*)addr(_load_header->entries_offset()); // [id, index]
876 _load_entries = (AOTCodeEntry*)(_search_entries + 2 * count);
877 log_info(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
878 }
879 uint preload_entries_count = _load_header->preload_entries_count();
880 if (preload_entries_count > 0) {
881 uint* entries_index = (uint*)addr(_load_header->preload_entries_offset());
882 log_info(aot, codecache, init)("Load %d preload entries from AOT Code Cache", preload_entries_count);
883 uint count = MIN2(preload_entries_count, SCLoadStop);
884 for (uint i = SCLoadStart; i < count; i++) {
885 uint index = entries_index[i];
886 AOTCodeEntry* entry = &(_load_entries[index]);
887 if (entry->not_entrant()) {
888 continue;
889 }
890 methodHandle mh(THREAD, entry->method());
891 assert((mh.not_null() && MetaspaceShared::is_in_shared_metaspace((address)mh())), "sanity");
892 if (skip_preload(mh)) {
893 continue; // Exclude preloading for this method
894 }
895 assert(mh->method_holder()->is_loaded(), "");
896 if (!mh->method_holder()->is_linked()) {
897 assert(!HAS_PENDING_EXCEPTION, "");
898 mh->method_holder()->link_class(THREAD);
899 if (HAS_PENDING_EXCEPTION) {
900 LogStreamHandle(Info, aot, codecache) log;
901 if (log.is_enabled()) {
902 ResourceMark rm;
903 log.print("Linkage failed for %s: ", mh->method_holder()->external_name());
904 THREAD->pending_exception()->print_value_on(&log);
905 if (log_is_enabled(Debug, aot, codecache)) {
906 THREAD->pending_exception()->print_on(&log);
907 }
908 }
909 CLEAR_PENDING_EXCEPTION;
910 }
911 }
912 if (mh->aot_code_entry() != nullptr) {
913 // Second C2 compilation of the same method could happen for
914 // different reasons without marking first entry as not entrant.
915 continue; // Keep old entry to avoid issues
916 }
917 mh->set_aot_code_entry(entry);
918 CompileBroker::compile_method(mh, InvocationEntryBci, CompLevel_full_optimization, methodHandle(), 0, false, CompileTask::Reason_Preload, CHECK);
919 }
920 }
921 }
922
923 static bool check_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level, uint decomp, AOTCodeEntry* entry) {
924 if (entry->kind() == kind) {
925 assert(entry->id() == id, "sanity");
926 if (kind != AOTCodeEntry::Code || (!entry->not_entrant() && !entry->has_clinit_barriers() &&
927 (entry->comp_level() == comp_level) &&
928 (entry->ignore_decompile() || entry->decompile() == decomp))) {
929 return true; // Found
930 }
931 }
932 return false;
933 }
934
935 AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id, uint comp_level, uint decomp) {
936 assert(_for_read, "sanity");
937 uint count = _load_header->entries_count();
938 if (_load_entries == nullptr) {
939 // Read it
940 _search_entries = (uint*)addr(_load_header->entries_offset()); // [id, index]
941 _load_entries = (AOTCodeEntry*)(_search_entries + 2 * count);
942 log_info(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset());
943 }
944 // Binary search
945 int l = 0;
946 int h = count - 1;
947 while (l <= h) {
948 int mid = (l + h) >> 1;
949 int ix = mid * 2;
950 uint is = _search_entries[ix];
951 if (is == id) {
952 int index = _search_entries[ix + 1];
953 AOTCodeEntry* entry = &(_load_entries[index]);
954 if (check_entry(kind, id, comp_level, decomp, entry)) {
955 return entry; // Found
956 }
957 // Leaner search around (could be the same nmethod with different decompile count)
958 for (int i = mid - 1; i >= l; i--) { // search back
959 ix = i * 2;
960 is = _search_entries[ix];
961 if (is != id) {
962 break;
963 }
964 index = _search_entries[ix + 1];
965 AOTCodeEntry* entry = &(_load_entries[index]);
966 if (check_entry(kind, id, comp_level, decomp, entry)) {
967 return entry; // Found
968 }
969 }
970 for (int i = mid + 1; i <= h; i++) { // search forward
971 ix = i * 2;
972 is = _search_entries[ix];
973 if (is != id) {
974 break;
975 }
976 index = _search_entries[ix + 1];
977 AOTCodeEntry* entry = &(_load_entries[index]);
978 if (check_entry(kind, id, comp_level, decomp, entry)) {
979 return entry; // Found
980 }
981 }
982 break; // Not found match (different decompile count or not_entrant state).
983 } else if (is < id) {
984 l = mid + 1;
985 } else {
986 h = mid - 1;
987 }
988 }
989 return nullptr;
990 }
991
992 void AOTCodeCache::invalidate_entry(AOTCodeEntry* entry) {
993 assert(entry!= nullptr, "all entries should be read already");
994 if (entry->not_entrant()) {
995 return; // Someone invalidated it already
996 }
997 #ifdef ASSERT
998 bool found = false;
999 if (_for_read) {
1000 uint count = _load_header->entries_count();
1001 uint i = 0;
1002 for(; i < count; i++) {
1003 if (entry == &(_load_entries[i])) {
1004 break;
1005 }
1006 }
1007 found = (i < count);
1008 }
1009 if (!found && _for_write) {
1010 uint count = _store_entries_cnt;
1011 uint i = 0;
1012 for(; i < count; i++) {
1013 if (entry == &(_store_entries[i])) {
1014 break;
1015 }
1016 }
1017 found = (i < count);
1018 }
1019 assert(found, "entry should exist");
1020 #endif
1021 entry->set_not_entrant();
1022 {
1023 uint name_offset = entry->offset() + entry->name_offset();
1024 const char* name;
1025 if (AOTCodeCache::is_loaded(entry)) {
1026 name = _load_buffer + name_offset;
1027 } else {
1028 name = _store_buffer + name_offset;
1029 }
1030 uint level = entry->comp_level();
1031 uint comp_id = entry->comp_id();
1032 uint decomp = entry->decompile();
1033 bool clinit_brs = entry->has_clinit_barriers();
1034 log_info(aot, codecache, nmethod)("Invalidated entry for '%s' (comp_id %d, comp_level %d, decomp: %d, hash: " UINT32_FORMAT_X_0 "%s)",
1035 name, comp_id, level, decomp, entry->id(), (clinit_brs ? ", has clinit barriers" : ""));
1036 }
1037 if (entry->next() != nullptr) {
1038 entry = entry->next();
1039 assert(entry->has_clinit_barriers(), "expecting only such entries here");
1040 invalidate_entry(entry);
1041 }
1042 }
1043
1044 static int uint_cmp(const void *i, const void *j) {
1045 uint a = *(uint *)i;
1046 uint b = *(uint *)j;
1047 return a > b ? 1 : a < b ? -1 : 0;
1048 }
1049
1050 AOTCodeStats AOTCodeStats::add_cached_code_stats(AOTCodeStats stats1, AOTCodeStats stats2) {
1051 AOTCodeStats result;
1052 for (int kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
1053 result.ccstats._kind_cnt[kind] = stats1.entry_count(kind) + stats2.entry_count(kind);
1054 }
1055
1056 for (int lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
1057 result.ccstats._nmethod_cnt[lvl] = stats1.nmethod_count(lvl) + stats2.nmethod_count(lvl);
1058 }
1059 result.ccstats._clinit_barriers_cnt = stats1.clinit_barriers_count() + stats2.clinit_barriers_count();
1060 return result;
1061 }
1062
1063 void AOTCodeCache::log_stats_on_exit() {
1064 LogStreamHandle(Info, aot, codecache, exit) log;
1065 if (log.is_enabled()) {
1066 AOTCodeStats prev_stats;
1067 AOTCodeStats current_stats;
1068 AOTCodeStats total_stats;
1069 uint max_size = 0;
1070
1071 uint load_count = (_load_header != nullptr) ? _load_header->entries_count() : 0;
1072
1073 for (uint i = 0; i < load_count; i++) {
1074 prev_stats.collect_entry_stats(&_load_entries[i]);
1075 if (max_size < _load_entries[i].size()) {
1076 max_size = _load_entries[i].size();
1077 }
1078 }
1079 for (uint i = 0; i < _store_entries_cnt; i++) {
1080 current_stats.collect_entry_stats(&_store_entries[i]);
1081 if (max_size < _store_entries[i].size()) {
1082 max_size = _store_entries[i].size();
1083 }
1084 }
1085 total_stats = AOTCodeStats::add_cached_code_stats(prev_stats, current_stats);
1086
1087 log.print_cr("Wrote %d AOTCodeEntry entries(%u max size) to AOT Code Cache",
1088 total_stats.total_count(), max_size);
1089 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
1090 if (total_stats.entry_count(kind) > 0) {
1091 log.print_cr(" %s: total=%u(old=%u+new=%u)",
1092 aot_code_entry_kind_name[kind], total_stats.entry_count(kind), prev_stats.entry_count(kind), current_stats.entry_count(kind));
1093 if (kind == AOTCodeEntry::Code) {
1094 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
1095 if (total_stats.nmethod_count(lvl) > 0) {
1096 log.print_cr(" Tier %d: total=%u(old=%u+new=%u)",
1097 lvl, total_stats.nmethod_count(lvl), prev_stats.nmethod_count(lvl), current_stats.nmethod_count(lvl));
1098 }
1099 }
1100 }
1101 }
1102 }
1103 log.print_cr("Total=%u(old=%u+new=%u)", total_stats.total_count(), prev_stats.total_count(), current_stats.total_count());
1104 }
1105 }
1106
1107 bool AOTCodeCache::finish_write() {
1108 if (!align_write()) {
1109 return false;
1110 }
1111 uint strings_offset = _write_position;
1112 int strings_count = store_strings();
1113 if (strings_count < 0) {
1114 return false;
1115 }
1116 if (!align_write()) {
1117 return false;
1118 }
1119 uint strings_size = _write_position - strings_offset;
1120
1121 uint entries_count = 0; // Number of entrant (useful) code entries
1122 uint entries_offset = _write_position;
1123
1124 uint store_count = _store_entries_cnt;
1125 if (store_count > 0) {
1126 _cached_code_directory = CachedCodeDirectory::create();
1127 assert(_cached_code_directory != nullptr, "Sanity check");
1128
1129 uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT);
1130 uint load_count = (_load_header != nullptr) ? _load_header->entries_count() : 0;
1131 uint code_count = store_count + load_count;
1132 uint search_count = code_count * 2;
1133 uint search_size = search_count * sizeof(uint);
1134 uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes
1135 uint preload_entries_cnt = 0;
1136 uint* preload_entries = NEW_C_HEAP_ARRAY(uint, code_count, mtCode);
1137 uint preload_entries_size = code_count * sizeof(uint);
1138 // _write_position should include code and strings
1139 uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it.
1140 uint total_size = _write_position + _load_size + header_size +
1141 code_alignment + search_size + preload_entries_size + entries_size;
1142
1143 assert(total_size < max_aot_code_size(), "Cached code region size (" UINT32_FORMAT " bytes) in AOT Code Cache is less than the required size (" UINT32_FORMAT " bytes).",
1144 total_size, max_aot_code_size());
1145
1146 // Create ordered search table for entries [id, index];
1147 uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode);
1148
1149 char* buffer = (char *)CDSAccess::allocate_from_code_cache(total_size + DATA_ALIGNMENT); // NEW_C_HEAP_ARRAY(char, total_size + DATA_ALIGNMENT, mtCode);
1150 char* start = align_up(buffer, DATA_ALIGNMENT);
1151 char* current = start + header_size; // Skip header
1152
1153 AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
1154
1155 // Add old entries first
1156 if (_for_read && (_load_header != nullptr)) {
1157 for(uint i = 0; i < load_count; i++) {
1158 if (_load_entries[i].load_fail()) {
1159 continue;
1160 }
1161 if (_load_entries[i].not_entrant()) {
1162 log_info(aot, codecache, exit)("Not entrant load entry id: %d, decomp: %d, hash: " UINT32_FORMAT_X_0, i, _load_entries[i].decompile(), _load_entries[i].id());
1163 if (_load_entries[i].for_preload()) {
1164 // Skip not entrant preload code:
1165 // we can't pre-load code which may have failing dependencies.
1166 continue;
1167 }
1168 _load_entries[i].set_entrant(); // Reset
1169 } else if (_load_entries[i].for_preload() && _load_entries[i].method() != nullptr) {
1170 // record entrant first version code for pre-loading
1171 preload_entries[preload_entries_cnt++] = entries_count;
1172 }
1173 {
1174 uint size = align_up(_load_entries[i].size(), DATA_ALIGNMENT);
1175 copy_bytes((_load_buffer + _load_entries[i].offset()), (address)current, size);
1176 _load_entries[i].set_offset(current - start); // New offset
1177 current += size;
1178 uint n = write_bytes(&(_load_entries[i]), sizeof(AOTCodeEntry));
1179 if (n != sizeof(AOTCodeEntry)) {
1180 FREE_C_HEAP_ARRAY(char, buffer);
1181 FREE_C_HEAP_ARRAY(uint, search);
1182 return false;
1183 }
1184 search[entries_count*2 + 0] = _load_entries[i].id();
1185 search[entries_count*2 + 1] = entries_count;
1186 entries_count++;
1187 }
1188 }
1189 }
1190 // AOTCodeEntry entries were allocated in reverse in store buffer.
1191 // Process them in reverse order to cache first code first.
1192 for (int i = store_count - 1; i >= 0; i--) {
1193 if (entries_address[i].load_fail()) {
1194 continue;
1195 }
1196 if (entries_address[i].not_entrant()) {
1197 log_info(aot, codecache, exit)("Not entrant new entry comp_id: %d, comp_level: %d, decomp: %d, hash: " UINT32_FORMAT_X_0 "%s", entries_address[i].comp_id(), entries_address[i].comp_level(), entries_address[i].decompile(), entries_address[i].id(), (entries_address[i].has_clinit_barriers() ? ", has clinit barriers" : ""));
1198 if (entries_address[i].for_preload()) {
1199 // Skip not entrant preload code:
1200 // we can't pre-load code which may have failing dependencies.
1201 continue;
1202 }
1203 entries_address[i].set_entrant(); // Reset
1204 } else if (entries_address[i].for_preload() && entries_address[i].method() != nullptr) {
1205 // record entrant first version code for pre-loading
1206 preload_entries[preload_entries_cnt++] = entries_count;
1207 }
1208 {
1209 entries_address[i].set_next(nullptr); // clear pointers before storing data
1210 uint size = align_up(entries_address[i].size(), DATA_ALIGNMENT);
1211 copy_bytes((_store_buffer + entries_address[i].offset()), (address)current, size);
1212 entries_address[i].set_offset(current - start); // New offset
1213 entries_address[i].update_method_for_writing();
1214 current += size;
1215 uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry));
1216 if (n != sizeof(AOTCodeEntry)) {
1217 FREE_C_HEAP_ARRAY(char, buffer);
1218 FREE_C_HEAP_ARRAY(uint, search);
1219 return false;
1220 }
1221 search[entries_count*2 + 0] = entries_address[i].id();
1222 search[entries_count*2 + 1] = entries_count;
1223 entries_count++;
1224 }
1225 }
1226
1227 if (entries_count == 0) {
1228 log_info(aot, codecache, exit)("No entires written to AOT Code Cache");
1229 FREE_C_HEAP_ARRAY(char, buffer);
1230 FREE_C_HEAP_ARRAY(uint, search);
1231 return true; // Nothing to write
1232 }
1233 assert(entries_count <= (store_count + load_count), "%d > (%d + %d)", entries_count, store_count, load_count);
1234 // Write strings
1235 if (strings_count > 0) {
1236 copy_bytes((_store_buffer + strings_offset), (address)current, strings_size);
1237 strings_offset = (current - start); // New offset
1238 current += strings_size;
1239 }
1240 uint preload_entries_offset = (current - start);
1241 preload_entries_size = preload_entries_cnt * sizeof(uint);
1242 if (preload_entries_size > 0) {
1243 copy_bytes((const char*)preload_entries, (address)current, preload_entries_size);
1244 current += preload_entries_size;
1245 log_info(aot, codecache, exit)("Wrote %d preload entries to AOT Code Cache", preload_entries_cnt);
1246 }
1247 if (preload_entries != nullptr) {
1248 FREE_C_HEAP_ARRAY(uint, preload_entries);
1249 }
1250
1251 uint new_entries_offset = (current - start); // New offset
1252 // Sort and store search table
1253 qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
1254 search_size = 2 * entries_count * sizeof(uint);
1255 copy_bytes((const char*)search, (address)current, search_size);
1256 FREE_C_HEAP_ARRAY(uint, search);
1257 current += search_size;
1258
1259 // Write entries
1260 entries_size = entries_count * sizeof(AOTCodeEntry); // New size
1261 copy_bytes((_store_buffer + entries_offset), (address)current, entries_size);
1262 current += entries_size;
1263
1264 log_stats_on_exit();
1265
1266 uint size = (current - start);
1267 assert(size <= total_size, "%d > %d", size , total_size);
1268
1269 // Finalize header
1270 AOTCodeCache::Header* header = (AOTCodeCache::Header*)start;
1271 header->init(size,
1272 (uint)strings_count, strings_offset,
1273 entries_count, new_entries_offset,
1274 preload_entries_cnt, preload_entries_offset,
1275 _use_meta_ptrs);
1276 log_info(aot, codecache, init)("Wrote AOTCodeCache header to AOT Code Cache");
1277 log_info(aot, codecache, exit)("Wrote %d bytes of data to AOT Code Cache", size);
1278
1279 _cached_code_directory->set_aot_code_data(size, start);
1280 }
1281 return true;
1282 }
1283
1284 bool AOTCodeCache::load_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1285 assert(start == cgen->assembler()->pc(), "wrong buffer");
1286 AOTCodeCache* cache = open_for_read();
1287 if (cache == nullptr) {
1288 return false;
1289 }
1290 AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Stub, (uint)id);
1291 if (entry == nullptr) {
1292 return false;
1293 }
1294 uint entry_position = entry->offset();
1295 // Read name
1296 uint name_offset = entry->name_offset() + entry_position;
1297 uint name_size = entry->name_size(); // Includes '/0'
1298 const char* saved_name = cache->addr(name_offset);
1299 if (strncmp(name, saved_name, (name_size - 1)) != 0) {
1300 log_warning(aot, codecache)("Saved stub's name '%s' is different from '%s' for id:%d", saved_name, name, (int)id);
1301 cache->set_failed();
1302 exit_vm_on_load_failure();
1303 return false;
1304 }
1305 log_info(aot, codecache,stubs)("Reading stub '%s' id:%d from AOT Code Cache", name, (int)id);
1306 // Read code
1307 uint code_offset = entry->code_offset() + entry_position;
1308 uint code_size = entry->code_size();
1309 copy_bytes(cache->addr(code_offset), start, code_size);
1310 cgen->assembler()->code_section()->set_end(start + code_size);
1311 log_info(aot, codecache,stubs)("Read stub '%s' id:%d from AOT Code Cache", name, (int)id);
1312 return true;
1313 }
1314
1315 bool AOTCodeCache::store_stub(StubCodeGenerator* cgen, vmIntrinsicID id, const char* name, address start) {
1316 AOTCodeCache* cache = open_for_write();
1317 if (cache == nullptr) {
1318 return false;
1319 }
1320 log_info(aot, codecache, stubs)("Writing stub '%s' id:%d to AOT Code Cache", name, (int)id);
1321 if (!cache->align_write()) {
1322 return false;
1323 }
1324 #ifdef ASSERT
1325 CodeSection* cs = cgen->assembler()->code_section();
1326 if (cs->has_locs()) {
1327 uint reloc_count = cs->locs_count();
1328 tty->print_cr("======== write stubs code section relocations [%d]:", reloc_count);
1329 // Collect additional data
1330 RelocIterator iter(cs);
1331 while (iter.next()) {
1332 switch (iter.type()) {
1333 case relocInfo::none:
1334 break;
1335 default: {
1336 iter.print_current_on(tty);
1337 fatal("stub's relocation %d unimplemented", (int)iter.type());
1338 break;
1339 }
1340 }
1341 }
1342 }
1343 #endif
1344 uint entry_position = cache->_write_position;
1345
1346 // Write code
1347 uint code_offset = 0;
1348 uint code_size = cgen->assembler()->pc() - start;
1349 uint n = cache->write_bytes(start, code_size);
1350 if (n != code_size) {
1351 return false;
1352 }
1353 // Write name
1354 uint name_offset = cache->_write_position - entry_position;
1355 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
1356 n = cache->write_bytes(name, name_size);
1357 if (n != name_size) {
1358 return false;
1359 }
1360 uint entry_size = cache->_write_position - entry_position;
1361 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
1362 code_offset, code_size, 0, 0,
1363 AOTCodeEntry::Stub, (uint32_t)id);
1364 log_info(aot, codecache, stubs)("Wrote stub '%s' id:%d to AOT Code Cache", name, (int)id);
1365 return true;
1366 }
1367
1368 Klass* AOTCodeReader::read_klass(const methodHandle& comp_method, bool shared) {
1369 uint code_offset = read_position();
1370 uint state = *(uint*)addr(code_offset);
1371 uint init_state = (state & 1);
1372 uint array_dim = (state >> 1);
1373 code_offset += sizeof(int);
1374 if (_cache->use_meta_ptrs() && shared) {
1375 uint klass_offset = *(uint*)addr(code_offset);
1376 code_offset += sizeof(uint);
1377 set_read_position(code_offset);
1378 Klass* k = (Klass*)((address)SharedBaseAddress + klass_offset);
1379 if (!MetaspaceShared::is_in_shared_metaspace((address)k)) {
1380 // Something changed in CDS
1381 set_lookup_failed();
1382 log_info(aot, codecache)("Lookup failed for shared klass: " INTPTR_FORMAT " is not in CDS ", p2i((address)k));
1383 return nullptr;
1384 }
1385 assert(k->is_klass(), "sanity");
1386 ResourceMark rm;
1387 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_loaded()) {
1388 set_lookup_failed();
1389 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for klass %s: not loaded",
1390 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
1391 return nullptr;
1392 } else
1393 // Allow not initialized klass which was uninitialized during code caching or for preload
1394 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_initialized() && (init_state == 1) && !_preload) {
1395 set_lookup_failed();
1396 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for klass %s: not initialized",
1397 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
1398 return nullptr;
1399 }
1400 if (array_dim > 0) {
1401 assert(k->is_instance_klass() || k->is_typeArray_klass(), "sanity check");
1402 Klass* ak = k->array_klass_or_null(array_dim);
1403 // FIXME: what would it take to create an array class on the fly?
1404 // Klass* ak = k->array_klass(dim, JavaThread::current());
1405 // guarantee(JavaThread::current()->pending_exception() == nullptr, "");
1406 if (ak == nullptr) {
1407 set_lookup_failed();
1408 log_info(aot, codecache)("%d (L%d): %d-dimension array klass lookup failed: %s",
1409 compile_id(), comp_level(), array_dim, k->external_name());
1410 }
1411 log_info(aot, codecache)("%d (L%d): Klass lookup: %s (object array)", compile_id(), comp_level(), k->external_name());
1412 return ak;
1413 } else {
1414 log_info(aot, codecache)("%d (L%d): Shared klass lookup: %s",
1415 compile_id(), comp_level(), k->external_name());
1416 return k;
1417 }
1418 }
1419 int name_length = *(int*)addr(code_offset);
1420 code_offset += sizeof(int);
1421 const char* dest = addr(code_offset);
1422 code_offset += name_length + 1;
1423 set_read_position(code_offset);
1424 TempNewSymbol klass_sym = SymbolTable::probe(&(dest[0]), name_length);
1425 if (klass_sym == nullptr) {
1426 set_lookup_failed();
1427 log_info(aot, codecache)("%d (L%d): Probe failed for class %s",
1428 compile_id(), comp_level(), &(dest[0]));
1429 return nullptr;
1430 }
1431 // Use class loader of compiled method.
1432 Thread* thread = Thread::current();
1433 Handle loader(thread, comp_method->method_holder()->class_loader());
1434 Klass* k = SystemDictionary::find_instance_or_array_klass(thread, klass_sym, loader);
1435 assert(!thread->has_pending_exception(), "should not throw");
1436 if (k == nullptr && !loader.is_null()) {
1437 // Try default loader and domain
1438 k = SystemDictionary::find_instance_or_array_klass(thread, klass_sym, Handle());
1439 assert(!thread->has_pending_exception(), "should not throw");
1440 }
1441 if (k != nullptr) {
1442 // Allow not initialized klass which was uninitialized during code caching
1443 if (k->is_instance_klass() && !InstanceKlass::cast(k)->is_initialized() && (init_state == 1)) {
1444 set_lookup_failed();
1445 log_info(aot, codecache)("%d (L%d): Lookup failed for klass %s: not initialized", compile_id(), comp_level(), &(dest[0]));
1446 return nullptr;
1447 }
1448 log_info(aot, codecache)("%d (L%d): Klass lookup %s", compile_id(), comp_level(), k->external_name());
1449 } else {
1450 set_lookup_failed();
1451 log_info(aot, codecache)("%d (L%d): Lookup failed for class %s", compile_id(), comp_level(), &(dest[0]));
1452 return nullptr;
1453 }
1454 return k;
1455 }
1456
1457 Method* AOTCodeReader::read_method(const methodHandle& comp_method, bool shared) {
1458 uint code_offset = read_position();
1459 if (_cache->use_meta_ptrs() && shared) {
1460 uint method_offset = *(uint*)addr(code_offset);
1461 code_offset += sizeof(uint);
1462 set_read_position(code_offset);
1463 Method* m = (Method*)((address)SharedBaseAddress + method_offset);
1464 if (!MetaspaceShared::is_in_shared_metaspace((address)m)) {
1465 // Something changed in CDS
1466 set_lookup_failed();
1467 log_info(aot, codecache)("Lookup failed for shared method: " INTPTR_FORMAT " is not in CDS ", p2i((address)m));
1468 return nullptr;
1469 }
1470 assert(m->is_method(), "sanity");
1471 ResourceMark rm;
1472 Klass* k = m->method_holder();
1473 if (!k->is_instance_klass()) {
1474 set_lookup_failed();
1475 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for holder %s: not instance klass",
1476 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
1477 return nullptr;
1478 } else if (!MetaspaceShared::is_in_shared_metaspace((address)k)) {
1479 set_lookup_failed();
1480 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for holder %s: not in CDS",
1481 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
1482 return nullptr;
1483 } else if (!InstanceKlass::cast(k)->is_loaded()) {
1484 set_lookup_failed();
1485 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for holder %s: not loaded",
1486 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name());
1487 return nullptr;
1488 } else if (!InstanceKlass::cast(k)->is_linked()) {
1489 set_lookup_failed();
1490 log_info(aot, codecache)("%d '%s' (L%d): Lookup failed for holder %s: not linked%s",
1491 compile_id(), comp_method->name_and_sig_as_C_string(), comp_level(), k->external_name(), (_preload ? " for code preload" : ""));
1492 return nullptr;
1493 }
1494 log_info(aot, codecache)("%d (L%d): Shared method lookup: %s",
1495 compile_id(), comp_level(), m->name_and_sig_as_C_string());
1496 return m;
1497 }
1498 int holder_length = *(int*)addr(code_offset);
1499 code_offset += sizeof(int);
1500 int name_length = *(int*)addr(code_offset);
1501 code_offset += sizeof(int);
1502 int signat_length = *(int*)addr(code_offset);
1503 code_offset += sizeof(int);
1504
1505 const char* dest = addr(code_offset);
1506 code_offset += holder_length + 1 + name_length + 1 + signat_length + 1;
1507 set_read_position(code_offset);
1508 TempNewSymbol klass_sym = SymbolTable::probe(&(dest[0]), holder_length);
1509 if (klass_sym == nullptr) {
1510 set_lookup_failed();
1511 log_info(aot, codecache)("%d (L%d): Probe failed for class %s", compile_id(), comp_level(), &(dest[0]));
1512 return nullptr;
1513 }
1514 // Use class loader of compiled method.
1515 Thread* thread = Thread::current();
1516 Handle loader(thread, comp_method->method_holder()->class_loader());
1517 Klass* k = SystemDictionary::find_instance_or_array_klass(thread, klass_sym, loader);
1518 assert(!thread->has_pending_exception(), "should not throw");
1519 if (k == nullptr && !loader.is_null()) {
1520 // Try default loader and domain
1521 k = SystemDictionary::find_instance_or_array_klass(thread, klass_sym, Handle());
1522 assert(!thread->has_pending_exception(), "should not throw");
1523 }
1524 if (k != nullptr) {
1525 if (!k->is_instance_klass()) {
1526 set_lookup_failed();
1527 log_info(aot, codecache)("%d (L%d): Lookup failed for holder %s: not instance klass",
1528 compile_id(), comp_level(), &(dest[0]));
1529 return nullptr;
1530 } else if (!InstanceKlass::cast(k)->is_linked()) {
1531 set_lookup_failed();
1532 log_info(aot, codecache)("%d (L%d): Lookup failed for holder %s: not linked",
1533 compile_id(), comp_level(), &(dest[0]));
1534 return nullptr;
1535 }
1536 log_info(aot, codecache)("%d (L%d): Holder lookup: %s", compile_id(), comp_level(), k->external_name());
1537 } else {
1538 set_lookup_failed();
1539 log_info(aot, codecache)("%d (L%d): Lookup failed for holder %s",
1540 compile_id(), comp_level(), &(dest[0]));
1541 return nullptr;
1542 }
1543 TempNewSymbol name_sym = SymbolTable::probe(&(dest[holder_length + 1]), name_length);
1544 int pos = holder_length + 1 + name_length + 1;
1545 TempNewSymbol sign_sym = SymbolTable::probe(&(dest[pos]), signat_length);
1546 if (name_sym == nullptr) {
1547 set_lookup_failed();
1548 log_info(aot, codecache)("%d (L%d): Probe failed for method name %s",
1549 compile_id(), comp_level(), &(dest[holder_length + 1]));
1550 return nullptr;
1551 }
1552 if (sign_sym == nullptr) {
1553 set_lookup_failed();
1554 log_info(aot, codecache)("%d (L%d): Probe failed for method signature %s",
1555 compile_id(), comp_level(), &(dest[pos]));
1556 return nullptr;
1557 }
1558 Method* m = InstanceKlass::cast(k)->find_method(name_sym, sign_sym);
1559 if (m != nullptr) {
1560 ResourceMark rm;
1561 log_info(aot, codecache)("%d (L%d): Method lookup: %s", compile_id(), comp_level(), m->name_and_sig_as_C_string());
1562 } else {
1563 set_lookup_failed();
1564 log_info(aot, codecache)("%d (L%d): Lookup failed for method %s::%s%s",
1565 compile_id(), comp_level(), &(dest[0]), &(dest[holder_length + 1]), &(dest[pos]));
1566 return nullptr;
1567 }
1568 return m;
1569 }
1570
1571 bool AOTCodeCache::write_klass(Klass* klass) {
1572 bool can_use_meta_ptrs = _use_meta_ptrs;
1573 uint array_dim = 0;
1574 if (klass->is_objArray_klass()) {
1575 array_dim = ObjArrayKlass::cast(klass)->dimension();
1576 klass = ObjArrayKlass::cast(klass)->bottom_klass(); // overwrites klass
1577 }
1578 uint init_state = 0;
1579 if (klass->is_instance_klass()) {
1580 InstanceKlass* ik = InstanceKlass::cast(klass);
1581 ClassLoaderData* cld = ik->class_loader_data();
1582 if (!cld->is_builtin_class_loader_data()) {
1583 set_lookup_failed();
1584 return false;
1585 }
1586 if (_for_preload && !CDSAccess::can_generate_cached_code(ik)) {
1587 _for_preload = false;
1588 // Bailout if code has clinit barriers:
1589 // method will be recompiled without them in any case
1590 if (_has_clinit_barriers) {
1591 set_lookup_failed();
1592 return false;
1593 }
1594 can_use_meta_ptrs = false;
1595 }
1596 init_state = (ik->is_initialized() ? 1 : 0);
1597 }
1598 ResourceMark rm;
1599 uint state = (array_dim << 1) | (init_state & 1);
1600 if (can_use_meta_ptrs && CDSAccess::can_generate_cached_code(klass)) {
1601 DataKind kind = DataKind::Klass_Shared;
1602 uint n = write_bytes(&kind, sizeof(int));
1603 if (n != sizeof(int)) {
1604 return false;
1605 }
1606 // Record state of instance klass initialization.
1607 n = write_bytes(&state, sizeof(int));
1608 if (n != sizeof(int)) {
1609 return false;
1610 }
1611 uint klass_offset = CDSAccess::delta_from_shared_address_base((address)klass);
1612 n = write_bytes(&klass_offset, sizeof(uint));
1613 if (n != sizeof(uint)) {
1614 return false;
1615 }
1616 log_info(aot, codecache)("%d (L%d): Wrote shared klass: %s%s%s @ 0x%08x", compile_id(), comp_level(), klass->external_name(),
1617 (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
1618 (array_dim > 0 ? " (object array)" : ""),
1619 klass_offset);
1620 return true;
1621 }
1622 // Bailout if code has clinit barriers:
1623 // method will be recompiled without them in any case
1624 if (_for_preload && _has_clinit_barriers) {
1625 set_lookup_failed();
1626 return false;
1627 }
1628 _for_preload = false;
1629 log_info(aot, codecache,cds)("%d (L%d): Not shared klass: %s", compile_id(), comp_level(), klass->external_name());
1630 if (klass->is_hidden()) { // Skip such nmethod
1631 set_lookup_failed();
1632 return false;
1633 }
1634 DataKind kind = DataKind::Klass;
1635 uint n = write_bytes(&kind, sizeof(int));
1636 if (n != sizeof(int)) {
1637 return false;
1638 }
1639 // Record state of instance klass initialization.
1640 n = write_bytes(&state, sizeof(int));
1641 if (n != sizeof(int)) {
1642 return false;
1643 }
1644 Symbol* name = klass->name();
1645 int name_length = name->utf8_length();
1646 int total_length = name_length + 1;
1647 char* dest = NEW_RESOURCE_ARRAY(char, total_length);
1648 name->as_C_string(dest, total_length);
1649 dest[total_length - 1] = '\0';
1650 LogTarget(Info, aot, codecache, loader) log;
1651 if (log.is_enabled()) {
1652 LogStream ls(log);
1653 oop loader = klass->class_loader();
1654 oop domain = klass->protection_domain();
1655 ls.print("Class %s loader: ", dest);
1656 if (loader == nullptr) {
1657 ls.print("nullptr");
1658 } else {
1659 loader->print_value_on(&ls);
1660 }
1661 ls.print(" domain: ");
1662 if (domain == nullptr) {
1663 ls.print("nullptr");
1664 } else {
1665 domain->print_value_on(&ls);
1666 }
1667 ls.cr();
1668 }
1669 n = write_bytes(&name_length, sizeof(int));
1670 if (n != sizeof(int)) {
1671 return false;
1672 }
1673 n = write_bytes(dest, total_length);
1674 if (n != (uint)total_length) {
1675 return false;
1676 }
1677 log_info(aot, codecache)("%d (L%d): Wrote klass: %s%s%s",
1678 compile_id(), comp_level(),
1679 dest, (!klass->is_instance_klass() ? "" : (init_state == 1 ? " (initialized)" : " (not-initialized)")),
1680 (array_dim > 0 ? " (object array)" : ""));
1681 return true;
1682 }
1683
1684 bool AOTCodeCache::write_method(Method* method) {
1685 bool can_use_meta_ptrs = _use_meta_ptrs;
1686 Klass* klass = method->method_holder();
1687 if (klass->is_instance_klass()) {
1688 InstanceKlass* ik = InstanceKlass::cast(klass);
1689 ClassLoaderData* cld = ik->class_loader_data();
1690 if (!cld->is_builtin_class_loader_data()) {
1691 set_lookup_failed();
1692 return false;
1693 }
1694 if (_for_preload && !CDSAccess::can_generate_cached_code(ik)) {
1695 _for_preload = false;
1696 // Bailout if code has clinit barriers:
1697 // method will be recompiled without them in any case
1698 if (_has_clinit_barriers) {
1699 set_lookup_failed();
1700 return false;
1701 }
1702 can_use_meta_ptrs = false;
1703 }
1704 }
1705 ResourceMark rm;
1706 if (can_use_meta_ptrs && CDSAccess::can_generate_cached_code(method)) {
1707 DataKind kind = DataKind::Method_Shared;
1708 uint n = write_bytes(&kind, sizeof(int));
1709 if (n != sizeof(int)) {
1710 return false;
1711 }
1712 uint method_offset = CDSAccess::delta_from_shared_address_base((address)method);
1713 n = write_bytes(&method_offset, sizeof(uint));
1714 if (n != sizeof(uint)) {
1715 return false;
1716 }
1717 log_info(aot, codecache)("%d (L%d): Wrote shared method: %s @ 0x%08x", compile_id(), comp_level(), method->name_and_sig_as_C_string(), method_offset);
1718 return true;
1719 }
1720 // Bailout if code has clinit barriers:
1721 // method will be recompiled without them in any case
1722 if (_for_preload && _has_clinit_barriers) {
1723 set_lookup_failed();
1724 return false;
1725 }
1726 _for_preload = false;
1727 log_info(aot, codecache,cds)("%d (L%d): Not shared method: %s", compile_id(), comp_level(), method->name_and_sig_as_C_string());
1728 if (method->is_hidden()) { // Skip such nmethod
1729 set_lookup_failed();
1730 return false;
1731 }
1732 DataKind kind = DataKind::Method;
1733 uint n = write_bytes(&kind, sizeof(int));
1734 if (n != sizeof(int)) {
1735 return false;
1736 }
1737 Symbol* name = method->name();
1738 Symbol* holder = method->klass_name();
1739 Symbol* signat = method->signature();
1740 int name_length = name->utf8_length();
1741 int holder_length = holder->utf8_length();
1742 int signat_length = signat->utf8_length();
1743
1744 // Write sizes and strings
1745 int total_length = holder_length + 1 + name_length + 1 + signat_length + 1;
1746 char* dest = NEW_RESOURCE_ARRAY(char, total_length);
1747 holder->as_C_string(dest, total_length);
1748 dest[holder_length] = '\0';
1749 int pos = holder_length + 1;
1750 name->as_C_string(&(dest[pos]), (total_length - pos));
1751 pos += name_length;
1752 dest[pos++] = '\0';
1753 signat->as_C_string(&(dest[pos]), (total_length - pos));
1754 dest[total_length - 1] = '\0';
1755
1756 LogTarget(Info, aot, codecache, loader) log;
1757 if (log.is_enabled()) {
1758 LogStream ls(log);
1759 oop loader = klass->class_loader();
1760 oop domain = klass->protection_domain();
1761 ls.print("Holder %s loader: ", dest);
1762 if (loader == nullptr) {
1763 ls.print("nullptr");
1764 } else {
1765 loader->print_value_on(&ls);
1766 }
1767 ls.print(" domain: ");
1768 if (domain == nullptr) {
1769 ls.print("nullptr");
1770 } else {
1771 domain->print_value_on(&ls);
1772 }
1773 ls.cr();
1774 }
1775
1776 n = write_bytes(&holder_length, sizeof(int));
1777 if (n != sizeof(int)) {
1778 return false;
1779 }
1780 n = write_bytes(&name_length, sizeof(int));
1781 if (n != sizeof(int)) {
1782 return false;
1783 }
1784 n = write_bytes(&signat_length, sizeof(int));
1785 if (n != sizeof(int)) {
1786 return false;
1787 }
1788 n = write_bytes(dest, total_length);
1789 if (n != (uint)total_length) {
1790 return false;
1791 }
1792 dest[holder_length] = ' ';
1793 dest[holder_length + 1 + name_length] = ' ';
1794 log_info(aot, codecache)("%d (L%d): Wrote method: %s", compile_id(), comp_level(), dest);
1795 return true;
1796 }
1797
1798 // Repair the pc relative information in the code after load
1799 bool AOTCodeReader::read_relocations(CodeBuffer* buffer, CodeBuffer* orig_buffer,
1800 OopRecorder* oop_recorder, ciMethod* target) {
1801 bool success = true;
1802 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
1803 uint code_offset = read_position();
1804 int reloc_count = *(int*)addr(code_offset);
1805 code_offset += sizeof(int);
1806 if (reloc_count == 0) {
1807 set_read_position(code_offset);
1808 continue;
1809 }
1810 // Read _locs_point (as offset from start)
1811 int locs_point_off = *(int*)addr(code_offset);
1812 code_offset += sizeof(int);
1813 uint reloc_size = reloc_count * sizeof(relocInfo);
1814 CodeSection* cs = buffer->code_section(i);
1815 if (cs->locs_capacity() < reloc_count) {
1816 cs->expand_locs(reloc_count);
1817 }
1818 relocInfo* reloc_start = cs->locs_start();
1819 copy_bytes(addr(code_offset), (address)reloc_start, reloc_size);
1820 code_offset += reloc_size;
1821 cs->set_locs_end(reloc_start + reloc_count);
1822 cs->set_locs_point(cs->start() + locs_point_off);
1823
1824 // Read additional relocation data: uint per relocation
1825 uint data_size = reloc_count * sizeof(uint);
1826 uint* reloc_data = (uint*)addr(code_offset);
1827 code_offset += data_size;
1828 set_read_position(code_offset);
1829 LogStreamHandle(Info, aot, codecache, reloc) log;
1830 if (log.is_enabled()) {
1831 log.print_cr("======== read code section %d relocations [%d]:", i, reloc_count);
1832 }
1833 RelocIterator iter(cs);
1834 int j = 0;
1835 while (iter.next()) {
1836 switch (iter.type()) {
1837 case relocInfo::none:
1838 break;
1839 case relocInfo::oop_type: {
1840 VM_ENTRY_MARK;
1841 oop_Relocation* r = (oop_Relocation*)iter.reloc();
1842 if (r->oop_is_immediate()) {
1843 assert(reloc_data[j] == (uint)j, "should be");
1844 methodHandle comp_method(THREAD, target->get_Method());
1845 oop obj = read_oop(THREAD, comp_method);
1846 jobject jo = JNIHandles::make_local(THREAD, obj);
1847 if (lookup_failed()) {
1848 success = false;
1849 break;
1850 }
1851 r->set_value((address)jo);
1852 } else if (false) {
1853 // Get already updated value from OopRecorder.
1854 assert(oop_recorder != nullptr, "sanity");
1855 int index = r->oop_index();
1856 jobject jo = oop_recorder->oop_at(index);
1857 oop obj = JNIHandles::resolve(jo);
1858 r->set_value(*reinterpret_cast<address*>(&obj));
1859 }
1860 break;
1861 }
1862 case relocInfo::metadata_type: {
1863 VM_ENTRY_MARK;
1864 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
1865 Metadata* m;
1866 if (r->metadata_is_immediate()) {
1867 assert(reloc_data[j] == (uint)j, "should be");
1868 methodHandle comp_method(THREAD, target->get_Method());
1869 m = read_metadata(comp_method);
1870 if (lookup_failed()) {
1871 success = false;
1872 break;
1873 }
1874 } else {
1875 // Get already updated value from OopRecorder.
1876 assert(oop_recorder != nullptr, "sanity");
1877 int index = r->metadata_index();
1878 m = oop_recorder->metadata_at(index);
1879 }
1880 r->set_value((address)m);
1881 break;
1882 }
1883 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
1884 case relocInfo::opt_virtual_call_type:
1885 case relocInfo::static_call_type: {
1886 address dest = _cache->address_for_id(reloc_data[j]);
1887 if (dest != (address)-1) {
1888 ((CallRelocation*)iter.reloc())->set_destination(dest);
1889 }
1890 break;
1891 }
1892 case relocInfo::trampoline_stub_type: {
1893 address dest = _cache->address_for_id(reloc_data[j]);
1894 if (dest != (address)-1) {
1895 ((trampoline_stub_Relocation*)iter.reloc())->set_destination(dest);
1896 }
1897 break;
1898 }
1899 case relocInfo::static_stub_type:
1900 break;
1901 case relocInfo::runtime_call_type: {
1902 address dest = _cache->address_for_id(reloc_data[j]);
1903 if (dest != (address)-1) {
1904 ((CallRelocation*)iter.reloc())->set_destination(dest);
1905 }
1906 break;
1907 }
1908 case relocInfo::runtime_call_w_cp_type:
1909 fatal("runtime_call_w_cp_type unimplemented");
1910 //address destination = iter.reloc()->value();
1911 break;
1912 case relocInfo::external_word_type: {
1913 address target = _cache->address_for_id(reloc_data[j]);
1914 // Add external address to global table
1915 int index = ExternalsRecorder::find_index(target);
1916 // Update index in relocation
1917 Relocation::add_jint(iter.data(), index);
1918 external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
1919 assert(reloc->target() == target, "sanity");
1920 reloc->set_value(target); // Patch address in the code
1921 iter.reloc()->fix_relocation_after_move(orig_buffer, buffer);
1922 break;
1923 }
1924 case relocInfo::internal_word_type:
1925 iter.reloc()->fix_relocation_after_move(orig_buffer, buffer);
1926 break;
1927 case relocInfo::section_word_type:
1928 iter.reloc()->fix_relocation_after_move(orig_buffer, buffer);
1929 break;
1930 case relocInfo::poll_type:
1931 break;
1932 case relocInfo::poll_return_type:
1933 break;
1934 case relocInfo::post_call_nop_type:
1935 break;
1936 case relocInfo::entry_guard_type:
1937 break;
1938 default:
1939 fatal("relocation %d unimplemented", (int)iter.type());
1940 break;
1941 }
1942 if (success && log.is_enabled()) {
1943 iter.print_current_on(&log);
1944 }
1945 j++;
1946 }
1947 assert(j <= (int)reloc_count, "sanity");
1948 }
1949 return success;
1950 }
1951
1952 bool AOTCodeReader::read_code(CodeBuffer* buffer, CodeBuffer* orig_buffer, uint code_offset) {
1953 assert(code_offset == align_up(code_offset, DATA_ALIGNMENT), "%d not aligned to %d", code_offset, DATA_ALIGNMENT);
1954 AOTCodeSection* aot_cs = (AOTCodeSection*)addr(code_offset);
1955 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
1956 CodeSection* cs = buffer->code_section(i);
1957 // Read original section size and address.
1958 uint orig_size = aot_cs[i]._size;
1959 log_debug(aot, codecache)("======== read code section %d [%d]:", i, orig_size);
1960 uint orig_size_align = align_up(orig_size, DATA_ALIGNMENT);
1961 if (i != (int)CodeBuffer::SECT_INSTS) {
1962 buffer->initialize_section_size(cs, orig_size_align);
1963 }
1964 if (orig_size_align > (uint)cs->capacity()) { // Will not fit
1965 log_info(aot, codecache)("%d (L%d): original code section %d size %d > current capacity %d",
1966 compile_id(), comp_level(), i, orig_size, cs->capacity());
1967 return false;
1968 }
1969 if (orig_size == 0) {
1970 assert(cs->size() == 0, "should match");
1971 continue; // skip trivial section
1972 }
1973 address orig_start = aot_cs[i]._origin_address;
1974
1975 // Populate fake original buffer (no code allocation in CodeCache).
1976 // It is used for relocations to calculate sections addesses delta.
1977 CodeSection* orig_cs = orig_buffer->code_section(i);
1978 assert(!orig_cs->is_allocated(), "This %d section should not be set", i);
1979 orig_cs->initialize(orig_start, orig_size);
1980
1981 // Load code to new buffer.
1982 address code_start = cs->start();
1983 copy_bytes(addr(aot_cs[i]._offset + code_offset), code_start, orig_size_align);
1984 cs->set_end(code_start + orig_size);
1985 }
1986
1987 return true;
1988 }
1989
1990 bool AOTCodeCache::load_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) {
1991 #ifdef ASSERT
1992 LogStreamHandle(Debug, aot, codecache, stubs) log;
1993 if (log.is_enabled()) {
1994 FlagSetting fs(PrintRelocations, true);
1995 buffer->print_on(&log);
1996 }
1997 #endif
1998 AOTCodeCache* cache = open_for_read();
1999 if (cache == nullptr) {
2000 return false;
2001 }
2002 log_info(aot, codecache, stubs)("Looking up adapter %s (0x%x) in AOT Code Cache", name, id);
2003 AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Adapter, id);
2004 if (entry == nullptr) {
2005 return false;
2006 }
2007 AOTCodeReader reader(cache, entry, nullptr);
2008 return reader.compile_adapter(buffer, name, offsets);
2009 }
2010 bool AOTCodeReader::compile_adapter(CodeBuffer* buffer, const char* name, uint32_t offsets[4]) {
2011 uint entry_position = _entry->offset();
2012 // Read name
2013 uint name_offset = entry_position + _entry->name_offset();
2014 uint name_size = _entry->name_size(); // Includes '/0'
2015 const char* stored_name = addr(name_offset);
2016 log_info(aot, codecache, stubs)("%d (L%d): Reading adapter '%s' from AOT Code Cache",
2017 compile_id(), comp_level(), name);
2018 if (strncmp(stored_name, name, (name_size - 1)) != 0) {
2019 log_warning(aot, codecache)("%d (L%d): Saved adapter's name '%s' is different from '%s'",
2020 compile_id(), comp_level(), stored_name, name);
2021 // n.b. this is not fatal -- we have just seen a hash id clash
2022 // so no need to call cache->set_failed()
2023 return false;
2024 }
2025 // Create fake original CodeBuffer
2026 CodeBuffer orig_buffer(name);
2027 // Read code
2028 uint code_offset = entry_position + _entry->code_offset();
2029 if (!read_code(buffer, &orig_buffer, code_offset)) {
2030 return false;
2031 }
2032 // Read relocations
2033 uint reloc_offset = entry_position + _entry->reloc_offset();
2034 set_read_position(reloc_offset);
2035 if (!read_relocations(buffer, &orig_buffer, nullptr, nullptr)) {
2036 return false;
2037 }
2038 uint offset = read_position();
2039 int offsets_count = *(int*)addr(offset);
2040 offset += sizeof(int);
2041 assert(offsets_count == 4, "wrong caller expectations");
2042 set_read_position(offset);
2043 for (int i = 0; i < offsets_count; i++) {
2044 uint32_t arg = *(uint32_t*)addr(offset);
2045 offset += sizeof(uint32_t);
2046 log_debug(aot, codecache, stubs)("%d (L%d): Reading adapter '%s' offsets[%d] == 0x%x from AOT Code Cache",
2047 compile_id(), comp_level(), stored_name, i, arg);
2048 offsets[i] = arg;
2049 }
2050 log_debug(aot, codecache, stubs)("%d (L%d): Read adapter '%s' with '%d' args from AOT Code Cache",
2051 compile_id(), comp_level(), stored_name, offsets_count);
2052 #ifdef ASSERT
2053 LogStreamHandle(Debug, aot, codecache, stubs) log;
2054 if (log.is_enabled()) {
2055 FlagSetting fs(PrintRelocations, true);
2056 buffer->print_on(&log);
2057 buffer->decode();
2058 }
2059 #endif
2060 // mark entry as loaded
2061 ((AOTCodeEntry *)_entry)->set_loaded();
2062 return true;
2063 }
2064
2065 bool AOTCodeCache::load_exception_blob(CodeBuffer* buffer, int* pc_offset) {
2066 #ifdef ASSERT
2067 LogStreamHandle(Debug, aot, codecache, nmethod) log;
2068 if (log.is_enabled()) {
2069 FlagSetting fs(PrintRelocations, true);
2070 buffer->print_on(&log);
2071 }
2072 #endif
2073 AOTCodeCache* cache = open_for_read();
2074 if (cache == nullptr) {
2075 return false;
2076 }
2077 AOTCodeEntry* entry = cache->find_entry(AOTCodeEntry::Blob, 999);
2078 if (entry == nullptr) {
2079 return false;
2080 }
2081 AOTCodeReader reader(cache, entry, nullptr);
2082 return reader.compile_blob(buffer, pc_offset);
2083 }
2084
2085 bool AOTCodeReader::compile_blob(CodeBuffer* buffer, int* pc_offset) {
2086 uint entry_position = _entry->offset();
2087
2088 // Read pc_offset
2089 *pc_offset = *(int*)addr(entry_position);
2090
2091 // Read name
2092 uint name_offset = entry_position + _entry->name_offset();
2093 uint name_size = _entry->name_size(); // Includes '/0'
2094 const char* name = addr(name_offset);
2095
2096 log_info(aot, codecache, stubs)("%d (L%d): Reading blob '%s' with pc_offset %d from AOT Code Cache",
2097 compile_id(), comp_level(), name, *pc_offset);
2098
2099 if (strncmp(buffer->name(), name, (name_size - 1)) != 0) {
2100 log_warning(aot, codecache)("%d (L%d): Saved blob's name '%s' is different from '%s'",
2101 compile_id(), comp_level(), name, buffer->name());
2102 ((AOTCodeCache*)_cache)->set_failed();
2103 exit_vm_on_load_failure();
2104 return false;
2105 }
2106
2107 // Create fake original CodeBuffer
2108 CodeBuffer orig_buffer(name);
2109
2110 // Read code
2111 uint code_offset = entry_position + _entry->code_offset();
2112 if (!read_code(buffer, &orig_buffer, code_offset)) {
2113 return false;
2114 }
2115
2116 // Read relocations
2117 uint reloc_offset = entry_position + _entry->reloc_offset();
2118 set_read_position(reloc_offset);
2119 if (!read_relocations(buffer, &orig_buffer, nullptr, nullptr)) {
2120 return false;
2121 }
2122
2123 log_info(aot, codecache, stubs)("%d (L%d): Read blob '%s' from AOT Code Cache",
2124 compile_id(), comp_level(), name);
2125 #ifdef ASSERT
2126 LogStreamHandle(Debug, aot, codecache, nmethod) log;
2127 if (log.is_enabled()) {
2128 FlagSetting fs(PrintRelocations, true);
2129 buffer->print_on(&log);
2130 buffer->decode();
2131 }
2132 #endif
2133 return true;
2134 }
2135
2136 bool AOTCodeCache::write_relocations(CodeBuffer* buffer, uint& all_reloc_size) {
2137 uint all_reloc_count = 0;
2138 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
2139 CodeSection* cs = buffer->code_section(i);
2140 uint reloc_count = cs->has_locs() ? cs->locs_count() : 0;
2141 all_reloc_count += reloc_count;
2142 }
2143 all_reloc_size = all_reloc_count * sizeof(relocInfo);
2144 bool success = true;
2145 uint* reloc_data = NEW_C_HEAP_ARRAY(uint, all_reloc_count, mtCode);
2146 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
2147 CodeSection* cs = buffer->code_section(i);
2148 int reloc_count = cs->has_locs() ? cs->locs_count() : 0;
2149 uint n = write_bytes(&reloc_count, sizeof(int));
2150 if (n != sizeof(int)) {
2151 success = false;
2152 break;
2153 }
2154 if (reloc_count == 0) {
2155 continue;
2156 }
2157 // Write _locs_point (as offset from start)
2158 int locs_point_off = cs->locs_point_off();
2159 n = write_bytes(&locs_point_off, sizeof(int));
2160 if (n != sizeof(int)) {
2161 success = false;
2162 break;
2163 }
2164 relocInfo* reloc_start = cs->locs_start();
2165 uint reloc_size = reloc_count * sizeof(relocInfo);
2166 n = write_bytes(reloc_start, reloc_size);
2167 if (n != reloc_size) {
2168 success = false;
2169 break;
2170 }
2171 LogStreamHandle(Info, aot, codecache, reloc) log;
2172 if (log.is_enabled()) {
2173 log.print_cr("======== write code section %d relocations [%d]:", i, reloc_count);
2174 }
2175 // Collect additional data
2176 RelocIterator iter(cs);
2177 bool has_immediate = false;
2178 int j = 0;
2179 while (iter.next()) {
2180 reloc_data[j] = 0; // initialize
2181 switch (iter.type()) {
2182 case relocInfo::none:
2183 break;
2184 case relocInfo::oop_type: {
2185 oop_Relocation* r = (oop_Relocation*)iter.reloc();
2186 if (r->oop_is_immediate()) {
2187 reloc_data[j] = (uint)j; // Indication that we need to restore immediate
2188 has_immediate = true;
2189 }
2190 break;
2191 }
2192 case relocInfo::metadata_type: {
2193 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
2194 if (r->metadata_is_immediate()) {
2195 reloc_data[j] = (uint)j; // Indication that we need to restore immediate
2196 has_immediate = true;
2197 }
2198 break;
2199 }
2200 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
2201 case relocInfo::opt_virtual_call_type:
2202 case relocInfo::static_call_type: {
2203 CallRelocation* r = (CallRelocation*)iter.reloc();
2204 address dest = r->destination();
2205 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2206 dest = (address)-1; // do nothing in this case when loading this relocation
2207 }
2208 reloc_data[j] = _table->id_for_address(dest, iter, buffer);
2209 break;
2210 }
2211 case relocInfo::trampoline_stub_type: {
2212 address dest = ((trampoline_stub_Relocation*)iter.reloc())->destination();
2213 reloc_data[j] = _table->id_for_address(dest, iter, buffer);
2214 break;
2215 }
2216 case relocInfo::static_stub_type:
2217 break;
2218 case relocInfo::runtime_call_type: {
2219 // Record offset of runtime destination
2220 CallRelocation* r = (CallRelocation*)iter.reloc();
2221 address dest = r->destination();
2222 if (dest == r->addr()) { // possible call via trampoline on Aarch64
2223 dest = (address)-1; // do nothing in this case when loading this relocation
2224 }
2225 reloc_data[j] = _table->id_for_address(dest, iter, buffer);
2226 break;
2227 }
2228 case relocInfo::runtime_call_w_cp_type:
2229 fatal("runtime_call_w_cp_type unimplemented");
2230 break;
2231 case relocInfo::external_word_type: {
2232 // Record offset of runtime target
2233 address target = ((external_word_Relocation*)iter.reloc())->target();
2234 reloc_data[j] = _table->id_for_address(target, iter, buffer);
2235 break;
2236 }
2237 case relocInfo::internal_word_type:
2238 break;
2239 case relocInfo::section_word_type:
2240 break;
2241 case relocInfo::poll_type:
2242 break;
2243 case relocInfo::poll_return_type:
2244 break;
2245 case relocInfo::post_call_nop_type:
2246 break;
2247 case relocInfo::entry_guard_type:
2248 break;
2249 default:
2250 fatal("relocation %d unimplemented", (int)iter.type());
2251 break;
2252 }
2253 if (log.is_enabled()) {
2254 iter.print_current_on(&log);
2255 }
2256 j++;
2257 }
2258 assert(j <= (int)reloc_count, "sanity");
2259 // Write additional relocation data: uint per relocation
2260 uint data_size = reloc_count * sizeof(uint);
2261 n = write_bytes(reloc_data, data_size);
2262 if (n != data_size) {
2263 success = false;
2264 break;
2265 }
2266 if (has_immediate) {
2267 // Save information about immediates in this Code Section
2268 RelocIterator iter_imm(cs);
2269 int j = 0;
2270 while (iter_imm.next()) {
2271 switch (iter_imm.type()) {
2272 case relocInfo::oop_type: {
2273 oop_Relocation* r = (oop_Relocation*)iter_imm.reloc();
2274 if (r->oop_is_immediate()) {
2275 assert(reloc_data[j] == (uint)j, "should be");
2276 jobject jo = *(jobject*)(r->oop_addr()); // Handle currently
2277 if (!write_oop(jo)) {
2278 success = false;
2279 }
2280 }
2281 break;
2282 }
2283 case relocInfo::metadata_type: {
2284 metadata_Relocation* r = (metadata_Relocation*)iter_imm.reloc();
2285 if (r->metadata_is_immediate()) {
2286 assert(reloc_data[j] == (uint)j, "should be");
2287 Metadata* m = r->metadata_value();
2288 if (!write_metadata(m)) {
2289 success = false;
2290 }
2291 }
2292 break;
2293 }
2294 default:
2295 break;
2296 }
2297 if (!success) {
2298 break;
2299 }
2300 j++;
2301 } // while (iter_imm.next())
2302 } // if (has_immediate)
2303 } // for(i < SECT_LIMIT)
2304 FREE_C_HEAP_ARRAY(uint, reloc_data);
2305 return success;
2306 }
2307
2308 bool AOTCodeCache::store_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) {
2309 assert(CDSConfig::is_dumping_adapters(), "must be");
2310 AOTCodeCache* cache = open_for_write();
2311 if (cache == nullptr) {
2312 return false;
2313 }
2314 log_info(aot, codecache, stubs)("Writing adapter '%s' (0x%x) to AOT Code Cache", name, id);
2315 #ifdef ASSERT
2316 LogStreamHandle(Debug, aot, codecache, stubs) log;
2317 if (log.is_enabled()) {
2318 FlagSetting fs(PrintRelocations, true);
2319 buffer->print_on(&log);
2320 buffer->decode();
2321 }
2322 #endif
2323 // we need to take a lock to stop main thread racing with C1 and C2 compiler threads to
2324 // write blobs in parallel with each other or with later nmethods
2325 MutexLocker ml(Compile_lock);
2326 if (!cache->align_write()) {
2327 return false;
2328 }
2329 uint entry_position = cache->_write_position;
2330 // Write name
2331 uint name_offset = cache->_write_position - entry_position;
2332 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
2333 uint n = cache->write_bytes(name, name_size);
2334 if (n != name_size) {
2335 return false;
2336 }
2337 // Write code section
2338 if (!cache->align_write()) {
2339 return false;
2340 }
2341 uint code_offset = cache->_write_position - entry_position;
2342 uint code_size = 0;
2343 if (!cache->write_code(buffer, code_size)) {
2344 return false;
2345 }
2346 // Write relocInfo array
2347 uint reloc_offset = cache->_write_position - entry_position;
2348 uint reloc_size = 0;
2349 if (!cache->write_relocations(buffer, reloc_size)) {
2350 return false;
2351 }
2352 int extras_count = 4;
2353 n = cache->write_bytes(&extras_count, sizeof(int));
2354 if (n != sizeof(int)) {
2355 return false;
2356 }
2357 for (int i = 0; i < 4; i++) {
2358 uint32_t arg = offsets[i];
2359 log_debug(aot, codecache, stubs)("Writing adapter '%s' (0x%x) offsets[%d] == 0x%x to AOT Code Cache", name, id, i, arg);
2360 n = cache->write_bytes(&arg, sizeof(uint32_t));
2361 if (n != sizeof(uint32_t)) {
2362 return false;
2363 }
2364 }
2365 uint entry_size = cache->_write_position - entry_position;
2366 AOTCodeEntry* entry = new (cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
2367 code_offset, code_size, reloc_offset, reloc_size,
2368 AOTCodeEntry::Adapter, id);
2369 log_info(aot, codecache, stubs)("Wrote adapter '%s' (0x%x) to AOT Code Cache", name, id);
2370 return true;
2371 }
2372
2373 bool AOTCodeCache::write_code(CodeBuffer* buffer, uint& code_size) {
2374 assert(_write_position == align_up(_write_position, DATA_ALIGNMENT), "%d not aligned to %d", _write_position, DATA_ALIGNMENT);
2375 //assert(buffer->blob() != nullptr, "sanity");
2376 uint code_offset = _write_position;
2377 uint cb_total_size = (uint)buffer->total_content_size();
2378 // Write information about Code sections first.
2379 AOTCodeSection aot_cs[CodeBuffer::SECT_LIMIT];
2380 uint aot_cs_size = (uint)(sizeof(AOTCodeSection) * CodeBuffer::SECT_LIMIT);
2381 uint offset = align_up(aot_cs_size, DATA_ALIGNMENT);
2382 uint total_size = 0;
2383 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
2384 const CodeSection* cs = buffer->code_section(i);
2385 assert(cs->mark() == nullptr, "CodeSection::_mark is not implemented");
2386 uint cs_size = (uint)cs->size();
2387 aot_cs[i]._size = cs_size;
2388 aot_cs[i]._origin_address = (cs_size == 0) ? nullptr : cs->start();
2389 aot_cs[i]._offset = (cs_size == 0) ? 0 : (offset + total_size);
2390 assert(cs->mark() == nullptr, "CodeSection::_mark is not implemented");
2391 total_size += align_up(cs_size, DATA_ALIGNMENT);
2392 }
2393 uint n = write_bytes(aot_cs, aot_cs_size);
2394 if (n != aot_cs_size) {
2395 return false;
2396 }
2397 if (!align_write()) {
2398 return false;
2399 }
2400 assert(_write_position == (code_offset + offset), "%d != (%d + %d)", _write_position, code_offset, offset);
2401 for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) {
2402 const CodeSection* cs = buffer->code_section(i);
2403 uint cs_size = (uint)cs->size();
2404 if (cs_size == 0) {
2405 continue; // skip trivial section
2406 }
2407 assert((_write_position - code_offset) == aot_cs[i]._offset, "%d != %d", _write_position, aot_cs[i]._offset);
2408 // Write code
2409 n = write_bytes(cs->start(), cs_size);
2410 if (n != cs_size) {
2411 return false;
2412 }
2413 if (!align_write()) {
2414 return false;
2415 }
2416 }
2417 assert((_write_position - code_offset) == (offset + total_size), "(%d - %d) != (%d + %d)", _write_position, code_offset, offset, total_size);
2418 code_size = total_size;
2419 return true;
2420 }
2421
2422 bool AOTCodeCache::store_exception_blob(CodeBuffer* buffer, int pc_offset) {
2423 AOTCodeCache* cache = open_for_write();
2424 if (cache == nullptr) {
2425 return false;
2426 }
2427 log_info(aot, codecache, stubs)("Writing blob '%s' to AOT Code Cache", buffer->name());
2428
2429 #ifdef ASSERT
2430 LogStreamHandle(Debug, aot, codecache, nmethod) log;
2431 if (log.is_enabled()) {
2432 FlagSetting fs(PrintRelocations, true);
2433 buffer->print_on(&log);
2434 buffer->decode();
2435 }
2436 #endif
2437 // we need to take a lock to prevent race between compiler thread generating blob and the main thread generating adapter
2438 MutexLocker ml(Compile_lock);
2439 if (!cache->align_write()) {
2440 return false;
2441 }
2442 uint entry_position = cache->_write_position;
2443
2444 // Write pc_offset
2445 uint n = cache->write_bytes(&pc_offset, sizeof(int));
2446 if (n != sizeof(int)) {
2447 return false;
2448 }
2449
2450 // Write name
2451 const char* name = buffer->name();
2452 uint name_offset = cache->_write_position - entry_position;
2453 uint name_size = (uint)strlen(name) + 1; // Includes '/0'
2454 n = cache->write_bytes(name, name_size);
2455 if (n != name_size) {
2456 return false;
2457 }
2458
2459 // Write code section
2460 if (!cache->align_write()) {
2461 return false;
2462 }
2463 uint code_offset = cache->_write_position - entry_position;
2464 uint code_size = 0;
2465 if (!cache->write_code(buffer, code_size)) {
2466 return false;
2467 }
2468 // Write relocInfo array
2469 uint reloc_offset = cache->_write_position - entry_position;
2470 uint reloc_size = 0;
2471 if (!cache->write_relocations(buffer, reloc_size)) {
2472 return false;
2473 }
2474
2475 uint entry_size = cache->_write_position - entry_position;
2476 AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
2477 code_offset, code_size, reloc_offset, reloc_size,
2478 AOTCodeEntry::Blob, (uint32_t)999);
2479 log_info(aot, codecache, stubs)("Wrote stub '%s' to AOT Code Cache", name);
2480 return true;
2481 }
2482
2483 DebugInformationRecorder* AOTCodeReader::read_debug_info(OopRecorder* oop_recorder) {
2484 uint code_offset = align_up(read_position(), DATA_ALIGNMENT);
2485 int data_size = *(int*)addr(code_offset);
2486 code_offset += sizeof(int);
2487 int pcs_length = *(int*)addr(code_offset);
2488 code_offset += sizeof(int);
2489
2490 log_debug(aot, codecache)("======== read DebugInfo [%d, %d]:", data_size, pcs_length);
2491
2492 // Aligned initial sizes
2493 int data_size_align = align_up(data_size, DATA_ALIGNMENT);
2494 int pcs_length_align = pcs_length + 1;
2495 assert(sizeof(PcDesc) > DATA_ALIGNMENT, "sanity");
2496 DebugInformationRecorder* recorder = new DebugInformationRecorder(oop_recorder, data_size_align, pcs_length);
2497
2498 copy_bytes(addr(code_offset), recorder->stream()->buffer(), data_size_align);
2499 recorder->stream()->set_position(data_size);
2500 code_offset += data_size;
2501
2502 uint pcs_size = pcs_length * sizeof(PcDesc);
2503 copy_bytes(addr(code_offset), (address)recorder->pcs(), pcs_size);
2504 code_offset += pcs_size;
2505 set_read_position(code_offset);
2506 return recorder;
2507 }
2508
2509 bool AOTCodeCache::write_debug_info(DebugInformationRecorder* recorder) {
2510 if (!align_write()) {
2511 return false;
2512 }
2513 // Don't call data_size() and pcs_size(). They will freeze OopRecorder.
2514 int data_size = recorder->stream()->position(); // In bytes
2515 uint n = write_bytes(&data_size, sizeof(int));
2516 if (n != sizeof(int)) {
2517 return false;
2518 }
2519 int pcs_length = recorder->pcs_length(); // In bytes
2520 n = write_bytes(&pcs_length, sizeof(int));
2521 if (n != sizeof(int)) {
2522 return false;
2523 }
2524 n = write_bytes(recorder->stream()->buffer(), data_size);
2525 if (n != (uint)data_size) {
2526 return false;
2527 }
2528 uint pcs_size = pcs_length * sizeof(PcDesc);
2529 n = write_bytes(recorder->pcs(), pcs_size);
2530 if (n != pcs_size) {
2531 return false;
2532 }
2533 return true;
2534 }
2535
2536 OopMapSet* AOTCodeReader::read_oop_maps() {
2537 uint code_offset = read_position();
2538 int om_count = *(int*)addr(code_offset);
2539 code_offset += sizeof(int);
2540
2541 log_debug(aot, codecache)("======== read oop maps [%d]:", om_count);
2542
2543 OopMapSet* oop_maps = new OopMapSet(om_count);
2544 for (int i = 0; i < (int)om_count; i++) {
2545 int data_size = *(int*)addr(code_offset);
2546 code_offset += sizeof(int);
2547
2548 OopMap* oop_map = new OopMap(data_size);
2549 // Preserve allocated stream
2550 CompressedWriteStream* stream = oop_map->write_stream();
2551
2552 // Read data which overwrites default data
2553 copy_bytes(addr(code_offset), (address)oop_map, sizeof(OopMap));
2554 code_offset += sizeof(OopMap);
2555 stream->set_position(data_size);
2556 oop_map->set_write_stream(stream);
2557 if (data_size > 0) {
2558 copy_bytes(addr(code_offset), (address)(oop_map->data()), (uint)data_size);
2559 code_offset += data_size;
2560 }
2561 #ifdef ASSERT
2562 oop_map->_locs_length = 0;
2563 oop_map->_locs_used = nullptr;
2564 #endif
2565 oop_maps->add(oop_map);
2566 }
2567 set_read_position(code_offset);
2568 return oop_maps;
2569 }
2570
2571 bool AOTCodeCache::write_oop_maps(OopMapSet* oop_maps) {
2572 uint om_count = oop_maps->size();
2573 uint n = write_bytes(&om_count, sizeof(int));
2574 if (n != sizeof(int)) {
2575 return false;
2576 }
2577 for (int i = 0; i < (int)om_count; i++) {
2578 OopMap* om = oop_maps->at(i);
2579 int data_size = om->data_size();
2580 n = write_bytes(&data_size, sizeof(int));
2581 if (n != sizeof(int)) {
2582 return false;
2583 }
2584 n = write_bytes(om, sizeof(OopMap));
2585 if (n != sizeof(OopMap)) {
2586 return false;
2587 }
2588 n = write_bytes(om->data(), (uint)data_size);
2589 if (n != (uint)data_size) {
2590 return false;
2591 }
2592 }
2593 return true;
2594 }
2595
2596 oop AOTCodeReader::read_oop(JavaThread* thread, const methodHandle& comp_method) {
2597 uint code_offset = read_position();
2598 oop obj = nullptr;
2599 DataKind kind = *(DataKind*)addr(code_offset);
2600 code_offset += sizeof(DataKind);
2601 set_read_position(code_offset);
2602 if (kind == DataKind::Null) {
2603 return nullptr;
2604 } else if (kind == DataKind::No_Data) {
2605 return cast_to_oop(Universe::non_oop_word());
2606 } else if (kind == DataKind::Klass || kind == DataKind::Klass_Shared) {
2607 Klass* k = read_klass(comp_method, (kind == DataKind::Klass_Shared));
2608 if (k == nullptr) {
2609 return nullptr;
2610 }
2611 obj = k->java_mirror();
2612 if (obj == nullptr) {
2613 set_lookup_failed();
2614 log_info(aot, codecache)("Lookup failed for java_mirror of klass %s", k->external_name());
2615 return nullptr;
2616 }
2617 } else if (kind == DataKind::Primitive) {
2618 code_offset = read_position();
2619 int t = *(int*)addr(code_offset);
2620 code_offset += sizeof(int);
2621 set_read_position(code_offset);
2622 BasicType bt = (BasicType)t;
2623 obj = java_lang_Class::primitive_mirror(bt);
2624 log_info(aot, codecache)("%d (L%d): Read primitive type klass: %s", compile_id(), comp_level(), type2name(bt));
2625 } else if (kind == DataKind::String_Shared) {
2626 code_offset = read_position();
2627 int k = *(int*)addr(code_offset);
2628 code_offset += sizeof(int);
2629 set_read_position(code_offset);
2630 obj = CDSAccess::get_archived_object(k);
2631 } else if (kind == DataKind::String) {
2632 code_offset = read_position();
2633 int length = *(int*)addr(code_offset);
2634 code_offset += sizeof(int);
2635 set_read_position(code_offset);
2636 const char* dest = addr(code_offset);
2637 set_read_position(code_offset + length);
2638 obj = StringTable::intern(&(dest[0]), thread);
2639 if (obj == nullptr) {
2640 set_lookup_failed();
2641 log_info(aot, codecache)("%d (L%d): Lookup failed for String %s",
2642 compile_id(), comp_level(), &(dest[0]));
2643 return nullptr;
2644 }
2645 assert(java_lang_String::is_instance(obj), "must be string");
2646 log_info(aot, codecache)("%d (L%d): Read String: %s", compile_id(), comp_level(), dest);
2647 } else if (kind == DataKind::SysLoader) {
2648 obj = SystemDictionary::java_system_loader();
2649 log_info(aot, codecache)("%d (L%d): Read java_system_loader", compile_id(), comp_level());
2650 } else if (kind == DataKind::PlaLoader) {
2651 obj = SystemDictionary::java_platform_loader();
2652 log_info(aot, codecache)("%d (L%d): Read java_platform_loader", compile_id(), comp_level());
2653 } else if (kind == DataKind::MH_Oop_Shared) {
2654 code_offset = read_position();
2655 int k = *(int*)addr(code_offset);
2656 code_offset += sizeof(int);
2657 set_read_position(code_offset);
2658 obj = CDSAccess::get_archived_object(k);
2659 } else {
2660 set_lookup_failed();
2661 log_info(aot, codecache)("%d (L%d): Unknown oop's kind: %d",
2662 compile_id(), comp_level(), (int)kind);
2663 return nullptr;
2664 }
2665 return obj;
2666 }
2667
2668 bool AOTCodeReader::read_oops(OopRecorder* oop_recorder, ciMethod* target) {
2669 uint code_offset = read_position();
2670 int oop_count = *(int*)addr(code_offset);
2671 code_offset += sizeof(int);
2672 set_read_position(code_offset);
2673 log_debug(aot, codecache)("======== read oops [%d]:", oop_count);
2674 if (oop_count == 0) {
2675 return true;
2676 }
2677 {
2678 VM_ENTRY_MARK;
2679 methodHandle comp_method(THREAD, target->get_Method());
2680 for (int i = 1; i < oop_count; i++) {
2681 oop obj = read_oop(THREAD, comp_method);
2682 if (lookup_failed()) {
2683 return false;
2684 }
2685 jobject jo = JNIHandles::make_local(THREAD, obj);
2686 if (oop_recorder->is_real(jo)) {
2687 oop_recorder->find_index(jo);
2688 } else {
2689 oop_recorder->allocate_oop_index(jo);
2690 }
2691 LogStreamHandle(Debug, aot, codecache, oops) log;
2692 if (log.is_enabled()) {
2693 log.print("%d: " INTPTR_FORMAT " ", i, p2i(jo));
2694 if (jo == (jobject)Universe::non_oop_word()) {
2695 log.print("non-oop word");
2696 } else if (jo == nullptr) {
2697 log.print("nullptr-oop");
2698 } else {
2699 JNIHandles::resolve(jo)->print_value_on(&log);
2700 }
2701 log.cr();
2702 }
2703 }
2704 }
2705 return true;
2706 }
2707
2708 Metadata* AOTCodeReader::read_metadata(const methodHandle& comp_method) {
2709 uint code_offset = read_position();
2710 Metadata* m = nullptr;
2711 DataKind kind = *(DataKind*)addr(code_offset);
2712 code_offset += sizeof(DataKind);
2713 set_read_position(code_offset);
2714 if (kind == DataKind::Null) {
2715 m = (Metadata*)nullptr;
2716 } else if (kind == DataKind::No_Data) {
2717 m = (Metadata*)Universe::non_oop_word();
2718 } else if (kind == DataKind::Klass || kind == DataKind::Klass_Shared) {
2719 m = (Metadata*)read_klass(comp_method, (kind == DataKind::Klass_Shared));
2720 } else if (kind == DataKind::Method || kind == DataKind::Method_Shared) {
2721 m = (Metadata*)read_method(comp_method, (kind == DataKind::Method_Shared));
2722 } else if (kind == DataKind::MethodCnts) {
2723 kind = *(DataKind*)addr(code_offset);
2724 bool shared = (kind == DataKind::Method_Shared);
2725 assert(kind == DataKind::Method || shared, "Sanity");
2726 code_offset += sizeof(DataKind);
2727 set_read_position(code_offset);
2728 m = (Metadata*)read_method(comp_method, shared);
2729 if (m != nullptr) {
2730 Method* method = (Method*)m;
2731 m = method->get_method_counters(Thread::current());
2732 if (m == nullptr) {
2733 set_lookup_failed();
2734 log_info(aot, codecache)("%d (L%d): Failed to get MethodCounters", compile_id(), comp_level());
2735 } else {
2736 log_info(aot, codecache)("%d (L%d): Read MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2737 }
2738 }
2739 } else {
2740 set_lookup_failed();
2741 log_info(aot, codecache)("%d (L%d): Unknown metadata's kind: %d", compile_id(), comp_level(), (int)kind);
2742 }
2743 return m;
2744 }
2745
2746 bool AOTCodeReader::read_metadata(OopRecorder* oop_recorder, ciMethod* target) {
2747 uint code_offset = read_position();
2748 int metadata_count = *(int*)addr(code_offset);
2749 code_offset += sizeof(int);
2750 set_read_position(code_offset);
2751
2752 log_debug(aot, codecache)("======== read metadata [%d]:", metadata_count);
2753
2754 if (metadata_count == 0) {
2755 return true;
2756 }
2757 {
2758 VM_ENTRY_MARK;
2759 methodHandle comp_method(THREAD, target->get_Method());
2760
2761 for (int i = 1; i < metadata_count; i++) {
2762 Metadata* m = read_metadata(comp_method);
2763 if (lookup_failed()) {
2764 return false;
2765 }
2766 if (oop_recorder->is_real(m)) {
2767 oop_recorder->find_index(m);
2768 } else {
2769 oop_recorder->allocate_metadata_index(m);
2770 }
2771 LogTarget(Debug, aot, codecache, metadata) log;
2772 if (log.is_enabled()) {
2773 LogStream ls(log);
2774 ls.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
2775 if (m == (Metadata*)Universe::non_oop_word()) {
2776 ls.print("non-metadata word");
2777 } else if (m == nullptr) {
2778 ls.print("nullptr-oop");
2779 } else {
2780 Metadata::print_value_on_maybe_null(&ls, m);
2781 }
2782 ls.cr();
2783 }
2784 }
2785 }
2786 return true;
2787 }
2788
2789 bool AOTCodeCache::write_oop(jobject& jo) {
2790 oop obj = JNIHandles::resolve(jo);
2791 return write_oop(obj);
2792 }
2793
2794 bool AOTCodeCache::write_oop(oop obj) {
2795 DataKind kind;
2796 uint n = 0;
2797 if (obj == nullptr) {
2798 kind = DataKind::Null;
2799 n = write_bytes(&kind, sizeof(int));
2800 if (n != sizeof(int)) {
2801 return false;
2802 }
2803 } else if (cast_from_oop<void *>(obj) == Universe::non_oop_word()) {
2804 kind = DataKind::No_Data;
2805 n = write_bytes(&kind, sizeof(int));
2806 if (n != sizeof(int)) {
2807 return false;
2808 }
2809 } else if (java_lang_Class::is_instance(obj)) {
2810 if (java_lang_Class::is_primitive(obj)) {
2811 int bt = (int)java_lang_Class::primitive_type(obj);
2812 kind = DataKind::Primitive;
2813 n = write_bytes(&kind, sizeof(int));
2814 if (n != sizeof(int)) {
2815 return false;
2816 }
2817 n = write_bytes(&bt, sizeof(int));
2818 if (n != sizeof(int)) {
2819 return false;
2820 }
2821 log_info(aot, codecache)("%d (L%d): Write primitive type klass: %s", compile_id(), comp_level(), type2name((BasicType)bt));
2822 } else {
2823 Klass* klass = java_lang_Class::as_Klass(obj);
2824 if (!write_klass(klass)) {
2825 return false;
2826 }
2827 }
2828 } else if (java_lang_String::is_instance(obj)) { // herere
2829 int k = CDSAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2830 if (k >= 0) {
2831 kind = DataKind::String_Shared;
2832 n = write_bytes(&kind, sizeof(int));
2833 if (n != sizeof(int)) {
2834 return false;
2835 }
2836 n = write_bytes(&k, sizeof(int));
2837 if (n != sizeof(int)) {
2838 return false;
2839 }
2840 return true;
2841 }
2842 kind = DataKind::String;
2843 n = write_bytes(&kind, sizeof(int));
2844 if (n != sizeof(int)) {
2845 return false;
2846 }
2847 ResourceMark rm;
2848 size_t length_sz = 0;
2849 const char* string = java_lang_String::as_utf8_string(obj, length_sz);
2850 int length = (int)length_sz; // FIXME -- cast
2851 length++; // write tailing '/0'
2852 n = write_bytes(&length, sizeof(int));
2853 if (n != sizeof(int)) {
2854 return false;
2855 }
2856 n = write_bytes(string, (uint)length);
2857 if (n != (uint)length) {
2858 return false;
2859 }
2860 log_info(aot, codecache)("%d (L%d): Write String: %s", compile_id(), comp_level(), string);
2861 } else if (java_lang_Module::is_instance(obj)) {
2862 fatal("Module object unimplemented");
2863 } else if (java_lang_ClassLoader::is_instance(obj)) {
2864 if (obj == SystemDictionary::java_system_loader()) {
2865 kind = DataKind::SysLoader;
2866 log_info(aot, codecache)("%d (L%d): Write ClassLoader: java_system_loader", compile_id(), comp_level());
2867 } else if (obj == SystemDictionary::java_platform_loader()) {
2868 kind = DataKind::PlaLoader;
2869 log_info(aot, codecache)("%d (L%d): Write ClassLoader: java_platform_loader", compile_id(), comp_level());
2870 } else {
2871 fatal("ClassLoader object unimplemented");
2872 return false;
2873 }
2874 n = write_bytes(&kind, sizeof(int));
2875 if (n != sizeof(int)) {
2876 return false;
2877 }
2878 } else { // herere
2879 int k = CDSAccess::get_archived_object_permanent_index(obj); // k >= 0 means obj is a "permanent heap object"
2880 if (k >= 0) {
2881 kind = DataKind::MH_Oop_Shared;
2882 n = write_bytes(&kind, sizeof(int));
2883 if (n != sizeof(int)) {
2884 return false;
2885 }
2886 n = write_bytes(&k, sizeof(int));
2887 if (n != sizeof(int)) {
2888 return false;
2889 }
2890 return true;
2891 }
2892 // Unhandled oop - bailout
2893 set_lookup_failed();
2894 log_info(aot, codecache, nmethod)("%d (L%d): Unhandled obj: " PTR_FORMAT " : %s",
2895 compile_id(), comp_level(), p2i(obj), obj->klass()->external_name());
2896 return false;
2897 }
2898 return true;
2899 }
2900
2901 bool AOTCodeCache::write_oops(OopRecorder* oop_recorder) {
2902 int oop_count = oop_recorder->oop_count();
2903 uint n = write_bytes(&oop_count, sizeof(int));
2904 if (n != sizeof(int)) {
2905 return false;
2906 }
2907 log_debug(aot, codecache)("======== write oops [%d]:", oop_count);
2908
2909 for (int i = 1; i < oop_count; i++) { // skip first virtual nullptr
2910 jobject jo = oop_recorder->oop_at(i);
2911 LogStreamHandle(Info, aot, codecache, oops) log;
2912 if (log.is_enabled()) {
2913 log.print("%d: " INTPTR_FORMAT " ", i, p2i(jo));
2914 if (jo == (jobject)Universe::non_oop_word()) {
2915 log.print("non-oop word");
2916 } else if (jo == nullptr) {
2917 log.print("nullptr-oop");
2918 } else {
2919 JNIHandles::resolve(jo)->print_value_on(&log);
2920 }
2921 log.cr();
2922 }
2923 if (!write_oop(jo)) {
2924 return false;
2925 }
2926 }
2927 return true;
2928 }
2929
2930 bool AOTCodeCache::write_metadata(Metadata* m) {
2931 uint n = 0;
2932 if (m == nullptr) {
2933 DataKind kind = DataKind::Null;
2934 n = write_bytes(&kind, sizeof(int));
2935 if (n != sizeof(int)) {
2936 return false;
2937 }
2938 } else if (m == (Metadata*)Universe::non_oop_word()) {
2939 DataKind kind = DataKind::No_Data;
2940 n = write_bytes(&kind, sizeof(int));
2941 if (n != sizeof(int)) {
2942 return false;
2943 }
2944 } else if (m->is_klass()) {
2945 if (!write_klass((Klass*)m)) {
2946 return false;
2947 }
2948 } else if (m->is_method()) {
2949 if (!write_method((Method*)m)) {
2950 return false;
2951 }
2952 } else if (m->is_methodCounters()) {
2953 DataKind kind = DataKind::MethodCnts;
2954 n = write_bytes(&kind, sizeof(int));
2955 if (n != sizeof(int)) {
2956 return false;
2957 }
2958 if (!write_method(((MethodCounters*)m)->method())) {
2959 return false;
2960 }
2961 log_info(aot, codecache)("%d (L%d): Write MethodCounters : " INTPTR_FORMAT, compile_id(), comp_level(), p2i(m));
2962 } else { // Not supported
2963 fatal("metadata : " INTPTR_FORMAT " unimplemented", p2i(m));
2964 return false;
2965 }
2966 return true;
2967 }
2968
2969 bool AOTCodeCache::write_metadata(OopRecorder* oop_recorder) {
2970 int metadata_count = oop_recorder->metadata_count();
2971 uint n = write_bytes(&metadata_count, sizeof(int));
2972 if (n != sizeof(int)) {
2973 return false;
2974 }
2975
2976 log_debug(aot, codecache)("======== write metadata [%d]:", metadata_count);
2977
2978 for (int i = 1; i < metadata_count; i++) { // skip first virtual nullptr
2979 Metadata* m = oop_recorder->metadata_at(i);
2980 LogStreamHandle(Debug, aot, codecache, metadata) log;
2981 if (log.is_enabled()) {
2982 log.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
2983 if (m == (Metadata*)Universe::non_oop_word()) {
2984 log.print("non-metadata word");
2985 } else if (m == nullptr) {
2986 log.print("nullptr-oop");
2987 } else {
2988 Metadata::print_value_on_maybe_null(&log, m);
2989 }
2990 log.cr();
2991 }
2992 if (!write_metadata(m)) {
2993 return false;
2994 }
2995 }
2996 return true;
2997 }
2998
2999 bool AOTCodeReader::read_dependencies(Dependencies* dependencies) {
3000 uint code_offset = read_position();
3001 int dependencies_size = *(int*)addr(code_offset);
3002
3003 log_debug(aot, codecache)("======== read dependencies [%d]:", dependencies_size);
3004
3005 code_offset += sizeof(int);
3006 code_offset = align_up(code_offset, DATA_ALIGNMENT);
3007 if (dependencies_size > 0) {
3008 dependencies->set_content((u_char*)addr(code_offset), dependencies_size);
3009 }
3010 code_offset += dependencies_size;
3011 set_read_position(code_offset);
3012 return true;
3013 }
3014
3015 bool AOTCodeCache::load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level) {
3016 assert(entry_bci == InvocationEntryBci, "unexpected entry_bci=%d", entry_bci);
3017 TraceTime t1("Total time to load AOT code", &_t_totalLoad, enable_timers(), false);
3018 CompileTask* task = env->task();
3019 task->mark_aot_load_start(os::elapsed_counter());
3020 AOTCodeEntry* entry = task->aot_code_entry();
3021 bool preload = task->preload();
3022 assert(entry != nullptr, "sanity");
3023 AOTCodeCache* cache = open_for_read();
3024 if (cache == nullptr) {
3025 return false;
3026 }
3027 if (log_is_enabled(Info, aot, codecache, nmethod)) {
3028 uint decomp = (target->method_data() == nullptr) ? 0 : target->method_data()->decompile_count();
3029 VM_ENTRY_MARK;
3030 ResourceMark rm;
3031 methodHandle method(THREAD, target->get_Method());
3032 const char* target_name = method->name_and_sig_as_C_string();
3033 uint hash = java_lang_String::hash_code((const jbyte*)target_name, (int)strlen(target_name));
3034 bool clinit_brs = entry->has_clinit_barriers();
3035 log_info(aot, codecache, nmethod)("%d (L%d): %s nmethod '%s' (decomp: %d, hash: " UINT32_FORMAT_X_0 "%s%s)",
3036 task->compile_id(), task->comp_level(), (preload ? "Preloading" : "Reading"),
3037 target_name, decomp, hash, (clinit_brs ? ", has clinit barriers" : ""),
3038 (entry->ignore_decompile() ? ", ignore_decomp" : ""));
3039 }
3040 ReadingMark rdmk;
3041 if (rdmk.failed()) {
3042 // Cache is closed, cannot touch anything.
3043 return false;
3044 }
3045
3046 AOTCodeReader reader(cache, entry, task);
3047 bool success = reader.compile_nmethod(env, target, compiler);
3048 if (success) {
3049 task->set_num_inlined_bytecodes(entry->num_inlined_bytecodes());
3050 } else {
3051 entry->set_load_fail();
3052 }
3053 task->mark_aot_load_finish(os::elapsed_counter());
3054 return success;
3055 }
3056
3057 AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry, CompileTask* task) {
3058 _cache = cache;
3059 _entry = entry;
3060 _load_buffer = cache->cache_buffer();
3061 _read_position = 0;
3062 if (task != nullptr) {
3063 _compile_id = task->compile_id();
3064 _comp_level = task->comp_level();
3065 _preload = task->preload();
3066 } else {
3067 _compile_id = 0;
3068 _comp_level = 0;
3069 _preload = false;
3070 }
3071 _lookup_failed = false;
3072 }
3073
3074 bool AOTCodeReader::read_oop_metadata_list(JavaThread* thread, ciMethod* target, GrowableArray<Handle> &oop_list, GrowableArray<Metadata*> &metadata_list, OopRecorder* oop_recorder) {
3075 methodHandle comp_method(JavaThread::current(), target->get_Method());
3076 JavaThread* current = JavaThread::current();
3077 uint offset = read_position();
3078 int count = *(int *)addr(offset);
3079 offset += sizeof(int);
3080 set_read_position(offset);
3081 for (int i = 0; i < count; i++) {
3082 oop obj = read_oop(current, comp_method);
3083 if (lookup_failed()) {
3084 return false;
3085 }
3086 Handle h(thread, obj);
3087 oop_list.append(h);
3088 if (oop_recorder != nullptr) {
3089 jobject jo = JNIHandles::make_local(thread, obj);
3090 if (oop_recorder->is_real(jo)) {
3091 oop_recorder->find_index(jo);
3092 } else {
3093 oop_recorder->allocate_oop_index(jo);
3094 }
3095 }
3096 LogStreamHandle(Debug, aot, codecache, oops) log;
3097 if (log.is_enabled()) {
3098 log.print("%d: " INTPTR_FORMAT " ", i, p2i(obj));
3099 if (obj == Universe::non_oop_word()) {
3100 log.print("non-oop word");
3101 } else if (obj == nullptr) {
3102 log.print("nullptr-oop");
3103 } else {
3104 obj->print_value_on(&log);
3105 }
3106 log.cr();
3107 }
3108 }
3109
3110 offset = read_position();
3111 count = *(int *)addr(offset);
3112 offset += sizeof(int);
3113 set_read_position(offset);
3114 for (int i = 0; i < count; i++) {
3115 Metadata* m = read_metadata(comp_method);
3116 if (lookup_failed()) {
3117 return false;
3118 }
3119 metadata_list.append(m);
3120 if (oop_recorder != nullptr) {
3121 if (oop_recorder->is_real(m)) {
3122 oop_recorder->find_index(m);
3123 } else {
3124 oop_recorder->allocate_metadata_index(m);
3125 }
3126 }
3127 LogTarget(Debug, aot, codecache, metadata) log;
3128 if (log.is_enabled()) {
3129 LogStream ls(log);
3130 ls.print("%d: " INTPTR_FORMAT " ", i, p2i(m));
3131 if (m == (Metadata*)Universe::non_oop_word()) {
3132 ls.print("non-metadata word");
3133 } else if (m == nullptr) {
3134 ls.print("nullptr-oop");
3135 } else {
3136 Metadata::print_value_on_maybe_null(&ls, m);
3137 }
3138 ls.cr();
3139 }
3140 }
3141 return true;
3142 }
3143
3144 ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
3145 uint offset = read_position();
3146 int size = *(int *)addr(offset);
3147 offset += sizeof(int);
3148 ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
3149 offset += size;
3150 set_read_position(offset);
3151 return oopmaps;
3152 }
3153
3154 bool AOTCodeReader::compile_nmethod(ciEnv* env, ciMethod* target, AbstractCompiler* compiler) {
3155 CompileTask* task = env->task();
3156 AOTCodeEntry *aot_code_entry = (AOTCodeEntry*)_entry;
3157 nmethod* nm = nullptr;
3158
3159 uint entry_position = aot_code_entry->offset();
3160 uint archived_nm_offset = entry_position + aot_code_entry->code_offset();
3161 nmethod* archived_nm = (nmethod*)addr(archived_nm_offset);
3162 set_read_position(archived_nm_offset + archived_nm->size());
3163
3164 OopRecorder* oop_recorder = new OopRecorder(env->arena());
3165 env->set_oop_recorder(oop_recorder);
3166
3167 uint offset;
3168
3169 #ifndef PRODUCT
3170 // Read asm remarks
3171 offset = read_position();
3172 uint count = *(uint *)addr(offset);
3173 offset += sizeof(uint);
3174 AsmRemarks asm_remarks;
3175 for (uint i = 0; i < count; i++) {
3176 uint remark_offset = *(uint *)addr(offset);
3177 offset += sizeof(uint);
3178 const char* remark = (const char*)addr(offset);
3179 offset += (uint)strlen(remark)+1;
3180 asm_remarks.insert(remark_offset, remark);
3181 }
3182 set_read_position(offset);
3183
3184 // Read dbg strings
3185 count = *(uint *)addr(offset);
3186 offset += sizeof(uint);
3187 DbgStrings dbg_strings;
3188 for (uint i = 0; i < count; i++) {
3189 const char* str = (const char*)addr(offset);
3190 offset += (uint)strlen(str)+1;
3191 dbg_strings.insert(str);
3192 }
3193 set_read_position(offset);
3194 #endif /* PRODUCT */
3195
3196 offset = read_position();
3197 address reloc_data = (address)addr(offset);
3198 offset += archived_nm->relocation_size();
3199 set_read_position(offset);
3200
3201 // Read oops and metadata
3202 VM_ENTRY_MARK
3203 GrowableArray<Handle> oop_list;
3204 GrowableArray<Metadata*> metadata_list;
3205
3206 if (!read_oop_metadata_list(THREAD, target, oop_list, metadata_list, oop_recorder)) {
3207 return false;
3208 }
3209
3210 ImmutableOopMapSet* oopmaps = read_oop_map_set();
3211
3212 offset = read_position();
3213 address immutable_data = (address)addr(offset);
3214 offset += archived_nm->immutable_data_size();
3215 set_read_position(offset);
3216
3217 GrowableArray<Handle> reloc_immediate_oop_list;
3218 GrowableArray<Metadata*> reloc_immediate_metadata_list;
3219 if (!read_oop_metadata_list(THREAD, target, reloc_immediate_oop_list, reloc_immediate_metadata_list, nullptr)) {
3220 return false;
3221 }
3222
3223 // Read Dependencies (compressed already)
3224 Dependencies* dependencies = new Dependencies(env);
3225 dependencies->set_content(immutable_data, archived_nm->dependencies_size());
3226 env->set_dependencies(dependencies);
3227
3228 if (VerifyCachedCode) {
3229 return false;
3230 }
3231
3232 TraceTime t1("Total time to register AOT nmethod", &_t_totalRegister, enable_timers(), false);
3233 env->register_aot_method(THREAD,
3234 target,
3235 compiler,
3236 archived_nm,
3237 reloc_data,
3238 oop_list,
3239 metadata_list,
3240 oopmaps,
3241 immutable_data,
3242 reloc_immediate_oop_list,
3243 reloc_immediate_metadata_list,
3244 NOT_PRODUCT_ARG(asm_remarks)
3245 NOT_PRODUCT_ARG(dbg_strings)
3246 this);
3247 bool success = task->is_success();
3248 if (success) {
3249 aot_code_entry->set_loaded();
3250 }
3251 return success;
3252 }
3253
3254 void AOTCodeReader::apply_relocations(nmethod* nm, GrowableArray<Handle> &oop_list, GrowableArray<Metadata*> &metadata_list) {
3255 LogStreamHandle(Info, aot, codecache, reloc) log;
3256 uint buffer_offset = read_position();
3257 int count = *(int*)addr(buffer_offset);
3258 buffer_offset += sizeof(int);
3259 if (log.is_enabled()) {
3260 log.print_cr("======== extra relocations count=%d", count);
3261 }
3262 uint* reloc_data = (uint*)addr(buffer_offset);
3263 buffer_offset += (count * sizeof(uint));
3264 set_read_position(buffer_offset);
3265
3266 RelocIterator iter(nm);
3267 int j = 0;
3268
3269 while (iter.next()) {
3270 switch (iter.type()) {
3271 case relocInfo::none:
3272 break;
3273 case relocInfo::oop_type: {
3274 oop_Relocation* r = (oop_Relocation*)iter.reloc();
3275 if (r->oop_is_immediate()) {
3276 Handle h = oop_list.at(reloc_data[j]);
3277 r->set_value(cast_from_oop<address>(h()));
3278 } else {
3279 r->fix_oop_relocation();
3280 }
3281 break;
3282 }
3283 case relocInfo::metadata_type: {
3284 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
3285 Metadata* m;
3286 if (r->metadata_is_immediate()) {
3287 m = metadata_list.at(reloc_data[j]);
3288 } else {
3289 // Get already updated value from nmethod.
3290 int index = r->metadata_index();
3291 m = nm->metadata_at(index);
3292 }
3293 r->set_value((address)m);
3294 break;
3295 }
3296 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
3297 case relocInfo::opt_virtual_call_type:
3298 case relocInfo::static_call_type: {
3299 address dest = _cache->address_for_id(reloc_data[j]);
3300 if (dest != (address)-1) {
3301 ((CallRelocation*)iter.reloc())->set_destination(dest);
3302 }
3303 break;
3304 }
3305 case relocInfo::trampoline_stub_type: {
3306 address dest = _cache->address_for_id(reloc_data[j]);
3307 if (dest != (address)-1) {
3308 ((trampoline_stub_Relocation*)iter.reloc())->set_destination(dest);
3309 }
3310 break;
3311 }
3312 case relocInfo::static_stub_type:
3313 break;
3314 case relocInfo::runtime_call_type: {
3315 address dest = _cache->address_for_id(reloc_data[j]);
3316 if (dest != (address)-1) {
3317 ((CallRelocation*)iter.reloc())->set_destination(dest);
3318 }
3319 break;
3320 }
3321 case relocInfo::runtime_call_w_cp_type:
3322 fatal("runtime_call_w_cp_type unimplemented");
3323 //address destination = iter.reloc()->value();
3324 break;
3325 case relocInfo::external_word_type: {
3326 address target = _cache->address_for_id(reloc_data[j]);
3327 // Add external address to global table
3328 int index = ExternalsRecorder::find_index(target);
3329 // Update index in relocation
3330 Relocation::add_jint(iter.data(), index);
3331 external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc();
3332 assert(reloc->target() == target, "sanity");
3333 reloc->set_value(target); // Patch address in the code
3334 break;
3335 }
3336 case relocInfo::internal_word_type: {
3337 internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc();
3338 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), nm->content_begin());
3339 break;
3340 }
3341 case relocInfo::section_word_type: {
3342 section_word_Relocation* r = (section_word_Relocation*)iter.reloc();
3343 r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), nm->content_begin());
3344 break;
3345 }
3346 case relocInfo::poll_type:
3347 break;
3348 case relocInfo::poll_return_type:
3349 break;
3350 case relocInfo::post_call_nop_type:
3351 break;
3352 case relocInfo::entry_guard_type:
3353 break;
3354 default:
3355 fatal("relocation %d unimplemented", (int)iter.type());
3356 break;
3357 }
3358 if (log.is_enabled()) {
3359 iter.print_current_on(&log);
3360 }
3361 j++;
3362 }
3363 assert(j == count, "must be");
3364 }
3365
3366 AOTCodeEntry* AOTCodeCache::store_nmethod(nmethod* nm, AbstractCompiler* compiler, bool for_preload) {
3367 if (!CDSConfig::is_dumping_cached_code()) {
3368 return nullptr; // The metadata and heap in the CDS image haven't been finalized yet.
3369 }
3370 if (nm->is_osr_method()) {
3371 return nullptr; // No OSR
3372 }
3373 if (!compiler->is_c1() && !compiler->is_c2()) {
3374 // Only c1 and c2 compilers
3375 return nullptr;
3376 }
3377 int comp_level = nm->comp_level();
3378 if (comp_level == CompLevel_full_profile) {
3379 // Do not cache C1 compiles with full profile i.e. tier3
3380 return nullptr;
3381 }
3382 assert(comp_level == CompLevel_simple || comp_level == CompLevel_limited_profile || comp_level == CompLevel_full_optimization, "must be");
3383
3384 TraceTime t1("Total time to store AOT code", &_t_totalStore, enable_timers(), false);
3385 AOTCodeCache* cache = open_for_write();
3386 if (cache == nullptr) {
3387 return nullptr; // Cache file is closed
3388 }
3389 AOTCodeEntry* entry = nullptr;
3390 entry = cache->write_nmethod(nm, for_preload);
3391 if (entry == nullptr) {
3392 log_info(aot, codecache, nmethod)("%d (L%d): nmethod store attempt failed", nm->compile_id(), comp_level);
3393 }
3394 return entry;
3395 }
3396
3397 bool AOTCodeCache::write_oops(nmethod* nm) {
3398 int count = nm->oops_count()-1;
3399 if (!write_bytes(&count, sizeof(int))) {
3400 return false;
3401 }
3402 for (oop* p = nm->oops_begin(); p < nm->oops_end(); p++) {
3403 if (!write_oop(*p)) {
3404 return false;
3405 }
3406 }
3407 return true;
3408 }
3409
3410 bool AOTCodeCache::write_metadata(nmethod* nm) {
3411 int count = nm->metadata_count()-1;
3412 if (!write_bytes(&count, sizeof(int))) {
3413 return false;
3414 }
3415 for (Metadata** p = nm->metadata_begin(); p < nm->metadata_end(); p++) {
3416 if (!write_metadata(*p)) {
3417 return false;
3418 }
3419 }
3420 return true;
3421 }
3422
3423 bool AOTCodeCache::write_oop_map_set(nmethod* nm) {
3424 ImmutableOopMapSet* oopmaps = nm->oop_maps();
3425 int oopmaps_size = oopmaps->nr_of_bytes();
3426 if (!write_bytes(&oopmaps_size, sizeof(int))) {
3427 return false;
3428 }
3429 uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes());
3430 if (n != (uint)oopmaps->nr_of_bytes()) {
3431 return false;
3432 }
3433 return true;
3434 }
3435
3436 AOTCodeEntry* AOTCodeCache::write_nmethod(nmethod* nm, bool for_preload) {
3437 assert(!nm->has_clinit_barriers() || _gen_preload_code, "sanity");
3438 uint comp_id = nm->compile_id();
3439 uint comp_level = nm->comp_level();
3440 Method* method = nm->method();
3441 bool method_in_cds = MetaspaceShared::is_in_shared_metaspace((address)method);
3442 InstanceKlass* holder = method->method_holder();
3443 bool klass_in_cds = holder->is_shared() && !holder->is_shared_unregistered_class();
3444 bool builtin_loader = holder->class_loader_data()->is_builtin_class_loader_data();
3445 if (!builtin_loader) {
3446 ResourceMark rm;
3447 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' loaded by custom class loader %s", comp_id, (int)comp_level, method->name_and_sig_as_C_string(), holder->class_loader_data()->loader_name());
3448 return nullptr;
3449 }
3450 if (for_preload && !(method_in_cds && klass_in_cds)) {
3451 ResourceMark rm;
3452 log_info(aot, codecache, nmethod)("%d (L%d): Skip method '%s' for preload: not in CDS", comp_id, (int)comp_level, method->name_and_sig_as_C_string());
3453 return nullptr;
3454 }
3455 assert(!for_preload || method_in_cds, "sanity");
3456 _for_preload = for_preload;
3457 _has_clinit_barriers = nm->has_clinit_barriers();
3458
3459 if (!align_write()) {
3460 return nullptr;
3461 }
3462
3463 uint entry_position = _write_position;
3464
3465 uint decomp = (method->method_data() == nullptr) ? 0 : method->method_data()->decompile_count();
3466
3467 // Is this one-step workflow assembly phase?
3468 // In this phase compilation is done based on saved profiling data
3469 // without application run. Ignore decompilation counters in such case.
3470 // Also ignore it for C1 code because it is decompiled unconditionally
3471 // when C2 generated code is published.
3472 bool ignore_decompile = (comp_level == CompLevel_limited_profile) ||
3473 CDSConfig::is_dumping_final_static_archive();
3474
3475 // Write name
3476 uint name_offset = 0;
3477 uint name_size = 0;
3478 uint hash = 0;
3479 uint n;
3480 {
3481 ResourceMark rm;
3482 const char* name = method->name_and_sig_as_C_string();
3483 log_info(aot, codecache, nmethod)("%d (L%d): Writing nmethod '%s' (comp level: %d, decomp: %d%s%s) to AOT Code Cache",
3484 comp_id, (int)comp_level, name, comp_level, decomp,
3485 (ignore_decompile ? ", ignore_decomp" : ""),
3486 (nm->has_clinit_barriers() ? ", has clinit barriers" : ""));
3487
3488 LogStreamHandle(Info, aot, codecache, loader) log;
3489 if (log.is_enabled()) {
3490 oop loader = holder->class_loader();
3491 oop domain = holder->protection_domain();
3492 log.print("Holder: ");
3493 holder->print_value_on(&log);
3494 log.print(" loader: ");
3495 if (loader == nullptr) {
3496 log.print("nullptr");
3497 } else {
3498 loader->print_value_on(&log);
3499 }
3500 log.print(" domain: ");
3501 if (domain == nullptr) {
3502 log.print("nullptr");
3503 } else {
3504 domain->print_value_on(&log);
3505 }
3506 log.cr();
3507 }
3508 name_offset = _write_position - entry_position;
3509 name_size = (uint)strlen(name) + 1; // Includes '/0'
3510 n = write_bytes(name, name_size);
3511 if (n != name_size) {
3512 return nullptr;
3513 }
3514 hash = java_lang_String::hash_code((const jbyte*)name, (int)strlen(name));
3515 }
3516
3517 uint archived_nm_offset = _write_position - entry_position;
3518 nmethod* archived_nm = (nmethod*)reserve_bytes(nm->size());
3519 if (archived_nm == nullptr) {
3520 return nullptr;
3521 }
3522 nm->copy_to((address)archived_nm);
3523
3524 archived_nm->prepare_for_archiving();
3525
3526 #ifndef PRODUCT
3527 // Write asm remarks
3528 uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
3529 if (count_ptr == nullptr) {
3530 return nullptr;
3531 }
3532 uint count = 0;
3533 bool result = nm->asm_remarks().iterate([&] (uint offset, const char* str) -> bool {
3534 log_info(aot, codecache, nmethod)("asm remark offset=%d, str=%s", offset, str);
3535 n = write_bytes(&offset, sizeof(uint));
3536 if (n != sizeof(uint)) {
3537 return false;
3538 }
3539 n = write_bytes(str, (uint)strlen(str) + 1);
3540 if (n != strlen(str) + 1) {
3541 return false;
3542 }
3543 count += 1;
3544 return true;
3545 });
3546 if (!result) {
3547 return nullptr;
3548 }
3549 *count_ptr = count;
3550
3551 // Write dbg strings
3552 count_ptr = (uint *)reserve_bytes(sizeof(uint));
3553 if (count_ptr == nullptr) {
3554 return nullptr;
3555 }
3556 count = 0;
3557 result = nm->dbg_strings().iterate([&] (const char* str) -> bool {
3558 log_info(aot, codecache, nmethod)("dbg string=%s", str);
3559 n = write_bytes(str, (uint)strlen(str) + 1);
3560 if (n != strlen(str) + 1) {
3561 return false;
3562 }
3563 count += 1;
3564 return true;
3565 });
3566 if (!result) {
3567 return nullptr;
3568 }
3569 *count_ptr = count;
3570 #endif /* PRODUCT */
3571
3572 uint reloc_data_size = nm->relocation_size();
3573 n = write_bytes((address)nm->relocation_begin(), reloc_data_size);
3574 if (n != reloc_data_size) {
3575 return nullptr;
3576 }
3577
3578 // Write oops and metadata present in the nmethod's data region
3579 if (!write_oops(nm)) {
3580 if (lookup_failed() && !failed()) {
3581 // Skip this method and reposition file
3582 set_write_position(entry_position);
3583 }
3584 return nullptr;
3585 }
3586 if (!write_metadata(nm)) {
3587 if (lookup_failed() && !failed()) {
3588 // Skip this method and reposition file
3589 set_write_position(entry_position);
3590 }
3591 return nullptr;
3592 }
3593
3594 if (!write_oop_map_set(nm)) {
3595 return nullptr;
3596 }
3597
3598 uint immutable_data_size = nm->immutable_data_size();
3599 n = write_bytes(nm->immutable_data_begin(), immutable_data_size);
3600 if (n != immutable_data_size) {
3601 return nullptr;
3602 }
3603
3604 JavaThread* thread = JavaThread::current();
3605 HandleMark hm(thread);
3606 GrowableArray<Handle> oop_list;
3607 GrowableArray<Metadata*> metadata_list;
3608
3609 nm->create_reloc_immediates_list(thread, oop_list, metadata_list);
3610 if (!write_nmethod_reloc_immediates(oop_list, metadata_list)) {
3611 if (lookup_failed() && !failed()) {
3612 // Skip this method and reposition file
3613 set_write_position(entry_position);
3614 }
3615 return nullptr;
3616 }
3617
3618 if (!write_nmethod_loadtime_relocations(thread, nm, oop_list, metadata_list)) {
3619 return nullptr;
3620 }
3621
3622 uint entry_size = _write_position - entry_position;
3623 AOTCodeEntry* entry = new (this) AOTCodeEntry(entry_position, entry_size, name_offset, name_size,
3624 archived_nm_offset, 0, 0, 0,
3625 AOTCodeEntry::Code, hash, nm->content_begin(), comp_level, comp_id, decomp,
3626 nm->has_clinit_barriers(), for_preload, ignore_decompile);
3627 if (method_in_cds) {
3628 entry->set_method(method);
3629 }
3630 #ifdef ASSERT
3631 if (nm->has_clinit_barriers() || for_preload) {
3632 assert(for_preload, "sanity");
3633 assert(entry->method() != nullptr, "sanity");
3634 }
3635 #endif
3636 {
3637 ResourceMark rm;
3638 const char* name = nm->method()->name_and_sig_as_C_string();
3639 log_info(aot, codecache, nmethod)("%d (L%d): Wrote nmethod '%s'%s to AOT Code Cache",
3640 comp_id, (int)comp_level, name, (for_preload ? " (for preload)" : ""));
3641 }
3642 if (VerifyCachedCode) {
3643 return nullptr;
3644 }
3645 return entry;
3646 }
3647
3648 bool AOTCodeCache::write_nmethod_loadtime_relocations(JavaThread* thread, nmethod* nm, GrowableArray<Handle>& oop_list, GrowableArray<Metadata*>& metadata_list) {
3649 LogStreamHandle(Info, aot, codecache, reloc) log;
3650 GrowableArray<uint> reloc_data;
3651 // Collect additional data
3652 RelocIterator iter(nm);
3653 bool has_immediate = false;
3654 while (iter.next()) {
3655 int idx = reloc_data.append(0); // default value
3656 switch (iter.type()) {
3657 case relocInfo::none:
3658 break;
3659 case relocInfo::oop_type: {
3660 oop_Relocation* r = (oop_Relocation*)iter.reloc();
3661 if (r->oop_is_immediate()) {
3662 // store index of oop in the reloc immediate oop list
3663 Handle h(thread, r->oop_value());
3664 int oop_idx = oop_list.find(h);
3665 assert(oop_idx != -1, "sanity check");
3666 reloc_data.at_put(idx, (uint)oop_idx);
3667 has_immediate = true;
3668 }
3669 break;
3670 }
3671 case relocInfo::metadata_type: {
3672 metadata_Relocation* r = (metadata_Relocation*)iter.reloc();
3673 if (r->metadata_is_immediate()) {
3674 // store index of metadata in the reloc immediate metadata list
3675 int metadata_idx = metadata_list.find(r->metadata_value());
3676 assert(metadata_idx != -1, "sanity check");
3677 reloc_data.at_put(idx, (uint)metadata_idx);
3678 has_immediate = true;
3679 }
3680 break;
3681 }
3682 case relocInfo::virtual_call_type: // Fall through. They all call resolve_*_call blobs.
3683 case relocInfo::opt_virtual_call_type:
3684 case relocInfo::static_call_type: {
3685 CallRelocation* r = (CallRelocation*)iter.reloc();
3686 address dest = r->destination();
3687 if (dest == r->addr()) { // possible call via trampoline on Aarch64
3688 dest = (address)-1; // do nothing in this case when loading this relocation
3689 }
3690 reloc_data.at_put(idx, _table->id_for_address(dest, iter, nullptr));
3691 break;
3692 }
3693 case relocInfo::trampoline_stub_type: {
3694 address dest = ((trampoline_stub_Relocation*)iter.reloc())->destination();
3695 reloc_data.at_put(idx, _table->id_for_address(dest, iter, nullptr));
3696 break;
3697 }
3698 case relocInfo::static_stub_type:
3699 break;
3700 case relocInfo::runtime_call_type: {
3701 // Record offset of runtime destination
3702 CallRelocation* r = (CallRelocation*)iter.reloc();
3703 address dest = r->destination();
3704 if (dest == r->addr()) { // possible call via trampoline on Aarch64
3705 dest = (address)-1; // do nothing in this case when loading this relocation
3706 }
3707 reloc_data.at_put(idx, _table->id_for_address(dest, iter, nullptr));
3708 break;
3709 }
3710 case relocInfo::runtime_call_w_cp_type:
3711 fatal("runtime_call_w_cp_type unimplemented");
3712 break;
3713 case relocInfo::external_word_type: {
3714 // Record offset of runtime target
3715 address target = ((external_word_Relocation*)iter.reloc())->target();
3716 reloc_data.at_put(idx, _table->id_for_address(target, iter, nullptr));
3717 break;
3718 }
3719 case relocInfo::internal_word_type:
3720 break;
3721 case relocInfo::section_word_type:
3722 break;
3723 case relocInfo::poll_type:
3724 break;
3725 case relocInfo::poll_return_type:
3726 break;
3727 case relocInfo::post_call_nop_type:
3728 break;
3729 case relocInfo::entry_guard_type:
3730 break;
3731 default:
3732 fatal("relocation %d unimplemented", (int)iter.type());
3733 break;
3734 }
3735 if (log.is_enabled()) {
3736 iter.print_current_on(&log);
3737 }
3738 }
3739
3740 // Write additional relocation data: uint per relocation
3741 // Write the count first
3742 int count = reloc_data.length();
3743 write_bytes(&count, sizeof(int));
3744 uint data_size = count * sizeof(uint);
3745 for (GrowableArrayIterator<uint> iter = reloc_data.begin();
3746 iter != reloc_data.end(); ++iter) {
3747 uint value = *iter;
3748 int n = write_bytes(&value, sizeof(uint));
3749 if (n != sizeof(uint)) {
3750 return false;
3751 break;
3752 }
3753 }
3754
3755 if (!align_write()) {
3756 return false;
3757 }
3758 return true; //success;
3759 }
3760
3761 bool AOTCodeCache::write_nmethod_reloc_immediates(GrowableArray<Handle>& oop_list, GrowableArray<Metadata*>& metadata_list) {
3762 int count = oop_list.length();
3763 if (!write_bytes(&count, sizeof(int))) {
3764 return false;
3765 }
3766 for (GrowableArrayIterator<Handle> iter = oop_list.begin();
3767 iter != oop_list.end(); ++iter) {
3768 Handle h = *iter;
3769 if (!write_oop(h())) {
3770 return false;
3771 }
3772 }
3773
3774 count = metadata_list.length();
3775 if (!write_bytes(&count, sizeof(int))) {
3776 return false;
3777 }
3778 for (GrowableArrayIterator<Metadata*> iter = metadata_list.begin();
3779 iter != metadata_list.end(); ++iter) {
3780 Metadata* m = *iter;
3781 if (!write_metadata(m)) {
3782 return false;
3783 }
3784 }
3785 return true;
3786 }
3787
3788 static void print_helper1(outputStream* st, const char* name, int count) {
3789 if (count > 0) {
3790 st->print(" %s=%d", name, count);
3791 }
3792 }
3793
3794 void AOTCodeCache::print_statistics_on(outputStream* st) {
3795 AOTCodeCache* cache = open_for_read();
3796 if (cache != nullptr) {
3797 ReadingMark rdmk;
3798 if (rdmk.failed()) {
3799 // Cache is closed, cannot touch anything.
3800 return;
3801 }
3802
3803 uint count = cache->_load_header->entries_count();
3804 uint* search_entries = (uint*)cache->addr(cache->_load_header->entries_offset()); // [id, index]
3805 AOTCodeEntry* load_entries = (AOTCodeEntry*)(search_entries + 2 * count);
3806
3807 AOTCodeStats stats;
3808 for (uint i = 0; i < count; i++) {
3809 stats.collect_all_stats(&load_entries[i]);
3810 }
3811
3812 for (uint kind = AOTCodeEntry::None; kind < AOTCodeEntry::Kind_count; kind++) {
3813 if (stats.entry_count(kind) > 0) {
3814 st->print(" %s:", aot_code_entry_kind_name[kind]);
3815 print_helper1(st, "total", stats.entry_count(kind));
3816 print_helper1(st, "loaded", stats.entry_loaded_count(kind));
3817 print_helper1(st, "invalidated", stats.entry_invalidated_count(kind));
3818 print_helper1(st, "failed", stats.entry_load_failed_count(kind));
3819 st->cr();
3820 }
3821 if (kind == AOTCodeEntry::Code) {
3822 for (uint lvl = CompLevel_none; lvl < AOTCompLevel_count; lvl++) {
3823 if (stats.nmethod_count(lvl) > 0) {
3824 st->print(" AOT Code T%d", lvl);
3825 print_helper1(st, "total", stats.nmethod_count(lvl));
3826 print_helper1(st, "loaded", stats.nmethod_loaded_count(lvl));
3827 print_helper1(st, "invalidated", stats.nmethod_invalidated_count(lvl));
3828 print_helper1(st, "failed", stats.nmethod_load_failed_count(lvl));
3829 if (lvl == AOTCompLevel_count-1) {
3830 print_helper1(st, "has_clinit_barriers", stats.clinit_barriers_count());
3831 }
3832 st->cr();
3833 }
3834 }
3835 }
3836 }
3837 } else {
3838 st->print_cr("failed to map code cache");
3839 }
3840 }
3841
3842 void AOTCodeCache::print_on(outputStream* st) {
3843 AOTCodeCache* cache = open_for_read();
3844 if (cache != nullptr) {
3845 ReadingMark rdmk;
3846 if (rdmk.failed()) {
3847 // Cache is closed, cannot touch anything.
3848 return;
3849 }
3850
3851 uint count = cache->_load_header->entries_count();
3852 uint* search_entries = (uint*)cache->addr(cache->_load_header->entries_offset()); // [id, index]
3853 AOTCodeEntry* load_entries = (AOTCodeEntry*)(search_entries + 2 * count);
3854
3855 for (uint i = 0; i < count; i++) {
3856 int index = search_entries[2*i + 1];
3857 AOTCodeEntry* entry = &(load_entries[index]);
3858
3859 st->print_cr("%4u: %4u: K%u L%u offset=%u decompile=%u size=%u code_size=%u%s%s%s%s",
3860 i, index, entry->kind(), entry->comp_level(), entry->offset(),
3861 entry->decompile(), entry->size(), entry->code_size(),
3862 entry->has_clinit_barriers() ? " has_clinit_barriers" : "",
3863 entry->for_preload() ? " for_preload" : "",
3864 entry->is_loaded() ? " loaded" : "",
3865 entry->not_entrant() ? " not_entrant" : "");
3866 st->print_raw(" ");
3867 AOTCodeReader reader(cache, entry, nullptr);
3868 reader.print_on(st);
3869 }
3870 } else {
3871 st->print_cr("failed to map code cache");
3872 }
3873 }
3874
3875 void AOTCodeCache::print_unused_entries_on(outputStream* st) {
3876 LogStreamHandle(Info, aot, codecache, init) info;
3877 if (info.is_enabled()) {
3878 AOTCodeCache::iterate([&](AOTCodeEntry* entry) {
3879 if (entry->is_code() && !entry->is_loaded()) {
3880 MethodTrainingData* mtd = MethodTrainingData::find(methodHandle(Thread::current(), entry->method()));
3881 if (mtd != nullptr) {
3882 if (mtd->has_holder()) {
3883 if (mtd->holder()->method_holder()->is_initialized()) {
3884 ResourceMark rm;
3885 mtd->iterate_compiles([&](CompileTrainingData* ctd) {
3886 if ((uint)ctd->level() == entry->comp_level()) {
3887 if (ctd->init_deps_left() == 0) {
3888 nmethod* nm = mtd->holder()->code();
3889 if (nm == nullptr) {
3890 if (mtd->holder()->queued_for_compilation()) {
3891 return; // scheduled for compilation
3892 }
3893 } else if ((uint)nm->comp_level() >= entry->comp_level()) {
3894 return; // already online compiled and superseded by a more optimal method
3895 }
3896 info.print("AOT Code Cache entry not loaded: ");
3897 ctd->print_on(&info);
3898 info.cr();
3899 }
3900 }
3901 });
3902 } else {
3903 // not yet initialized
3904 }
3905 } else {
3906 info.print("AOT Code Cache entry doesn't have a holder: ");
3907 mtd->print_on(&info);
3908 info.cr();
3909 }
3910 }
3911 }
3912 });
3913 }
3914 }
3915
3916 void AOTCodeReader::print_on(outputStream* st) {
3917 uint entry_position = _entry->offset();
3918 set_read_position(entry_position);
3919
3920 // Read name
3921 uint name_offset = entry_position + _entry->name_offset();
3922 uint name_size = _entry->name_size(); // Includes '/0'
3923 const char* name = addr(name_offset);
3924
3925 st->print_cr(" name: %s", name);
3926 }
3927
3928 // address table ids for generated routines, external addresses and C
3929 // string addresses are partitioned into positive integer ranges
3930 // defined by the following positive base and max values
3931 // i.e. [_extrs_base, _extrs_base + _extrs_max -1],
3932 // [_stubs_base, _stubs_base + _stubs_max -1],
3933 // ...
3934 // [_c_str_base, _c_str_base + _c_str_max -1],
3935 #define _extrs_max 80
3936 #define _stubs_max 120
3937 #define _all_blobs_max 100
3938 #define _blobs_max 24
3939 #define _C2_blobs_max 25
3940 #define _C1_blobs_max (_all_blobs_max - _blobs_max - _C2_blobs_max)
3941 #define _all_max 300
3942
3943 #define _c_str_max MAX_STR_COUNT
3944 #define _extrs_base 0
3945 #define _stubs_base (_extrs_base + _extrs_max)
3946 #define _blobs_base (_stubs_base + _stubs_max)
3947 #define _C1_blobs_base (_blobs_base + _blobs_max)
3948 #define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max)
3949 #if (_C2_blobs_base >= _all_max)
3950 #error AOTCodeAddressTable ranges need adjusting
3951 #endif
3952 #define _c_str_base _all_max
3953
3954 #define SET_ADDRESS(type, addr) \
3955 { \
3956 type##_addr[type##_length++] = (address) (addr); \
3957 assert(type##_length <= type##_max, "increase size"); \
3958 }
3959
3960 static bool initializing_extrs = false;
3961 void AOTCodeAddressTable::init_extrs() {
3962 if (_extrs_complete || initializing_extrs) return; // Done already
3963 initializing_extrs = true;
3964 _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode);
3965
3966 _extrs_length = 0;
3967 _stubs_length = 0;
3968
3969 // Runtime methods
3970 #ifdef COMPILER2
3971 SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C);
3972 #endif
3973 #ifdef COMPILER1
3974 SET_ADDRESS(_extrs, Runtime1::is_instance_of);
3975 SET_ADDRESS(_extrs, Runtime1::trace_block_entry);
3976 #endif
3977
3978 SET_ADDRESS(_extrs, CompressedOops::base_addr());
3979 #if INCLUDE_G1GC
3980 SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_post_entry);
3981 SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry);
3982 #endif
3983
3984 #if INCLUDE_SHENANDOAHGC
3985 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_oop);
3986 SET_ADDRESS(_extrs, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
3987 SET_ADDRESS(_extrs, ShenandoahRuntime::write_ref_field_pre);
3988 SET_ADDRESS(_extrs, ShenandoahRuntime::clone_barrier);
3989 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong);
3990 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_strong_narrow);
3991 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak);
3992 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_weak_narrow);
3993 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom);
3994 SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
3995 #endif
3996 SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite);
3997
3998 SET_ADDRESS(_extrs, SharedRuntime::log_jni_monitor_still_held);
3999 SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry);
4000 SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages);
4001 SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit);
4002
4003 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method);
4004 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract);
4005 SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss);
4006 SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C);
4007 SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C);
4008 SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C);
4009
4010 SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C);
4011 SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone);
4012 #if defined(AMD64) && !defined(ZERO)
4013 SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply);
4014 SET_ADDRESS(_extrs, SharedRuntime::montgomery_square);
4015 #endif // AMD64
4016 SET_ADDRESS(_extrs, SharedRuntime::d2f);
4017 SET_ADDRESS(_extrs, SharedRuntime::d2i);
4018 SET_ADDRESS(_extrs, SharedRuntime::d2l);
4019 SET_ADDRESS(_extrs, SharedRuntime::dcos);
4020 SET_ADDRESS(_extrs, SharedRuntime::dexp);
4021 SET_ADDRESS(_extrs, SharedRuntime::dlog);
4022 SET_ADDRESS(_extrs, SharedRuntime::dlog10);
4023 SET_ADDRESS(_extrs, SharedRuntime::dpow);
4024 SET_ADDRESS(_extrs, SharedRuntime::dsin);
4025 SET_ADDRESS(_extrs, SharedRuntime::dtan);
4026 SET_ADDRESS(_extrs, SharedRuntime::f2i);
4027 SET_ADDRESS(_extrs, SharedRuntime::f2l);
4028 #ifndef ZERO
4029 SET_ADDRESS(_extrs, SharedRuntime::drem);
4030 SET_ADDRESS(_extrs, SharedRuntime::frem);
4031 #endif
4032 SET_ADDRESS(_extrs, SharedRuntime::l2d);
4033 SET_ADDRESS(_extrs, SharedRuntime::l2f);
4034 SET_ADDRESS(_extrs, SharedRuntime::ldiv);
4035 SET_ADDRESS(_extrs, SharedRuntime::lmul);
4036 SET_ADDRESS(_extrs, SharedRuntime::lrem);
4037 #if INCLUDE_JVMTI
4038 SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc);
4039 #endif /* INCLUDE_JVMTI */
4040 BarrierSet* bs = BarrierSet::barrier_set();
4041 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
4042 SET_ADDRESS(_extrs, ci_card_table_address_as<address>());
4043 }
4044 SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset());
4045 SET_ADDRESS(_extrs, Thread::current);
4046
4047 SET_ADDRESS(_extrs, os::javaTimeMillis);
4048 SET_ADDRESS(_extrs, os::javaTimeNanos);
4049
4050 #if INCLUDE_JVMTI
4051 SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events);
4052 #endif /* INCLUDE_JVMTI */
4053 SET_ADDRESS(_extrs, StubRoutines::crc_table_addr());
4054 #ifndef PRODUCT
4055 SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr);
4056 SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure);
4057 #endif
4058
4059 #ifndef ZERO
4060 #if defined(AMD64) || defined(AARCH64) || defined(RISCV64)
4061 SET_ADDRESS(_extrs, MacroAssembler::debug64);
4062 #endif
4063 #if defined(AMD64)
4064 SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31());
4065 #endif
4066 #endif
4067
4068 #ifdef COMPILER1
4069 #ifdef X86
4070 SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool);
4071 SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool);
4072 SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool);
4073 SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool);
4074 #endif
4075 #endif
4076
4077 // addresses of fields in AOT runtime constants area
4078 address* p = AOTRuntimeConstants::field_addresses_list();
4079 while (*p != nullptr) {
4080 SET_ADDRESS(_extrs, *p++);
4081 }
4082
4083 _extrs_complete = true;
4084 log_info(aot, codecache,init)("External addresses recorded");
4085 }
4086
4087 static bool initializing_early_stubs = false;
4088 void AOTCodeAddressTable::init_early_stubs() {
4089 if (_complete || initializing_early_stubs) return; // Done already
4090 initializing_early_stubs = true;
4091 _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode);
4092 _stubs_length = 0;
4093 SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry());
4094 _early_stubs_complete = true;
4095 log_info(aot, codecache,init)("early stubs recorded");
4096 }
4097
4098 static bool initializing_shared_blobs = false;
4099 void AOTCodeAddressTable::init_shared_blobs() {
4100 if (_complete || initializing_shared_blobs) return; // Done already
4101 initializing_shared_blobs = true;
4102 _blobs_addr = NEW_C_HEAP_ARRAY(address, _all_blobs_max, mtCode);
4103
4104 // Divide _blobs_addr array to chunks because they could be initialized in parrallel
4105 _C1_blobs_addr = _blobs_addr + _blobs_max;// C1 blobs addresses stored after shared blobs
4106 _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs
4107
4108 _blobs_length = 0; // for shared blobs
4109 _C1_blobs_length = 0;
4110 _C2_blobs_length = 0;
4111
4112 // Blobs
4113 SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub());
4114 SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub());
4115 SET_ADDRESS(_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub());
4116 SET_ADDRESS(_blobs, SharedRuntime::get_resolve_virtual_call_stub());
4117 SET_ADDRESS(_blobs, SharedRuntime::get_resolve_static_call_stub());
4118 SET_ADDRESS(_blobs, SharedRuntime::deopt_blob()->entry_point());
4119 SET_ADDRESS(_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point());
4120 SET_ADDRESS(_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point());
4121 #ifdef COMPILER2
4122 SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point());
4123 #endif
4124
4125 assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length);
4126 log_info(aot, codecache,init)("Early shared blobs recorded");
4127 }
4128
4129 static bool initializing_stubs = false;
4130 void AOTCodeAddressTable::init_stubs() {
4131 if (_complete || initializing_stubs) return; // Done already
4132 initializing_stubs = true;
4133 // final blobs
4134 SET_ADDRESS(_blobs, SharedRuntime::throw_AbstractMethodError_entry());
4135 SET_ADDRESS(_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry());
4136 SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry());
4137 SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry());
4138 SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry());
4139
4140 assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length);
4141
4142 _shared_blobs_complete = true;
4143 log_info(aot, codecache,init)("All shared blobs recorded");
4144
4145 // Stubs
4146 SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier());
4147 /*
4148 SET_ADDRESS(_stubs, StubRoutines::throw_AbstractMethodError_entry());
4149 SET_ADDRESS(_stubs, StubRoutines::throw_IncompatibleClassChangeError_entry());
4150 SET_ADDRESS(_stubs, StubRoutines::throw_NullPointerException_at_call_entry());
4151 SET_ADDRESS(_stubs, StubRoutines::throw_StackOverflowError_entry());
4152 SET_ADDRESS(_stubs, StubRoutines::throw_delayed_StackOverflowError_entry());
4153 */
4154 SET_ADDRESS(_stubs, StubRoutines::atomic_xchg_entry());
4155 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_entry());
4156 SET_ADDRESS(_stubs, StubRoutines::atomic_cmpxchg_long_entry());
4157 SET_ADDRESS(_stubs, StubRoutines::atomic_add_entry());
4158 SET_ADDRESS(_stubs, StubRoutines::fence_entry());
4159
4160 SET_ADDRESS(_stubs, StubRoutines::cont_thaw());
4161 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrier());
4162 SET_ADDRESS(_stubs, StubRoutines::cont_returnBarrierExc());
4163
4164 JFR_ONLY(SET_ADDRESS(_stubs, SharedRuntime::jfr_write_checkpoint());)
4165
4166
4167 SET_ADDRESS(_stubs, StubRoutines::jbyte_arraycopy());
4168 SET_ADDRESS(_stubs, StubRoutines::jshort_arraycopy());
4169 SET_ADDRESS(_stubs, StubRoutines::jint_arraycopy());
4170 SET_ADDRESS(_stubs, StubRoutines::jlong_arraycopy());
4171 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy);
4172 SET_ADDRESS(_stubs, StubRoutines::_oop_arraycopy_uninit);
4173
4174 SET_ADDRESS(_stubs, StubRoutines::jbyte_disjoint_arraycopy());
4175 SET_ADDRESS(_stubs, StubRoutines::jshort_disjoint_arraycopy());
4176 SET_ADDRESS(_stubs, StubRoutines::jint_disjoint_arraycopy());
4177 SET_ADDRESS(_stubs, StubRoutines::jlong_disjoint_arraycopy());
4178 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy);
4179 SET_ADDRESS(_stubs, StubRoutines::_oop_disjoint_arraycopy_uninit);
4180
4181 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_arraycopy());
4182 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_arraycopy());
4183 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_arraycopy());
4184 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_arraycopy());
4185 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy);
4186 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_arraycopy_uninit);
4187
4188 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_disjoint_arraycopy());
4189 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_disjoint_arraycopy());
4190 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_disjoint_arraycopy());
4191 SET_ADDRESS(_stubs, StubRoutines::arrayof_jlong_disjoint_arraycopy());
4192 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy);
4193 SET_ADDRESS(_stubs, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit);
4194
4195 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy);
4196 SET_ADDRESS(_stubs, StubRoutines::_checkcast_arraycopy_uninit);
4197
4198 SET_ADDRESS(_stubs, StubRoutines::unsafe_arraycopy());
4199 SET_ADDRESS(_stubs, StubRoutines::generic_arraycopy());
4200
4201 SET_ADDRESS(_stubs, StubRoutines::jbyte_fill());
4202 SET_ADDRESS(_stubs, StubRoutines::jshort_fill());
4203 SET_ADDRESS(_stubs, StubRoutines::jint_fill());
4204 SET_ADDRESS(_stubs, StubRoutines::arrayof_jbyte_fill());
4205 SET_ADDRESS(_stubs, StubRoutines::arrayof_jshort_fill());
4206 SET_ADDRESS(_stubs, StubRoutines::arrayof_jint_fill());
4207
4208 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback());
4209 SET_ADDRESS(_stubs, StubRoutines::data_cache_writeback_sync());
4210
4211 SET_ADDRESS(_stubs, StubRoutines::aescrypt_encryptBlock());
4212 SET_ADDRESS(_stubs, StubRoutines::aescrypt_decryptBlock());
4213 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_encryptAESCrypt());
4214 SET_ADDRESS(_stubs, StubRoutines::cipherBlockChaining_decryptAESCrypt());
4215 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_encryptAESCrypt());
4216 SET_ADDRESS(_stubs, StubRoutines::electronicCodeBook_decryptAESCrypt());
4217 SET_ADDRESS(_stubs, StubRoutines::poly1305_processBlocks());
4218 SET_ADDRESS(_stubs, StubRoutines::counterMode_AESCrypt());
4219 SET_ADDRESS(_stubs, StubRoutines::ghash_processBlocks());
4220 SET_ADDRESS(_stubs, StubRoutines::chacha20Block());
4221 SET_ADDRESS(_stubs, StubRoutines::base64_encodeBlock());
4222 SET_ADDRESS(_stubs, StubRoutines::base64_decodeBlock());
4223 SET_ADDRESS(_stubs, StubRoutines::md5_implCompress());
4224 SET_ADDRESS(_stubs, StubRoutines::md5_implCompressMB());
4225 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompress());
4226 SET_ADDRESS(_stubs, StubRoutines::sha1_implCompressMB());
4227 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompress());
4228 SET_ADDRESS(_stubs, StubRoutines::sha256_implCompressMB());
4229 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompress());
4230 SET_ADDRESS(_stubs, StubRoutines::sha512_implCompressMB());
4231 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompress());
4232 SET_ADDRESS(_stubs, StubRoutines::sha3_implCompressMB());
4233
4234 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32());
4235
4236 SET_ADDRESS(_stubs, StubRoutines::crc32c_table_addr());
4237 SET_ADDRESS(_stubs, StubRoutines::updateBytesCRC32C());
4238 SET_ADDRESS(_stubs, StubRoutines::updateBytesAdler32());
4239
4240 SET_ADDRESS(_stubs, StubRoutines::multiplyToLen());
4241 SET_ADDRESS(_stubs, StubRoutines::squareToLen());
4242 SET_ADDRESS(_stubs, StubRoutines::mulAdd());
4243 SET_ADDRESS(_stubs, StubRoutines::montgomeryMultiply());
4244 SET_ADDRESS(_stubs, StubRoutines::montgomerySquare());
4245 SET_ADDRESS(_stubs, StubRoutines::bigIntegerRightShift());
4246 SET_ADDRESS(_stubs, StubRoutines::bigIntegerLeftShift());
4247 SET_ADDRESS(_stubs, StubRoutines::galoisCounterMode_AESCrypt());
4248
4249 SET_ADDRESS(_stubs, StubRoutines::vectorizedMismatch());
4250
4251 SET_ADDRESS(_stubs, StubRoutines::dexp());
4252 SET_ADDRESS(_stubs, StubRoutines::dlog());
4253 SET_ADDRESS(_stubs, StubRoutines::dlog10());
4254 SET_ADDRESS(_stubs, StubRoutines::dpow());
4255 SET_ADDRESS(_stubs, StubRoutines::dsin());
4256 SET_ADDRESS(_stubs, StubRoutines::dcos());
4257 SET_ADDRESS(_stubs, StubRoutines::dlibm_reduce_pi04l());
4258 SET_ADDRESS(_stubs, StubRoutines::dlibm_sin_cos_huge());
4259 SET_ADDRESS(_stubs, StubRoutines::dlibm_tan_cot_huge());
4260 SET_ADDRESS(_stubs, StubRoutines::dtan());
4261
4262 SET_ADDRESS(_stubs, StubRoutines::f2hf_adr());
4263 SET_ADDRESS(_stubs, StubRoutines::hf2f_adr());
4264
4265 #if defined(AMD64) && !defined(ZERO)
4266 SET_ADDRESS(_stubs, StubRoutines::x86::d2i_fixup());
4267 SET_ADDRESS(_stubs, StubRoutines::x86::f2i_fixup());
4268 SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup());
4269 SET_ADDRESS(_stubs, StubRoutines::x86::f2l_fixup());
4270 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_mask());
4271 SET_ADDRESS(_stubs, StubRoutines::x86::float_sign_flip());
4272 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_mask());
4273 SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip());
4274 SET_ADDRESS(_stubs, StubRoutines::x86::vector_popcount_lut());
4275 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_mask());
4276 SET_ADDRESS(_stubs, StubRoutines::x86::vector_float_sign_flip());
4277 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_mask());
4278 SET_ADDRESS(_stubs, StubRoutines::x86::vector_double_sign_flip());
4279 // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
4280 // See C2_MacroAssembler::load_iota_indices().
4281 for (int i = 0; i < 6; i++) {
4282 SET_ADDRESS(_stubs, StubRoutines::x86::vector_iota_indices() + i * 64);
4283 }
4284 #endif
4285 #if defined(AARCH64) && !defined(ZERO)
4286 SET_ADDRESS(_stubs, StubRoutines::aarch64::zero_blocks());
4287 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives());
4288 SET_ADDRESS(_stubs, StubRoutines::aarch64::count_positives_long());
4289 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_array_equals());
4290 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LL());
4291 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UU());
4292 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_LU());
4293 SET_ADDRESS(_stubs, StubRoutines::aarch64::compare_long_string_UL());
4294 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ul());
4295 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_ll());
4296 SET_ADDRESS(_stubs, StubRoutines::aarch64::string_indexof_linear_uu());
4297 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_byte_array_inflate());
4298 SET_ADDRESS(_stubs, StubRoutines::aarch64::spin_wait());
4299
4300 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BOOLEAN));
4301 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_BYTE));
4302 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_SHORT));
4303 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_CHAR));
4304 SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_INT));
4305 #endif
4306
4307 _complete = true;
4308 log_info(aot, codecache,init)("Stubs recorded");
4309 }
4310
4311 void AOTCodeAddressTable::init_opto() {
4312 #ifdef COMPILER2
4313 // OptoRuntime Blobs
4314 SET_ADDRESS(_C2_blobs, OptoRuntime::uncommon_trap_blob()->entry_point());
4315 SET_ADDRESS(_C2_blobs, OptoRuntime::exception_blob()->entry_point());
4316 SET_ADDRESS(_C2_blobs, OptoRuntime::new_instance_Java());
4317 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_Java());
4318 SET_ADDRESS(_C2_blobs, OptoRuntime::new_array_nozero_Java());
4319 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray2_Java());
4320 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray3_Java());
4321 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray4_Java());
4322 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarray5_Java());
4323 SET_ADDRESS(_C2_blobs, OptoRuntime::multianewarrayN_Java());
4324 SET_ADDRESS(_C2_blobs, OptoRuntime::vtable_must_compile_stub());
4325 SET_ADDRESS(_C2_blobs, OptoRuntime::complete_monitor_locking_Java());
4326 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notify_Java());
4327 SET_ADDRESS(_C2_blobs, OptoRuntime::monitor_notifyAll_Java());
4328 SET_ADDRESS(_C2_blobs, OptoRuntime::rethrow_stub());
4329 SET_ADDRESS(_C2_blobs, OptoRuntime::slow_arraycopy_Java());
4330 SET_ADDRESS(_C2_blobs, OptoRuntime::register_finalizer_Java());
4331 SET_ADDRESS(_C2_blobs, OptoRuntime::class_init_barrier_Java());
4332 #if INCLUDE_JVMTI
4333 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_start());
4334 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_end());
4335 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_mount());
4336 SET_ADDRESS(_C2_blobs, OptoRuntime::notify_jvmti_vthread_unmount());
4337 #endif /* INCLUDE_JVMTI */
4338 #endif
4339
4340 assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length);
4341 _opto_complete = true;
4342 log_info(aot, codecache,init)("OptoRuntime Blobs recorded");
4343 }
4344
4345 void AOTCodeAddressTable::init_c1() {
4346 #ifdef COMPILER1
4347 // Runtime1 Blobs
4348 for (int i = 0; i < (int)(C1StubId::NUM_STUBIDS); i++) {
4349 C1StubId id = (C1StubId)i;
4350 if (Runtime1::blob_for(id) == nullptr) {
4351 log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id));
4352 continue;
4353 }
4354 if (Runtime1::entry_for(id) == nullptr) {
4355 log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id));
4356 continue;
4357 }
4358 address entry = Runtime1::entry_for(id);
4359 SET_ADDRESS(_C1_blobs, entry);
4360 }
4361 #if INCLUDE_G1GC
4362 if (UseG1GC) {
4363 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
4364 address entry = bs->pre_barrier_c1_runtime_code_blob()->code_begin();
4365 SET_ADDRESS(_C1_blobs, entry);
4366 entry = bs->post_barrier_c1_runtime_code_blob()->code_begin();
4367 SET_ADDRESS(_C1_blobs, entry);
4368 }
4369 #endif // INCLUDE_G1GC
4370 #if INCLUDE_ZGC
4371 if (UseZGC) {
4372 ZBarrierSetC1* bs = (ZBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
4373 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_oop_field_preloaded_runtime_stub);
4374 SET_ADDRESS(_C1_blobs, bs->_load_barrier_on_weak_oop_field_preloaded_runtime_stub);
4375 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_with_healing);
4376 SET_ADDRESS(_C1_blobs, bs->_store_barrier_on_oop_field_without_healing);
4377 }
4378 #endif // INCLUDE_ZGC
4379 #if INCLUDE_SHENANDOAHGC
4380 if (UseShenandoahGC) {
4381 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
4382 SET_ADDRESS(_C1_blobs, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
4383 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_rt_code_blob()->code_begin());
4384 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin());
4385 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_weak_rt_code_blob()->code_begin());
4386 SET_ADDRESS(_C1_blobs, bs->load_reference_barrier_phantom_rt_code_blob()->code_begin());
4387 }
4388 #endif // INCLUDE_SHENANDOAHGC
4389 #endif // COMPILER1
4390
4391 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length);
4392 _c1_complete = true;
4393 log_info(aot, codecache,init)("Runtime1 Blobs recorded");
4394 }
4395
4396 #undef SET_ADDRESS
4397
4398 AOTCodeAddressTable::~AOTCodeAddressTable() {
4399 if (_extrs_addr != nullptr) {
4400 FREE_C_HEAP_ARRAY(address, _extrs_addr);
4401 }
4402 if (_stubs_addr != nullptr) {
4403 FREE_C_HEAP_ARRAY(address, _stubs_addr);
4404 }
4405 if (_blobs_addr != nullptr) {
4406 FREE_C_HEAP_ARRAY(address, _blobs_addr);
4407 }
4408 }
4409
4410 #ifdef PRODUCT
4411 #define MAX_STR_COUNT 200
4412 #else
4413 #define MAX_STR_COUNT 500
4414 #endif
4415 static const char* _C_strings[MAX_STR_COUNT] = {nullptr};
4416 static int _C_strings_count = 0;
4417 static int _C_strings_s[MAX_STR_COUNT] = {0};
4418 static int _C_strings_id[MAX_STR_COUNT] = {0};
4419 static int _C_strings_len[MAX_STR_COUNT] = {0};
4420 static int _C_strings_hash[MAX_STR_COUNT] = {0};
4421 static int _C_strings_used = 0;
4422
4423 void AOTCodeCache::load_strings() {
4424 uint strings_count = _load_header->strings_count();
4425 if (strings_count == 0) {
4426 return;
4427 }
4428 uint strings_offset = _load_header->strings_offset();
4429 uint strings_size = _load_header->entries_offset() - strings_offset;
4430 uint data_size = (uint)(strings_count * sizeof(uint));
4431 uint* sizes = (uint*)addr(strings_offset);
4432 uint* hashs = (uint*)addr(strings_offset + data_size);
4433 strings_size -= 2 * data_size;
4434 // We have to keep cached strings longer than _cache buffer
4435 // because they are refernced from compiled code which may
4436 // still be executed on VM exit after _cache is freed.
4437 char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode);
4438 memcpy(p, addr(strings_offset + 2 * data_size), strings_size);
4439 _C_strings_buf = p;
4440 assert(strings_count <= MAX_STR_COUNT, "sanity");
4441 for (uint i = 0; i < strings_count; i++) {
4442 _C_strings[i] = p;
4443 uint len = sizes[i];
4444 _C_strings_s[i] = i;
4445 _C_strings_id[i] = i;
4446 _C_strings_len[i] = len;
4447 _C_strings_hash[i] = hashs[i];
4448 p += len;
4449 }
4450 assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size);
4451 _C_strings_count = strings_count;
4452 _C_strings_used = strings_count;
4453 log_info(aot, codecache, init)("Load %d C strings at offset %d from AOT Code Cache", _C_strings_count, strings_offset);
4454 }
4455
4456 int AOTCodeCache::store_strings() {
4457 uint offset = _write_position;
4458 uint length = 0;
4459 if (_C_strings_used > 0) {
4460 // Write sizes first
4461 for (int i = 0; i < _C_strings_used; i++) {
4462 uint len = _C_strings_len[i] + 1; // Include 0
4463 length += len;
4464 assert(len < 1000, "big string: %s", _C_strings[i]);
4465 uint n = write_bytes(&len, sizeof(uint));
4466 if (n != sizeof(uint)) {
4467 return -1;
4468 }
4469 }
4470 // Write hashs
4471 for (int i = 0; i < _C_strings_used; i++) {
4472 uint n = write_bytes(&(_C_strings_hash[i]), sizeof(uint));
4473 if (n != sizeof(uint)) {
4474 return -1;
4475 }
4476 }
4477 for (int i = 0; i < _C_strings_used; i++) {
4478 uint len = _C_strings_len[i] + 1; // Include 0
4479 uint n = write_bytes(_C_strings[_C_strings_s[i]], len);
4480 if (n != len) {
4481 return -1;
4482 }
4483 }
4484 log_info(aot, codecache, exit)("Wrote %d C strings of total length %d at offset %d to AOT Code Cache",
4485 _C_strings_used, length, offset);
4486 }
4487 return _C_strings_used;
4488 }
4489
4490 void AOTCodeCache::add_new_C_string(const char* str) {
4491 assert(for_write(), "only when storing code");
4492 _table->add_C_string(str);
4493 }
4494
4495 void AOTCodeAddressTable::add_C_string(const char* str) {
4496 if (str != nullptr && _extrs_complete) {
4497 // Check previous strings address
4498 for (int i = 0; i < _C_strings_count; i++) {
4499 if (_C_strings[i] == str) {
4500 return; // Found existing one
4501 }
4502 }
4503 // Add new one
4504 if (_C_strings_count < MAX_STR_COUNT) {
4505 log_trace(aot, codecache)("add_C_string: [%d] " INTPTR_FORMAT " %s", _C_strings_count, p2i(str), str);
4506 _C_strings_id[_C_strings_count] = -1; // Init
4507 _C_strings[_C_strings_count++] = str;
4508 } else {
4509 if (Thread::current()->is_Compiler_thread()) {
4510 CompileTask* task = ciEnv::current()->task();
4511 log_info(aot, codecache)("%d (L%d): Number of C strings > max %d %s",
4512 task->compile_id(), task->comp_level(), MAX_STR_COUNT, str);
4513 }
4514 }
4515 }
4516 }
4517
4518 int AOTCodeAddressTable::id_for_C_string(address str) {
4519 for (int i = 0; i < _C_strings_count; i++) {
4520 if (_C_strings[i] == (const char*)str) { // found
4521 int id = _C_strings_id[i];
4522 if (id >= 0) {
4523 assert(id < _C_strings_used, "%d >= %d", id , _C_strings_used);
4524 return id; // Found recorded
4525 }
4526 // Search for the same string content
4527 int len = (int)strlen((const char*)str);
4528 int hash = java_lang_String::hash_code((const jbyte*)str, len);
4529 for (int j = 0; j < _C_strings_used; j++) {
4530 if ((_C_strings_len[j] == len) && (_C_strings_hash[j] == hash)) {
4531 _C_strings_id[i] = j; // Found match
4532 return j;
4533 }
4534 }
4535 // Not found in recorded, add new
4536 id = _C_strings_used++;
4537 _C_strings_s[id] = i;
4538 _C_strings_id[i] = id;
4539 _C_strings_len[id] = len;
4540 _C_strings_hash[id] = hash;
4541 return id;
4542 }
4543 }
4544 return -1;
4545 }
4546
4547 address AOTCodeAddressTable::address_for_C_string(int idx) {
4548 assert(idx < _C_strings_count, "sanity");
4549 return (address)_C_strings[idx];
4550 }
4551
4552 int search_address(address addr, address* table, uint length) {
4553 for (int i = 0; i < (int)length; i++) {
4554 if (table[i] == addr) {
4555 return i;
4556 }
4557 }
4558 return -1;
4559 }
4560
4561 address AOTCodeAddressTable::address_for_id(int idx) {
4562 if (!_extrs_complete) {
4563 fatal("AOT Code Cache VM runtime addresses table is not complete");
4564 }
4565 if (idx == -1) {
4566 return (address)-1;
4567 }
4568 uint id = (uint)idx;
4569 // special case for symbols based relative to os::init
4570 if (id > (_c_str_base + _c_str_max)) {
4571 return (address)os::init + idx;
4572 }
4573 if (idx < 0) {
4574 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
4575 }
4576 // no need to compare unsigned id against 0
4577 if (/* id >= _extrs_base && */ id < _extrs_length) {
4578 return _extrs_addr[id - _extrs_base];
4579 }
4580 if (id >= _stubs_base && id < _stubs_base + _stubs_length) {
4581 return _stubs_addr[id - _stubs_base];
4582 }
4583 if (id >= _blobs_base && id < _blobs_base + _blobs_length) {
4584 return _blobs_addr[id - _blobs_base];
4585 }
4586 if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) {
4587 return _C1_blobs_addr[id - _C1_blobs_base];
4588 }
4589 if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) {
4590 return _C2_blobs_addr[id - _C2_blobs_base];
4591 }
4592 if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) {
4593 return address_for_C_string(id - _c_str_base);
4594 }
4595 fatal("Incorrect id %d for AOT Code Cache addresses table", id);
4596 return nullptr;
4597 }
4598
4599 int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer* buffer) {
4600 if (!_extrs_complete) {
4601 fatal("AOT Code Cache VM runtime addresses table is not complete");
4602 }
4603 int id = -1;
4604 if (addr == (address)-1) { // Static call stub has jump to itself
4605 return id;
4606 }
4607 // Seach for C string
4608 id = id_for_C_string(addr);
4609 if (id >=0) {
4610 return id + _c_str_base;
4611 }
4612 if (StubRoutines::contains(addr)) {
4613 // Search in stubs
4614 id = search_address(addr, _stubs_addr, _stubs_length);
4615 if (id < 0) {
4616 StubCodeDesc* desc = StubCodeDesc::desc_for(addr);
4617 if (desc == nullptr) {
4618 desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset);
4619 }
4620 const char* sub_name = (desc != nullptr) ? desc->name() : "<unknown>";
4621 fatal("Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name);
4622 } else {
4623 return _stubs_base + id;
4624 }
4625 } else {
4626 CodeBlob* cb = CodeCache::find_blob(addr);
4627 if (cb != nullptr) {
4628 int id_base = _blobs_base;
4629 // Search in code blobs
4630 id = search_address(addr, _blobs_addr, _blobs_length);
4631 if (id == -1) {
4632 id_base = _C1_blobs_base;
4633 // search C1 blobs
4634 id = search_address(addr, _C1_blobs_addr, _C1_blobs_length);
4635 }
4636 if (id == -1) {
4637 id_base = _C2_blobs_base;
4638 // search C2 blobs
4639 id = search_address(addr, _C2_blobs_addr, _C2_blobs_length);
4640 }
4641 if (id < 0) {
4642 fatal("Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name());
4643 } else {
4644 return id_base + id;
4645 }
4646 } else {
4647 // Search in runtime functions
4648 id = search_address(addr, _extrs_addr, _extrs_length);
4649 if (id < 0) {
4650 ResourceMark rm;
4651 const int buflen = 1024;
4652 char* func_name = NEW_RESOURCE_ARRAY(char, buflen);
4653 int offset = 0;
4654 if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) {
4655 if (offset > 0) {
4656 // Could be address of C string
4657 uint dist = (uint)pointer_delta(addr, (address)os::init, 1);
4658 CompileTask* task = ciEnv::current()->task();
4659 uint compile_id = 0;
4660 uint comp_level =0;
4661 if (task != nullptr) { // this could be called from compiler runtime initialization (compiler blobs)
4662 compile_id = task->compile_id();
4663 comp_level = task->comp_level();
4664 }
4665 log_info(aot, codecache)("%d (L%d): Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table",
4666 compile_id, comp_level, p2i(addr), dist, (const char*)addr);
4667 assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance");
4668 return dist;
4669 }
4670 fatal("Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset);
4671 } else {
4672 os::print_location(tty, p2i(addr), true);
4673 reloc.print_current_on(tty);
4674 #ifndef PRODUCT
4675 buffer->print_on(tty);
4676 buffer->decode();
4677 #endif // !PRODUCT
4678 fatal("Address " INTPTR_FORMAT " for <unknown> is missing in AOT Code Cache addresses table", p2i(addr));
4679 }
4680 } else {
4681 return _extrs_base + id;
4682 }
4683 }
4684 }
4685 return id;
4686 }
4687
4688 #undef _extrs_max
4689 #undef _stubs_max
4690 #undef _all_blobs_max
4691 #undef _blobs_max
4692 #undef _C1_blobs_max
4693 #undef _C2_blobs_max
4694 #undef _extrs_base
4695 #undef _stubs_base
4696 #undef _blobs_base
4697 #undef _C1_blobs_base
4698 #undef _C2_blobs_base
4699 #undef _c_str_base
4700
4701 void AOTRuntimeConstants::initialize_from_runtime() {
4702 BarrierSet* bs = BarrierSet::barrier_set();
4703 if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
4704 CardTableBarrierSet* ctbs = ((CardTableBarrierSet*)bs);
4705 _aot_runtime_constants._grain_shift = ctbs->grain_shift();
4706 _aot_runtime_constants._card_shift = ctbs->card_shift();
4707 }
4708 }
4709
4710 AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
4711
4712 address AOTRuntimeConstants::_field_addresses_list[] = {
4713 grain_shift_address(),
4714 card_shift_address(),
4715 nullptr
4716 };
4717
4718
4719 void AOTCodeCache::wait_for_no_nmethod_readers() {
4720 while (true) {
4721 int cur = Atomic::load(&_nmethod_readers);
4722 int upd = -(cur + 1);
4723 if (cur >= 0 && Atomic::cmpxchg(&_nmethod_readers, cur, upd) == cur) {
4724 // Success, no new readers should appear.
4725 break;
4726 }
4727 }
4728
4729 // Now wait for all readers to leave.
4730 SpinYield w;
4731 while (Atomic::load(&_nmethod_readers) != -1) {
4732 w.wait();
4733 }
4734 }
4735
4736 AOTCodeCache::ReadingMark::ReadingMark() {
4737 while (true) {
4738 int cur = Atomic::load(&_nmethod_readers);
4739 if (cur < 0) {
4740 // Cache is already closed, cannot proceed.
4741 _failed = true;
4742 return;
4743 }
4744 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
4745 // Successfully recorded ourselves as entered.
4746 _failed = false;
4747 return;
4748 }
4749 }
4750 }
4751
4752 AOTCodeCache::ReadingMark::~ReadingMark() {
4753 if (_failed) {
4754 return;
4755 }
4756 while (true) {
4757 int cur = Atomic::load(&_nmethod_readers);
4758 if (cur > 0) {
4759 // Cache is open, we are counting down towards 0.
4760 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur - 1) == cur) {
4761 return;
4762 }
4763 } else {
4764 // Cache is closed, we are counting up towards -1.
4765 if (Atomic::cmpxchg(&_nmethod_readers, cur, cur + 1) == cur) {
4766 return;
4767 }
4768 }
4769 }
4770 }