1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "cds/aotMetaspace.hpp"
26 #include "cds/aotStreamedHeapLoader.hpp"
27 #include "cds/aotThread.hpp"
28 #include "cds/cdsConfig.hpp"
29 #include "cds/filemap.hpp"
30 #include "cds/heapShared.inline.hpp"
31 #include "classfile/classLoaderDataShared.hpp"
32 #include "classfile/javaClasses.inline.hpp"
33 #include "classfile/stringTable.hpp"
34 #include "classfile/vmClasses.hpp"
35 #include "gc/shared/collectedHeap.inline.hpp"
36 #include "gc/shared/oopStorage.inline.hpp"
37 #include "gc/shared/oopStorageSet.inline.hpp"
38 #include "logging/log.hpp"
39 #include "memory/iterator.inline.hpp"
40 #include "memory/oopFactory.hpp"
41 #include "oops/access.inline.hpp"
42 #include "oops/objArrayOop.inline.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/globals.hpp"
45 #include "runtime/globals_extension.hpp"
46 #include "runtime/handles.inline.hpp"
47 #include "runtime/java.hpp"
48 #include "runtime/mutex.hpp"
49 #include "runtime/thread.hpp"
50 #include "utilities/bitMap.inline.hpp"
51 #include "utilities/exceptions.hpp"
52 #include "utilities/globalDefinitions.hpp"
53 #include "utilities/stack.inline.hpp"
54 #include "utilities/ticks.hpp"
55
56 #include <type_traits>
57
58 #if INCLUDE_CDS_JAVA_HEAP
59
60 FileMapRegion* AOTStreamedHeapLoader::_heap_region;
61 FileMapRegion* AOTStreamedHeapLoader::_bitmap_region;
62 int* AOTStreamedHeapLoader::_roots_archive;
63 OopHandle AOTStreamedHeapLoader::_roots;
64 BitMapView AOTStreamedHeapLoader::_oopmap;
65 bool AOTStreamedHeapLoader::_is_in_use;
66 int AOTStreamedHeapLoader::_previous_batch_last_object_index;
67 int AOTStreamedHeapLoader::_current_batch_last_object_index;
68 int AOTStreamedHeapLoader::_current_root_index;
69 size_t AOTStreamedHeapLoader::_allocated_words;
70 bool AOTStreamedHeapLoader::_allow_gc;
71 bool AOTStreamedHeapLoader::_objects_are_handles;
72 size_t AOTStreamedHeapLoader::_num_archived_objects;
73 int AOTStreamedHeapLoader::_num_roots;
74 size_t AOTStreamedHeapLoader::_heap_region_used;
75 bool AOTStreamedHeapLoader::_loading_all_objects;
76
77 size_t* AOTStreamedHeapLoader::_object_index_to_buffer_offset_table;
78 void** AOTStreamedHeapLoader::_object_index_to_heap_object_table;
79 int* AOTStreamedHeapLoader::_root_highest_object_index_table;
80
81 bool AOTStreamedHeapLoader::_waiting_for_iterator;
82 bool AOTStreamedHeapLoader::_swapping_root_format;
83
84 static uint64_t _early_materialization_time_ns = 0;
85 static uint64_t _late_materialization_time_ns = 0;
86 static uint64_t _final_materialization_time_ns = 0;
87 static uint64_t _cleanup_materialization_time_ns = 0;
88 static volatile uint64_t _accumulated_lazy_materialization_time_ns = 0;
89 static Ticks _materialization_start_ticks;
90
91 int AOTStreamedHeapLoader::object_index_for_root_index(int root_index) {
92 return _roots_archive[root_index];
93 }
94
95 int AOTStreamedHeapLoader::highest_object_index_for_root_index(int root_index) {
96 return _root_highest_object_index_table[root_index];
97 }
98
99 size_t AOTStreamedHeapLoader::buffer_offset_for_object_index(int object_index) {
100 return _object_index_to_buffer_offset_table[object_index];
101 }
102
103 oopDesc* AOTStreamedHeapLoader::archive_object_for_object_index(int object_index) {
104 size_t buffer_offset = buffer_offset_for_object_index(object_index);
105 address bottom = (address)_heap_region->mapped_base();
106 return (oopDesc*)(bottom + buffer_offset);
107 }
108
109 size_t AOTStreamedHeapLoader::buffer_offset_for_archive_object(oopDesc* archive_object) {
110 address bottom = (address)_heap_region->mapped_base();
111 return size_t(archive_object) - size_t(bottom);
112 }
113
114 template <bool use_coops>
115 BitMap::idx_t AOTStreamedHeapLoader::obj_bit_idx_for_buffer_offset(size_t buffer_offset) {
116 if constexpr (use_coops) {
117 return BitMap::idx_t(buffer_offset / sizeof(narrowOop));
118 } else {
119 return BitMap::idx_t(buffer_offset / sizeof(HeapWord));
120 }
121 }
122
123 oop AOTStreamedHeapLoader::heap_object_for_object_index(int object_index) {
124 assert(object_index >= 0 && object_index <= (int)_num_archived_objects,
125 "Heap object reference out of index: %d", object_index);
126
127 if (_objects_are_handles) {
128 oop* handle = (oop*)_object_index_to_heap_object_table[object_index];
129 if (handle == nullptr) {
130 return nullptr;
131 }
132 return NativeAccess<>::oop_load(handle);
133 } else {
134 return cast_to_oop(_object_index_to_heap_object_table[object_index]);
135 }
136 }
137
138 void AOTStreamedHeapLoader::set_heap_object_for_object_index(int object_index, oop heap_object) {
139 assert(heap_object_for_object_index(object_index) == nullptr, "Should only set once with this API");
140 if (_objects_are_handles) {
141 oop* handle = Universe::vm_global()->allocate();
142 NativeAccess<>::oop_store(handle, heap_object);
143 _object_index_to_heap_object_table[object_index] = (void*)handle;
144 } else {
145 _object_index_to_heap_object_table[object_index] = cast_from_oop<void*>(heap_object);
146 }
147 }
148
149 int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) {
150 assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string");
151 address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset();
152 return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr;
153 }
154
155 static int archive_array_length(oopDesc* archive_array) {
156 return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes());
157 }
158
159 // archive_object lives in CDS mapped memory, not in the GC heap.
160 // Calling expand_for_hash() converts the raw oopDesc* to an oop, which
161 // triggers oop verification. ZGC's verifier rejects non-heap addresses,
162 // so we must suspend the check for that call.
163 #ifdef CHECK_UNHANDLED_OOPS
164 class SuspendCheckOopFunction : public StackObj {
165 CheckOopFunctionPointer _saved;
166 public:
167 SuspendCheckOopFunction() : _saved(check_oop_function) { check_oop_function = nullptr; }
168 ~SuspendCheckOopFunction() { check_oop_function = _saved; }
169 };
170 #endif
171
172 static bool archive_expand_for_hash(Klass* klass, oopDesc* archive_object) {
173 CHECK_UNHANDLED_OOPS_ONLY(SuspendCheckOopFunction suspend;)
174 return klass->expand_for_hash(archive_object, archive_object->mark());
175 }
176
177 static size_t archive_object_size(oopDesc* archive_object) {
178 Klass* klass = archive_object->klass();
179 int lh = klass->layout_helper();
180
181 if (Klass::layout_helper_is_instance(lh)) {
182 // Instance
183 if (Klass::layout_helper_needs_slow_path(lh)) {
184 return ((size_t*)(archive_object))[-1];
185 } else {
186 size_t size = (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize;
187 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && archive_expand_for_hash(klass, archive_object)) {
188 size = align_object_size(size + 1);
189 }
190 return size;
191 }
192 } else if (Klass::layout_helper_is_array(lh)) {
193 // Array
194 size_t size_in_bytes;
195 size_t array_length = (size_t)archive_array_length(archive_object);
196 size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh);
197 size_in_bytes += (size_t)Klass::layout_helper_header_size(lh);
198
199 size_t size = align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize;
200 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && archive_expand_for_hash(klass, archive_object)) {
201 size = align_object_size(size + 1);
202 }
203 return size;
204 } else {
205 // Other
206 return ((size_t*)(archive_object))[-1];
207 }
208 }
209
210 oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) {
211 assert(!archive_object->is_stackChunk(), "no such objects are archived");
212
213 NoJvmtiEventsMark njem;
214 oop heap_object;
215
216 Klass* klass = archive_object->klass();
217 assert(!(UseCompactObjectHeaders && mark.is_hashed_not_expanded()), "Must not be hashed/not-expanded");
218 if (klass->is_mirror_instance_klass()) {
219 // The oop_size field must hold the *un-expanded* base size: oopDesc::size()
220 // re-adds the identity-hash expansion word for an expanded mark, so passing
221 // the expanded total (`size`) here leaves the mirror transiently oversized
222 // (size() > allocated) between allocation and the payload copy. A GC during
223 // CDS heap load (far more likely at large ObjectAlignmentInBytes, which bloats
224 // the heap) can observe that and desync the heap walk. The archive object's
225 // own oop_size field already holds the correct base size.
226 size_t base_size = (size_t)java_lang_Class::oop_size(cast_to_oop(archive_object));
227 assert(!(UseCompactObjectHeaders && mark.is_not_hashed_expanded()), "should not happen");
228 heap_object = Universe::heap()->class_allocate(klass, size, base_size, CHECK_NULL);
229 } else if (klass->is_instance_klass()) {
230 heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL);
231 } else {
232 assert(klass->is_array_klass(), "must be");
233 int length = archive_array_length(archive_object);
234 bool do_zero = klass->is_objArray_klass();
235 heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL);
236 }
237
238 heap_object->set_mark(mark);
239
240 return heap_object;
241 }
242
243 void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) {
244 objArrayOop roots = objArrayOop(_roots.resolve());
245 OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock
246 roots->obj_at_put(root_index, heap_object);
247 }
248
249 void AOTStreamedHeapLoader::TracingObjectLoader::wait_for_iterator() {
250 if (JavaThread::current()->is_active_Java_thread()) {
251 // When the main thread has bootstrapped past the point of allowing safepoints,
252 // we can and indeed have to use safepoint checking waiting.
253 AOTHeapLoading_lock->wait();
254 } else {
255 // If we have no bootstrapped the main thread far enough, then we cannot and
256 // indeed also don't need to perform safepoint checking waiting.
257 AOTHeapLoading_lock->wait_without_safepoint_check();
258 }
259 }
260
261 // Link object after copying in-place
262 template <typename LinkerT>
263 class AOTStreamedHeapLoader::InPlaceLinkingOopClosure : public BasicOopIterateClosure {
264 private:
265 oop _obj;
266 LinkerT _linker;
267
268 public:
269 InPlaceLinkingOopClosure(oop obj, LinkerT linker)
270 : _obj(obj),
271 _linker(linker) {
272 }
273
274 virtual void do_oop(oop* p) { do_oop_work(p, (int)*(intptr_t*)p); }
275 virtual void do_oop(narrowOop* p) { do_oop_work(p, *(int*)p); }
276
277 template <typename T>
278 void do_oop_work(T* p, int object_index) {
279 int p_offset = pointer_delta_as_int((address)p, cast_from_oop<address>(_obj));
280 oop pointee = _linker(p_offset, object_index);
281 if (pointee != nullptr) {
282 _obj->obj_field_put_access<IS_DEST_UNINITIALIZED>((int)p_offset, pointee);
283 }
284 }
285 };
286
287 template <bool use_coops, typename LinkerT>
288 void AOTStreamedHeapLoader::copy_payload_carefully(oopDesc* archive_object,
289 oop heap_object,
290 BitMap::idx_t header_bit,
291 BitMap::idx_t start_bit,
292 BitMap::idx_t end_bit,
293 LinkerT linker) {
294 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
295 using OopElementT = std::conditional_t<use_coops, narrowOop, oop>;
296
297 BitMap::idx_t unfinished_bit = start_bit;
298 BitMap::idx_t next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
299
300 // Fill in heap object bytes
301 while (unfinished_bit < end_bit) {
302 assert(unfinished_bit >= start_bit && unfinished_bit < end_bit, "out of bounds copying");
303
304 // This is the address of the pointee inside the input stream
305 size_t payload_offset = unfinished_bit - header_bit;
306 RawElementT* archive_payload_addr = ((RawElementT*)archive_object) + payload_offset;
307 RawElementT* heap_payload_addr = cast_from_oop<RawElementT*>(heap_object) + payload_offset;
308
309 assert(heap_payload_addr >= cast_from_oop<RawElementT*>(heap_object) &&
310 (HeapWord*)heap_payload_addr < cast_from_oop<HeapWord*>(heap_object) + heap_object->size(),
311 "Out of bounds copying");
312
313 if (next_reference_bit > unfinished_bit) {
314 // Primitive bytes available
315 size_t primitive_elements = next_reference_bit - unfinished_bit;
316 size_t primitive_bytes = primitive_elements * sizeof(RawElementT);
317 ::memcpy(heap_payload_addr, archive_payload_addr, primitive_bytes);
318
319 unfinished_bit = next_reference_bit;
320 } else {
321 // Encountered reference
322 RawElementT* archive_p = (RawElementT*)archive_payload_addr;
323 OopElementT* heap_p = (OopElementT*)heap_payload_addr;
324 int pointee_object_index = (int)*archive_p;
325 int heap_p_offset = pointer_delta_as_int((address)heap_p, cast_from_oop<address>(heap_object));
326
327 // The object index is retrieved from the archive, not the heap object. This is
328 // important after GC is enabled. Concurrent GC threads may scan references in the
329 // heap for various reasons after this point. Therefore, it is not okay to first copy
330 // the object index from a reference location in the archived object payload to a
331 // corresponding location in the heap object payload, and then fix it up afterwards to
332 // refer to a heap object. This is why this code iterates carefully over object references
333 // in the archived object, linking them one by one, without clobbering the reference
334 // locations in the heap objects with anything other than transitions from null to the
335 // intended linked object.
336 oop obj = linker(heap_p_offset, pointee_object_index);
337 if (obj != nullptr) {
338 heap_object->obj_field_put(heap_p_offset, obj);
339 }
340
341 unfinished_bit++;
342 next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
343 }
344 }
345 }
346
347 template <bool use_coops, typename LinkerT>
348 void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object,
349 oop heap_object,
350 size_t size,
351 LinkerT linker) {
352 if (!_allow_gc) {
353 // Without concurrent GC running, we can copy incorrect object references
354 // and metadata references into the heap object and then fix them up in-place.
355 size_t offset = 1;
356 size_t payload_size = size - offset;
357 HeapWord* archive_start = ((HeapWord*)archive_object);
358 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
359
360 Copy::disjoint_words(archive_start + offset, heap_start + offset, payload_size);
361
362 if (UseCompactObjectHeaders) {
363 // The copying might have missed the first 4 bytes of payload/arraylength, copy that also.
364 *(reinterpret_cast<jint*>(heap_start) + 1) = *(reinterpret_cast<jint*>(archive_start) + 1);
365 }
366
367 // In-place linking fixes up object indices from references of the heap object,
368 // and patches them up to refer to objects. This can be done because we just copied
369 // the payload of the object from the archive to the heap object, including the
370 // reference object indices. However, this is only okay to do before the GC can run.
371 // A concurrent GC thread might racingly read the object payload after GC is enabled.
372 InPlaceLinkingOopClosure cl(heap_object, linker);
373 heap_object->oop_iterate(&cl);
374 HeapShared::remap_loaded_metadata(heap_object);
375 return;
376 }
377
378 // When a concurrent GC may be running, we take care not to copy incorrect oops,
379 // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the
380 // intended runtime linked values only.
381 size_t word_scale = use_coops ? 2 : 1;
382 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
383
384 // Skip the markWord; it is set at allocation time
385 size_t header_size = (UseCompactObjectHeaders && use_coops) ? 1 : word_scale;
386
387 size_t buffer_offset = buffer_offset_for_archive_object(archive_object);
388 const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset<use_coops>(buffer_offset);
389 const BitMap::idx_t start_bit = header_bit + header_size;
390 const BitMap::idx_t end_bit = header_bit + size * word_scale;
391
392 BitMap::idx_t curr_bit = start_bit;
393
394 if (UseCompactObjectHeaders && !use_coops) {
395 // Copy first 4 primitive bytes.
396 jint* archive_start = reinterpret_cast<jint*>(archive_object);
397 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
398 *(reinterpret_cast<jint*>(heap_start) + 1) = *(archive_start + 1);
399 }
400
401 // We are a bit paranoid about GC or other safepointing operations observing
402 // shady metadata fields from the archive that do not point at real metadata.
403 // We deal with this by explicitly reading the requested address from the
404 // archive and fixing it to real Metadata before writing it into the heap object.
405 HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) {
406 BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT);
407 BitMap::idx_t skip = word_scale;
408 assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit,
409 "Metadata field out of bounds");
410
411 // Copy payload before metadata field
412 copy_payload_carefully<use_coops>(archive_object,
413 heap_object,
414 header_bit,
415 curr_bit,
416 metadata_field_idx,
417 linker);
418
419 // Copy metadata field
420 Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset);
421 Metadata* const runtime_metadata = archive_metadata != nullptr
422 ? (Metadata*)(address(archive_metadata) + AOTMetaspace::relocation_delta())
423 : nullptr;
424 assert(runtime_metadata == nullptr || AOTMetaspace::in_aot_cache(runtime_metadata), "Invalid metadata pointer");
425 DEBUG_ONLY(Metadata* const previous_metadata = heap_object->metadata_field(metadata_offset);)
426 assert(previous_metadata == nullptr || previous_metadata == runtime_metadata, "Should not observe transient values");
427 heap_object->metadata_field_put(metadata_offset, runtime_metadata);
428 curr_bit = metadata_field_idx + skip;
429 });
430
431 // Copy trailing metadata after the last metadata word. This is usually doing
432 // all the copying.
433 copy_payload_carefully<use_coops>(archive_object,
434 heap_object,
435 header_bit,
436 curr_bit,
437 end_bit,
438 linker);
439 }
440
441 void AOTStreamedHeapLoader::copy_object_eager_linking(oopDesc* archive_object, oop heap_object, size_t size) {
442 auto linker = [&](int p_offset, int pointee_object_index) {
443 oop obj = AOTStreamedHeapLoader::heap_object_for_object_index(pointee_object_index);
444 assert(pointee_object_index == 0 || obj != nullptr, "Eager object loading should only encounter already allocated links");
445 return obj;
446 };
447 if (UseCompressedOops) {
448 copy_object_impl<true>(archive_object, heap_object, size, linker);
449 } else {
450 copy_object_impl<false>(archive_object, heap_object, size, linker);
451 }
452 }
453
454 void AOTStreamedHeapLoader::TracingObjectLoader::copy_object_lazy_linking(int object_index,
455 oopDesc* archive_object,
456 oop heap_object,
457 size_t size,
458 Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack) {
459 auto linker = [&](int p_offset, int pointee_object_index) {
460 dfs_stack.push({pointee_object_index, object_index, p_offset});
461
462 // The tracing linker is a bit lazy and mutates the reference fields in its traversal.
463 // Returning null means don't link now.
464 return oop(nullptr);
465 };
466 if (UseCompressedOops) {
467 copy_object_impl<true>(archive_object, heap_object, size, linker);
468 } else {
469 copy_object_impl<false>(archive_object, heap_object, size, linker);
470 }
471 }
472
473 oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object_inner(int object_index, Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack, TRAPS) {
474 // Allocate object
475 oopDesc* archive_object = archive_object_for_object_index(object_index);
476 size_t size = archive_object_size(archive_object);
477 markWord mark = archive_object->mark();
478
479 // The markWord is marked if the object is a String and it should be interned,
480 // make sure to unmark it before allocating memory for the object.
481 bool string_intern = mark.is_marked();
482 mark = mark.set_unmarked();
483
484 oop heap_object;
485
486 if (string_intern) {
487 int value_object_index = archived_string_value_object_index(archive_object);
488
489 // Materialize the value object.
490 (void)materialize_object(value_object_index, dfs_stack, CHECK_NULL);
491
492 // Allocate and link the string.
493 heap_object = allocate_object(archive_object, mark, size, CHECK_NULL);
494 copy_object_eager_linking(archive_object, heap_object, size);
495
496 assert(java_lang_String::value(heap_object) == heap_object_for_object_index(value_object_index), "Linker should have linked this correctly");
497
498 // Replace the string with interned string
499 heap_object = StringTable::intern(heap_object, CHECK_NULL);
500 } else {
501 heap_object = allocate_object(archive_object, mark, size, CHECK_NULL);
502
503 // Fill in object contents
504 copy_object_lazy_linking(object_index, archive_object, heap_object, size, dfs_stack);
505 }
506
507 // Install forwarding
508 set_heap_object_for_object_index(object_index, heap_object);
509
510 return heap_object;
511 }
512
513 oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object(int object_index, Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack, TRAPS) {
514 if (object_index <= _previous_batch_last_object_index) {
515 // The transitive closure of this object has been materialized; no need to do anything
516 return heap_object_for_object_index(object_index);
517 }
518
519 if (object_index <= _current_batch_last_object_index) {
520 // The AOTThread is currently materializing this object and its transitive closure; only need to wait for it to complete
521 _waiting_for_iterator = true;
522 while (object_index > _previous_batch_last_object_index) {
523 wait_for_iterator();
524 }
525 _waiting_for_iterator = false;
526
527 // Notify the AOT thread if it is waiting for tracing to finish
528 AOTHeapLoading_lock->notify_all();
529 return heap_object_for_object_index(object_index);;
530 }
531
532 oop heap_object = heap_object_for_object_index(object_index);
533 if (heap_object != nullptr) {
534 // Already materialized by mutator
535 return heap_object;
536 }
537
538 return materialize_object_inner(object_index, dfs_stack, THREAD);
539 }
540
541 void AOTStreamedHeapLoader::TracingObjectLoader::drain_stack(Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack, TRAPS) {
542 while (!dfs_stack.is_empty()) {
543 AOTHeapTraversalEntry entry = dfs_stack.pop();
544 int pointee_object_index = entry._pointee_object_index;
545 oop pointee_heap_object = materialize_object(pointee_object_index, dfs_stack, CHECK);
546 oop heap_object = heap_object_for_object_index(entry._base_object_index);
547 if (_allow_gc) {
548 heap_object->obj_field_put(entry._heap_field_offset_bytes, pointee_heap_object);
549 } else {
550 heap_object->obj_field_put_access<IS_DEST_UNINITIALIZED>(entry._heap_field_offset_bytes, pointee_heap_object);
551 }
552 }
553 }
554
555 oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object_transitive(int object_index, Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack, TRAPS) {
556 assert_locked_or_safepoint(AOTHeapLoading_lock);
557 while (_waiting_for_iterator) {
558 wait_for_iterator();
559 }
560
561 auto handlized_materialize_object = [&](TRAPS) {
562 oop obj = materialize_object(object_index, dfs_stack, CHECK_(Handle()));
563 return Handle(THREAD, obj);
564 };
565
566 Handle result = handlized_materialize_object(CHECK_NULL);
567 drain_stack(dfs_stack, CHECK_NULL);
568
569 return result();
570 }
571
572 oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_root(int root_index, Stack<AOTHeapTraversalEntry, mtClassShared>& dfs_stack, TRAPS) {
573 int root_object_index = object_index_for_root_index(root_index);
574 oop root = materialize_object_transitive(root_object_index, dfs_stack, CHECK_NULL);
575 install_root(root_index, root);
576
577 return root;
578 }
579
580 int oop_handle_cmp(const void* left, const void* right) {
581 oop* left_handle = *(oop**)left;
582 oop* right_handle = *(oop**)right;
583
584 if (right_handle > left_handle) {
585 return -1;
586 } else if (left_handle > right_handle) {
587 return 1;
588 }
589
590 return 0;
591 }
592
593 // The range is inclusive
594 void AOTStreamedHeapLoader::IterativeObjectLoader::initialize_range(int first_object_index, int last_object_index, TRAPS) {
595 for (int i = first_object_index; i <= last_object_index; ++i) {
596 oopDesc* archive_object = archive_object_for_object_index(i);
597 markWord mark = archive_object->mark();
598 bool string_intern = mark.is_marked();
599 if (string_intern) {
600 int value_object_index = archived_string_value_object_index(archive_object);
601 if (value_object_index == i + 1) {
602 // Interned strings are eagerly materialized in the allocation phase, so there is
603 // nothing else to do for interned strings here for the string nor its value array.
604 i++;
605 }
606 continue;
607 }
608 size_t size = archive_object_size(archive_object);
609 oop heap_object = heap_object_for_object_index(i);
610 copy_object_eager_linking(archive_object, heap_object, size);
611 }
612 }
613
614 // The range is inclusive
615 size_t AOTStreamedHeapLoader::IterativeObjectLoader::materialize_range(int first_object_index, int last_object_index, TRAPS) {
616 GrowableArrayCHeap<int, mtClassShared> lazy_object_indices(0);
617 size_t materialized_words = 0;
618
619 for (int i = first_object_index; i <= last_object_index; ++i) {
620 oopDesc* archive_object = archive_object_for_object_index(i);
621 markWord mark = archive_object->mark();
622
623 // The markWord is marked if the object is a String and it should be interned,
624 // make sure to unmark it before allocating memory for the object.
625 bool string_intern = mark.is_marked();
626 mark = mark.set_unmarked();
627
628 size_t size = archive_object_size(archive_object);
629 materialized_words += size;
630
631 oop heap_object = heap_object_for_object_index(i);
632 if (heap_object != nullptr) {
633 // Lazy loading has already initialized the object; we must not mutate it
634 lazy_object_indices.append(i);
635 continue;
636 }
637
638 if (!string_intern) {
639 // The normal case; no lazy loading have loaded the object yet
640 heap_object = allocate_object(archive_object, mark, size, CHECK_0);
641 set_heap_object_for_object_index(i, heap_object);
642 continue;
643 }
644
645 // Eagerly materialize interned strings to ensure that objects earlier than the string
646 // in a batch get linked to the intended interned string, and not a copy.
647 int value_object_index = archived_string_value_object_index(archive_object);
648
649 bool is_normal_interned_string = value_object_index == i + 1;
650
651 if (value_object_index < first_object_index) {
652 // If materialized in a previous batch, the value should already be allocated and initialized.
653 assert(heap_object_for_object_index(value_object_index) != nullptr, "should be materialized");
654 } else {
655 // Materialize the value object.
656 oopDesc* archive_value_object = archive_object_for_object_index(value_object_index);
657 markWord value_mark = archive_value_object->mark();
658 size_t value_size = archive_object_size(archive_value_object);
659 oop value_heap_object;
660
661 if (is_normal_interned_string) {
662 // The common case: the value is next to the string. This happens when only the interned
663 // string points to its value character array.
664 assert(value_object_index <= last_object_index, "Must be within this batch: %d <= %d", value_object_index, last_object_index);
665 value_heap_object = allocate_object(archive_value_object, value_mark, value_size, CHECK_0);
666 set_heap_object_for_object_index(value_object_index, value_heap_object);
667 materialized_words += value_size;
668 } else {
669 // In the uncommon case, multiple strings point to the value of an interned string.
670 // The string can then be earlier in the batch.
671 assert(value_object_index < i, "surprising index");
672 value_heap_object = heap_object_for_object_index(value_object_index);
673 }
674
675 copy_object_eager_linking(archive_value_object, value_heap_object, value_size);
676 }
677 // Allocate and link the string.
678 heap_object = allocate_object(archive_object, mark, size, CHECK_0);
679 copy_object_eager_linking(archive_object, heap_object, size);
680
681 assert(java_lang_String::value(heap_object) == heap_object_for_object_index(value_object_index), "Linker should have linked this correctly");
682
683 // Replace the string with interned string
684 heap_object = StringTable::intern(heap_object, CHECK_0);
685 set_heap_object_for_object_index(i, heap_object);
686
687 if (is_normal_interned_string) {
688 // Skip over the string value, already materialized
689 i++;
690 }
691 }
692
693 if (lazy_object_indices.is_empty()) {
694 // Normal case; no sprinkled lazy objects in the root subgraph
695 initialize_range(first_object_index, last_object_index, CHECK_0);
696 } else {
697 // The user lazy initialized some objects that are already initialized; we have to initialize around them
698 // to make sure they are not mutated.
699 int previous_object_index = first_object_index - 1; // Exclusive start of initialization slice
700 for (int i = 0; i < lazy_object_indices.length(); ++i) {
701 int lazy_object_index = lazy_object_indices.at(i);
702 int slice_start_object_index = previous_object_index;
703 int slice_end_object_index = lazy_object_index;
704
705 if (slice_end_object_index - slice_start_object_index > 1) { // Both markers are exclusive
706 initialize_range(slice_start_object_index + 1, slice_end_object_index - 1, CHECK_0);
707 }
708 previous_object_index = lazy_object_index;
709 }
710 // Process tail range
711 if (last_object_index - previous_object_index > 0) {
712 initialize_range(previous_object_index + 1, last_object_index, CHECK_0);
713 }
714 }
715
716 return materialized_words;
717 }
718
719 bool AOTStreamedHeapLoader::IterativeObjectLoader::has_more() {
720 return _current_root_index < _num_roots;
721 }
722
723 void AOTStreamedHeapLoader::IterativeObjectLoader::materialize_next_batch(TRAPS) {
724 assert(has_more(), "only materialize if there is something to materialize");
725
726 int min_batch_objects = 128;
727 int from_root_index = _current_root_index;
728 int max_to_root_index = _num_roots - 1;
729 int until_root_index = from_root_index;
730 int highest_object_index;
731
732 // Expand the batch size from one root, to N roots until we cross 128 objects in total
733 for (;;) {
734 highest_object_index = highest_object_index_for_root_index(until_root_index);
735 if (highest_object_index - _previous_batch_last_object_index >= min_batch_objects) {
736 break;
737 }
738 if (until_root_index == max_to_root_index) {
739 break;
740 }
741 until_root_index++;
742 }
743
744 oop root = nullptr;
745
746 // Materialize objects of necessary, representing the transitive closure of the root
747 if (highest_object_index > _previous_batch_last_object_index) {
748 while (_swapping_root_format) {
749 // When the roots are being upgraded to use handles, it is not safe to racingly
750 // iterate over the object; we must wait. Setting the current batch last object index
751 // to something other than the previous batch last object index indicates to the
752 // root swapping that there is current iteration ongoing.
753 AOTHeapLoading_lock->wait();
754 }
755 int first_object_index = _previous_batch_last_object_index + 1;
756 _current_batch_last_object_index = highest_object_index;
757 size_t allocated_words;
758 {
759 MutexUnlocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
760 allocated_words = materialize_range(first_object_index, highest_object_index, CHECK);
761 }
762 _allocated_words += allocated_words;
763 _previous_batch_last_object_index = _current_batch_last_object_index;
764 if (_waiting_for_iterator) {
765 // If tracer is waiting, let it know at the next point of unlocking that the root
766 // set it waited for has been processed now.
767 AOTHeapLoading_lock->notify_all();
768 }
769 }
770
771 // Install the root
772 for (int i = from_root_index; i <= until_root_index; ++i) {
773 int root_object_index = object_index_for_root_index(i);
774 root = heap_object_for_object_index(root_object_index);
775 install_root(i, root);
776 ++_current_root_index;
777 }
778 }
779
780 bool AOTStreamedHeapLoader::materialize_early(TRAPS) {
781 Ticks start = Ticks::now();
782
783 // Only help with early materialization from the AOT thread if the heap archive can be allocated
784 // without the need for a GC. Otherwise, do lazy loading until GC is enabled later in the bootstrapping.
785 size_t bootstrap_max_memory = Universe::heap()->bootstrap_max_memory();
786 size_t bootstrap_min_memory = MAX2(_heap_region_used, 2 * M);
787
788 size_t before_gc_materialize_budget_bytes = (bootstrap_max_memory > bootstrap_min_memory) ? bootstrap_max_memory - bootstrap_min_memory : 0;
789 size_t before_gc_materialize_budget_words = before_gc_materialize_budget_bytes / HeapWordSize;
790
791 log_info(aot, heap)("Max bootstrapping memory: %zuM, min bootstrapping memory: %zuM, selected budget: %zuM",
792 bootstrap_max_memory / M, bootstrap_min_memory / M, before_gc_materialize_budget_bytes / M);
793
794 while (IterativeObjectLoader::has_more()) {
795 if (_allow_gc || _allocated_words > before_gc_materialize_budget_words) {
796 log_info(aot, heap)("Early object materialization interrupted at root %d", _current_root_index);
797 break;
798 }
799
800 IterativeObjectLoader::materialize_next_batch(CHECK_false);
801 }
802
803 _early_materialization_time_ns = (Ticks::now() - start).nanoseconds();
804
805 bool finished_before_gc_allowed = !_allow_gc && !IterativeObjectLoader::has_more();
806
807 return finished_before_gc_allowed;
808 }
809
810 void AOTStreamedHeapLoader::materialize_late(TRAPS) {
811 Ticks start = Ticks::now();
812
813 // Continue materializing with GC allowed
814
815 while (IterativeObjectLoader::has_more()) {
816 IterativeObjectLoader::materialize_next_batch(CHECK);
817 }
818
819 _late_materialization_time_ns = (Ticks::now() - start).nanoseconds();
820 }
821
822 void AOTStreamedHeapLoader::cleanup() {
823 // First ensure there is no concurrent tracing going on
824 while (_waiting_for_iterator) {
825 AOTHeapLoading_lock->wait();
826 }
827
828 Ticks start = Ticks::now();
829
830 // Remove OopStorage roots
831 if (_objects_are_handles) {
832 size_t num_handles = _num_archived_objects;
833 // Skip the null entry
834 oop** handles = ((oop**)_object_index_to_heap_object_table) + 1;
835 // Sort the handles so that oop storage can release them faster
836 qsort(handles, num_handles, sizeof(oop*), (int (*)(const void*, const void*))oop_handle_cmp);
837 size_t num_null_handles = 0;
838 for (size_t handles_remaining = num_handles; handles_remaining != 0; --handles_remaining) {
839 oop* handle = handles[handles_remaining - 1];
840 if (handle == nullptr) {
841 num_null_handles = handles_remaining;
842 break;
843 }
844 NativeAccess<>::oop_store(handle, nullptr);
845 }
846 Universe::vm_global()->release(&handles[num_null_handles], num_handles - num_null_handles);
847 }
848
849 FREE_C_HEAP_ARRAY(_object_index_to_heap_object_table);
850
851 // Unmap regions
852 FileMapInfo::current_info()->unmap_region(AOTMetaspace::hp);
853 FileMapInfo::current_info()->unmap_region(AOTMetaspace::bm);
854
855 _cleanup_materialization_time_ns = (Ticks::now() - start).nanoseconds();
856
857 log_statistics();
858 }
859
860 void AOTStreamedHeapLoader::log_statistics() {
861 uint64_t total_duration_us = (Ticks::now() - _materialization_start_ticks).microseconds();
862 const bool is_async = _loading_all_objects && !AOTEagerlyLoadObjects;
863 const char* const async_or_sync = is_async ? "async" : "sync";
864 log_info(aot, heap)("start to finish materialization time: " UINT64_FORMAT "us",
865 total_duration_us);
866 log_info(aot, heap)("early object materialization time (%s): " UINT64_FORMAT "us",
867 async_or_sync, _early_materialization_time_ns / 1000);
868 log_info(aot, heap)("late object materialization time (%s): " UINT64_FORMAT "us",
869 async_or_sync, _late_materialization_time_ns / 1000);
870 log_info(aot, heap)("object materialization cleanup time (%s): " UINT64_FORMAT "us",
871 async_or_sync, _cleanup_materialization_time_ns / 1000);
872 log_info(aot, heap)("final object materialization time stall (sync): " UINT64_FORMAT "us",
873 _final_materialization_time_ns / 1000);
874 log_info(aot, heap)("bootstrapping lazy materialization time (sync): " UINT64_FORMAT "us",
875 _accumulated_lazy_materialization_time_ns / 1000);
876
877 uint64_t sync_time = _final_materialization_time_ns + _accumulated_lazy_materialization_time_ns;
878 uint64_t async_time = _early_materialization_time_ns + _late_materialization_time_ns + _cleanup_materialization_time_ns;
879
880 if (!is_async) {
881 sync_time += async_time;
882 async_time = 0;
883 }
884
885 log_info(aot, heap)("sync materialization time: " UINT64_FORMAT "us",
886 sync_time / 1000);
887
888 log_info(aot, heap)("async materialization time: " UINT64_FORMAT "us",
889 async_time / 1000);
890
891 uint64_t iterative_time = (uint64_t)(is_async ? async_time : sync_time);
892 uint64_t materialized_bytes = _allocated_words * HeapWordSize;
893 log_info(aot, heap)("%s materialized " UINT64_FORMAT "K (" UINT64_FORMAT "M/s)", async_or_sync,
894 materialized_bytes / 1024, uint64_t(materialized_bytes * UCONST64(1'000'000'000) / M / iterative_time));
895 }
896
897 void AOTStreamedHeapLoader::materialize_objects() {
898 // We cannot handle any exception when materializing roots. Exits the VM.
899 EXCEPTION_MARK
900
901 // Objects are laid out in DFS order; DFS traverse the roots by linearly walking all objects
902 HandleMark hm(THREAD);
903
904 // Early materialization with a budget before GC is allowed
905 MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
906
907 materialize_early(CHECK);
908 await_gc_enabled();
909 materialize_late(CHECK);
910 // Notify materialization is done
911 AOTHeapLoading_lock->notify_all();
912 cleanup();
913 }
914
915 void AOTStreamedHeapLoader::switch_object_index_to_handle(int object_index) {
916 oop heap_object = cast_to_oop(_object_index_to_heap_object_table[object_index]);
917 if (heap_object == nullptr) {
918 return;
919 }
920
921 oop* handle = Universe::vm_global()->allocate();
922 NativeAccess<>::oop_store(handle, heap_object);
923 _object_index_to_heap_object_table[object_index] = handle;
924 }
925
926 void AOTStreamedHeapLoader::enable_gc() {
927 if (AOTEagerlyLoadObjects && !IterativeObjectLoader::has_more()) {
928 // Everything was loaded eagerly at early startup
929 return;
930 }
931
932 // We cannot handle any exception when materializing roots. Exits the VM.
933 EXCEPTION_MARK
934
935 MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
936
937 // First wait until no tracing is active
938 while (_waiting_for_iterator) {
939 AOTHeapLoading_lock->wait();
940 }
941
942 // Lock further tracing from starting
943 _waiting_for_iterator = true;
944
945 // Record iterator progress
946 int num_handles = (int)_num_archived_objects;
947
948 // Lock further iteration from starting
949 _swapping_root_format = true;
950
951 // Then wait for the iterator to stop
952 while (_previous_batch_last_object_index != _current_batch_last_object_index) {
953 AOTHeapLoading_lock->wait();
954 }
955
956 if (IterativeObjectLoader::has_more()) {
957 // If there is more to be materialized, we have to upgrade the object index
958 // to object mapping to use handles. If there isn't more to materialize, the
959 // handle will no longer e used; they are only used to materialize objects.
960
961 for (int i = 1; i <= num_handles; ++i) {
962 // Upgrade the roots to use handles
963 switch_object_index_to_handle(i);
964 }
965
966 // From now on, accessing the object table must be done through a handle.
967 _objects_are_handles = true;
968 }
969
970 // Unlock tracing
971 _waiting_for_iterator = false;
972
973 // Unlock iteration
974 _swapping_root_format = false;
975
976 _allow_gc = true;
977
978 AOTHeapLoading_lock->notify_all();
979
980 if (AOTEagerlyLoadObjects && IterativeObjectLoader::has_more()) {
981 materialize_late(CHECK);
982 cleanup();
983 }
984 }
985
986 void AOTStreamedHeapLoader::materialize_thread_object() {
987 AOTThread::materialize_thread_object();
988 }
989
990 void AOTStreamedHeapLoader::finish_materialize_objects() {
991 Ticks start = Ticks::now();
992
993 if (_loading_all_objects) {
994 MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
995 // Wait for the AOT thread to finish
996 while (IterativeObjectLoader::has_more()) {
997 AOTHeapLoading_lock->wait();
998 }
999 } else {
1000 assert(!AOTEagerlyLoadObjects, "sanity");
1001 assert(_current_root_index == 0, "sanity");
1002 // Without the full module graph we have done only lazy tracing materialization.
1003 // Ensure all roots are processed here by triggering root loading on every root.
1004 for (int i = 0; i < _num_roots; ++i) {
1005 get_root(i);
1006 }
1007 cleanup();
1008 }
1009
1010 _final_materialization_time_ns = (Ticks::now() - start).nanoseconds();
1011 }
1012
1013 void account_lazy_materialization_time_ns(uint64_t time, const char* description, int index) {
1014 AtomicAccess::add(&_accumulated_lazy_materialization_time_ns, time);
1015 log_debug(aot, heap)("Lazy materialization of %s: %d end (" UINT64_FORMAT " us of " UINT64_FORMAT " us)", description, index, time / 1000, _accumulated_lazy_materialization_time_ns / 1000);
1016 }
1017
1018 // Initialize an empty array of AOT heap roots; materialize them lazily
1019 void AOTStreamedHeapLoader::initialize() {
1020 EXCEPTION_MARK
1021
1022 _materialization_start_ticks = Ticks::now();
1023
1024 FileMapInfo::current_info()->map_bitmap_region();
1025
1026 _heap_region = FileMapInfo::current_info()->region_at(AOTMetaspace::hp);
1027 _bitmap_region = FileMapInfo::current_info()->region_at(AOTMetaspace::bm);
1028
1029 assert(_heap_region->used() > 0, "empty heap archive?");
1030
1031 _is_in_use = true;
1032
1033 // archived roots are at this offset in the stream.
1034 size_t roots_offset = FileMapInfo::current_info()->streamed_heap()->roots_offset();
1035 size_t forwarding_offset = FileMapInfo::current_info()->streamed_heap()->forwarding_offset();
1036 size_t root_highest_object_index_table_offset = FileMapInfo::current_info()->streamed_heap()->root_highest_object_index_table_offset();
1037 _num_archived_objects = FileMapInfo::current_info()->streamed_heap()->num_archived_objects();
1038
1039 // The first int is the length of the array
1040 _roots_archive = ((int*)(((address)_heap_region->mapped_base()) + roots_offset)) + 1;
1041 _num_roots = _roots_archive[-1];
1042 _heap_region_used = _heap_region->used();
1043
1044 // We can't retire a TLAB until the filler klass is set; set it to the archived object klass.
1045 CollectedHeap::set_filler_object_klass(vmClasses::Object_klass());
1046
1047 objArrayOop roots = oopFactory::new_objectArray(_num_roots, CHECK);
1048 _roots = OopHandle(Universe::vm_global(), roots);
1049
1050 _object_index_to_buffer_offset_table = (size_t*)(((address)_heap_region->mapped_base()) + forwarding_offset);
1051 // We allocate the first entry for "null"
1052 _object_index_to_heap_object_table = NEW_C_HEAP_ARRAY(void*, _num_archived_objects + 1, mtClassShared);
1053 Copy::zero_to_bytes(_object_index_to_heap_object_table, (_num_archived_objects + 1) * sizeof(void*));
1054
1055 _root_highest_object_index_table = (int*)(((address)_heap_region->mapped_base()) + root_highest_object_index_table_offset);
1056
1057 address start = (address)(_bitmap_region->mapped_base()) + _heap_region->oopmap_offset();
1058 _oopmap = BitMapView((BitMap::bm_word_t*)start, _heap_region->oopmap_size_in_bits());
1059
1060
1061 if (FLAG_IS_DEFAULT(AOTEagerlyLoadObjects)) {
1062 // Concurrency will not help much if there are no extra cores available.
1063 FLAG_SET_ERGO(AOTEagerlyLoadObjects, os::initial_active_processor_count() <= 1);
1064 }
1065
1066 // If the full module graph is not available or the JVMTI class file load hook is on, we
1067 // will prune the object graph to not include cached objects in subgraphs that are not intended
1068 // to be loaded.
1069 _loading_all_objects = CDSConfig::is_using_full_module_graph() && !JvmtiExport::should_post_class_file_load_hook();
1070 if (!_loading_all_objects) {
1071 // When not using FMG, fall back to tracing materialization
1072 FLAG_SET_ERGO(AOTEagerlyLoadObjects, false);
1073 return;
1074 }
1075
1076 if (AOTEagerlyLoadObjects) {
1077 // Objects are laid out in DFS order; DFS traverse the roots by linearly walking all objects
1078 HandleMark hm(THREAD);
1079
1080 // Early materialization with a budget before GC is allowed
1081 MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
1082
1083 bool finished_before_gc_allowed = materialize_early(CHECK);
1084 if (finished_before_gc_allowed) {
1085 cleanup();
1086 }
1087 } else {
1088 AOTThread::initialize();
1089 }
1090 }
1091
1092 oop AOTStreamedHeapLoader::materialize_root(int root_index) {
1093 Ticks start = Ticks::now();
1094 // We cannot handle any exception when materializing a root. Exits the VM.
1095 EXCEPTION_MARK
1096 Stack<AOTHeapTraversalEntry, mtClassShared> dfs_stack;
1097 HandleMark hm(THREAD);
1098
1099 oop result;
1100 {
1101 MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag);
1102
1103 oop root = objArrayOop(_roots.resolve())->obj_at(root_index);
1104
1105 if (root != nullptr) {
1106 // The root has already been materialized
1107 result = root;
1108 } else {
1109 // The root has not been materialized, start tracing materialization
1110 result = TracingObjectLoader::materialize_root(root_index, dfs_stack, CHECK_NULL);
1111 }
1112 }
1113
1114 uint64_t duration = (Ticks::now() - start).nanoseconds();
1115
1116 account_lazy_materialization_time_ns(duration, "root", root_index);
1117
1118 return result;
1119 }
1120
1121 oop AOTStreamedHeapLoader::get_root(int index) {
1122 oop result = objArrayOop(_roots.resolve())->obj_at(index);
1123 if (result == nullptr) {
1124 // Materialize root
1125 result = materialize_root(index);
1126 }
1127 if (result == _roots.resolve()) {
1128 // A self-reference to the roots array acts as a sentinel object for null,
1129 // indicating that the root has been cleared.
1130 result = nullptr;
1131 }
1132 // Acquire the root transitive object payload
1133 OrderAccess::acquire();
1134 return result;
1135 }
1136
1137 void AOTStreamedHeapLoader::clear_root(int index) {
1138 // Self-reference to the roots array acts as a sentinel object for null,
1139 // indicating that the root has been cleared.
1140 objArrayOop(_roots.resolve())->obj_at_put(index, _roots.resolve());
1141 }
1142
1143 void AOTStreamedHeapLoader::await_gc_enabled() {
1144 while (!_allow_gc) {
1145 AOTHeapLoading_lock->wait();
1146 }
1147 }
1148
1149 void AOTStreamedHeapLoader::finish_initialization(FileMapInfo* static_mapinfo) {
1150 static_mapinfo->stream_heap_region();
1151 }
1152
1153 AOTMapLogger::OopDataIterator* AOTStreamedHeapLoader::oop_iterator(FileMapInfo* info, address buffer_start, address buffer_end) {
1154 class StreamedLoaderOopIterator : public AOTStreamedHeapOopIterator {
1155 public:
1156 StreamedLoaderOopIterator(address buffer_start,
1157 int num_archived_objects)
1158 : AOTStreamedHeapOopIterator(buffer_start, num_archived_objects) {}
1159
1160 AOTMapLogger::OopData capture(int dfs_index) override {
1161 size_t buffered_offset = buffer_offset_for_object_index(dfs_index);
1162 address buffered_addr = _buffer_start + buffered_offset;
1163 oopDesc* raw_oop = (oopDesc*)buffered_addr;
1164 size_t size = archive_object_size(raw_oop);
1165
1166 intptr_t target_location = (intptr_t)buffered_offset;
1167 uint32_t narrow_location = checked_cast<uint32_t>(dfs_index);
1168 Klass* klass = raw_oop->klass();
1169
1170 address requested_addr = (address)buffered_offset;
1171
1172 return { buffered_addr,
1173 requested_addr,
1174 target_location,
1175 narrow_location,
1176 raw_oop,
1177 klass,
1178 size,
1179 false };
1180 }
1181
1182 GrowableArrayCHeap<AOTMapLogger::OopData, mtClass>* roots() override {
1183 GrowableArrayCHeap<AOTMapLogger::OopData, mtClass>* result = new GrowableArrayCHeap<AOTMapLogger::OopData, mtClass>();
1184
1185 for (int i = 0; i < _num_roots; ++i) {
1186 int object_index = object_index_for_root_index(i);
1187 result->append(capture(object_index));
1188 }
1189
1190 return result;
1191 }
1192 };
1193
1194 assert(_is_in_use, "printing before initializing?");
1195
1196 return new StreamedLoaderOopIterator(buffer_start, (int)info->streamed_heap()->num_archived_objects());
1197 }
1198
1199 #endif // INCLUDE_CDS_JAVA_HEAP