1 /*
2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "cds/archiveHeapLoader.inline.hpp"
27 #include "cds/cdsConfig.hpp"
28 #include "cds/heapShared.hpp"
29 #include "cds/metaspaceShared.hpp"
30 #include "classfile/classLoaderDataShared.hpp"
31 #include "classfile/systemDictionaryShared.hpp"
32 #include "classfile/vmClasses.hpp"
33 #include "gc/shared/collectedHeap.hpp"
34 #include "logging/log.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "memory/universe.hpp"
38 #include "sanitizers/ub.hpp"
39 #include "utilities/bitMap.inline.hpp"
40 #include "utilities/copy.hpp"
41
42 #if INCLUDE_CDS_JAVA_HEAP
43
44 bool ArchiveHeapLoader::_is_mapped = false;
45 bool ArchiveHeapLoader::_is_loaded = false;
46
47 bool ArchiveHeapLoader::_narrow_oop_base_initialized = false;
48 address ArchiveHeapLoader::_narrow_oop_base;
49 int ArchiveHeapLoader::_narrow_oop_shift;
50
51 // Support for loaded heap.
52 uintptr_t ArchiveHeapLoader::_loaded_heap_bottom = 0;
53 uintptr_t ArchiveHeapLoader::_loaded_heap_top = 0;
54 uintptr_t ArchiveHeapLoader::_dumptime_base = UINTPTR_MAX;
55 uintptr_t ArchiveHeapLoader::_dumptime_top = 0;
56 intx ArchiveHeapLoader::_runtime_offset = 0;
57 bool ArchiveHeapLoader::_loading_failed = false;
58
59 // Support for mapped heap.
60 uintptr_t ArchiveHeapLoader::_mapped_heap_bottom = 0;
61 bool ArchiveHeapLoader::_mapped_heap_relocation_initialized = false;
62 ptrdiff_t ArchiveHeapLoader::_mapped_heap_delta = 0;
63
64 // Every mapped region is offset by _mapped_heap_delta from its requested address.
65 // See FileMapInfo::heap_region_requested_address().
66 ATTRIBUTE_NO_UBSAN
67 void ArchiveHeapLoader::init_mapped_heap_info(address mapped_heap_bottom, ptrdiff_t delta, int dumptime_oop_shift) {
68 assert(!_mapped_heap_relocation_initialized, "only once");
69 if (!UseCompressedOops) {
70 assert(dumptime_oop_shift == 0, "sanity");
71 }
72 assert(can_map(), "sanity");
73 init_narrow_oop_decoding(CompressedOops::base() + delta, dumptime_oop_shift);
74 _mapped_heap_bottom = (intptr_t)mapped_heap_bottom;
75 _mapped_heap_delta = delta;
76 _mapped_heap_relocation_initialized = true;
77 }
78
79 void ArchiveHeapLoader::init_narrow_oop_decoding(address base, int shift) {
80 assert(!_narrow_oop_base_initialized, "only once");
81 _narrow_oop_base_initialized = true;
82 _narrow_oop_base = base;
83 _narrow_oop_shift = shift;
84 }
85
86 void ArchiveHeapLoader::fixup_region() {
87 FileMapInfo* mapinfo = FileMapInfo::current_info();
88 if (is_mapped()) {
89 mapinfo->fixup_mapped_heap_region();
90 } else if (_loading_failed) {
91 fill_failed_loaded_heap();
92 }
93 if (is_in_use()) {
94 if (!CDSConfig::is_using_full_module_graph()) {
95 // Need to remove all the archived java.lang.Module objects from HeapShared::roots().
96 ClassLoaderDataShared::clear_archived_oops();
97 }
98 }
99 }
100
101 // ------------------ Support for Region MAPPING -----------------------------------------
102
103 // Patch all the embedded oop pointers inside an archived heap region,
104 // to be consistent with the runtime oop encoding.
105 class PatchCompressedEmbeddedPointers: public BitMapClosure {
106 narrowOop* _start;
107
108 public:
109 PatchCompressedEmbeddedPointers(narrowOop* start) : _start(start) {}
110
111 bool do_bit(size_t offset) {
112 narrowOop* p = _start + offset;
113 narrowOop v = *p;
114 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
115 oop o = ArchiveHeapLoader::decode_from_mapped_archive(v);
116 RawAccess<IS_NOT_NULL>::oop_store(p, o);
117 return true;
118 }
119 };
120
121 class PatchCompressedEmbeddedPointersQuick: public BitMapClosure {
122 narrowOop* _start;
123 uint32_t _delta;
124
125 public:
126 PatchCompressedEmbeddedPointersQuick(narrowOop* start, uint32_t delta) : _start(start), _delta(delta) {}
127
128 bool do_bit(size_t offset) {
129 narrowOop* p = _start + offset;
130 narrowOop v = *p;
131 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
132 narrowOop new_v = CompressedOops::narrow_oop_cast(CompressedOops::narrow_oop_value(v) + _delta);
133 assert(!CompressedOops::is_null(new_v), "should never relocate to narrowOop(0)");
134 #ifdef ASSERT
135 oop o1 = ArchiveHeapLoader::decode_from_mapped_archive(v);
136 oop o2 = CompressedOops::decode_not_null(new_v);
137 assert(o1 == o2, "quick delta must work");
138 #endif
139 RawAccess<IS_NOT_NULL>::oop_store(p, new_v);
140 return true;
141 }
142 };
143
144 class PatchUncompressedEmbeddedPointers: public BitMapClosure {
145 oop* _start;
146 intptr_t _delta;
147
148 public:
149 PatchUncompressedEmbeddedPointers(oop* start, intx runtime_offset) :
150 _start(start),
151 _delta(runtime_offset) {}
152
153 PatchUncompressedEmbeddedPointers(oop* start) :
154 _start(start),
155 _delta(ArchiveHeapLoader::mapped_heap_delta()) {}
156
157 bool do_bit(size_t offset) {
158 oop* p = _start + offset;
159 intptr_t dumptime_oop = (intptr_t)((void*)*p);
160 assert(dumptime_oop != 0, "null oops should have been filtered out at dump time");
161 intptr_t runtime_oop = dumptime_oop + _delta;
162 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(runtime_oop));
163 return true;
164 }
165 };
166
167 void ArchiveHeapLoader::patch_compressed_embedded_pointers(BitMapView bm,
168 FileMapInfo* info,
169 MemRegion region) {
170 narrowOop dt_encoded_bottom = info->encoded_heap_region_dumptime_address();
171 narrowOop rt_encoded_bottom = CompressedOops::encode_not_null(cast_to_oop(region.start()));
172 log_info(cds)("patching heap embedded pointers: narrowOop 0x%8x -> 0x%8x",
173 (uint)dt_encoded_bottom, (uint)rt_encoded_bottom);
174
175 // Optimization: if dumptime shift is the same as runtime shift, we can perform a
176 // quick conversion from "dumptime narrowOop" -> "runtime narrowOop".
177 narrowOop* patching_start = (narrowOop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos();
178 if (_narrow_oop_shift == CompressedOops::shift()) {
179 uint32_t quick_delta = (uint32_t)rt_encoded_bottom - (uint32_t)dt_encoded_bottom;
180 log_info(cds)("CDS heap data relocation quick delta = 0x%x", quick_delta);
181 if (quick_delta == 0) {
182 log_info(cds)("CDS heap data relocation unnecessary, quick_delta = 0");
183 } else {
184 PatchCompressedEmbeddedPointersQuick patcher(patching_start, quick_delta);
185 bm.iterate(&patcher);
186 }
187 } else {
188 log_info(cds)("CDS heap data quick relocation not possible");
189 PatchCompressedEmbeddedPointers patcher(patching_start);
190 bm.iterate(&patcher);
191 }
192 }
193
194 // Patch all the non-null pointers that are embedded in the archived heap objects
195 // in this (mapped) region
196 void ArchiveHeapLoader::patch_embedded_pointers(FileMapInfo* info,
197 MemRegion region, address oopmap,
198 size_t oopmap_size_in_bits) {
199 BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits);
200 if (UseCompressedOops) {
201 patch_compressed_embedded_pointers(bm, info, region);
202 } else {
203 PatchUncompressedEmbeddedPointers patcher((oop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos());
204 bm.iterate(&patcher);
205 }
206 }
207
208 // ------------------ Support for Region LOADING -----------------------------------------
209
210 // The CDS archive remembers each heap object by its address at dump time, but
211 // the heap object may be loaded at a different address at run time. This structure is used
212 // to translate the dump time addresses for all objects in FileMapInfo::space_at(region_index)
213 // to their runtime addresses.
214 struct LoadedArchiveHeapRegion {
215 int _region_index; // index for FileMapInfo::space_at(index)
216 size_t _region_size; // number of bytes in this region
217 uintptr_t _dumptime_base; // The dump-time (decoded) address of the first object in this region
218 intx _runtime_offset; // If an object's dump time address P is within in this region, its
219 // runtime address is P + _runtime_offset
220 uintptr_t top() {
221 return _dumptime_base + _region_size;
222 }
223 };
224
225 void ArchiveHeapLoader::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loaded_region) {
226 _dumptime_base = loaded_region->_dumptime_base;
227 _dumptime_top = loaded_region->top();
228 _runtime_offset = loaded_region->_runtime_offset;
229 }
230
231 bool ArchiveHeapLoader::can_load() {
232 return Universe::heap()->can_load_archived_objects();
233 }
234
235 class ArchiveHeapLoader::PatchLoadedRegionPointers: public BitMapClosure {
236 narrowOop* _start;
237 intx _offset;
238 uintptr_t _base;
239 uintptr_t _top;
240
241 public:
242 PatchLoadedRegionPointers(narrowOop* start, LoadedArchiveHeapRegion* loaded_region)
243 : _start(start),
244 _offset(loaded_region->_runtime_offset),
245 _base(loaded_region->_dumptime_base),
246 _top(loaded_region->top()) {}
247
248 bool do_bit(size_t offset) {
249 assert(UseCompressedOops, "PatchLoadedRegionPointers for uncompressed oops is unimplemented");
250 narrowOop* p = _start + offset;
251 narrowOop v = *p;
252 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
253 uintptr_t o = cast_from_oop<uintptr_t>(ArchiveHeapLoader::decode_from_archive(v));
254 assert(_base <= o && o < _top, "must be");
255
256 o += _offset;
257 ArchiveHeapLoader::assert_in_loaded_heap(o);
258 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(o));
259 return true;
260 }
261 };
262
263 bool ArchiveHeapLoader::init_loaded_region(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region,
264 MemRegion& archive_space) {
265 size_t total_bytes = 0;
266 FileMapRegion* r = mapinfo->region_at(MetaspaceShared::hp);
267 r->assert_is_heap_region();
268 if (r->used() == 0) {
269 return false;
270 }
271
272 assert(is_aligned(r->used(), HeapWordSize), "must be");
273 total_bytes += r->used();
274 loaded_region->_region_index = MetaspaceShared::hp;
275 loaded_region->_region_size = r->used();
276 loaded_region->_dumptime_base = (uintptr_t)mapinfo->heap_region_dumptime_address();
277
278 assert(is_aligned(total_bytes, HeapWordSize), "must be");
279 size_t word_size = total_bytes / HeapWordSize;
280 HeapWord* buffer = Universe::heap()->allocate_loaded_archive_space(word_size);
281 if (buffer == nullptr) {
282 return false;
283 }
284
285 archive_space = MemRegion(buffer, word_size);
286 _loaded_heap_bottom = (uintptr_t)archive_space.start();
287 _loaded_heap_top = _loaded_heap_bottom + total_bytes;
288
289 loaded_region->_runtime_offset = _loaded_heap_bottom - loaded_region->_dumptime_base;
290
291 return true;
292 }
293
294 bool ArchiveHeapLoader::load_heap_region_impl(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region,
295 uintptr_t load_address) {
296 uintptr_t bitmap_base = (uintptr_t)mapinfo->map_bitmap_region();
297 if (bitmap_base == 0) {
298 _loading_failed = true;
299 return false; // OOM or CRC error
300 }
301
302 FileMapRegion* r = mapinfo->region_at(loaded_region->_region_index);
303 if (!mapinfo->read_region(loaded_region->_region_index, (char*)load_address, r->used(), /* do_commit = */ false)) {
304 // There's no easy way to free the buffer, so we will fill it with zero later
305 // in fill_failed_loaded_heap(), and it will eventually be GC'ed.
306 log_warning(cds)("Loading of heap region %d has failed. Archived objects are disabled", loaded_region->_region_index);
307 _loading_failed = true;
308 return false;
309 }
310 assert(r->mapped_base() == (char*)load_address, "sanity");
311 log_info(cds)("Loaded heap region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT
312 " size " SIZE_FORMAT_W(6) " delta " INTX_FORMAT,
313 loaded_region->_region_index, load_address, load_address + loaded_region->_region_size,
314 loaded_region->_region_size, loaded_region->_runtime_offset);
315
316 uintptr_t oopmap = bitmap_base + r->oopmap_offset();
317 BitMapView bm((BitMap::bm_word_t*)oopmap, r->oopmap_size_in_bits());
318
319 if (UseCompressedOops) {
320 PatchLoadedRegionPointers patcher((narrowOop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region);
321 bm.iterate(&patcher);
322 } else {
323 PatchUncompressedEmbeddedPointers patcher((oop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region->_runtime_offset);
324 bm.iterate(&patcher);
325 }
326 return true;
327 }
328
329 bool ArchiveHeapLoader::load_heap_region(FileMapInfo* mapinfo) {
330 assert(can_load(), "loaded heap for must be supported");
331 init_narrow_oop_decoding(mapinfo->narrow_oop_base(), mapinfo->narrow_oop_shift());
332
333 LoadedArchiveHeapRegion loaded_region;
334 memset(&loaded_region, 0, sizeof(loaded_region));
335
336 MemRegion archive_space;
337 if (!init_loaded_region(mapinfo, &loaded_region, archive_space)) {
338 return false;
339 }
340
341 if (!load_heap_region_impl(mapinfo, &loaded_region, (uintptr_t)archive_space.start())) {
342 assert(_loading_failed, "must be");
343 return false;
344 }
345
346 init_loaded_heap_relocation(&loaded_region);
347 _is_loaded = true;
348
349 return true;
350 }
351
352 class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure {
353 ResourceHashtable<uintptr_t, bool>* _table;
354
355 public:
356 VerifyLoadedHeapEmbeddedPointers(ResourceHashtable<uintptr_t, bool>* table) : _table(table) {}
357
358 virtual void do_oop(narrowOop* p) {
359 // This should be called before the loaded region is modified, so all the embedded pointers
360 // must be null, or must point to a valid object in the loaded region.
361 narrowOop v = *p;
362 if (!CompressedOops::is_null(v)) {
363 oop o = CompressedOops::decode_not_null(v);
364 uintptr_t u = cast_from_oop<uintptr_t>(o);
365 ArchiveHeapLoader::assert_in_loaded_heap(u);
366 guarantee(_table->contains(u), "must point to beginning of object in loaded archived region");
367 }
368 }
369 virtual void do_oop(oop* p) {
370 oop v = *p;
371 if(v != nullptr) {
372 uintptr_t u = cast_from_oop<uintptr_t>(v);
373 ArchiveHeapLoader::assert_in_loaded_heap(u);
374 guarantee(_table->contains(u), "must point to beginning of object in loaded archived region");
375 }
376 }
377 };
378
379 void ArchiveHeapLoader::finish_initialization() {
380 if (is_loaded()) {
381 // These operations are needed only when the heap is loaded (not mapped).
382 finish_loaded_heap();
383 if (VerifyArchivedFields > 0) {
384 verify_loaded_heap();
385 }
386 }
387 if (is_in_use()) {
388 patch_native_pointers();
389 intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom;
390
391 // The heap roots are stored in one or more segments that are laid out consecutively.
392 // The size of each segment (except for the last one) is max_size_in_{elems,bytes}.
393 HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments();
394 HeapShared::init_root_segment_sizes(segments.max_size_in_elems());
395 intptr_t first_segment_addr = bottom + segments.base_offset();
396 for (size_t c = 0; c < segments.count(); c++) {
397 oop segment_oop = cast_to_oop(first_segment_addr + (c * segments.max_size_in_bytes()));
398 assert(segment_oop->is_objArray(), "Must be");
399 HeapShared::add_root_segment((objArrayOop)segment_oop);
400 }
401 }
402 }
403
404 void ArchiveHeapLoader::finish_loaded_heap() {
405 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
406 HeapWord* top = (HeapWord*)_loaded_heap_top;
407
408 MemRegion archive_space = MemRegion(bottom, top);
409 Universe::heap()->complete_loaded_archive_space(archive_space);
410 }
411
412 void ArchiveHeapLoader::verify_loaded_heap() {
413 log_info(cds, heap)("Verify all oops and pointers in loaded heap");
414
415 ResourceMark rm;
416 ResourceHashtable<uintptr_t, bool> table;
417 VerifyLoadedHeapEmbeddedPointers verifier(&table);
418 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
419 HeapWord* top = (HeapWord*)_loaded_heap_top;
420
421 for (HeapWord* p = bottom; p < top; ) {
422 oop o = cast_to_oop(p);
423 table.put(cast_from_oop<uintptr_t>(o), true);
424 p += o->size();
425 }
426
427 for (HeapWord* p = bottom; p < top; ) {
428 oop o = cast_to_oop(p);
429 o->oop_iterate(&verifier);
430 p += o->size();
431 }
432 }
433
434 void ArchiveHeapLoader::fill_failed_loaded_heap() {
435 assert(_loading_failed, "must be");
436 if (_loaded_heap_bottom != 0) {
437 assert(_loaded_heap_top != 0, "must be");
438 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
439 HeapWord* top = (HeapWord*)_loaded_heap_top;
440 Universe::heap()->fill_with_objects(bottom, top - bottom);
441 }
442 }
443
444 oop ArchiveHeapLoader::oop_from_offset(int offset) {
445 // Once GC starts, the offsets saved in CachedCodeDirectoryInternal::_permanent_oop_offsets
446 // will become invalid. I don't know what function can check if GCs are allowed, but surely
447 // GCs can't happen before the Object class is loaded.
448 assert(CDSConfig::is_using_archive(), "sanity");
449 assert(vmClasses::Object_klass()->class_loader_data() == nullptr,
450 "can be called only very early during VM start-up");
451 if (is_loaded()) {
452 return cast_to_oop(_loaded_heap_bottom + offset);
453 } else {
454 assert(is_mapped(), "must be");
455 return cast_to_oop(_mapped_heap_bottom + offset);
456 }
457 }
458
459 class PatchNativePointers: public BitMapClosure {
460 Metadata** _start;
461
462 public:
463 PatchNativePointers(Metadata** start) : _start(start) {}
464
465 bool do_bit(size_t offset) {
466 Metadata** p = _start + offset;
467 *p = (Metadata*)(address(*p) + MetaspaceShared::relocation_delta());
468 return true;
469 }
470 };
471
472 void ArchiveHeapLoader::patch_native_pointers() {
473 if (MetaspaceShared::relocation_delta() == 0) {
474 return;
475 }
476
477 FileMapRegion* r = FileMapInfo::current_info()->region_at(MetaspaceShared::hp);
478 if (r->mapped_base() != nullptr && r->has_ptrmap()) {
479 log_info(cds, heap)("Patching native pointers in heap region");
480 BitMapView bm = FileMapInfo::current_info()->ptrmap_view(MetaspaceShared::hp);
481 PatchNativePointers patcher((Metadata**)r->mapped_base() + FileMapInfo::current_info()->heap_ptrmap_start_pos());
482 bm.iterate(&patcher);
483 }
484 }
485 #endif // INCLUDE_CDS_JAVA_HEAP