1 /* 2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "cds/archiveHeapLoader.inline.hpp" 27 #include "cds/cdsConfig.hpp" 28 #include "cds/heapShared.hpp" 29 #include "cds/metaspaceShared.hpp" 30 #include "classfile/classLoaderDataShared.hpp" 31 #include "classfile/systemDictionaryShared.hpp" 32 #include "gc/shared/collectedHeap.hpp" 33 #include "logging/log.hpp" 34 #include "memory/iterator.inline.hpp" 35 #include "memory/resourceArea.hpp" 36 #include "memory/universe.hpp" 37 #include "sanitizers/ub.hpp" 38 #include "utilities/bitMap.inline.hpp" 39 #include "utilities/copy.hpp" 40 41 #if INCLUDE_CDS_JAVA_HEAP 42 43 bool ArchiveHeapLoader::_is_mapped = false; 44 bool ArchiveHeapLoader::_is_loaded = false; 45 46 bool ArchiveHeapLoader::_narrow_oop_base_initialized = false; 47 address ArchiveHeapLoader::_narrow_oop_base; 48 int ArchiveHeapLoader::_narrow_oop_shift; 49 50 // Support for loaded heap. 51 uintptr_t ArchiveHeapLoader::_loaded_heap_bottom = 0; 52 uintptr_t ArchiveHeapLoader::_loaded_heap_top = 0; 53 uintptr_t ArchiveHeapLoader::_dumptime_base = UINTPTR_MAX; 54 uintptr_t ArchiveHeapLoader::_dumptime_top = 0; 55 intx ArchiveHeapLoader::_runtime_offset = 0; 56 bool ArchiveHeapLoader::_loading_failed = false; 57 58 // Support for mapped heap. 59 uintptr_t ArchiveHeapLoader::_mapped_heap_bottom = 0; 60 bool ArchiveHeapLoader::_mapped_heap_relocation_initialized = false; 61 ptrdiff_t ArchiveHeapLoader::_mapped_heap_delta = 0; 62 63 // Every mapped region is offset by _mapped_heap_delta from its requested address. 64 // See FileMapInfo::heap_region_requested_address(). 65 ATTRIBUTE_NO_UBSAN 66 void ArchiveHeapLoader::init_mapped_heap_info(address mapped_heap_bottom, ptrdiff_t delta, int dumptime_oop_shift) { 67 assert(!_mapped_heap_relocation_initialized, "only once"); 68 if (!UseCompressedOops) { 69 assert(dumptime_oop_shift == 0, "sanity"); 70 } 71 assert(can_map(), "sanity"); 72 init_narrow_oop_decoding(CompressedOops::base() + delta, dumptime_oop_shift); 73 _mapped_heap_bottom = (intptr_t)mapped_heap_bottom; 74 _mapped_heap_delta = delta; 75 _mapped_heap_relocation_initialized = true; 76 } 77 78 void ArchiveHeapLoader::init_narrow_oop_decoding(address base, int shift) { 79 assert(!_narrow_oop_base_initialized, "only once"); 80 _narrow_oop_base_initialized = true; 81 _narrow_oop_base = base; 82 _narrow_oop_shift = shift; 83 } 84 85 void ArchiveHeapLoader::fixup_region() { 86 FileMapInfo* mapinfo = FileMapInfo::current_info(); 87 if (is_mapped()) { 88 mapinfo->fixup_mapped_heap_region(); 89 } else if (_loading_failed) { 90 fill_failed_loaded_heap(); 91 } 92 if (is_in_use()) { 93 if (!CDSConfig::is_using_full_module_graph()) { 94 // Need to remove all the archived java.lang.Module objects from HeapShared::roots(). 95 ClassLoaderDataShared::clear_archived_oops(); 96 } 97 } 98 } 99 100 // ------------------ Support for Region MAPPING ----------------------------------------- 101 102 // Patch all the embedded oop pointers inside an archived heap region, 103 // to be consistent with the runtime oop encoding. 104 class PatchCompressedEmbeddedPointers: public BitMapClosure { 105 narrowOop* _start; 106 107 public: 108 PatchCompressedEmbeddedPointers(narrowOop* start) : _start(start) {} 109 110 bool do_bit(size_t offset) { 111 narrowOop* p = _start + offset; 112 narrowOop v = *p; 113 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time"); 114 oop o = ArchiveHeapLoader::decode_from_mapped_archive(v); 115 RawAccess<IS_NOT_NULL>::oop_store(p, o); 116 return true; 117 } 118 }; 119 120 class PatchCompressedEmbeddedPointersQuick: public BitMapClosure { 121 narrowOop* _start; 122 uint32_t _delta; 123 124 public: 125 PatchCompressedEmbeddedPointersQuick(narrowOop* start, uint32_t delta) : _start(start), _delta(delta) {} 126 127 bool do_bit(size_t offset) { 128 narrowOop* p = _start + offset; 129 narrowOop v = *p; 130 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time"); 131 narrowOop new_v = CompressedOops::narrow_oop_cast(CompressedOops::narrow_oop_value(v) + _delta); 132 assert(!CompressedOops::is_null(new_v), "should never relocate to narrowOop(0)"); 133 #ifdef ASSERT 134 oop o1 = ArchiveHeapLoader::decode_from_mapped_archive(v); 135 oop o2 = CompressedOops::decode_not_null(new_v); 136 assert(o1 == o2, "quick delta must work"); 137 #endif 138 RawAccess<IS_NOT_NULL>::oop_store(p, new_v); 139 return true; 140 } 141 }; 142 143 class PatchUncompressedEmbeddedPointers: public BitMapClosure { 144 oop* _start; 145 146 public: 147 PatchUncompressedEmbeddedPointers(oop* start) : _start(start) {} 148 149 bool do_bit(size_t offset) { 150 oop* p = _start + offset; 151 intptr_t dumptime_oop = (intptr_t)((void*)*p); 152 assert(dumptime_oop != 0, "null oops should have been filtered out at dump time"); 153 intptr_t runtime_oop = dumptime_oop + ArchiveHeapLoader::mapped_heap_delta(); 154 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(runtime_oop)); 155 return true; 156 } 157 }; 158 159 void ArchiveHeapLoader::patch_compressed_embedded_pointers(BitMapView bm, 160 FileMapInfo* info, 161 MemRegion region) { 162 narrowOop dt_encoded_bottom = info->encoded_heap_region_dumptime_address(); 163 narrowOop rt_encoded_bottom = CompressedOops::encode_not_null(cast_to_oop(region.start())); 164 log_info(cds)("patching heap embedded pointers: narrowOop 0x%8x -> 0x%8x", 165 (uint)dt_encoded_bottom, (uint)rt_encoded_bottom); 166 167 // Optimization: if dumptime shift is the same as runtime shift, we can perform a 168 // quick conversion from "dumptime narrowOop" -> "runtime narrowOop". 169 narrowOop* patching_start = (narrowOop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos(); 170 if (_narrow_oop_shift == CompressedOops::shift()) { 171 uint32_t quick_delta = (uint32_t)rt_encoded_bottom - (uint32_t)dt_encoded_bottom; 172 log_info(cds)("CDS heap data relocation quick delta = 0x%x", quick_delta); 173 if (quick_delta == 0) { 174 log_info(cds)("CDS heap data relocation unnecessary, quick_delta = 0"); 175 } else { 176 PatchCompressedEmbeddedPointersQuick patcher(patching_start, quick_delta); 177 bm.iterate(&patcher); 178 } 179 } else { 180 log_info(cds)("CDS heap data quick relocation not possible"); 181 PatchCompressedEmbeddedPointers patcher(patching_start); 182 bm.iterate(&patcher); 183 } 184 } 185 186 // Patch all the non-null pointers that are embedded in the archived heap objects 187 // in this (mapped) region 188 void ArchiveHeapLoader::patch_embedded_pointers(FileMapInfo* info, 189 MemRegion region, address oopmap, 190 size_t oopmap_size_in_bits) { 191 BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits); 192 if (UseCompressedOops) { 193 patch_compressed_embedded_pointers(bm, info, region); 194 } else { 195 PatchUncompressedEmbeddedPointers patcher((oop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos()); 196 bm.iterate(&patcher); 197 } 198 } 199 200 // ------------------ Support for Region LOADING ----------------------------------------- 201 202 // The CDS archive remembers each heap object by its address at dump time, but 203 // the heap object may be loaded at a different address at run time. This structure is used 204 // to translate the dump time addresses for all objects in FileMapInfo::space_at(region_index) 205 // to their runtime addresses. 206 struct LoadedArchiveHeapRegion { 207 int _region_index; // index for FileMapInfo::space_at(index) 208 size_t _region_size; // number of bytes in this region 209 uintptr_t _dumptime_base; // The dump-time (decoded) address of the first object in this region 210 intx _runtime_offset; // If an object's dump time address P is within in this region, its 211 // runtime address is P + _runtime_offset 212 uintptr_t top() { 213 return _dumptime_base + _region_size; 214 } 215 }; 216 217 void ArchiveHeapLoader::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loaded_region) { 218 _dumptime_base = loaded_region->_dumptime_base; 219 _dumptime_top = loaded_region->top(); 220 _runtime_offset = loaded_region->_runtime_offset; 221 } 222 223 bool ArchiveHeapLoader::can_load() { 224 if (!UseCompressedOops) { 225 // Pointer relocation for uncompressed oops is unimplemented. 226 return false; 227 } 228 return Universe::heap()->can_load_archived_objects(); 229 } 230 231 class ArchiveHeapLoader::PatchLoadedRegionPointers: public BitMapClosure { 232 narrowOop* _start; 233 intx _offset; 234 uintptr_t _base; 235 uintptr_t _top; 236 237 public: 238 PatchLoadedRegionPointers(narrowOop* start, LoadedArchiveHeapRegion* loaded_region) 239 : _start(start), 240 _offset(loaded_region->_runtime_offset), 241 _base(loaded_region->_dumptime_base), 242 _top(loaded_region->top()) {} 243 244 bool do_bit(size_t offset) { 245 assert(UseCompressedOops, "PatchLoadedRegionPointers for uncompressed oops is unimplemented"); 246 narrowOop* p = _start + offset; 247 narrowOop v = *p; 248 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time"); 249 uintptr_t o = cast_from_oop<uintptr_t>(ArchiveHeapLoader::decode_from_archive(v)); 250 assert(_base <= o && o < _top, "must be"); 251 252 o += _offset; 253 ArchiveHeapLoader::assert_in_loaded_heap(o); 254 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(o)); 255 return true; 256 } 257 }; 258 259 bool ArchiveHeapLoader::init_loaded_region(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region, 260 MemRegion& archive_space) { 261 size_t total_bytes = 0; 262 FileMapRegion* r = mapinfo->region_at(MetaspaceShared::hp); 263 r->assert_is_heap_region(); 264 if (r->used() == 0) { 265 return false; 266 } 267 268 assert(is_aligned(r->used(), HeapWordSize), "must be"); 269 total_bytes += r->used(); 270 loaded_region->_region_index = MetaspaceShared::hp; 271 loaded_region->_region_size = r->used(); 272 loaded_region->_dumptime_base = (uintptr_t)mapinfo->heap_region_dumptime_address(); 273 274 assert(is_aligned(total_bytes, HeapWordSize), "must be"); 275 size_t word_size = total_bytes / HeapWordSize; 276 HeapWord* buffer = Universe::heap()->allocate_loaded_archive_space(word_size); 277 if (buffer == nullptr) { 278 return false; 279 } 280 281 archive_space = MemRegion(buffer, word_size); 282 _loaded_heap_bottom = (uintptr_t)archive_space.start(); 283 _loaded_heap_top = _loaded_heap_bottom + total_bytes; 284 285 loaded_region->_runtime_offset = _loaded_heap_bottom - loaded_region->_dumptime_base; 286 287 return true; 288 } 289 290 bool ArchiveHeapLoader::load_heap_region_impl(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region, 291 uintptr_t load_address) { 292 uintptr_t bitmap_base = (uintptr_t)mapinfo->map_bitmap_region(); 293 if (bitmap_base == 0) { 294 _loading_failed = true; 295 return false; // OOM or CRC error 296 } 297 298 FileMapRegion* r = mapinfo->region_at(loaded_region->_region_index); 299 if (!mapinfo->read_region(loaded_region->_region_index, (char*)load_address, r->used(), /* do_commit = */ false)) { 300 // There's no easy way to free the buffer, so we will fill it with zero later 301 // in fill_failed_loaded_heap(), and it will eventually be GC'ed. 302 log_warning(cds)("Loading of heap region %d has failed. Archived objects are disabled", loaded_region->_region_index); 303 _loading_failed = true; 304 return false; 305 } 306 assert(r->mapped_base() == (char*)load_address, "sanity"); 307 log_info(cds)("Loaded heap region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT 308 " size " SIZE_FORMAT_W(6) " delta " INTX_FORMAT, 309 loaded_region->_region_index, load_address, load_address + loaded_region->_region_size, 310 loaded_region->_region_size, loaded_region->_runtime_offset); 311 312 uintptr_t oopmap = bitmap_base + r->oopmap_offset(); 313 BitMapView bm((BitMap::bm_word_t*)oopmap, r->oopmap_size_in_bits()); 314 315 PatchLoadedRegionPointers patcher((narrowOop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region); 316 bm.iterate(&patcher); 317 return true; 318 } 319 320 bool ArchiveHeapLoader::load_heap_region(FileMapInfo* mapinfo) { 321 assert(UseCompressedOops, "loaded heap for !UseCompressedOops is unimplemented"); 322 init_narrow_oop_decoding(mapinfo->narrow_oop_base(), mapinfo->narrow_oop_shift()); 323 324 LoadedArchiveHeapRegion loaded_region; 325 memset(&loaded_region, 0, sizeof(loaded_region)); 326 327 MemRegion archive_space; 328 if (!init_loaded_region(mapinfo, &loaded_region, archive_space)) { 329 return false; 330 } 331 332 if (!load_heap_region_impl(mapinfo, &loaded_region, (uintptr_t)archive_space.start())) { 333 assert(_loading_failed, "must be"); 334 return false; 335 } 336 337 init_loaded_heap_relocation(&loaded_region); 338 _is_loaded = true; 339 340 return true; 341 } 342 343 class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure { 344 ResourceHashtable<uintptr_t, bool>* _table; 345 346 public: 347 VerifyLoadedHeapEmbeddedPointers(ResourceHashtable<uintptr_t, bool>* table) : _table(table) {} 348 349 virtual void do_oop(narrowOop* p) { 350 // This should be called before the loaded region is modified, so all the embedded pointers 351 // must be null, or must point to a valid object in the loaded region. 352 narrowOop v = *p; 353 if (!CompressedOops::is_null(v)) { 354 oop o = CompressedOops::decode_not_null(v); 355 uintptr_t u = cast_from_oop<uintptr_t>(o); 356 ArchiveHeapLoader::assert_in_loaded_heap(u); 357 guarantee(_table->contains(u), "must point to beginning of object in loaded archived region"); 358 } 359 } 360 virtual void do_oop(oop* p) { 361 // Uncompressed oops are not supported by loaded heaps. 362 Unimplemented(); 363 } 364 }; 365 366 void ArchiveHeapLoader::finish_initialization() { 367 if (is_loaded()) { 368 // These operations are needed only when the heap is loaded (not mapped). 369 finish_loaded_heap(); 370 if (VerifyArchivedFields > 0) { 371 verify_loaded_heap(); 372 } 373 } 374 if (is_in_use()) { 375 patch_native_pointers(); 376 intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom; 377 378 // The heap roots are stored in one or more segments that are laid out consecutively. 379 // The size of each segment (except for the last one) is max_size_in_{elems,bytes}. 380 HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments(); 381 HeapShared::init_root_segment_sizes(segments.max_size_in_elems()); 382 intptr_t first_segment_addr = bottom + segments.base_offset(); 383 for (size_t c = 0; c < segments.count(); c++) { 384 oop segment_oop = cast_to_oop(first_segment_addr + (c * segments.max_size_in_bytes())); 385 assert(segment_oop->is_objArray(), "Must be"); 386 HeapShared::add_root_segment((objArrayOop)segment_oop); 387 } 388 } 389 } 390 391 void ArchiveHeapLoader::finish_loaded_heap() { 392 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom; 393 HeapWord* top = (HeapWord*)_loaded_heap_top; 394 395 MemRegion archive_space = MemRegion(bottom, top); 396 Universe::heap()->complete_loaded_archive_space(archive_space); 397 } 398 399 void ArchiveHeapLoader::verify_loaded_heap() { 400 log_info(cds, heap)("Verify all oops and pointers in loaded heap"); 401 402 ResourceMark rm; 403 ResourceHashtable<uintptr_t, bool> table; 404 VerifyLoadedHeapEmbeddedPointers verifier(&table); 405 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom; 406 HeapWord* top = (HeapWord*)_loaded_heap_top; 407 408 for (HeapWord* p = bottom; p < top; ) { 409 oop o = cast_to_oop(p); 410 table.put(cast_from_oop<uintptr_t>(o), true); 411 p += o->size(); 412 } 413 414 for (HeapWord* p = bottom; p < top; ) { 415 oop o = cast_to_oop(p); 416 o->oop_iterate(&verifier); 417 p += o->size(); 418 } 419 } 420 421 void ArchiveHeapLoader::fill_failed_loaded_heap() { 422 assert(_loading_failed, "must be"); 423 if (_loaded_heap_bottom != 0) { 424 assert(_loaded_heap_top != 0, "must be"); 425 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom; 426 HeapWord* top = (HeapWord*)_loaded_heap_top; 427 Universe::heap()->fill_with_objects(bottom, top - bottom); 428 } 429 } 430 431 class PatchNativePointers: public BitMapClosure { 432 Metadata** _start; 433 434 public: 435 PatchNativePointers(Metadata** start) : _start(start) {} 436 437 bool do_bit(size_t offset) { 438 Metadata** p = _start + offset; 439 *p = (Metadata*)(address(*p) + MetaspaceShared::relocation_delta()); 440 return true; 441 } 442 }; 443 444 void ArchiveHeapLoader::patch_native_pointers() { 445 if (MetaspaceShared::relocation_delta() == 0) { 446 return; 447 } 448 449 FileMapRegion* r = FileMapInfo::current_info()->region_at(MetaspaceShared::hp); 450 if (r->mapped_base() != nullptr && r->has_ptrmap()) { 451 log_info(cds, heap)("Patching native pointers in heap region"); 452 BitMapView bm = FileMapInfo::current_info()->ptrmap_view(MetaspaceShared::hp); 453 PatchNativePointers patcher((Metadata**)r->mapped_base() + FileMapInfo::current_info()->heap_ptrmap_start_pos()); 454 bm.iterate(&patcher); 455 } 456 } 457 #endif // INCLUDE_CDS_JAVA_HEAP