1 /*
  2  * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "cds/archiveHeapLoader.inline.hpp"
 27 #include "cds/cdsConfig.hpp"
 28 #include "cds/heapShared.hpp"
 29 #include "cds/metaspaceShared.hpp"
 30 #include "classfile/classLoaderDataShared.hpp"
 31 #include "classfile/systemDictionaryShared.hpp"
 32 #include "gc/shared/collectedHeap.hpp"
 33 #include "logging/log.hpp"
 34 #include "memory/iterator.inline.hpp"
 35 #include "memory/resourceArea.hpp"
 36 #include "memory/universe.hpp"
 37 #include "sanitizers/ub.hpp"
 38 #include "utilities/bitMap.inline.hpp"
 39 #include "utilities/copy.hpp"
 40 
 41 #if INCLUDE_CDS_JAVA_HEAP
 42 
 43 bool ArchiveHeapLoader::_is_mapped = false;
 44 bool ArchiveHeapLoader::_is_loaded = false;
 45 
 46 bool    ArchiveHeapLoader::_narrow_oop_base_initialized = false;
 47 address ArchiveHeapLoader::_narrow_oop_base;
 48 int     ArchiveHeapLoader::_narrow_oop_shift;
 49 
 50 // Support for loaded heap.
 51 uintptr_t ArchiveHeapLoader::_loaded_heap_bottom = 0;
 52 uintptr_t ArchiveHeapLoader::_loaded_heap_top = 0;
 53 uintptr_t ArchiveHeapLoader::_dumptime_base = UINTPTR_MAX;
 54 uintptr_t ArchiveHeapLoader::_dumptime_top = 0;
 55 intx ArchiveHeapLoader::_runtime_offset = 0;
 56 bool ArchiveHeapLoader::_loading_failed = false;
 57 
 58 // Support for mapped heap.
 59 uintptr_t ArchiveHeapLoader::_mapped_heap_bottom = 0;
 60 bool      ArchiveHeapLoader::_mapped_heap_relocation_initialized = false;
 61 ptrdiff_t ArchiveHeapLoader::_mapped_heap_delta = 0;
 62 
 63 // Every mapped region is offset by _mapped_heap_delta from its requested address.
 64 // See FileMapInfo::heap_region_requested_address().
 65 ATTRIBUTE_NO_UBSAN
 66 void ArchiveHeapLoader::init_mapped_heap_info(address mapped_heap_bottom, ptrdiff_t delta, int dumptime_oop_shift) {
 67   assert(!_mapped_heap_relocation_initialized, "only once");
 68   if (!UseCompressedOops) {
 69     assert(dumptime_oop_shift == 0, "sanity");
 70   }
 71   assert(can_map(), "sanity");
 72   init_narrow_oop_decoding(CompressedOops::base() + delta, dumptime_oop_shift);
 73   _mapped_heap_bottom = (intptr_t)mapped_heap_bottom;
 74   _mapped_heap_delta = delta;
 75   _mapped_heap_relocation_initialized = true;
 76 }
 77 
 78 void ArchiveHeapLoader::init_narrow_oop_decoding(address base, int shift) {
 79   assert(!_narrow_oop_base_initialized, "only once");
 80   _narrow_oop_base_initialized = true;
 81   _narrow_oop_base = base;
 82   _narrow_oop_shift = shift;
 83 }
 84 
 85 void ArchiveHeapLoader::fixup_region() {
 86   FileMapInfo* mapinfo = FileMapInfo::current_info();
 87   if (is_mapped()) {
 88     mapinfo->fixup_mapped_heap_region();
 89   } else if (_loading_failed) {
 90     fill_failed_loaded_heap();
 91   }
 92   if (is_in_use()) {
 93     if (!CDSConfig::is_using_full_module_graph()) {
 94       // Need to remove all the archived java.lang.Module objects from HeapShared::roots().
 95       ClassLoaderDataShared::clear_archived_oops();
 96     }
 97   }
 98 }
 99 
100 // ------------------ Support for Region MAPPING -----------------------------------------
101 
102 // Patch all the embedded oop pointers inside an archived heap region,
103 // to be consistent with the runtime oop encoding.
104 class PatchCompressedEmbeddedPointers: public BitMapClosure {
105   narrowOop* _start;
106 
107  public:
108   PatchCompressedEmbeddedPointers(narrowOop* start) : _start(start) {}
109 
110   bool do_bit(size_t offset) {
111     narrowOop* p = _start + offset;
112     narrowOop v = *p;
113     assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
114     oop o = ArchiveHeapLoader::decode_from_mapped_archive(v);
115     RawAccess<IS_NOT_NULL>::oop_store(p, o);
116     return true;
117   }
118 };
119 
120 class PatchCompressedEmbeddedPointersQuick: public BitMapClosure {
121   narrowOop* _start;
122   uint32_t _delta;
123 
124  public:
125   PatchCompressedEmbeddedPointersQuick(narrowOop* start, uint32_t delta) : _start(start), _delta(delta) {}
126 
127   bool do_bit(size_t offset) {
128     narrowOop* p = _start + offset;
129     narrowOop v = *p;
130     assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
131     narrowOop new_v = CompressedOops::narrow_oop_cast(CompressedOops::narrow_oop_value(v) + _delta);
132     assert(!CompressedOops::is_null(new_v), "should never relocate to narrowOop(0)");
133 #ifdef ASSERT
134     oop o1 = ArchiveHeapLoader::decode_from_mapped_archive(v);
135     oop o2 = CompressedOops::decode_not_null(new_v);
136     assert(o1 == o2, "quick delta must work");
137 #endif
138     RawAccess<IS_NOT_NULL>::oop_store(p, new_v);
139     return true;
140   }
141 };
142 
143 class PatchUncompressedEmbeddedPointers: public BitMapClosure {
144   oop* _start;
145 
146  public:
147   PatchUncompressedEmbeddedPointers(oop* start) : _start(start) {}
148 
149   bool do_bit(size_t offset) {
150     oop* p = _start + offset;
151     intptr_t dumptime_oop = (intptr_t)((void*)*p);
152     assert(dumptime_oop != 0, "null oops should have been filtered out at dump time");
153     intptr_t runtime_oop = dumptime_oop + ArchiveHeapLoader::mapped_heap_delta();
154     RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(runtime_oop));
155     return true;
156   }
157 };
158 
159 void ArchiveHeapLoader::patch_compressed_embedded_pointers(BitMapView bm,
160                                                   FileMapInfo* info,
161                                                   MemRegion region) {
162   narrowOop dt_encoded_bottom = info->encoded_heap_region_dumptime_address();
163   narrowOop rt_encoded_bottom = CompressedOops::encode_not_null(cast_to_oop(region.start()));
164   log_info(cds)("patching heap embedded pointers: narrowOop 0x%8x -> 0x%8x",
165                   (uint)dt_encoded_bottom, (uint)rt_encoded_bottom);
166 
167   // Optimization: if dumptime shift is the same as runtime shift, we can perform a
168   // quick conversion from "dumptime narrowOop" -> "runtime narrowOop".
169   narrowOop* patching_start = (narrowOop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos();
170   if (_narrow_oop_shift == CompressedOops::shift()) {
171     uint32_t quick_delta = (uint32_t)rt_encoded_bottom - (uint32_t)dt_encoded_bottom;
172     log_info(cds)("CDS heap data relocation quick delta = 0x%x", quick_delta);
173     if (quick_delta == 0) {
174       log_info(cds)("CDS heap data relocation unnecessary, quick_delta = 0");
175     } else {
176       PatchCompressedEmbeddedPointersQuick patcher(patching_start, quick_delta);
177       bm.iterate(&patcher);
178     }
179   } else {
180     log_info(cds)("CDS heap data quick relocation not possible");
181     PatchCompressedEmbeddedPointers patcher(patching_start);
182     bm.iterate(&patcher);
183   }
184 }
185 
186 // Patch all the non-null pointers that are embedded in the archived heap objects
187 // in this (mapped) region
188 void ArchiveHeapLoader::patch_embedded_pointers(FileMapInfo* info,
189                                                 MemRegion region, address oopmap,
190                                                 size_t oopmap_size_in_bits) {
191   BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits);
192   if (UseCompressedOops) {
193     patch_compressed_embedded_pointers(bm, info, region);
194   } else {
195     PatchUncompressedEmbeddedPointers patcher((oop*)region.start() + FileMapInfo::current_info()->heap_oopmap_start_pos());
196     bm.iterate(&patcher);
197   }
198 }
199 
200 // ------------------ Support for Region LOADING -----------------------------------------
201 
202 // The CDS archive remembers each heap object by its address at dump time, but
203 // the heap object may be loaded at a different address at run time. This structure is used
204 // to translate the dump time addresses for all objects in FileMapInfo::space_at(region_index)
205 // to their runtime addresses.
206 struct LoadedArchiveHeapRegion {
207   int       _region_index;   // index for FileMapInfo::space_at(index)
208   size_t    _region_size;    // number of bytes in this region
209   uintptr_t _dumptime_base;  // The dump-time (decoded) address of the first object in this region
210   intx      _runtime_offset; // If an object's dump time address P is within in this region, its
211                              // runtime address is P + _runtime_offset
212   uintptr_t top() {
213     return _dumptime_base + _region_size;
214   }
215 };
216 
217 void ArchiveHeapLoader::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loaded_region) {
218   _dumptime_base = loaded_region->_dumptime_base;
219   _dumptime_top = loaded_region->top();
220   _runtime_offset = loaded_region->_runtime_offset;
221 }
222 
223 bool ArchiveHeapLoader::can_load() {
224   if (!UseCompressedOops) {
225     // Pointer relocation for uncompressed oops is unimplemented.
226     return false;
227   }
228   return Universe::heap()->can_load_archived_objects();
229 }
230 
231 class ArchiveHeapLoader::PatchLoadedRegionPointers: public BitMapClosure {
232   narrowOop* _start;
233   intx _offset;
234   uintptr_t _base;
235   uintptr_t _top;
236 
237  public:
238   PatchLoadedRegionPointers(narrowOop* start, LoadedArchiveHeapRegion* loaded_region)
239     : _start(start),
240       _offset(loaded_region->_runtime_offset),
241       _base(loaded_region->_dumptime_base),
242       _top(loaded_region->top()) {}
243 
244   bool do_bit(size_t offset) {
245     assert(UseCompressedOops, "PatchLoadedRegionPointers for uncompressed oops is unimplemented");
246     narrowOop* p = _start + offset;
247     narrowOop v = *p;
248     assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
249     uintptr_t o = cast_from_oop<uintptr_t>(ArchiveHeapLoader::decode_from_archive(v));
250     assert(_base <= o && o < _top, "must be");
251 
252     o += _offset;
253     ArchiveHeapLoader::assert_in_loaded_heap(o);
254     RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(o));
255     return true;
256   }
257 };
258 
259 bool ArchiveHeapLoader::init_loaded_region(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region,
260                                            MemRegion& archive_space) {
261   size_t total_bytes = 0;
262   FileMapRegion* r = mapinfo->region_at(MetaspaceShared::hp);
263   r->assert_is_heap_region();
264   if (r->used() == 0) {
265     return false;
266   }
267 
268   assert(is_aligned(r->used(), HeapWordSize), "must be");
269   total_bytes += r->used();
270   loaded_region->_region_index = MetaspaceShared::hp;
271   loaded_region->_region_size = r->used();
272   loaded_region->_dumptime_base = (uintptr_t)mapinfo->heap_region_dumptime_address();
273 
274   assert(is_aligned(total_bytes, HeapWordSize), "must be");
275   size_t word_size = total_bytes / HeapWordSize;
276   HeapWord* buffer = Universe::heap()->allocate_loaded_archive_space(word_size);
277   if (buffer == nullptr) {
278     return false;
279   }
280 
281   archive_space = MemRegion(buffer, word_size);
282   _loaded_heap_bottom = (uintptr_t)archive_space.start();
283   _loaded_heap_top    = _loaded_heap_bottom + total_bytes;
284 
285   loaded_region->_runtime_offset = _loaded_heap_bottom - loaded_region->_dumptime_base;
286 
287   return true;
288 }
289 
290 bool ArchiveHeapLoader::load_heap_region_impl(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_region,
291                                               uintptr_t load_address) {
292   uintptr_t bitmap_base = (uintptr_t)mapinfo->map_bitmap_region();
293   if (bitmap_base == 0) {
294     _loading_failed = true;
295     return false; // OOM or CRC error
296   }
297 
298   FileMapRegion* r = mapinfo->region_at(loaded_region->_region_index);
299   if (!mapinfo->read_region(loaded_region->_region_index, (char*)load_address, r->used(), /* do_commit = */ false)) {
300     // There's no easy way to free the buffer, so we will fill it with zero later
301     // in fill_failed_loaded_heap(), and it will eventually be GC'ed.
302     log_warning(cds)("Loading of heap region %d has failed. Archived objects are disabled", loaded_region->_region_index);
303     _loading_failed = true;
304     return false;
305   }
306   assert(r->mapped_base() == (char*)load_address, "sanity");
307   log_info(cds)("Loaded heap    region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT
308                 " size " SIZE_FORMAT_W(6) " delta " INTX_FORMAT,
309                 loaded_region->_region_index, load_address, load_address + loaded_region->_region_size,
310                 loaded_region->_region_size, loaded_region->_runtime_offset);
311 
312   uintptr_t oopmap = bitmap_base + r->oopmap_offset();
313   BitMapView bm((BitMap::bm_word_t*)oopmap, r->oopmap_size_in_bits());
314 
315   PatchLoadedRegionPointers patcher((narrowOop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region);
316   bm.iterate(&patcher);
317   return true;
318 }
319 
320 bool ArchiveHeapLoader::load_heap_region(FileMapInfo* mapinfo) {
321   assert(UseCompressedOops, "loaded heap for !UseCompressedOops is unimplemented");
322   init_narrow_oop_decoding(mapinfo->narrow_oop_base(), mapinfo->narrow_oop_shift());
323 
324   LoadedArchiveHeapRegion loaded_region;
325   memset(&loaded_region, 0, sizeof(loaded_region));
326 
327   MemRegion archive_space;
328   if (!init_loaded_region(mapinfo, &loaded_region, archive_space)) {
329     return false;
330   }
331 
332   if (!load_heap_region_impl(mapinfo, &loaded_region, (uintptr_t)archive_space.start())) {
333     assert(_loading_failed, "must be");
334     return false;
335   }
336 
337   init_loaded_heap_relocation(&loaded_region);
338   _is_loaded = true;
339 
340   return true;
341 }
342 
343 class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure {
344   ResourceHashtable<uintptr_t, bool>* _table;
345 
346  public:
347   VerifyLoadedHeapEmbeddedPointers(ResourceHashtable<uintptr_t, bool>* table) : _table(table) {}
348 
349   virtual void do_oop(narrowOop* p) {
350     // This should be called before the loaded region is modified, so all the embedded pointers
351     // must be null, or must point to a valid object in the loaded region.
352     narrowOop v = *p;
353     if (!CompressedOops::is_null(v)) {
354       oop o = CompressedOops::decode_not_null(v);
355       uintptr_t u = cast_from_oop<uintptr_t>(o);
356       ArchiveHeapLoader::assert_in_loaded_heap(u);
357       guarantee(_table->contains(u), "must point to beginning of object in loaded archived region");
358     }
359   }
360   virtual void do_oop(oop* p) {
361     // Uncompressed oops are not supported by loaded heaps.
362     Unimplemented();
363   }
364 };
365 
366 void ArchiveHeapLoader::finish_initialization() {
367   if (is_loaded()) {
368     // These operations are needed only when the heap is loaded (not mapped).
369     finish_loaded_heap();
370     if (VerifyArchivedFields > 0) {
371       verify_loaded_heap();
372     }
373   }
374   if (is_in_use()) {
375     patch_native_pointers();
376     intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom;
377 
378     // The heap roots are stored in one or more segments that are laid out consecutively.
379     // The size of each segment (except for the last one) is max_size_in_{elems,bytes}.
380     HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments();
381     HeapShared::init_root_segment_sizes(segments.max_size_in_elems());
382     intptr_t first_segment_addr = bottom + segments.base_offset();
383     for (size_t c = 0; c < segments.count(); c++) {
384       oop segment_oop = cast_to_oop(first_segment_addr + (c * segments.max_size_in_bytes()));
385       assert(segment_oop->is_objArray(), "Must be");
386       HeapShared::add_root_segment((objArrayOop)segment_oop);
387     }
388   }
389 }
390 
391 void ArchiveHeapLoader::finish_loaded_heap() {
392   HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
393   HeapWord* top    = (HeapWord*)_loaded_heap_top;
394 
395   MemRegion archive_space = MemRegion(bottom, top);
396   Universe::heap()->complete_loaded_archive_space(archive_space);
397 }
398 
399 void ArchiveHeapLoader::verify_loaded_heap() {
400   log_info(cds, heap)("Verify all oops and pointers in loaded heap");
401 
402   ResourceMark rm;
403   ResourceHashtable<uintptr_t, bool> table;
404   VerifyLoadedHeapEmbeddedPointers verifier(&table);
405   HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
406   HeapWord* top    = (HeapWord*)_loaded_heap_top;
407 
408   for (HeapWord* p = bottom; p < top; ) {
409     oop o = cast_to_oop(p);
410     table.put(cast_from_oop<uintptr_t>(o), true);
411     p += o->size();
412   }
413 
414   for (HeapWord* p = bottom; p < top; ) {
415     oop o = cast_to_oop(p);
416     o->oop_iterate(&verifier);
417     p += o->size();
418   }
419 }
420 
421 void ArchiveHeapLoader::fill_failed_loaded_heap() {
422   assert(_loading_failed, "must be");
423   if (_loaded_heap_bottom != 0) {
424     assert(_loaded_heap_top != 0, "must be");
425     HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
426     HeapWord* top = (HeapWord*)_loaded_heap_top;
427     Universe::heap()->fill_with_objects(bottom, top - bottom);
428   }
429 }
430 
431 class PatchNativePointers: public BitMapClosure {
432   Metadata** _start;
433 
434  public:
435   PatchNativePointers(Metadata** start) : _start(start) {}
436 
437   bool do_bit(size_t offset) {
438     Metadata** p = _start + offset;
439     *p = (Metadata*)(address(*p) + MetaspaceShared::relocation_delta());
440     return true;
441   }
442 };
443 
444 void ArchiveHeapLoader::patch_native_pointers() {
445   if (MetaspaceShared::relocation_delta() == 0) {
446     return;
447   }
448 
449   FileMapRegion* r = FileMapInfo::current_info()->region_at(MetaspaceShared::hp);
450   if (r->mapped_base() != nullptr && r->has_ptrmap()) {
451     log_info(cds, heap)("Patching native pointers in heap region");
452     BitMapView bm = FileMapInfo::current_info()->ptrmap_view(MetaspaceShared::hp);
453     PatchNativePointers patcher((Metadata**)r->mapped_base() + FileMapInfo::current_info()->heap_ptrmap_start_pos());
454     bm.iterate(&patcher);
455   }
456 }
457 #endif // INCLUDE_CDS_JAVA_HEAP