1 /*
2 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "cds/archiveBuilder.hpp"
27 #include "cds/archiveUtils.hpp"
28 #include "cds/cdsHeapVerifier.hpp"
29 #include "cds/filemap.hpp"
30 #include "cds/heapShared.inline.hpp"
31 #include "cds/metaspaceShared.hpp"
32 #include "classfile/classLoaderData.hpp"
33 #include "classfile/classLoaderDataShared.hpp"
34 #include "classfile/javaClasses.inline.hpp"
35 #include "classfile/moduleEntry.hpp"
36 #include "classfile/stringTable.hpp"
37 #include "classfile/symbolTable.hpp"
38 #include "classfile/systemDictionary.hpp"
39 #include "classfile/systemDictionaryShared.hpp"
40 #include "classfile/vmClasses.hpp"
41 #include "classfile/vmSymbols.hpp"
42 #include "gc/shared/collectedHeap.hpp"
43 #include "gc/shared/gcLocker.hpp"
44 #include "gc/shared/gcVMOperations.hpp"
45 #include "logging/log.hpp"
46 #include "logging/logStream.hpp"
47 #include "memory/iterator.inline.hpp"
48 #include "memory/metadataFactory.hpp"
49 #include "memory/metaspaceClosure.hpp"
50 #include "memory/resourceArea.hpp"
51 #include "memory/universe.hpp"
52 #include "oops/compressedOops.inline.hpp"
53 #include "oops/fieldStreams.inline.hpp"
54 #include "oops/objArrayOop.inline.hpp"
55 #include "oops/oop.inline.hpp"
56 #include "oops/typeArrayOop.inline.hpp"
57 #include "prims/jvmtiExport.hpp"
58 #include "runtime/fieldDescriptor.inline.hpp"
59 #include "runtime/globals_extension.hpp"
60 #include "runtime/init.hpp"
61 #include "runtime/java.hpp"
62 #include "runtime/javaCalls.hpp"
63 #include "runtime/safepoint.hpp"
64 #include "runtime/safepointVerifiers.hpp"
65 #include "utilities/bitMap.inline.hpp"
66 #include "utilities/copy.hpp"
67 #if INCLUDE_G1GC
68 #include "gc/g1/g1CollectedHeap.hpp"
69 #endif
70
71 #if INCLUDE_CDS_JAVA_HEAP
72
73 bool HeapShared::_closed_regions_mapped = false;
74 bool HeapShared::_open_regions_mapped = false;
75 bool HeapShared::_is_loaded = false;
76 bool HeapShared::_disable_writing = false;
77 address HeapShared::_narrow_oop_base;
78 int HeapShared::_narrow_oop_shift;
79 DumpedInternedStrings *HeapShared::_dumped_interned_strings = NULL;
80
81 // Support for loaded heap.
82 uintptr_t HeapShared::_loaded_heap_bottom = 0;
83 uintptr_t HeapShared::_loaded_heap_top = 0;
84 uintptr_t HeapShared::_dumptime_base_0 = UINTPTR_MAX;
85 uintptr_t HeapShared::_dumptime_base_1 = UINTPTR_MAX;
86 uintptr_t HeapShared::_dumptime_base_2 = UINTPTR_MAX;
87 uintptr_t HeapShared::_dumptime_base_3 = UINTPTR_MAX;
88 uintptr_t HeapShared::_dumptime_top = 0;
89 intx HeapShared::_runtime_offset_0 = 0;
90 intx HeapShared::_runtime_offset_1 = 0;
91 intx HeapShared::_runtime_offset_2 = 0;
92 intx HeapShared::_runtime_offset_3 = 0;
93 bool HeapShared::_loading_failed = false;
94
95 // Support for mapped heap (!UseCompressedOops only)
96 ptrdiff_t HeapShared::_runtime_delta = 0;
97
98 //
99 // If you add new entries to the following tables, you should know what you're doing!
100 //
101
102 // Entry fields for shareable subgraphs archived in the closed archive heap
103 // region. Warning: Objects in the subgraphs should not have reference fields
104 // assigned at runtime.
105 static ArchivableStaticFieldInfo closed_archive_subgraph_entry_fields[] = {
106 {"java/lang/Integer$IntegerCache", "archivedCache"},
107 {"java/lang/Long$LongCache", "archivedCache"},
108 {"java/lang/Byte$ByteCache", "archivedCache"},
109 {"java/lang/Short$ShortCache", "archivedCache"},
110 {"java/lang/Character$CharacterCache", "archivedCache"},
111 {"java/util/jar/Attributes$Name", "KNOWN_NAMES"},
112 {"sun/util/locale/BaseLocale", "constantBaseLocales"},
113 };
114 // Entry fields for subgraphs archived in the open archive heap region.
115 static ArchivableStaticFieldInfo open_archive_subgraph_entry_fields[] = {
116 {"jdk/internal/module/ArchivedModuleGraph", "archivedModuleGraph"},
117 {"java/util/ImmutableCollections", "archivedObjects"},
118 {"java/lang/ModuleLayer", "EMPTY_LAYER"},
119 {"java/lang/module/Configuration", "EMPTY_CONFIGURATION"},
120 {"jdk/internal/math/FDBigInteger", "archivedCaches"},
121 };
122
123 // Entry fields for subgraphs archived in the open archive heap region (full module graph).
124 static ArchivableStaticFieldInfo fmg_open_archive_subgraph_entry_fields[] = {
125 {"jdk/internal/loader/ArchivedClassLoaders", "archivedClassLoaders"},
126 {"jdk/internal/module/ArchivedBootLayer", "archivedBootLayer"},
127 {"java/lang/Module$ArchivedData", "archivedData"},
128 };
129
130 const static int num_closed_archive_subgraph_entry_fields =
131 sizeof(closed_archive_subgraph_entry_fields) / sizeof(ArchivableStaticFieldInfo);
132 const static int num_open_archive_subgraph_entry_fields =
133 sizeof(open_archive_subgraph_entry_fields) / sizeof(ArchivableStaticFieldInfo);
134 const static int num_fmg_open_archive_subgraph_entry_fields =
135 sizeof(fmg_open_archive_subgraph_entry_fields) / sizeof(ArchivableStaticFieldInfo);
136
137 GrowableArrayCHeap<oop, mtClassShared>* HeapShared::_pending_roots = NULL;
138 OopHandle HeapShared::_roots;
139
140 #ifdef ASSERT
141 bool HeapShared::is_archived_object_during_dumptime(oop p) {
142 assert(HeapShared::can_write(), "must be");
143 assert(DumpSharedSpaces, "this function is only used with -Xshare:dump");
144 return Universe::heap()->is_archived_object(p);
145 }
146 #endif
147
148 static bool is_subgraph_root_class_of(ArchivableStaticFieldInfo fields[], int num, InstanceKlass* ik) {
149 for (int i = 0; i < num; i++) {
150 if (fields[i].klass == ik) {
151 return true;
152 }
153 }
154 return false;
155 }
156
157 bool HeapShared::is_subgraph_root_class(InstanceKlass* ik) {
158 return is_subgraph_root_class_of(closed_archive_subgraph_entry_fields,
159 num_closed_archive_subgraph_entry_fields, ik) ||
160 is_subgraph_root_class_of(open_archive_subgraph_entry_fields,
161 num_open_archive_subgraph_entry_fields, ik) ||
162 is_subgraph_root_class_of(fmg_open_archive_subgraph_entry_fields,
163 num_fmg_open_archive_subgraph_entry_fields, ik);
164 }
165
166 void HeapShared::fixup_regions() {
167 FileMapInfo* mapinfo = FileMapInfo::current_info();
168 if (is_mapped()) {
169 mapinfo->fixup_mapped_heap_regions();
170 } else if (_loading_failed) {
171 fill_failed_loaded_region();
172 }
173 if (is_fully_available()) {
174 if (!MetaspaceShared::use_full_module_graph()) {
175 // Need to remove all the archived java.lang.Module objects from HeapShared::roots().
176 ClassLoaderDataShared::clear_archived_oops();
177 }
178 }
179 SystemDictionaryShared::update_archived_mirror_native_pointers();
180 }
181
182 unsigned HeapShared::oop_hash(oop const& p) {
183 // Do not call p->identity_hash() as that will update the
184 // object header.
185 return primitive_hash(cast_from_oop<intptr_t>(p));
186 }
187
188 static void reset_states(oop obj, TRAPS) {
189 Handle h_obj(THREAD, obj);
190 InstanceKlass* klass = InstanceKlass::cast(obj->klass());
191 TempNewSymbol method_name = SymbolTable::new_symbol("resetArchivedStates");
192 Symbol* method_sig = vmSymbols::void_method_signature();
193
194 while (klass != NULL) {
195 Method* method = klass->find_method(method_name, method_sig);
196 if (method != NULL) {
197 assert(method->is_private(), "must be");
198 if (log_is_enabled(Debug, cds)) {
199 ResourceMark rm(THREAD);
200 log_debug(cds)(" calling %s", method->name_and_sig_as_C_string());
201 }
202 JavaValue result(T_VOID);
203 JavaCalls::call_special(&result, h_obj, klass,
204 method_name, method_sig, CHECK);
205 }
206 klass = klass->java_super();
207 }
208 }
209
210 void HeapShared::reset_archived_object_states(TRAPS) {
211 assert(DumpSharedSpaces, "dump-time only");
212 log_debug(cds)("Resetting platform loader");
213 reset_states(SystemDictionary::java_platform_loader(), CHECK);
214 log_debug(cds)("Resetting system loader");
215 reset_states(SystemDictionary::java_system_loader(), CHECK);
216
217 // Clean up jdk.internal.loader.ClassLoaders::bootLoader(), which is not
218 // directly used for class loading, but rather is used by the core library
219 // to keep track of resources, etc, loaded by the null class loader.
220 //
221 // Note, this object is non-null, and is not the same as
222 // ClassLoaderData::the_null_class_loader_data()->class_loader(),
223 // which is null.
224 log_debug(cds)("Resetting boot loader");
225 JavaValue result(T_OBJECT);
226 JavaCalls::call_static(&result,
227 vmClasses::jdk_internal_loader_ClassLoaders_klass(),
228 vmSymbols::bootLoader_name(),
229 vmSymbols::void_BuiltinClassLoader_signature(),
230 CHECK);
231 Handle boot_loader(THREAD, result.get_oop());
232 reset_states(boot_loader(), CHECK);
233 }
234
235 HeapShared::ArchivedObjectCache* HeapShared::_archived_object_cache = NULL;
236 HeapShared::OriginalObjectTable* HeapShared::_original_object_table = NULL;
237 oop HeapShared::find_archived_heap_object(oop obj) {
238 assert(DumpSharedSpaces, "dump-time only");
239 ArchivedObjectCache* cache = archived_object_cache();
240 CachedOopInfo* p = cache->get(obj);
241 if (p != NULL) {
242 return p->_obj;
243 } else {
244 return NULL;
245 }
246 }
247
248 int HeapShared::append_root(oop obj) {
249 assert(DumpSharedSpaces, "dump-time only");
250
251 // No GC should happen since we aren't scanning _pending_roots.
252 assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
253
254 if (_pending_roots == NULL) {
255 _pending_roots = new GrowableArrayCHeap<oop, mtClassShared>(500);
256 }
257
258 return _pending_roots->append(obj);
259 }
260
261 objArrayOop HeapShared::roots() {
262 if (DumpSharedSpaces) {
263 assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
264 if (!HeapShared::can_write()) {
265 return NULL;
266 }
267 } else {
268 assert(UseSharedSpaces, "must be");
269 }
270
271 objArrayOop roots = (objArrayOop)_roots.resolve();
272 assert(roots != NULL, "should have been initialized");
273 return roots;
274 }
275
276 // Returns an objArray that contains all the roots of the archived objects
277 oop HeapShared::get_root(int index, bool clear) {
278 assert(index >= 0, "sanity");
279 if (DumpSharedSpaces) {
280 assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
281 assert(_pending_roots != NULL, "sanity");
282 return _pending_roots->at(index);
283 } else {
284 assert(UseSharedSpaces, "must be");
285 assert(!_roots.is_empty(), "must have loaded shared heap");
286 oop result = roots()->obj_at(index);
287 if (clear) {
288 clear_root(index);
289 }
290 return result;
291 }
292 }
293
294 void HeapShared::clear_root(int index) {
295 assert(index >= 0, "sanity");
296 assert(UseSharedSpaces, "must be");
297 if (is_fully_available()) {
298 if (log_is_enabled(Debug, cds, heap)) {
299 oop old = roots()->obj_at(index);
300 log_debug(cds, heap)("Clearing root %d: was " PTR_FORMAT, index, p2i(old));
301 }
302 roots()->obj_at_put(index, NULL);
303 }
304 }
305
306 oop HeapShared::archive_object(oop obj) {
307 assert(DumpSharedSpaces, "dump-time only");
308
309 assert(!obj->is_stackChunk(), "do not archive stack chunks");
310
311 oop ao = find_archived_heap_object(obj);
312 if (ao != NULL) {
313 // already archived
314 return ao;
315 }
316
317 int len = obj->size();
318 if (G1CollectedHeap::heap()->is_archive_alloc_too_large(len)) {
319 log_debug(cds, heap)("Cannot archive, object (" PTR_FORMAT ") is too large: " SIZE_FORMAT,
320 p2i(obj), (size_t)obj->size());
321 return NULL;
322 }
323
324 oop archived_oop = cast_to_oop(G1CollectedHeap::heap()->archive_mem_allocate(len));
325 if (archived_oop != NULL) {
326 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(obj), cast_from_oop<HeapWord*>(archived_oop), len);
327 // Reinitialize markword to remove age/marking/locking/etc.
328 //
329 // We need to retain the identity_hash, because it may have been used by some hashtables
330 // in the shared heap. This also has the side effect of pre-initializing the
331 // identity_hash for all shared objects, so they are less likely to be written
332 // into during run time, increasing the potential of memory sharing.
333 int hash_original = obj->identity_hash();
334
335 assert(SafepointSynchronize::is_at_safepoint(), "resolving displaced headers only at safepoint");
336 markWord mark = obj->mark();
337 if (mark.has_displaced_mark_helper()) {
338 mark = mark.displaced_mark_helper();
339 }
340 narrowKlass nklass = mark.narrow_klass();
341 archived_oop->set_mark(markWord::prototype().copy_set_hash(hash_original) LP64_ONLY(.set_narrow_klass(nklass)));
342 assert(archived_oop->mark().is_unlocked(), "sanity");
343
344 DEBUG_ONLY(int hash_archived = archived_oop->identity_hash());
345 assert(hash_original == hash_archived, "Different hash codes: original %x, archived %x", hash_original, hash_archived);
346
347 ArchivedObjectCache* cache = archived_object_cache();
348 CachedOopInfo info = make_cached_oop_info(archived_oop);
349 cache->put(obj, info);
350 if (_original_object_table != NULL) {
351 _original_object_table->put(archived_oop, obj);
352 }
353 if (log_is_enabled(Debug, cds, heap)) {
354 ResourceMark rm;
355 log_debug(cds, heap)("Archived heap object " PTR_FORMAT " ==> " PTR_FORMAT " : %s",
356 p2i(obj), p2i(archived_oop), obj->klass()->external_name());
357 }
358 } else {
359 log_error(cds, heap)(
360 "Cannot allocate space for object " PTR_FORMAT " in archived heap region",
361 p2i(obj));
362 log_error(cds)("Out of memory. Please run with a larger Java heap, current MaxHeapSize = "
363 SIZE_FORMAT "M", MaxHeapSize/M);
364 os::_exit(-1);
365 }
366 return archived_oop;
367 }
368
369 void HeapShared::archive_klass_objects() {
370 GrowableArray<Klass*>* klasses = ArchiveBuilder::current()->klasses();
371 assert(klasses != NULL, "sanity");
372 for (int i = 0; i < klasses->length(); i++) {
373 Klass* k = ArchiveBuilder::get_relocated_klass(klasses->at(i));
374
375 // archive mirror object
376 java_lang_Class::archive_mirror(k);
377
378 // archive the resolved_referenes array
379 if (k->is_instance_klass()) {
380 InstanceKlass* ik = InstanceKlass::cast(k);
381 ik->constants()->archive_resolved_references();
382 }
383 }
384 }
385
386 // -- Handling of Enum objects
387 // Java Enum classes have synthetic <clinit> methods that look like this
388 // enum MyEnum {FOO, BAR}
389 // MyEnum::<clinint> {
390 // /*static final MyEnum*/ MyEnum::FOO = new MyEnum("FOO");
391 // /*static final MyEnum*/ MyEnum::BAR = new MyEnum("BAR");
392 // }
393 //
394 // If MyEnum::FOO object is referenced by any of the archived subgraphs, we must
395 // ensure the archived value equals (in object address) to the runtime value of
396 // MyEnum::FOO.
397 //
398 // However, since MyEnum::<clinint> is synthetically generated by javac, there's
399 // no way of programmatically handling this inside the Java code (as you would handle
400 // ModuleLayer::EMPTY_LAYER, for example).
401 //
402 // Instead, we archive all static field of such Enum classes. At runtime,
403 // HeapShared::initialize_enum_klass() will skip the <clinit> method and pull
404 // the static fields out of the archived heap.
405 void HeapShared::check_enum_obj(int level,
406 KlassSubGraphInfo* subgraph_info,
407 oop orig_obj,
408 bool is_closed_archive) {
409 Klass* k = orig_obj->klass();
410 Klass* relocated_k = ArchiveBuilder::get_relocated_klass(k);
411 if (!k->is_instance_klass()) {
412 return;
413 }
414 InstanceKlass* ik = InstanceKlass::cast(k);
415 if (ik->java_super() == vmClasses::Enum_klass() && !ik->has_archived_enum_objs()) {
416 ResourceMark rm;
417 ik->set_has_archived_enum_objs();
418 relocated_k->set_has_archived_enum_objs();
419 oop mirror = ik->java_mirror();
420
421 for (JavaFieldStream fs(ik); !fs.done(); fs.next()) {
422 if (fs.access_flags().is_static()) {
423 fieldDescriptor& fd = fs.field_descriptor();
424 if (fd.field_type() != T_OBJECT && fd.field_type() != T_ARRAY) {
425 guarantee(false, "static field %s::%s must be T_OBJECT or T_ARRAY",
426 ik->external_name(), fd.name()->as_C_string());
427 }
428 oop oop_field = mirror->obj_field(fd.offset());
429 if (oop_field == NULL) {
430 guarantee(false, "static field %s::%s must not be null",
431 ik->external_name(), fd.name()->as_C_string());
432 } else if (oop_field->klass() != ik && oop_field->klass() != ik->array_klass_or_null()) {
433 guarantee(false, "static field %s::%s is of the wrong type",
434 ik->external_name(), fd.name()->as_C_string());
435 }
436 oop archived_oop_field = archive_reachable_objects_from(level, subgraph_info, oop_field, is_closed_archive);
437 int root_index = append_root(archived_oop_field);
438 log_info(cds, heap)("Archived enum obj @%d %s::%s (" INTPTR_FORMAT " -> " INTPTR_FORMAT ")",
439 root_index, ik->external_name(), fd.name()->as_C_string(),
440 p2i((oopDesc*)oop_field), p2i((oopDesc*)archived_oop_field));
441 SystemDictionaryShared::add_enum_klass_static_field(ik, root_index);
442 }
443 }
444 }
445 }
446
447 // See comments in HeapShared::check_enum_obj()
448 bool HeapShared::initialize_enum_klass(InstanceKlass* k, TRAPS) {
449 if (!is_fully_available()) {
450 return false;
451 }
452
453 RunTimeClassInfo* info = RunTimeClassInfo::get_for(k);
454 assert(info != NULL, "sanity");
455
456 if (log_is_enabled(Info, cds, heap)) {
457 ResourceMark rm;
458 log_info(cds, heap)("Initializing Enum class: %s", k->external_name());
459 }
460
461 oop mirror = k->java_mirror();
462 int i = 0;
463 for (JavaFieldStream fs(k); !fs.done(); fs.next()) {
464 if (fs.access_flags().is_static()) {
465 int root_index = info->enum_klass_static_field_root_index_at(i++);
466 fieldDescriptor& fd = fs.field_descriptor();
467 assert(fd.field_type() == T_OBJECT || fd.field_type() == T_ARRAY, "must be");
468 mirror->obj_field_put(fd.offset(), get_root(root_index, /*clear=*/true));
469 }
470 }
471 return true;
472 }
473
474 void HeapShared::run_full_gc_in_vm_thread() {
475 if (HeapShared::can_write()) {
476 // Avoid fragmentation while archiving heap objects.
477 // We do this inside a safepoint, so that no further allocation can happen after GC
478 // has finished.
479 if (GCLocker::is_active()) {
480 // Just checking for safety ...
481 // This should not happen during -Xshare:dump. If you see this, probably the Java core lib
482 // has been modified such that JNI code is executed in some clean up threads after
483 // we have finished class loading.
484 log_warning(cds)("GC locker is held, unable to start extra compacting GC. This may produce suboptimal results.");
485 } else {
486 log_info(cds)("Run GC ...");
487 Universe::heap()->collect_as_vm_thread(GCCause::_archive_time_gc);
488 log_info(cds)("Run GC done");
489 }
490 }
491 }
492
493 void HeapShared::archive_objects(GrowableArray<MemRegion>* closed_regions,
494 GrowableArray<MemRegion>* open_regions) {
495
496 G1HeapVerifier::verify_ready_for_archiving();
497
498 {
499 NoSafepointVerifier nsv;
500
501 // Cache for recording where the archived objects are copied to
502 create_archived_object_cache(log_is_enabled(Info, cds, map));
503
504 log_info(cds)("Heap range = [" PTR_FORMAT " - " PTR_FORMAT "]",
505 UseCompressedOops ? p2i(CompressedOops::begin()) :
506 p2i((address)G1CollectedHeap::heap()->reserved().start()),
507 UseCompressedOops ? p2i(CompressedOops::end()) :
508 p2i((address)G1CollectedHeap::heap()->reserved().end()));
509 log_info(cds)("Dumping objects to closed archive heap region ...");
510 copy_closed_objects(closed_regions);
511
512 log_info(cds)("Dumping objects to open archive heap region ...");
513 copy_open_objects(open_regions);
514
515 CDSHeapVerifier::verify();
516 }
517
518 G1HeapVerifier::verify_archive_regions();
519 }
520
521 void HeapShared::copy_closed_objects(GrowableArray<MemRegion>* closed_regions) {
522 assert(HeapShared::can_write(), "must be");
523
524 G1CollectedHeap::heap()->begin_archive_alloc_range();
525
526 // Archive interned string objects
527 StringTable::write_to_archive(_dumped_interned_strings);
528
529 archive_object_subgraphs(closed_archive_subgraph_entry_fields,
530 num_closed_archive_subgraph_entry_fields,
531 true /* is_closed_archive */,
532 false /* is_full_module_graph */);
533
534 G1CollectedHeap::heap()->end_archive_alloc_range(closed_regions,
535 os::vm_allocation_granularity());
536 }
537
538 void HeapShared::copy_open_objects(GrowableArray<MemRegion>* open_regions) {
539 assert(HeapShared::can_write(), "must be");
540
541 G1CollectedHeap::heap()->begin_archive_alloc_range(true /* open */);
542
543 java_lang_Class::archive_basic_type_mirrors();
544
545 archive_klass_objects();
546
547 archive_object_subgraphs(open_archive_subgraph_entry_fields,
548 num_open_archive_subgraph_entry_fields,
549 false /* is_closed_archive */,
550 false /* is_full_module_graph */);
551 if (MetaspaceShared::use_full_module_graph()) {
552 archive_object_subgraphs(fmg_open_archive_subgraph_entry_fields,
553 num_fmg_open_archive_subgraph_entry_fields,
554 false /* is_closed_archive */,
555 true /* is_full_module_graph */);
556 ClassLoaderDataShared::init_archived_oops();
557 }
558
559 copy_roots();
560
561 G1CollectedHeap::heap()->end_archive_alloc_range(open_regions,
562 os::vm_allocation_granularity());
563 }
564
565 // Copy _pending_archive_roots into an objArray
566 void HeapShared::copy_roots() {
567 // HeapShared::roots() points into an ObjArray in the open archive region. A portion of the
568 // objects in this array are discovered during HeapShared::archive_objects(). For example,
569 // in HeapShared::archive_reachable_objects_from() -> HeapShared::check_enum_obj().
570 // However, HeapShared::archive_objects() happens inside a safepoint, so we can't
571 // allocate a "regular" ObjArray and pass the result to HeapShared::archive_object().
572 // Instead, we have to roll our own alloc/copy routine here.
573 int length = _pending_roots != NULL ? _pending_roots->length() : 0;
574 size_t size = objArrayOopDesc::object_size(length);
575 Klass* k = Universe::objectArrayKlassObj(); // already relocated to point to archived klass
576 HeapWord* mem = G1CollectedHeap::heap()->archive_mem_allocate(size);
577
578 memset(mem, 0, size * BytesPerWord);
579 {
580 // This is copied from MemAllocator::finish
581 oopDesc::set_mark(mem, k->prototype_header());
582 #ifndef _LP64
583 oopDesc::release_set_klass(mem, k);
584 #endif
585 }
586 {
587 // This is copied from ObjArrayAllocator::initialize
588 arrayOopDesc::set_length(mem, length);
589 }
590
591 _roots = OopHandle(Universe::vm_global(), cast_to_oop(mem));
592 for (int i = 0; i < length; i++) {
593 roots()->obj_at_put(i, _pending_roots->at(i));
594 }
595 log_info(cds)("archived obj roots[%d] = " SIZE_FORMAT " words, klass = %p, obj = %p", length, size, k, mem);
596 }
597
598 void HeapShared::init_narrow_oop_decoding(address base, int shift) {
599 _narrow_oop_base = base;
600 _narrow_oop_shift = shift;
601 }
602
603 //
604 // Subgraph archiving support
605 //
606 HeapShared::DumpTimeKlassSubGraphInfoTable* HeapShared::_dump_time_subgraph_info_table = NULL;
607 HeapShared::RunTimeKlassSubGraphInfoTable HeapShared::_run_time_subgraph_info_table;
608
609 // Get the subgraph_info for Klass k. A new subgraph_info is created if
610 // there is no existing one for k. The subgraph_info records the relocated
611 // Klass* of the original k.
612 KlassSubGraphInfo* HeapShared::init_subgraph_info(Klass* k, bool is_full_module_graph) {
613 assert(DumpSharedSpaces, "dump time only");
614 bool created;
615 Klass* relocated_k = ArchiveBuilder::get_relocated_klass(k);
616 KlassSubGraphInfo* info =
617 _dump_time_subgraph_info_table->put_if_absent(k, KlassSubGraphInfo(relocated_k, is_full_module_graph),
618 &created);
619 assert(created, "must not initialize twice");
620 return info;
621 }
622
623 KlassSubGraphInfo* HeapShared::get_subgraph_info(Klass* k) {
624 assert(DumpSharedSpaces, "dump time only");
625 KlassSubGraphInfo* info = _dump_time_subgraph_info_table->get(k);
626 assert(info != NULL, "must have been initialized");
627 return info;
628 }
629
630 // Add an entry field to the current KlassSubGraphInfo.
631 void KlassSubGraphInfo::add_subgraph_entry_field(
632 int static_field_offset, oop v, bool is_closed_archive) {
633 assert(DumpSharedSpaces, "dump time only");
634 if (_subgraph_entry_fields == NULL) {
635 _subgraph_entry_fields =
636 new(ResourceObj::C_HEAP, mtClass) GrowableArray<int>(10, mtClass);
637 }
638 _subgraph_entry_fields->append(static_field_offset);
639 _subgraph_entry_fields->append(HeapShared::append_root(v));
640 }
641
642 // Add the Klass* for an object in the current KlassSubGraphInfo's subgraphs.
643 // Only objects of boot classes can be included in sub-graph.
644 void KlassSubGraphInfo::add_subgraph_object_klass(Klass* orig_k) {
645 assert(DumpSharedSpaces, "dump time only");
646 Klass* relocated_k = ArchiveBuilder::get_relocated_klass(orig_k);
647
648 if (_subgraph_object_klasses == NULL) {
649 _subgraph_object_klasses =
650 new(ResourceObj::C_HEAP, mtClass) GrowableArray<Klass*>(50, mtClass);
651 }
652
653 assert(ArchiveBuilder::current()->is_in_buffer_space(relocated_k), "must be a shared class");
654
655 if (_k == relocated_k) {
656 // Don't add the Klass containing the sub-graph to it's own klass
657 // initialization list.
658 return;
659 }
660
661 if (relocated_k->is_instance_klass()) {
662 assert(InstanceKlass::cast(relocated_k)->is_shared_boot_class(),
663 "must be boot class");
664 // vmClasses::xxx_klass() are not updated, need to check
665 // the original Klass*
666 if (orig_k == vmClasses::String_klass() ||
667 orig_k == vmClasses::Object_klass()) {
668 // Initialized early during VM initialization. No need to be added
669 // to the sub-graph object class list.
670 return;
671 }
672 } else if (relocated_k->is_objArray_klass()) {
673 Klass* abk = ObjArrayKlass::cast(relocated_k)->bottom_klass();
674 if (abk->is_instance_klass()) {
675 assert(InstanceKlass::cast(abk)->is_shared_boot_class(),
676 "must be boot class");
677 }
678 if (relocated_k == Universe::objectArrayKlassObj()) {
679 // Initialized early during Universe::genesis. No need to be added
680 // to the list.
681 return;
682 }
683 } else {
684 assert(relocated_k->is_typeArray_klass(), "must be");
685 // Primitive type arrays are created early during Universe::genesis.
686 return;
687 }
688
689 if (log_is_enabled(Debug, cds, heap)) {
690 if (!_subgraph_object_klasses->contains(relocated_k)) {
691 ResourceMark rm;
692 log_debug(cds, heap)("Adding klass %s", orig_k->external_name());
693 }
694 }
695
696 _subgraph_object_klasses->append_if_missing(relocated_k);
697 _has_non_early_klasses |= is_non_early_klass(orig_k);
698 }
699
700 bool KlassSubGraphInfo::is_non_early_klass(Klass* k) {
701 if (k->is_objArray_klass()) {
702 k = ObjArrayKlass::cast(k)->bottom_klass();
703 }
704 if (k->is_instance_klass()) {
705 if (!SystemDictionaryShared::is_early_klass(InstanceKlass::cast(k))) {
706 ResourceMark rm;
707 log_info(cds, heap)("non-early: %s", k->external_name());
708 return true;
709 } else {
710 return false;
711 }
712 } else {
713 return false;
714 }
715 }
716
717 // Initialize an archived subgraph_info_record from the given KlassSubGraphInfo.
718 void ArchivedKlassSubGraphInfoRecord::init(KlassSubGraphInfo* info) {
719 _k = info->klass();
720 _entry_field_records = NULL;
721 _subgraph_object_klasses = NULL;
722 _is_full_module_graph = info->is_full_module_graph();
723
724 if (_is_full_module_graph) {
725 // Consider all classes referenced by the full module graph as early -- we will be
726 // allocating objects of these classes during JVMTI early phase, so they cannot
727 // be processed by (non-early) JVMTI ClassFileLoadHook
728 _has_non_early_klasses = false;
729 } else {
730 _has_non_early_klasses = info->has_non_early_klasses();
731 }
732
733 if (_has_non_early_klasses) {
734 ResourceMark rm;
735 log_info(cds, heap)(
736 "Subgraph of klass %s has non-early klasses and cannot be used when JVMTI ClassFileLoadHook is enabled",
737 _k->external_name());
738 }
739
740 // populate the entry fields
741 GrowableArray<int>* entry_fields = info->subgraph_entry_fields();
742 if (entry_fields != NULL) {
743 int num_entry_fields = entry_fields->length();
744 assert(num_entry_fields % 2 == 0, "sanity");
745 _entry_field_records =
746 ArchiveBuilder::new_ro_array<int>(num_entry_fields);
747 for (int i = 0 ; i < num_entry_fields; i++) {
748 _entry_field_records->at_put(i, entry_fields->at(i));
749 }
750 }
751
752 // the Klasses of the objects in the sub-graphs
753 GrowableArray<Klass*>* subgraph_object_klasses = info->subgraph_object_klasses();
754 if (subgraph_object_klasses != NULL) {
755 int num_subgraphs_klasses = subgraph_object_klasses->length();
756 _subgraph_object_klasses =
757 ArchiveBuilder::new_ro_array<Klass*>(num_subgraphs_klasses);
758 for (int i = 0; i < num_subgraphs_klasses; i++) {
759 Klass* subgraph_k = subgraph_object_klasses->at(i);
760 if (log_is_enabled(Info, cds, heap)) {
761 ResourceMark rm;
762 log_info(cds, heap)(
763 "Archived object klass %s (%2d) => %s",
764 _k->external_name(), i, subgraph_k->external_name());
765 }
766 _subgraph_object_klasses->at_put(i, subgraph_k);
767 ArchivePtrMarker::mark_pointer(_subgraph_object_klasses->adr_at(i));
768 }
769 }
770
771 ArchivePtrMarker::mark_pointer(&_k);
772 ArchivePtrMarker::mark_pointer(&_entry_field_records);
773 ArchivePtrMarker::mark_pointer(&_subgraph_object_klasses);
774 }
775
776 struct CopyKlassSubGraphInfoToArchive : StackObj {
777 CompactHashtableWriter* _writer;
778 CopyKlassSubGraphInfoToArchive(CompactHashtableWriter* writer) : _writer(writer) {}
779
780 bool do_entry(Klass* klass, KlassSubGraphInfo& info) {
781 if (info.subgraph_object_klasses() != NULL || info.subgraph_entry_fields() != NULL) {
782 ArchivedKlassSubGraphInfoRecord* record =
783 (ArchivedKlassSubGraphInfoRecord*)ArchiveBuilder::ro_region_alloc(sizeof(ArchivedKlassSubGraphInfoRecord));
784 record->init(&info);
785
786 Klass* relocated_k = ArchiveBuilder::get_relocated_klass(klass);
787 unsigned int hash = SystemDictionaryShared::hash_for_shared_dictionary((address)relocated_k);
788 u4 delta = ArchiveBuilder::current()->any_to_offset_u4(record);
789 _writer->add(hash, delta);
790 }
791 return true; // keep on iterating
792 }
793 };
794
795 // Build the records of archived subgraph infos, which include:
796 // - Entry points to all subgraphs from the containing class mirror. The entry
797 // points are static fields in the mirror. For each entry point, the field
798 // offset, value and is_closed_archive flag are recorded in the sub-graph
799 // info. The value is stored back to the corresponding field at runtime.
800 // - A list of klasses that need to be loaded/initialized before archived
801 // java object sub-graph can be accessed at runtime.
802 void HeapShared::write_subgraph_info_table() {
803 // Allocate the contents of the hashtable(s) inside the RO region of the CDS archive.
804 DumpTimeKlassSubGraphInfoTable* d_table = _dump_time_subgraph_info_table;
805 CompactHashtableStats stats;
806
807 _run_time_subgraph_info_table.reset();
808
809 CompactHashtableWriter writer(d_table->_count, &stats);
810 CopyKlassSubGraphInfoToArchive copy(&writer);
811 d_table->iterate(©);
812 writer.dump(&_run_time_subgraph_info_table, "subgraphs");
813 }
814
815 void HeapShared::serialize(SerializeClosure* soc) {
816 oop roots_oop = NULL;
817
818 if (soc->reading()) {
819 soc->do_oop(&roots_oop); // read from archive
820 assert(oopDesc::is_oop_or_null(roots_oop), "is oop");
821 // Create an OopHandle only if we have actually mapped or loaded the roots
822 if (roots_oop != NULL) {
823 assert(HeapShared::is_fully_available(), "must be");
824 _roots = OopHandle(Universe::vm_global(), roots_oop);
825 }
826 } else {
827 // writing
828 roots_oop = roots();
829 soc->do_oop(&roots_oop); // write to archive
830 }
831
832 _run_time_subgraph_info_table.serialize_header(soc);
833 }
834
835 static void verify_the_heap(Klass* k, const char* which) {
836 if (VerifyArchivedFields > 0) {
837 ResourceMark rm;
838 log_info(cds, heap)("Verify heap %s initializing static field(s) in %s",
839 which, k->external_name());
840
841 VM_Verify verify_op;
842 VMThread::execute(&verify_op);
843
844 if (VerifyArchivedFields > 1 && is_init_completed()) {
845 // At this time, the oop->klass() of some archived objects in the heap may not
846 // have been loaded into the system dictionary yet. Nevertheless, oop->klass() should
847 // have enough information (object size, oop maps, etc) so that a GC can be safely
848 // performed.
849 //
850 // -XX:VerifyArchivedFields=2 force a GC to happen in such an early stage
851 // to check for GC safety.
852 log_info(cds, heap)("Trigger GC %s initializing static field(s) in %s",
853 which, k->external_name());
854 FlagSetting fs1(VerifyBeforeGC, true);
855 FlagSetting fs2(VerifyDuringGC, true);
856 FlagSetting fs3(VerifyAfterGC, true);
857 Universe::heap()->collect(GCCause::_java_lang_system_gc);
858 }
859 }
860 }
861
862 // Before GC can execute, we must ensure that all oops reachable from HeapShared::roots()
863 // have a valid klass. I.e., oopDesc::klass() must have already been resolved.
864 //
865 // Note: if a ArchivedKlassSubGraphInfoRecord contains non-early classes, and JVMTI
866 // ClassFileLoadHook is enabled, it's possible for this class to be dynamically replaced. In
867 // this case, we will not load the ArchivedKlassSubGraphInfoRecord and will clear its roots.
868 void HeapShared::resolve_classes(JavaThread* THREAD) {
869 if (!is_fully_available()) {
870 return; // nothing to do
871 }
872 resolve_classes_for_subgraphs(closed_archive_subgraph_entry_fields,
873 num_closed_archive_subgraph_entry_fields,
874 THREAD);
875 resolve_classes_for_subgraphs(open_archive_subgraph_entry_fields,
876 num_open_archive_subgraph_entry_fields,
877 THREAD);
878 resolve_classes_for_subgraphs(fmg_open_archive_subgraph_entry_fields,
879 num_fmg_open_archive_subgraph_entry_fields,
880 THREAD);
881 }
882
883 void HeapShared::resolve_classes_for_subgraphs(ArchivableStaticFieldInfo fields[],
884 int num, JavaThread* THREAD) {
885 for (int i = 0; i < num; i++) {
886 ArchivableStaticFieldInfo* info = &fields[i];
887 TempNewSymbol klass_name = SymbolTable::new_symbol(info->klass_name);
888 InstanceKlass* k = SystemDictionaryShared::find_builtin_class(klass_name);
889 assert(k != NULL && k->is_shared_boot_class(), "sanity");
890 resolve_classes_for_subgraph_of(k, THREAD);
891 }
892 }
893
894 void HeapShared::resolve_classes_for_subgraph_of(Klass* k, JavaThread* THREAD) {
895 ExceptionMark em(THREAD);
896 const ArchivedKlassSubGraphInfoRecord* record =
897 resolve_or_init_classes_for_subgraph_of(k, /*do_init=*/false, THREAD);
898 if (HAS_PENDING_EXCEPTION) {
899 CLEAR_PENDING_EXCEPTION;
900 }
901 if (record == NULL) {
902 clear_archived_roots_of(k);
903 }
904 }
905
906 void HeapShared::initialize_from_archived_subgraph(Klass* k, JavaThread* THREAD) {
907 if (!is_fully_available()) {
908 return; // nothing to do
909 }
910
911 ExceptionMark em(THREAD);
912 const ArchivedKlassSubGraphInfoRecord* record =
913 resolve_or_init_classes_for_subgraph_of(k, /*do_init=*/true, THREAD);
914
915 if (HAS_PENDING_EXCEPTION) {
916 CLEAR_PENDING_EXCEPTION;
917 // None of the field value will be set if there was an exception when initializing the classes.
918 // The java code will not see any of the archived objects in the
919 // subgraphs referenced from k in this case.
920 return;
921 }
922
923 if (record != NULL) {
924 init_archived_fields_for(k, record);
925 }
926 }
927
928 const ArchivedKlassSubGraphInfoRecord*
929 HeapShared::resolve_or_init_classes_for_subgraph_of(Klass* k, bool do_init, TRAPS) {
930 assert(!DumpSharedSpaces, "Should not be called with DumpSharedSpaces");
931
932 if (!k->is_shared()) {
933 return NULL;
934 }
935 unsigned int hash = SystemDictionaryShared::hash_for_shared_dictionary_quick(k);
936 const ArchivedKlassSubGraphInfoRecord* record = _run_time_subgraph_info_table.lookup(k, hash, 0);
937
938 // Initialize from archived data. Currently this is done only
939 // during VM initialization time. No lock is needed.
940 if (record != NULL) {
941 if (record->is_full_module_graph() && !MetaspaceShared::use_full_module_graph()) {
942 if (log_is_enabled(Info, cds, heap)) {
943 ResourceMark rm(THREAD);
944 log_info(cds, heap)("subgraph %s cannot be used because full module graph is disabled",
945 k->external_name());
946 }
947 return NULL;
948 }
949
950 if (record->has_non_early_klasses() && JvmtiExport::should_post_class_file_load_hook()) {
951 if (log_is_enabled(Info, cds, heap)) {
952 ResourceMark rm(THREAD);
953 log_info(cds, heap)("subgraph %s cannot be used because JVMTI ClassFileLoadHook is enabled",
954 k->external_name());
955 }
956 return NULL;
957 }
958
959 resolve_or_init(k, do_init, CHECK_NULL);
960
961 // Load/link/initialize the klasses of the objects in the subgraph.
962 // NULL class loader is used.
963 Array<Klass*>* klasses = record->subgraph_object_klasses();
964 if (klasses != NULL) {
965 for (int i = 0; i < klasses->length(); i++) {
966 Klass* klass = klasses->at(i);
967 if (!klass->is_shared()) {
968 return NULL;
969 }
970 resolve_or_init(klass, do_init, CHECK_NULL);
971 }
972 }
973 }
974
975 return record;
976 }
977
978 void HeapShared::resolve_or_init(Klass* k, bool do_init, TRAPS) {
979 if (!do_init) {
980 if (k->class_loader_data() == NULL) {
981 Klass* resolved_k = SystemDictionary::resolve_or_null(k->name(), CHECK);
982 assert(resolved_k == k, "classes used by archived heap must not be replaced by JVMTI ClassFileLoadHook");
983 }
984 } else {
985 assert(k->class_loader_data() != NULL, "must have been resolved by HeapShared::resolve_classes");
986 if (k->is_instance_klass()) {
987 InstanceKlass* ik = InstanceKlass::cast(k);
988 ik->initialize(CHECK);
989 } else if (k->is_objArray_klass()) {
990 ObjArrayKlass* oak = ObjArrayKlass::cast(k);
991 oak->initialize(CHECK);
992 }
993 }
994 }
995
996 void HeapShared::init_archived_fields_for(Klass* k, const ArchivedKlassSubGraphInfoRecord* record) {
997 verify_the_heap(k, "before");
998
999 // Load the subgraph entry fields from the record and store them back to
1000 // the corresponding fields within the mirror.
1001 oop m = k->java_mirror();
1002 Array<int>* entry_field_records = record->entry_field_records();
1003 if (entry_field_records != NULL) {
1004 int efr_len = entry_field_records->length();
1005 assert(efr_len % 2 == 0, "sanity");
1006 for (int i = 0; i < efr_len; i += 2) {
1007 int field_offset = entry_field_records->at(i);
1008 int root_index = entry_field_records->at(i+1);
1009 oop v = get_root(root_index, /*clear=*/true);
1010 m->obj_field_put(field_offset, v);
1011 log_debug(cds, heap)(" " PTR_FORMAT " init field @ %2d = " PTR_FORMAT, p2i(k), field_offset, p2i(v));
1012 }
1013
1014 // Done. Java code can see the archived sub-graphs referenced from k's
1015 // mirror after this point.
1016 if (log_is_enabled(Info, cds, heap)) {
1017 ResourceMark rm;
1018 log_info(cds, heap)("initialize_from_archived_subgraph %s " PTR_FORMAT "%s",
1019 k->external_name(), p2i(k), JvmtiExport::is_early_phase() ? " (early)" : "");
1020 }
1021 }
1022
1023 verify_the_heap(k, "after ");
1024 }
1025
1026 void HeapShared::clear_archived_roots_of(Klass* k) {
1027 unsigned int hash = SystemDictionaryShared::hash_for_shared_dictionary_quick(k);
1028 const ArchivedKlassSubGraphInfoRecord* record = _run_time_subgraph_info_table.lookup(k, hash, 0);
1029 if (record != NULL) {
1030 Array<int>* entry_field_records = record->entry_field_records();
1031 if (entry_field_records != NULL) {
1032 int efr_len = entry_field_records->length();
1033 assert(efr_len % 2 == 0, "sanity");
1034 for (int i = 0; i < efr_len; i += 2) {
1035 int root_index = entry_field_records->at(i+1);
1036 clear_root(root_index);
1037 }
1038 }
1039 }
1040 }
1041
1042 class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
1043 int _level;
1044 bool _is_closed_archive;
1045 bool _record_klasses_only;
1046 KlassSubGraphInfo* _subgraph_info;
1047 oop _orig_referencing_obj;
1048 oop _archived_referencing_obj;
1049
1050 // The following are for maintaining a stack for determining
1051 // CachedOopInfo::_referrer
1052 static WalkOopAndArchiveClosure* _current;
1053 WalkOopAndArchiveClosure* _last;
1054 public:
1055 WalkOopAndArchiveClosure(int level,
1056 bool is_closed_archive,
1057 bool record_klasses_only,
1058 KlassSubGraphInfo* subgraph_info,
1059 oop orig, oop archived) :
1060 _level(level), _is_closed_archive(is_closed_archive),
1061 _record_klasses_only(record_klasses_only),
1062 _subgraph_info(subgraph_info),
1063 _orig_referencing_obj(orig), _archived_referencing_obj(archived) {
1064 _last = _current;
1065 _current = this;
1066 }
1067 ~WalkOopAndArchiveClosure() {
1068 _current = _last;
1069 }
1070 void do_oop(narrowOop *p) { WalkOopAndArchiveClosure::do_oop_work(p); }
1071 void do_oop( oop *p) { WalkOopAndArchiveClosure::do_oop_work(p); }
1072
1073 protected:
1074 template <class T> void do_oop_work(T *p) {
1075 oop obj = RawAccess<>::oop_load(p);
1076 if (!CompressedOops::is_null(obj)) {
1077 assert(!HeapShared::is_archived_object_during_dumptime(obj),
1078 "original objects must not point to archived objects");
1079
1080 size_t field_delta = pointer_delta(p, _orig_referencing_obj, sizeof(char));
1081 T* new_p = (T*)(cast_from_oop<address>(_archived_referencing_obj) + field_delta);
1082
1083 if (!_record_klasses_only && log_is_enabled(Debug, cds, heap)) {
1084 ResourceMark rm;
1085 log_debug(cds, heap)("(%d) %s[" SIZE_FORMAT "] ==> " PTR_FORMAT " size " SIZE_FORMAT " %s", _level,
1086 _orig_referencing_obj->klass()->external_name(), field_delta,
1087 p2i(obj), obj->size() * HeapWordSize, obj->klass()->external_name());
1088 LogTarget(Trace, cds, heap) log;
1089 LogStream out(log);
1090 obj->print_on(&out);
1091 }
1092
1093 oop archived = HeapShared::archive_reachable_objects_from(
1094 _level + 1, _subgraph_info, obj, _is_closed_archive);
1095 assert(archived != NULL, "VM should have exited with unarchivable objects for _level > 1");
1096 assert(HeapShared::is_archived_object_during_dumptime(archived), "must be");
1097
1098 if (!_record_klasses_only) {
1099 // Update the reference in the archived copy of the referencing object.
1100 log_debug(cds, heap)("(%d) updating oop @[" PTR_FORMAT "] " PTR_FORMAT " ==> " PTR_FORMAT,
1101 _level, p2i(new_p), p2i(obj), p2i(archived));
1102 RawAccess<IS_NOT_NULL>::oop_store(new_p, archived);
1103 }
1104 }
1105 }
1106
1107 public:
1108 static WalkOopAndArchiveClosure* current() { return _current; }
1109 oop orig_referencing_obj() { return _orig_referencing_obj; }
1110 KlassSubGraphInfo* subgraph_info() { return _subgraph_info; }
1111 };
1112
1113 WalkOopAndArchiveClosure* WalkOopAndArchiveClosure::_current = NULL;
1114
1115 HeapShared::CachedOopInfo HeapShared::make_cached_oop_info(oop orig_obj) {
1116 CachedOopInfo info;
1117 WalkOopAndArchiveClosure* walker = WalkOopAndArchiveClosure::current();
1118
1119 info._subgraph_info = (walker == NULL) ? NULL : walker->subgraph_info();
1120 info._referrer = (walker == NULL) ? NULL : walker->orig_referencing_obj();
1121 info._obj = orig_obj;
1122
1123 return info;
1124 }
1125
1126 void HeapShared::check_closed_region_object(InstanceKlass* k) {
1127 // Check fields in the object
1128 for (JavaFieldStream fs(k); !fs.done(); fs.next()) {
1129 if (!fs.access_flags().is_static()) {
1130 BasicType ft = fs.field_descriptor().field_type();
1131 if (!fs.access_flags().is_final() && is_reference_type(ft)) {
1132 ResourceMark rm;
1133 log_warning(cds, heap)(
1134 "Please check reference field in %s instance in closed archive heap region: %s %s",
1135 k->external_name(), (fs.name())->as_C_string(),
1136 (fs.signature())->as_C_string());
1137 }
1138 }
1139 }
1140 }
1141
1142 void HeapShared::check_module_oop(oop orig_module_obj) {
1143 assert(DumpSharedSpaces, "must be");
1144 assert(java_lang_Module::is_instance(orig_module_obj), "must be");
1145 ModuleEntry* orig_module_ent = java_lang_Module::module_entry_raw(orig_module_obj);
1146 if (orig_module_ent == NULL) {
1147 // These special Module objects are created in Java code. They are not
1148 // defined via Modules::define_module(), so they don't have a ModuleEntry:
1149 // java.lang.Module::ALL_UNNAMED_MODULE
1150 // java.lang.Module::EVERYONE_MODULE
1151 // jdk.internal.loader.ClassLoaders$BootClassLoader::unnamedModule
1152 assert(java_lang_Module::name(orig_module_obj) == NULL, "must be unnamed");
1153 log_info(cds, heap)("Module oop with No ModuleEntry* @[" PTR_FORMAT "]", p2i(orig_module_obj));
1154 } else {
1155 ClassLoaderData* loader_data = orig_module_ent->loader_data();
1156 assert(loader_data->is_builtin_class_loader_data(), "must be");
1157 }
1158 }
1159
1160
1161 // (1) If orig_obj has not been archived yet, archive it.
1162 // (2) If orig_obj has not been seen yet (since start_recording_subgraph() was called),
1163 // trace all objects that are reachable from it, and make sure these objects are archived.
1164 // (3) Record the klasses of all orig_obj and all reachable objects.
1165 oop HeapShared::archive_reachable_objects_from(int level,
1166 KlassSubGraphInfo* subgraph_info,
1167 oop orig_obj,
1168 bool is_closed_archive) {
1169 assert(orig_obj != NULL, "must be");
1170 assert(!is_archived_object_during_dumptime(orig_obj), "sanity");
1171
1172 if (!JavaClasses::is_supported_for_archiving(orig_obj)) {
1173 // This object has injected fields that cannot be supported easily, so we disallow them for now.
1174 // If you get an error here, you probably made a change in the JDK library that has added
1175 // these objects that are referenced (directly or indirectly) by static fields.
1176 ResourceMark rm;
1177 log_error(cds, heap)("Cannot archive object of class %s", orig_obj->klass()->external_name());
1178 os::_exit(1);
1179 }
1180
1181 // java.lang.Class instances cannot be included in an archived object sub-graph. We only support
1182 // them as Klass::_archived_mirror because they need to be specially restored at run time.
1183 //
1184 // If you get an error here, you probably made a change in the JDK library that has added a Class
1185 // object that is referenced (directly or indirectly) by static fields.
1186 if (java_lang_Class::is_instance(orig_obj)) {
1187 log_error(cds, heap)("(%d) Unknown java.lang.Class object is in the archived sub-graph", level);
1188 os::_exit(1);
1189 }
1190
1191 oop archived_obj = find_archived_heap_object(orig_obj);
1192 if (java_lang_String::is_instance(orig_obj) && archived_obj != NULL) {
1193 // To save time, don't walk strings that are already archived. They just contain
1194 // pointers to a type array, whose klass doesn't need to be recorded.
1195 return archived_obj;
1196 }
1197
1198 if (has_been_seen_during_subgraph_recording(orig_obj)) {
1199 // orig_obj has already been archived and traced. Nothing more to do.
1200 return archived_obj;
1201 } else {
1202 set_has_been_seen_during_subgraph_recording(orig_obj);
1203 }
1204
1205 bool record_klasses_only = (archived_obj != NULL);
1206 if (archived_obj == NULL) {
1207 ++_num_new_archived_objs;
1208 archived_obj = archive_object(orig_obj);
1209 if (archived_obj == NULL) {
1210 // Skip archiving the sub-graph referenced from the current entry field.
1211 ResourceMark rm;
1212 log_error(cds, heap)(
1213 "Cannot archive the sub-graph referenced from %s object ("
1214 PTR_FORMAT ") size " SIZE_FORMAT ", skipped.",
1215 orig_obj->klass()->external_name(), p2i(orig_obj), orig_obj->size() * HeapWordSize);
1216 if (level == 1) {
1217 // Don't archive a subgraph root that's too big. For archives static fields, that's OK
1218 // as the Java code will take care of initializing this field dynamically.
1219 return NULL;
1220 } else {
1221 // We don't know how to handle an object that has been archived, but some of its reachable
1222 // objects cannot be archived. Bail out for now. We might need to fix this in the future if
1223 // we have a real use case.
1224 os::_exit(1);
1225 }
1226 }
1227
1228 if (java_lang_Module::is_instance(orig_obj)) {
1229 check_module_oop(orig_obj);
1230 java_lang_Module::set_module_entry(archived_obj, NULL);
1231 java_lang_Module::set_loader(archived_obj, NULL);
1232 } else if (java_lang_ClassLoader::is_instance(orig_obj)) {
1233 // class_data will be restored explicitly at run time.
1234 guarantee(orig_obj == SystemDictionary::java_platform_loader() ||
1235 orig_obj == SystemDictionary::java_system_loader() ||
1236 java_lang_ClassLoader::loader_data(orig_obj) == NULL, "must be");
1237 java_lang_ClassLoader::release_set_loader_data(archived_obj, NULL);
1238 }
1239 }
1240
1241 assert(archived_obj != NULL, "must be");
1242 Klass *orig_k = orig_obj->klass();
1243 subgraph_info->add_subgraph_object_klass(orig_k);
1244
1245 WalkOopAndArchiveClosure walker(level, is_closed_archive, record_klasses_only,
1246 subgraph_info, orig_obj, archived_obj);
1247 orig_obj->oop_iterate(&walker);
1248 if (is_closed_archive && orig_k->is_instance_klass()) {
1249 check_closed_region_object(InstanceKlass::cast(orig_k));
1250 }
1251
1252 check_enum_obj(level + 1, subgraph_info, orig_obj, is_closed_archive);
1253 return archived_obj;
1254 }
1255
1256 //
1257 // Start from the given static field in a java mirror and archive the
1258 // complete sub-graph of java heap objects that are reached directly
1259 // or indirectly from the starting object by following references.
1260 // Sub-graph archiving restrictions (current):
1261 //
1262 // - All classes of objects in the archived sub-graph (including the
1263 // entry class) must be boot class only.
1264 // - No java.lang.Class instance (java mirror) can be included inside
1265 // an archived sub-graph. Mirror can only be the sub-graph entry object.
1266 //
1267 // The Java heap object sub-graph archiving process (see
1268 // WalkOopAndArchiveClosure):
1269 //
1270 // 1) Java object sub-graph archiving starts from a given static field
1271 // within a Class instance (java mirror). If the static field is a
1272 // reference field and points to a non-null java object, proceed to
1273 // the next step.
1274 //
1275 // 2) Archives the referenced java object. If an archived copy of the
1276 // current object already exists, updates the pointer in the archived
1277 // copy of the referencing object to point to the current archived object.
1278 // Otherwise, proceed to the next step.
1279 //
1280 // 3) Follows all references within the current java object and recursively
1281 // archive the sub-graph of objects starting from each reference.
1282 //
1283 // 4) Updates the pointer in the archived copy of referencing object to
1284 // point to the current archived object.
1285 //
1286 // 5) The Klass of the current java object is added to the list of Klasses
1287 // for loading and initializing before any object in the archived graph can
1288 // be accessed at runtime.
1289 //
1290 void HeapShared::archive_reachable_objects_from_static_field(InstanceKlass *k,
1291 const char* klass_name,
1292 int field_offset,
1293 const char* field_name,
1294 bool is_closed_archive) {
1295 assert(DumpSharedSpaces, "dump time only");
1296 assert(k->is_shared_boot_class(), "must be boot class");
1297
1298 oop m = k->java_mirror();
1299
1300 KlassSubGraphInfo* subgraph_info = get_subgraph_info(k);
1301 oop f = m->obj_field(field_offset);
1302
1303 log_debug(cds, heap)("Start archiving from: %s::%s (" PTR_FORMAT ")", klass_name, field_name, p2i(f));
1304
1305 if (!CompressedOops::is_null(f)) {
1306 if (log_is_enabled(Trace, cds, heap)) {
1307 LogTarget(Trace, cds, heap) log;
1308 LogStream out(log);
1309 f->print_on(&out);
1310 }
1311
1312 oop af = archive_reachable_objects_from(1, subgraph_info, f, is_closed_archive);
1313
1314 if (af == NULL) {
1315 log_error(cds, heap)("Archiving failed %s::%s (some reachable objects cannot be archived)",
1316 klass_name, field_name);
1317 } else {
1318 // Note: the field value is not preserved in the archived mirror.
1319 // Record the field as a new subGraph entry point. The recorded
1320 // information is restored from the archive at runtime.
1321 subgraph_info->add_subgraph_entry_field(field_offset, af, is_closed_archive);
1322 log_info(cds, heap)("Archived field %s::%s => " PTR_FORMAT, klass_name, field_name, p2i(af));
1323 }
1324 } else {
1325 // The field contains null, we still need to record the entry point,
1326 // so it can be restored at runtime.
1327 subgraph_info->add_subgraph_entry_field(field_offset, NULL, false);
1328 }
1329 }
1330
1331 #ifndef PRODUCT
1332 class VerifySharedOopClosure: public BasicOopIterateClosure {
1333 private:
1334 bool _is_archived;
1335
1336 public:
1337 VerifySharedOopClosure(bool is_archived) : _is_archived(is_archived) {}
1338
1339 void do_oop(narrowOop *p) { VerifySharedOopClosure::do_oop_work(p); }
1340 void do_oop( oop *p) { VerifySharedOopClosure::do_oop_work(p); }
1341
1342 protected:
1343 template <class T> void do_oop_work(T *p) {
1344 oop obj = RawAccess<>::oop_load(p);
1345 if (!CompressedOops::is_null(obj)) {
1346 HeapShared::verify_reachable_objects_from(obj, _is_archived);
1347 }
1348 }
1349 };
1350
1351 void HeapShared::verify_subgraph_from_static_field(InstanceKlass* k, int field_offset) {
1352 assert(DumpSharedSpaces, "dump time only");
1353 assert(k->is_shared_boot_class(), "must be boot class");
1354
1355 oop m = k->java_mirror();
1356 oop f = m->obj_field(field_offset);
1357 if (!CompressedOops::is_null(f)) {
1358 verify_subgraph_from(f);
1359 }
1360 }
1361
1362 void HeapShared::verify_subgraph_from(oop orig_obj) {
1363 oop archived_obj = find_archived_heap_object(orig_obj);
1364 if (archived_obj == NULL) {
1365 // It's OK for the root of a subgraph to be not archived. See comments in
1366 // archive_reachable_objects_from().
1367 return;
1368 }
1369
1370 // Verify that all objects reachable from orig_obj are archived.
1371 init_seen_objects_table();
1372 verify_reachable_objects_from(orig_obj, false);
1373 delete_seen_objects_table();
1374
1375 // Note: we could also verify that all objects reachable from the archived
1376 // copy of orig_obj can only point to archived objects, with:
1377 // init_seen_objects_table();
1378 // verify_reachable_objects_from(archived_obj, true);
1379 // init_seen_objects_table();
1380 // but that's already done in G1HeapVerifier::verify_archive_regions so we
1381 // won't do it here.
1382 }
1383
1384 void HeapShared::verify_reachable_objects_from(oop obj, bool is_archived) {
1385 _num_total_verifications ++;
1386 if (!has_been_seen_during_subgraph_recording(obj)) {
1387 set_has_been_seen_during_subgraph_recording(obj);
1388
1389 if (is_archived) {
1390 assert(is_archived_object_during_dumptime(obj), "must be");
1391 assert(find_archived_heap_object(obj) == NULL, "must be");
1392 } else {
1393 assert(!is_archived_object_during_dumptime(obj), "must be");
1394 assert(find_archived_heap_object(obj) != NULL, "must be");
1395 }
1396
1397 VerifySharedOopClosure walker(is_archived);
1398 obj->oop_iterate(&walker);
1399 }
1400 }
1401 #endif
1402
1403 HeapShared::SeenObjectsTable* HeapShared::_seen_objects_table = NULL;
1404 int HeapShared::_num_new_walked_objs;
1405 int HeapShared::_num_new_archived_objs;
1406 int HeapShared::_num_old_recorded_klasses;
1407
1408 int HeapShared::_num_total_subgraph_recordings = 0;
1409 int HeapShared::_num_total_walked_objs = 0;
1410 int HeapShared::_num_total_archived_objs = 0;
1411 int HeapShared::_num_total_recorded_klasses = 0;
1412 int HeapShared::_num_total_verifications = 0;
1413
1414 bool HeapShared::has_been_seen_during_subgraph_recording(oop obj) {
1415 return _seen_objects_table->get(obj) != NULL;
1416 }
1417
1418 void HeapShared::set_has_been_seen_during_subgraph_recording(oop obj) {
1419 assert(!has_been_seen_during_subgraph_recording(obj), "sanity");
1420 _seen_objects_table->put(obj, true);
1421 ++ _num_new_walked_objs;
1422 }
1423
1424 void HeapShared::start_recording_subgraph(InstanceKlass *k, const char* class_name, bool is_full_module_graph) {
1425 log_info(cds, heap)("Start recording subgraph(s) for archived fields in %s", class_name);
1426 init_subgraph_info(k, is_full_module_graph);
1427 init_seen_objects_table();
1428 _num_new_walked_objs = 0;
1429 _num_new_archived_objs = 0;
1430 _num_old_recorded_klasses = get_subgraph_info(k)->num_subgraph_object_klasses();
1431 }
1432
1433 void HeapShared::done_recording_subgraph(InstanceKlass *k, const char* class_name) {
1434 int num_new_recorded_klasses = get_subgraph_info(k)->num_subgraph_object_klasses() -
1435 _num_old_recorded_klasses;
1436 log_info(cds, heap)("Done recording subgraph(s) for archived fields in %s: "
1437 "walked %d objs, archived %d new objs, recorded %d classes",
1438 class_name, _num_new_walked_objs, _num_new_archived_objs,
1439 num_new_recorded_klasses);
1440
1441 delete_seen_objects_table();
1442
1443 _num_total_subgraph_recordings ++;
1444 _num_total_walked_objs += _num_new_walked_objs;
1445 _num_total_archived_objs += _num_new_archived_objs;
1446 _num_total_recorded_klasses += num_new_recorded_klasses;
1447 }
1448
1449 class ArchivableStaticFieldFinder: public FieldClosure {
1450 InstanceKlass* _ik;
1451 Symbol* _field_name;
1452 bool _found;
1453 int _offset;
1454 public:
1455 ArchivableStaticFieldFinder(InstanceKlass* ik, Symbol* field_name) :
1456 _ik(ik), _field_name(field_name), _found(false), _offset(-1) {}
1457
1458 virtual void do_field(fieldDescriptor* fd) {
1459 if (fd->name() == _field_name) {
1460 assert(!_found, "fields cannot be overloaded");
1461 assert(is_reference_type(fd->field_type()), "can archive only fields that are references");
1462 _found = true;
1463 _offset = fd->offset();
1464 }
1465 }
1466 bool found() { return _found; }
1467 int offset() { return _offset; }
1468 };
1469
1470 void HeapShared::init_subgraph_entry_fields(ArchivableStaticFieldInfo fields[],
1471 int num, TRAPS) {
1472 for (int i = 0; i < num; i++) {
1473 ArchivableStaticFieldInfo* info = &fields[i];
1474 TempNewSymbol klass_name = SymbolTable::new_symbol(info->klass_name);
1475 TempNewSymbol field_name = SymbolTable::new_symbol(info->field_name);
1476
1477 Klass* k = SystemDictionary::resolve_or_fail(klass_name, true, CHECK);
1478 InstanceKlass* ik = InstanceKlass::cast(k);
1479 assert(InstanceKlass::cast(ik)->is_shared_boot_class(),
1480 "Only support boot classes");
1481 ik->initialize(CHECK);
1482
1483 ArchivableStaticFieldFinder finder(ik, field_name);
1484 ik->do_local_static_fields(&finder);
1485 assert(finder.found(), "field must exist");
1486
1487 info->klass = ik;
1488 info->offset = finder.offset();
1489 }
1490 }
1491
1492 void HeapShared::init_subgraph_entry_fields(TRAPS) {
1493 assert(HeapShared::can_write(), "must be");
1494 _dump_time_subgraph_info_table = new (ResourceObj::C_HEAP, mtClass)DumpTimeKlassSubGraphInfoTable();
1495 init_subgraph_entry_fields(closed_archive_subgraph_entry_fields,
1496 num_closed_archive_subgraph_entry_fields,
1497 CHECK);
1498 init_subgraph_entry_fields(open_archive_subgraph_entry_fields,
1499 num_open_archive_subgraph_entry_fields,
1500 CHECK);
1501 if (MetaspaceShared::use_full_module_graph()) {
1502 init_subgraph_entry_fields(fmg_open_archive_subgraph_entry_fields,
1503 num_fmg_open_archive_subgraph_entry_fields,
1504 CHECK);
1505 }
1506 }
1507
1508 void HeapShared::init_for_dumping(TRAPS) {
1509 if (HeapShared::can_write()) {
1510 _dumped_interned_strings = new (ResourceObj::C_HEAP, mtClass)DumpedInternedStrings();
1511 init_subgraph_entry_fields(CHECK);
1512 }
1513 }
1514
1515 void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
1516 int num, bool is_closed_archive,
1517 bool is_full_module_graph) {
1518 _num_total_subgraph_recordings = 0;
1519 _num_total_walked_objs = 0;
1520 _num_total_archived_objs = 0;
1521 _num_total_recorded_klasses = 0;
1522 _num_total_verifications = 0;
1523
1524 // For each class X that has one or more archived fields:
1525 // [1] Dump the subgraph of each archived field
1526 // [2] Create a list of all the class of the objects that can be reached
1527 // by any of these static fields.
1528 // At runtime, these classes are initialized before X's archived fields
1529 // are restored by HeapShared::initialize_from_archived_subgraph().
1530 int i;
1531 for (i = 0; i < num; ) {
1532 ArchivableStaticFieldInfo* info = &fields[i];
1533 const char* klass_name = info->klass_name;
1534 start_recording_subgraph(info->klass, klass_name, is_full_module_graph);
1535
1536 // If you have specified consecutive fields of the same klass in
1537 // fields[], these will be archived in the same
1538 // {start_recording_subgraph ... done_recording_subgraph} pass to
1539 // save time.
1540 for (; i < num; i++) {
1541 ArchivableStaticFieldInfo* f = &fields[i];
1542 if (f->klass_name != klass_name) {
1543 break;
1544 }
1545
1546 archive_reachable_objects_from_static_field(f->klass, f->klass_name,
1547 f->offset, f->field_name,
1548 is_closed_archive);
1549 }
1550 done_recording_subgraph(info->klass, klass_name);
1551 }
1552
1553 log_info(cds, heap)("Archived subgraph records in %s archive heap region = %d",
1554 is_closed_archive ? "closed" : "open",
1555 _num_total_subgraph_recordings);
1556 log_info(cds, heap)(" Walked %d objects", _num_total_walked_objs);
1557 log_info(cds, heap)(" Archived %d objects", _num_total_archived_objs);
1558 log_info(cds, heap)(" Recorded %d klasses", _num_total_recorded_klasses);
1559
1560 #ifndef PRODUCT
1561 for (int i = 0; i < num; i++) {
1562 ArchivableStaticFieldInfo* f = &fields[i];
1563 verify_subgraph_from_static_field(f->klass, f->offset);
1564 }
1565 log_info(cds, heap)(" Verified %d references", _num_total_verifications);
1566 #endif
1567 }
1568
1569 // Not all the strings in the global StringTable are dumped into the archive, because
1570 // some of those strings may be only referenced by classes that are excluded from
1571 // the archive. We need to explicitly mark the strings that are:
1572 // [1] used by classes that WILL be archived;
1573 // [2] included in the SharedArchiveConfigFile.
1574 void HeapShared::add_to_dumped_interned_strings(oop string) {
1575 assert_at_safepoint(); // DumpedInternedStrings uses raw oops
1576 bool created;
1577 _dumped_interned_strings->put_if_absent(string, true, &created);
1578 }
1579
1580 // At dump-time, find the location of all the non-null oop pointers in an archived heap
1581 // region. This way we can quickly relocate all the pointers without using
1582 // BasicOopIterateClosure at runtime.
1583 class FindEmbeddedNonNullPointers: public BasicOopIterateClosure {
1584 void* _start;
1585 BitMap *_oopmap;
1586 int _num_total_oops;
1587 int _num_null_oops;
1588 public:
1589 FindEmbeddedNonNullPointers(void* start, BitMap* oopmap)
1590 : _start(start), _oopmap(oopmap), _num_total_oops(0), _num_null_oops(0) {}
1591
1592 virtual void do_oop(narrowOop* p) {
1593 assert(UseCompressedOops, "sanity");
1594 _num_total_oops ++;
1595 narrowOop v = *p;
1596 if (!CompressedOops::is_null(v)) {
1597 // Note: HeapShared::to_requested_address() is not necessary because
1598 // the heap always starts at a deterministic address with UseCompressedOops==true.
1599 size_t idx = p - (narrowOop*)_start;
1600 _oopmap->set_bit(idx);
1601 } else {
1602 _num_null_oops ++;
1603 }
1604 }
1605 virtual void do_oop(oop* p) {
1606 assert(!UseCompressedOops, "sanity");
1607 _num_total_oops ++;
1608 if ((*p) != NULL) {
1609 size_t idx = p - (oop*)_start;
1610 _oopmap->set_bit(idx);
1611 if (DumpSharedSpaces) {
1612 // Make heap content deterministic.
1613 *p = HeapShared::to_requested_address(*p);
1614 }
1615 } else {
1616 _num_null_oops ++;
1617 }
1618 }
1619 int num_total_oops() const { return _num_total_oops; }
1620 int num_null_oops() const { return _num_null_oops; }
1621 };
1622
1623
1624 address HeapShared::to_requested_address(address dumptime_addr) {
1625 assert(DumpSharedSpaces, "static dump time only");
1626 if (dumptime_addr == NULL || UseCompressedOops) {
1627 return dumptime_addr;
1628 }
1629
1630 // With UseCompressedOops==false, actual_base is selected by the OS so
1631 // it's different across -Xshare:dump runs.
1632 address actual_base = (address)G1CollectedHeap::heap()->reserved().start();
1633 address actual_end = (address)G1CollectedHeap::heap()->reserved().end();
1634 assert(actual_base <= dumptime_addr && dumptime_addr <= actual_end, "must be an address in the heap");
1635
1636 // We always write the objects as if the heap started at this address. This
1637 // makes the heap content deterministic.
1638 //
1639 // Note that at runtime, the heap address is also selected by the OS, so
1640 // the archive heap will not be mapped at 0x10000000. Instead, we will call
1641 // HeapShared::patch_embedded_pointers() to relocate the heap contents
1642 // accordingly.
1643 const address REQUESTED_BASE = (address)0x10000000;
1644 intx delta = REQUESTED_BASE - actual_base;
1645
1646 address requested_addr = dumptime_addr + delta;
1647 assert(REQUESTED_BASE != 0 && requested_addr != NULL, "sanity");
1648 return requested_addr;
1649 }
1650
1651 ResourceBitMap HeapShared::calculate_oopmap(MemRegion region) {
1652 size_t num_bits = region.byte_size() / (UseCompressedOops ? sizeof(narrowOop) : sizeof(oop));
1653 ResourceBitMap oopmap(num_bits);
1654
1655 HeapWord* p = region.start();
1656 HeapWord* end = region.end();
1657 FindEmbeddedNonNullPointers finder((void*)p, &oopmap);
1658 ArchiveBuilder* builder = DumpSharedSpaces ? ArchiveBuilder::current() : NULL;
1659
1660 int num_objs = 0;
1661 while (p < end) {
1662 oop o = cast_to_oop(p);
1663 o->oop_iterate(&finder);
1664 p += o->size();
1665 if (DumpSharedSpaces) {
1666 builder->relocate_klass_ptr(o);
1667 }
1668 ++ num_objs;
1669 }
1670
1671 log_info(cds, heap)("calculate_oopmap: objects = %6d, oop fields = %7d (nulls = %7d)",
1672 num_objs, finder.num_total_oops(), finder.num_null_oops());
1673 return oopmap;
1674 }
1675
1676 // Patch all the embedded oop pointers inside an archived heap region,
1677 // to be consistent with the runtime oop encoding.
1678 class PatchCompressedEmbeddedPointers: public BitMapClosure {
1679 narrowOop* _start;
1680
1681 public:
1682 PatchCompressedEmbeddedPointers(narrowOop* start) : _start(start) {}
1683
1684 bool do_bit(size_t offset) {
1685 narrowOop* p = _start + offset;
1686 narrowOop v = *p;
1687 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
1688 oop o = HeapShared::decode_from_archive(v);
1689 RawAccess<IS_NOT_NULL>::oop_store(p, o);
1690 return true;
1691 }
1692 };
1693
1694 class PatchUncompressedEmbeddedPointers: public BitMapClosure {
1695 oop* _start;
1696
1697 public:
1698 PatchUncompressedEmbeddedPointers(oop* start) : _start(start) {}
1699
1700 bool do_bit(size_t offset) {
1701 oop* p = _start + offset;
1702 intptr_t dumptime_oop = (intptr_t)((void*)*p);
1703 assert(dumptime_oop != 0, "null oops should have been filtered out at dump time");
1704 intptr_t runtime_oop = dumptime_oop + HeapShared::runtime_delta();
1705 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(runtime_oop));
1706 return true;
1707 }
1708 };
1709
1710 // Patch all the non-null pointers that are embedded in the archived heap objects
1711 // in this region
1712 void HeapShared::patch_embedded_pointers(MemRegion region, address oopmap,
1713 size_t oopmap_size_in_bits) {
1714 BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits);
1715
1716 #ifndef PRODUCT
1717 ResourceMark rm;
1718 ResourceBitMap checkBm = calculate_oopmap(region);
1719 assert(bm.is_same(checkBm), "sanity");
1720 #endif
1721
1722 if (UseCompressedOops) {
1723 PatchCompressedEmbeddedPointers patcher((narrowOop*)region.start());
1724 bm.iterate(&patcher);
1725 } else {
1726 PatchUncompressedEmbeddedPointers patcher((oop*)region.start());
1727 bm.iterate(&patcher);
1728 }
1729 }
1730
1731 // The CDS archive remembers each heap object by its address at dump time, but
1732 // the heap object may be loaded at a different address at run time. This structure is used
1733 // to translate the dump time addresses for all objects in FileMapInfo::space_at(region_index)
1734 // to their runtime addresses.
1735 struct LoadedArchiveHeapRegion {
1736 int _region_index; // index for FileMapInfo::space_at(index)
1737 size_t _region_size; // number of bytes in this region
1738 uintptr_t _dumptime_base; // The dump-time (decoded) address of the first object in this region
1739 intx _runtime_offset; // If an object's dump time address P is within in this region, its
1740 // runtime address is P + _runtime_offset
1741
1742 static int comparator(const void* a, const void* b) {
1743 LoadedArchiveHeapRegion* reg_a = (LoadedArchiveHeapRegion*)a;
1744 LoadedArchiveHeapRegion* reg_b = (LoadedArchiveHeapRegion*)b;
1745 if (reg_a->_dumptime_base < reg_b->_dumptime_base) {
1746 return -1;
1747 } else if (reg_a->_dumptime_base == reg_b->_dumptime_base) {
1748 return 0;
1749 } else {
1750 return 1;
1751 }
1752 }
1753
1754 uintptr_t top() {
1755 return _dumptime_base + _region_size;
1756 }
1757 };
1758
1759 void HeapShared::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loaded_regions,
1760 int num_loaded_regions) {
1761 _dumptime_base_0 = loaded_regions[0]._dumptime_base;
1762 _dumptime_base_1 = loaded_regions[1]._dumptime_base;
1763 _dumptime_base_2 = loaded_regions[2]._dumptime_base;
1764 _dumptime_base_3 = loaded_regions[3]._dumptime_base;
1765 _dumptime_top = loaded_regions[num_loaded_regions-1].top();
1766
1767 _runtime_offset_0 = loaded_regions[0]._runtime_offset;
1768 _runtime_offset_1 = loaded_regions[1]._runtime_offset;
1769 _runtime_offset_2 = loaded_regions[2]._runtime_offset;
1770 _runtime_offset_3 = loaded_regions[3]._runtime_offset;
1771
1772 assert(2 <= num_loaded_regions && num_loaded_regions <= 4, "must be");
1773 if (num_loaded_regions < 4) {
1774 _dumptime_base_3 = UINTPTR_MAX;
1775 }
1776 if (num_loaded_regions < 3) {
1777 _dumptime_base_2 = UINTPTR_MAX;
1778 }
1779 }
1780
1781 bool HeapShared::can_load() {
1782 return Universe::heap()->can_load_archived_objects();
1783 }
1784
1785 template <int NUM_LOADED_REGIONS>
1786 class PatchLoadedRegionPointers: public BitMapClosure {
1787 narrowOop* _start;
1788 intx _offset_0;
1789 intx _offset_1;
1790 intx _offset_2;
1791 intx _offset_3;
1792 uintptr_t _base_0;
1793 uintptr_t _base_1;
1794 uintptr_t _base_2;
1795 uintptr_t _base_3;
1796 uintptr_t _top;
1797
1798 static_assert(MetaspaceShared::max_num_heap_regions == 4, "can't handle more than 4 regions");
1799 static_assert(NUM_LOADED_REGIONS >= 2, "we have at least 2 loaded regions");
1800 static_assert(NUM_LOADED_REGIONS <= 4, "we have at most 4 loaded regions");
1801
1802 public:
1803 PatchLoadedRegionPointers(narrowOop* start, LoadedArchiveHeapRegion* loaded_regions)
1804 : _start(start),
1805 _offset_0(loaded_regions[0]._runtime_offset),
1806 _offset_1(loaded_regions[1]._runtime_offset),
1807 _offset_2(loaded_regions[2]._runtime_offset),
1808 _offset_3(loaded_regions[3]._runtime_offset),
1809 _base_0(loaded_regions[0]._dumptime_base),
1810 _base_1(loaded_regions[1]._dumptime_base),
1811 _base_2(loaded_regions[2]._dumptime_base),
1812 _base_3(loaded_regions[3]._dumptime_base) {
1813 _top = loaded_regions[NUM_LOADED_REGIONS-1].top();
1814 }
1815
1816 bool do_bit(size_t offset) {
1817 narrowOop* p = _start + offset;
1818 narrowOop v = *p;
1819 assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time");
1820 uintptr_t o = cast_from_oop<uintptr_t>(HeapShared::decode_from_archive(v));
1821 assert(_base_0 <= o && o < _top, "must be");
1822
1823
1824 // We usually have only 2 regions for the default archive. Use template to avoid unnecessary comparisons.
1825 if (NUM_LOADED_REGIONS > 3 && o >= _base_3) {
1826 o += _offset_3;
1827 } else if (NUM_LOADED_REGIONS > 2 && o >= _base_2) {
1828 o += _offset_2;
1829 } else if (o >= _base_1) {
1830 o += _offset_1;
1831 } else {
1832 o += _offset_0;
1833 }
1834 HeapShared::assert_in_loaded_heap(o);
1835 RawAccess<IS_NOT_NULL>::oop_store(p, cast_to_oop(o));
1836 return true;
1837 }
1838 };
1839
1840 int HeapShared::init_loaded_regions(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_regions,
1841 MemRegion& archive_space) {
1842 size_t total_bytes = 0;
1843 int num_loaded_regions = 0;
1844 for (int i = MetaspaceShared::first_archive_heap_region;
1845 i <= MetaspaceShared::last_archive_heap_region; i++) {
1846 FileMapRegion* r = mapinfo->space_at(i);
1847 r->assert_is_heap_region();
1848 if (r->used() > 0) {
1849 assert(is_aligned(r->used(), HeapWordSize), "must be");
1850 total_bytes += r->used();
1851 LoadedArchiveHeapRegion* ri = &loaded_regions[num_loaded_regions++];
1852 ri->_region_index = i;
1853 ri->_region_size = r->used();
1854 ri->_dumptime_base = (uintptr_t)mapinfo->start_address_as_decoded_from_archive(r);
1855 }
1856 }
1857
1858 assert(is_aligned(total_bytes, HeapWordSize), "must be");
1859 size_t word_size = total_bytes / HeapWordSize;
1860 HeapWord* buffer = Universe::heap()->allocate_loaded_archive_space(word_size);
1861 if (buffer == nullptr) {
1862 return 0;
1863 }
1864
1865 archive_space = MemRegion(buffer, word_size);
1866 _loaded_heap_bottom = (uintptr_t)archive_space.start();
1867 _loaded_heap_top = _loaded_heap_bottom + total_bytes;
1868
1869 return num_loaded_regions;
1870 }
1871
1872 void HeapShared::sort_loaded_regions(LoadedArchiveHeapRegion* loaded_regions, int num_loaded_regions,
1873 uintptr_t buffer) {
1874 // Find the relocation offset of the pointers in each region
1875 qsort(loaded_regions, num_loaded_regions, sizeof(LoadedArchiveHeapRegion),
1876 LoadedArchiveHeapRegion::comparator);
1877
1878 uintptr_t p = buffer;
1879 for (int i = 0; i < num_loaded_regions; i++) {
1880 // This region will be loaded at p, so all objects inside this
1881 // region will be shifted by ri->offset
1882 LoadedArchiveHeapRegion* ri = &loaded_regions[i];
1883 ri->_runtime_offset = p - ri->_dumptime_base;
1884 p += ri->_region_size;
1885 }
1886 assert(p == _loaded_heap_top, "must be");
1887 }
1888
1889 bool HeapShared::load_regions(FileMapInfo* mapinfo, LoadedArchiveHeapRegion* loaded_regions,
1890 int num_loaded_regions, uintptr_t buffer) {
1891 uintptr_t bitmap_base = (uintptr_t)mapinfo->map_bitmap_region();
1892 if (bitmap_base == 0) {
1893 _loading_failed = true;
1894 return false; // OOM or CRC error
1895 }
1896 uintptr_t load_address = buffer;
1897 for (int i = 0; i < num_loaded_regions; i++) {
1898 LoadedArchiveHeapRegion* ri = &loaded_regions[i];
1899 FileMapRegion* r = mapinfo->space_at(ri->_region_index);
1900
1901 if (!mapinfo->read_region(ri->_region_index, (char*)load_address, r->used(), /* do_commit = */ false)) {
1902 // There's no easy way to free the buffer, so we will fill it with zero later
1903 // in fill_failed_loaded_region(), and it will eventually be GC'ed.
1904 log_warning(cds)("Loading of heap region %d has failed. Archived objects are disabled", i);
1905 _loading_failed = true;
1906 return false;
1907 }
1908 log_info(cds)("Loaded heap region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT
1909 " size " SIZE_FORMAT_W(6) " delta " INTX_FORMAT,
1910 ri->_region_index, load_address, load_address + ri->_region_size,
1911 ri->_region_size, ri->_runtime_offset);
1912
1913 uintptr_t oopmap = bitmap_base + r->oopmap_offset();
1914 BitMapView bm((BitMap::bm_word_t*)oopmap, r->oopmap_size_in_bits());
1915
1916 if (num_loaded_regions == 4) {
1917 PatchLoadedRegionPointers<4> patcher((narrowOop*)load_address, loaded_regions);
1918 bm.iterate(&patcher);
1919 } else if (num_loaded_regions == 3) {
1920 PatchLoadedRegionPointers<3> patcher((narrowOop*)load_address, loaded_regions);
1921 bm.iterate(&patcher);
1922 } else {
1923 assert(num_loaded_regions == 2, "must be");
1924 PatchLoadedRegionPointers<2> patcher((narrowOop*)load_address, loaded_regions);
1925 bm.iterate(&patcher);
1926 }
1927
1928 load_address += r->used();
1929 }
1930
1931 return true;
1932 }
1933
1934 bool HeapShared::load_heap_regions(FileMapInfo* mapinfo) {
1935 init_narrow_oop_decoding(mapinfo->narrow_oop_base(), mapinfo->narrow_oop_shift());
1936
1937 LoadedArchiveHeapRegion loaded_regions[MetaspaceShared::max_num_heap_regions];
1938 memset(loaded_regions, 0, sizeof(loaded_regions));
1939
1940 MemRegion archive_space;
1941 int num_loaded_regions = init_loaded_regions(mapinfo, loaded_regions, archive_space);
1942 if (num_loaded_regions <= 0) {
1943 return false;
1944 }
1945 sort_loaded_regions(loaded_regions, num_loaded_regions, (uintptr_t)archive_space.start());
1946 if (!load_regions(mapinfo, loaded_regions, num_loaded_regions, (uintptr_t)archive_space.start())) {
1947 assert(_loading_failed, "must be");
1948 return false;
1949 }
1950
1951 init_loaded_heap_relocation(loaded_regions, num_loaded_regions);
1952 _is_loaded = true;
1953
1954 return true;
1955 }
1956
1957 class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure {
1958 ResourceHashtable<uintptr_t, bool>* _table;
1959
1960 public:
1961 VerifyLoadedHeapEmbeddedPointers(ResourceHashtable<uintptr_t, bool>* table) : _table(table) {}
1962
1963 virtual void do_oop(narrowOop* p) {
1964 // This should be called before the loaded regions are modified, so all the embedded pointers
1965 // must be NULL, or must point to a valid object in the loaded regions.
1966 narrowOop v = *p;
1967 if (!CompressedOops::is_null(v)) {
1968 oop o = CompressedOops::decode_not_null(v);
1969 uintptr_t u = cast_from_oop<uintptr_t>(o);
1970 HeapShared::assert_in_loaded_heap(u);
1971 guarantee(_table->contains(u), "must point to beginning of object in loaded archived regions");
1972 }
1973 }
1974 virtual void do_oop(oop* p) {
1975 ShouldNotReachHere();
1976 }
1977 };
1978
1979 void HeapShared::finish_initialization() {
1980 if (is_loaded()) {
1981 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
1982 HeapWord* top = (HeapWord*)_loaded_heap_top;
1983
1984 MemRegion archive_space = MemRegion(bottom, top);
1985 Universe::heap()->complete_loaded_archive_space(archive_space);
1986 }
1987
1988 if (VerifyArchivedFields <= 0 || !is_loaded()) {
1989 return;
1990 }
1991
1992 log_info(cds, heap)("Verify all oops and pointers in loaded heap");
1993
1994 ResourceMark rm;
1995 ResourceHashtable<uintptr_t, bool> table;
1996 VerifyLoadedHeapEmbeddedPointers verifier(&table);
1997 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
1998 HeapWord* top = (HeapWord*)_loaded_heap_top;
1999
2000 for (HeapWord* p = bottom; p < top; ) {
2001 oop o = cast_to_oop(p);
2002 table.put(cast_from_oop<uintptr_t>(o), true);
2003 p += o->size();
2004 }
2005
2006 for (HeapWord* p = bottom; p < top; ) {
2007 oop o = cast_to_oop(p);
2008 o->oop_iterate(&verifier);
2009 p += o->size();
2010 }
2011 }
2012
2013 void HeapShared::fill_failed_loaded_region() {
2014 assert(_loading_failed, "must be");
2015 if (_loaded_heap_bottom != 0) {
2016 assert(_loaded_heap_top != 0, "must be");
2017 HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
2018 HeapWord* top = (HeapWord*)_loaded_heap_top;
2019 Universe::heap()->fill_with_objects(bottom, top - bottom);
2020 }
2021 }
2022
2023 #endif // INCLUDE_CDS_JAVA_HEAP