47 #include "runtime/mutexLocker.hpp"
48 #include "utilities/bitMap.inline.hpp"
49 #if INCLUDE_G1GC
50 #include "gc/g1/g1CollectedHeap.hpp"
51 #include "gc/g1/g1HeapRegion.hpp"
52 #endif
53
54 #if INCLUDE_CDS_JAVA_HEAP
55
56 GrowableArrayCHeap<u1, mtClassShared>* AOTMappedHeapWriter::_buffer = nullptr;
57
58 bool AOTMappedHeapWriter::_is_writing_deterministic_heap = false;
59 size_t AOTMappedHeapWriter::_buffer_used;
60
61 // Heap root segments
62 HeapRootSegments AOTMappedHeapWriter::_heap_root_segments;
63
64 address AOTMappedHeapWriter::_requested_bottom;
65 address AOTMappedHeapWriter::_requested_top;
66
67 GrowableArrayCHeap<AOTMappedHeapWriter::NativePointerInfo, mtClassShared>* AOTMappedHeapWriter::_native_pointers;
68 GrowableArrayCHeap<oop, mtClassShared>* AOTMappedHeapWriter::_source_objs;
69 GrowableArrayCHeap<AOTMappedHeapWriter::HeapObjOrder, mtClassShared>* AOTMappedHeapWriter::_source_objs_order;
70
71 AOTMappedHeapWriter::BufferOffsetToSourceObjectTable*
72 AOTMappedHeapWriter::_buffer_offset_to_source_obj_table = nullptr;
73
74 DumpedInternedStrings *AOTMappedHeapWriter::_dumped_interned_strings = nullptr;
75
76 typedef HashTable<
77 size_t, // offset of a filler from AOTMappedHeapWriter::buffer_bottom()
78 size_t, // size of this filler (in bytes)
79 127, // prime number
80 AnyObj::C_HEAP,
81 mtClassShared> FillersTable;
82 static FillersTable* _fillers;
83 static int _num_native_ptrs = 0;
84
85 void AOTMappedHeapWriter::init() {
86 if (CDSConfig::is_dumping_heap()) {
87 Universe::heap()->collect(GCCause::_java_lang_system_gc);
88
89 _buffer_offset_to_source_obj_table = new (mtClassShared) BufferOffsetToSourceObjectTable(/*size (prime)*/36137, /*max size*/1 * M);
90 _dumped_interned_strings = new (mtClass)DumpedInternedStrings(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE);
91 _fillers = new (mtClassShared) FillersTable();
92 _requested_bottom = nullptr;
93 _requested_top = nullptr;
94
95 _native_pointers = new GrowableArrayCHeap<NativePointerInfo, mtClassShared>(2048);
96 _source_objs = new GrowableArrayCHeap<oop, mtClassShared>(10000);
97
98 guarantee(MIN_GC_REGION_ALIGNMENT <= G1HeapRegion::min_region_size_in_words() * HeapWordSize, "must be");
99
100 if (CDSConfig::old_cds_flags_used()) {
101 // With the old CDS workflow, we can guatantee determninistic output: given
102 // the same classlist file, we can generate the same static CDS archive.
103 // To ensure determinism, we always use the same compressed oop encoding
104 // (zero-based, no shift). See set_requested_address_range().
105 _is_writing_deterministic_heap = true;
106 } else {
107 // Determninistic output is not supported by the new AOT workflow, so
108 // we don't force the (zero-based, no shift) encoding. This way, it is more
109 // likely that we can avoid oop relocation in the production run.
110 _is_writing_deterministic_heap = false;
124
125 address AOTMappedHeapWriter::narrow_oop_base() {
126 if (is_writing_deterministic_heap()) {
127 return nullptr;
128 } else {
129 return CompressedOops::base();
130 }
131 }
132
133 int AOTMappedHeapWriter::narrow_oop_shift() {
134 if (is_writing_deterministic_heap()) {
135 return 0;
136 } else {
137 return CompressedOops::shift();
138 }
139 }
140
141 void AOTMappedHeapWriter::delete_tables_with_raw_oops() {
142 delete _source_objs;
143 _source_objs = nullptr;
144
145 delete _dumped_interned_strings;
146 _dumped_interned_strings = nullptr;
147 }
148
149 void AOTMappedHeapWriter::add_source_obj(oop src_obj) {
150 _source_objs->append(src_obj);
151 }
152
153 void AOTMappedHeapWriter::write(GrowableArrayCHeap<oop, mtClassShared>* roots,
154 AOTMappedHeapInfo* heap_info) {
155 assert(CDSConfig::is_dumping_heap(), "sanity");
156 allocate_buffer();
157 copy_source_objs_to_buffer(roots);
158 set_requested_address_range(heap_info);
159 relocate_embedded_oops(roots, heap_info);
160 }
161
162 bool AOTMappedHeapWriter::is_too_large_to_archive(oop o) {
163 return is_too_large_to_archive(o->size());
164 }
165
166 bool AOTMappedHeapWriter::is_string_too_large_to_archive(oop string) {
167 typeArrayOop value = java_lang_String::value_no_keepalive(string);
168 return is_too_large_to_archive(value);
169 }
170
171 bool AOTMappedHeapWriter::is_too_large_to_archive(size_t size) {
172 assert(size > 0, "no zero-size object");
173 assert(size * HeapWordSize > size, "no overflow");
174 static_assert(MIN_GC_REGION_ALIGNMENT > 0, "must be positive");
175
176 size_t byte_size = size * HeapWordSize;
177 if (byte_size > size_t(MIN_GC_REGION_ALIGNMENT)) {
178 return true;
179 } else {
180 return false;
181 }
182 }
183
184 // Keep track of the contents of the archived interned string table. This table
185 // is used only by CDSHeapVerifier.
186 void AOTMappedHeapWriter::add_to_dumped_interned_strings(oop string) {
187 assert_at_safepoint(); // DumpedInternedStrings uses raw oops
188 assert(!is_string_too_large_to_archive(string), "must be");
189 bool created;
190 _dumped_interned_strings->put_if_absent(string, true, &created);
191 if (created) {
192 // Prevent string deduplication from changing the value field to
193 // something not in the archive.
194 java_lang_String::set_deduplication_forbidden(string);
195 _dumped_interned_strings->maybe_grow();
196 }
197 }
198
199 bool AOTMappedHeapWriter::is_dumped_interned_string(oop o) {
200 return _dumped_interned_strings->get(o) != nullptr;
201 }
202
203 // Various lookup functions between source_obj, buffered_obj and requested_obj
204 bool AOTMappedHeapWriter::is_in_requested_range(oop o) {
205 assert(_requested_bottom != nullptr, "do not call before _requested_bottom is initialized");
206 address a = cast_from_oop<address>(o);
207 return (_requested_bottom <= a && a < _requested_top);
208 }
209
210 oop AOTMappedHeapWriter::requested_obj_from_buffer_offset(size_t offset) {
211 oop req_obj = cast_to_oop(_requested_bottom + offset);
212 assert(is_in_requested_range(req_obj), "must be");
213 return req_obj;
214 }
215
216 oop AOTMappedHeapWriter::source_obj_to_requested_obj(oop src_obj) {
217 assert(CDSConfig::is_dumping_heap(), "dump-time only");
218 HeapShared::CachedOopInfo* p = HeapShared::get_cached_oop_info(src_obj);
219 if (p != nullptr) {
220 return requested_obj_from_buffer_offset(p->buffer_offset());
221 } else {
222 return nullptr;
413 _source_objs_order->append(os);
414 }
415 log_info(aot)("computed ranks");
416 _source_objs_order->sort(compare_objs_by_oop_fields);
417 log_info(aot)("sorting heap objects done");
418 }
419
420 void AOTMappedHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
421 // There could be multiple root segments, which we want to be aligned by region.
422 // Putting them ahead of objects makes sure we waste no space.
423 copy_roots_to_buffer(roots);
424
425 sort_source_objs();
426 for (int i = 0; i < _source_objs_order->length(); i++) {
427 int src_obj_index = _source_objs_order->at(i)._index;
428 oop src_obj = _source_objs->at(src_obj_index);
429 HeapShared::CachedOopInfo* info = HeapShared::get_cached_oop_info(src_obj);
430 assert(info != nullptr, "must be");
431 size_t buffer_offset = copy_one_source_obj_to_buffer(src_obj);
432 info->set_buffer_offset(buffer_offset);
433
434 OopHandle handle(Universe::vm_global(), src_obj);
435 _buffer_offset_to_source_obj_table->put_when_absent(buffer_offset, handle);
436 _buffer_offset_to_source_obj_table->maybe_grow();
437
438 if (java_lang_Module::is_instance(src_obj)) {
439 Modules::check_archived_module_oop(src_obj);
440 }
441 }
442
443 log_info(aot)("Size of heap region = %zu bytes, %d objects, %d roots, %d native ptrs",
444 _buffer_used, _source_objs->length() + 1, roots->length(), _num_native_ptrs);
445 }
446
447 size_t AOTMappedHeapWriter::filler_array_byte_size(int length) {
448 size_t byte_size = objArrayOopDesc::object_size(length) * HeapWordSize;
449 return byte_size;
450 }
451
452 int AOTMappedHeapWriter::filler_array_length(size_t fill_bytes) {
453 assert(is_object_aligned(fill_bytes), "must be");
454 size_t elemSize = (UseCompressedOops ? sizeof(narrowOop) : sizeof(oop));
455
456 int initial_length = to_array_length(fill_bytes / elemSize);
457 for (int length = initial_length; length >= 0; length --) {
458 size_t array_byte_size = filler_array_byte_size(length);
459 if (array_byte_size == fill_bytes) {
460 return length;
461 }
462 }
463
464 ShouldNotReachHere();
513 _fillers->put(buffered_address_to_offset((address)filler), fill_bytes);
514 }
515 }
516
517 size_t AOTMappedHeapWriter::get_filler_size_at(address buffered_addr) {
518 size_t* p = _fillers->get(buffered_address_to_offset(buffered_addr));
519 if (p != nullptr) {
520 assert(*p > 0, "filler must be larger than zero bytes");
521 return *p;
522 } else {
523 return 0; // buffered_addr is not a filler
524 }
525 }
526
527 template <typename T>
528 void update_buffered_object_field(address buffered_obj, int field_offset, T value) {
529 T* field_addr = cast_to_oop(buffered_obj)->field_addr<T>(field_offset);
530 *field_addr = value;
531 }
532
533 size_t AOTMappedHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
534 assert(!is_too_large_to_archive(src_obj), "already checked");
535 size_t byte_size = src_obj->size() * HeapWordSize;
536 assert(byte_size > 0, "no zero-size objects");
537
538 // For region-based collectors such as G1, the archive heap may be mapped into
539 // multiple regions. We need to make sure that we don't have an object that can possible
540 // span across two regions.
541 maybe_fill_gc_region_gap(byte_size);
542
543 size_t new_used = _buffer_used + byte_size;
544 assert(new_used > _buffer_used, "no wrap around");
545
546 size_t cur_min_region_bottom = align_down(_buffer_used, MIN_GC_REGION_ALIGNMENT);
547 size_t next_min_region_bottom = align_down(new_used, MIN_GC_REGION_ALIGNMENT);
548 assert(cur_min_region_bottom == next_min_region_bottom, "no object should cross minimal GC region boundaries");
549
550 ensure_buffer_space(new_used);
551
552 address from = cast_from_oop<address>(src_obj);
553 address to = offset_to_buffered_address<address>(_buffer_used);
|
47 #include "runtime/mutexLocker.hpp"
48 #include "utilities/bitMap.inline.hpp"
49 #if INCLUDE_G1GC
50 #include "gc/g1/g1CollectedHeap.hpp"
51 #include "gc/g1/g1HeapRegion.hpp"
52 #endif
53
54 #if INCLUDE_CDS_JAVA_HEAP
55
56 GrowableArrayCHeap<u1, mtClassShared>* AOTMappedHeapWriter::_buffer = nullptr;
57
58 bool AOTMappedHeapWriter::_is_writing_deterministic_heap = false;
59 size_t AOTMappedHeapWriter::_buffer_used;
60
61 // Heap root segments
62 HeapRootSegments AOTMappedHeapWriter::_heap_root_segments;
63
64 address AOTMappedHeapWriter::_requested_bottom;
65 address AOTMappedHeapWriter::_requested_top;
66
67 static size_t _num_strings = 0;
68 static size_t _string_bytes = 0;
69 static size_t _num_packages = 0;
70 static size_t _num_protection_domains = 0;
71
72 GrowableArrayCHeap<AOTMappedHeapWriter::NativePointerInfo, mtClassShared>* AOTMappedHeapWriter::_native_pointers;
73 GrowableArrayCHeap<oop, mtClassShared>* AOTMappedHeapWriter::_source_objs;
74 GrowableArrayCHeap<AOTMappedHeapWriter::HeapObjOrder, mtClassShared>* AOTMappedHeapWriter::_source_objs_order;
75
76 AOTMappedHeapWriter::BufferOffsetToSourceObjectTable*
77 AOTMappedHeapWriter::_buffer_offset_to_source_obj_table = nullptr;
78
79 typedef HashTable<
80 size_t, // offset of a filler from AOTMappedHeapWriter::buffer_bottom()
81 size_t, // size of this filler (in bytes)
82 127, // prime number
83 AnyObj::C_HEAP,
84 mtClassShared> FillersTable;
85 static FillersTable* _fillers;
86 static int _num_native_ptrs = 0;
87
88 void AOTMappedHeapWriter::init() {
89 if (CDSConfig::is_dumping_heap()) {
90 Universe::heap()->collect(GCCause::_java_lang_system_gc);
91
92 _buffer_offset_to_source_obj_table = new (mtClassShared) BufferOffsetToSourceObjectTable(/*size (prime)*/36137, /*max size*/1 * M);
93 _fillers = new (mtClassShared) FillersTable();
94 _requested_bottom = nullptr;
95 _requested_top = nullptr;
96
97 _native_pointers = new GrowableArrayCHeap<NativePointerInfo, mtClassShared>(2048);
98 _source_objs = new GrowableArrayCHeap<oop, mtClassShared>(10000);
99
100 guarantee(MIN_GC_REGION_ALIGNMENT <= G1HeapRegion::min_region_size_in_words() * HeapWordSize, "must be");
101
102 if (CDSConfig::old_cds_flags_used()) {
103 // With the old CDS workflow, we can guatantee determninistic output: given
104 // the same classlist file, we can generate the same static CDS archive.
105 // To ensure determinism, we always use the same compressed oop encoding
106 // (zero-based, no shift). See set_requested_address_range().
107 _is_writing_deterministic_heap = true;
108 } else {
109 // Determninistic output is not supported by the new AOT workflow, so
110 // we don't force the (zero-based, no shift) encoding. This way, it is more
111 // likely that we can avoid oop relocation in the production run.
112 _is_writing_deterministic_heap = false;
126
127 address AOTMappedHeapWriter::narrow_oop_base() {
128 if (is_writing_deterministic_heap()) {
129 return nullptr;
130 } else {
131 return CompressedOops::base();
132 }
133 }
134
135 int AOTMappedHeapWriter::narrow_oop_shift() {
136 if (is_writing_deterministic_heap()) {
137 return 0;
138 } else {
139 return CompressedOops::shift();
140 }
141 }
142
143 void AOTMappedHeapWriter::delete_tables_with_raw_oops() {
144 delete _source_objs;
145 _source_objs = nullptr;
146 }
147
148 void AOTMappedHeapWriter::add_source_obj(oop src_obj) {
149 _source_objs->append(src_obj);
150 }
151
152 void AOTMappedHeapWriter::write(GrowableArrayCHeap<oop, mtClassShared>* roots,
153 AOTMappedHeapInfo* heap_info) {
154 assert(CDSConfig::is_dumping_heap(), "sanity");
155 allocate_buffer();
156 copy_source_objs_to_buffer(roots);
157 set_requested_address_range(heap_info);
158 relocate_embedded_oops(roots, heap_info);
159 }
160
161 bool AOTMappedHeapWriter::is_too_large_to_archive(oop o) {
162 return is_too_large_to_archive(o->size());
163 }
164
165 bool AOTMappedHeapWriter::is_string_too_large_to_archive(oop string) {
166 typeArrayOop value = java_lang_String::value_no_keepalive(string);
167 return is_too_large_to_archive(value);
168 }
169
170 bool AOTMappedHeapWriter::is_too_large_to_archive(size_t size) {
171 assert(size > 0, "no zero-size object");
172 assert(size * HeapWordSize > size, "no overflow");
173 static_assert(MIN_GC_REGION_ALIGNMENT > 0, "must be positive");
174
175 size_t byte_size = size * HeapWordSize;
176 if (byte_size > size_t(MIN_GC_REGION_ALIGNMENT)) {
177 return true;
178 } else {
179 return false;
180 }
181 }
182
183 // Various lookup functions between source_obj, buffered_obj and requested_obj
184 bool AOTMappedHeapWriter::is_in_requested_range(oop o) {
185 assert(_requested_bottom != nullptr, "do not call before _requested_bottom is initialized");
186 address a = cast_from_oop<address>(o);
187 return (_requested_bottom <= a && a < _requested_top);
188 }
189
190 oop AOTMappedHeapWriter::requested_obj_from_buffer_offset(size_t offset) {
191 oop req_obj = cast_to_oop(_requested_bottom + offset);
192 assert(is_in_requested_range(req_obj), "must be");
193 return req_obj;
194 }
195
196 oop AOTMappedHeapWriter::source_obj_to_requested_obj(oop src_obj) {
197 assert(CDSConfig::is_dumping_heap(), "dump-time only");
198 HeapShared::CachedOopInfo* p = HeapShared::get_cached_oop_info(src_obj);
199 if (p != nullptr) {
200 return requested_obj_from_buffer_offset(p->buffer_offset());
201 } else {
202 return nullptr;
393 _source_objs_order->append(os);
394 }
395 log_info(aot)("computed ranks");
396 _source_objs_order->sort(compare_objs_by_oop_fields);
397 log_info(aot)("sorting heap objects done");
398 }
399
400 void AOTMappedHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
401 // There could be multiple root segments, which we want to be aligned by region.
402 // Putting them ahead of objects makes sure we waste no space.
403 copy_roots_to_buffer(roots);
404
405 sort_source_objs();
406 for (int i = 0; i < _source_objs_order->length(); i++) {
407 int src_obj_index = _source_objs_order->at(i)._index;
408 oop src_obj = _source_objs->at(src_obj_index);
409 HeapShared::CachedOopInfo* info = HeapShared::get_cached_oop_info(src_obj);
410 assert(info != nullptr, "must be");
411 size_t buffer_offset = copy_one_source_obj_to_buffer(src_obj);
412 info->set_buffer_offset(buffer_offset);
413 assert(buffer_offset <= 0x7fffffff, "sanity");
414
415 OopHandle handle(Universe::vm_global(), src_obj);
416 _buffer_offset_to_source_obj_table->put_when_absent(buffer_offset, handle);
417 _buffer_offset_to_source_obj_table->maybe_grow();
418
419 if (java_lang_Module::is_instance(src_obj)) {
420 Modules::check_archived_module_oop(src_obj);
421 }
422 }
423
424 log_info(aot)("Size of heap region = %zu bytes, %d objects, %d roots, %d native ptrs",
425 _buffer_used, _source_objs->length() + 1, roots->length(), _num_native_ptrs);
426 log_info(cds)(" strings = %8zu (%zu bytes)", _num_strings, _string_bytes);
427 log_info(cds)(" packages = %8zu", _num_packages);
428 log_info(cds)(" protection domains = %8zu", _num_protection_domains);
429 }
430
431 size_t AOTMappedHeapWriter::filler_array_byte_size(int length) {
432 size_t byte_size = objArrayOopDesc::object_size(length) * HeapWordSize;
433 return byte_size;
434 }
435
436 int AOTMappedHeapWriter::filler_array_length(size_t fill_bytes) {
437 assert(is_object_aligned(fill_bytes), "must be");
438 size_t elemSize = (UseCompressedOops ? sizeof(narrowOop) : sizeof(oop));
439
440 int initial_length = to_array_length(fill_bytes / elemSize);
441 for (int length = initial_length; length >= 0; length --) {
442 size_t array_byte_size = filler_array_byte_size(length);
443 if (array_byte_size == fill_bytes) {
444 return length;
445 }
446 }
447
448 ShouldNotReachHere();
497 _fillers->put(buffered_address_to_offset((address)filler), fill_bytes);
498 }
499 }
500
501 size_t AOTMappedHeapWriter::get_filler_size_at(address buffered_addr) {
502 size_t* p = _fillers->get(buffered_address_to_offset(buffered_addr));
503 if (p != nullptr) {
504 assert(*p > 0, "filler must be larger than zero bytes");
505 return *p;
506 } else {
507 return 0; // buffered_addr is not a filler
508 }
509 }
510
511 template <typename T>
512 void update_buffered_object_field(address buffered_obj, int field_offset, T value) {
513 T* field_addr = cast_to_oop(buffered_obj)->field_addr<T>(field_offset);
514 *field_addr = value;
515 }
516
517 void AOTMappedHeapWriter::update_stats(oop src_obj) {
518 if (java_lang_String::is_instance(src_obj)) {
519 _num_strings ++;
520 _string_bytes += src_obj->size() * HeapWordSize;
521 _string_bytes += java_lang_String::value(src_obj)->size() * HeapWordSize;
522 } else {
523 Klass* k = src_obj->klass();
524 Symbol* name = k->name();
525 if (name->equals("java/lang/NamedPackage") || name->equals("java/lang/Package")) {
526 _num_packages ++;
527 } else if (name->equals("java/security/ProtectionDomain")) {
528 _num_protection_domains ++;
529 }
530 }
531 }
532
533 size_t AOTMappedHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
534 update_stats(src_obj);
535
536 assert(!is_too_large_to_archive(src_obj), "already checked");
537 size_t byte_size = src_obj->size() * HeapWordSize;
538 assert(byte_size > 0, "no zero-size objects");
539
540 // For region-based collectors such as G1, the archive heap may be mapped into
541 // multiple regions. We need to make sure that we don't have an object that can possible
542 // span across two regions.
543 maybe_fill_gc_region_gap(byte_size);
544
545 size_t new_used = _buffer_used + byte_size;
546 assert(new_used > _buffer_used, "no wrap around");
547
548 size_t cur_min_region_bottom = align_down(_buffer_used, MIN_GC_REGION_ALIGNMENT);
549 size_t next_min_region_bottom = align_down(new_used, MIN_GC_REGION_ALIGNMENT);
550 assert(cur_min_region_bottom == next_min_region_bottom, "no object should cross minimal GC region boundaries");
551
552 ensure_buffer_space(new_used);
553
554 address from = cast_from_oop<address>(src_obj);
555 address to = offset_to_buffered_address<address>(_buffer_used);
|