148
149 int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) {
150 assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string");
151 address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset();
152 return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr;
153 }
154
155 static int archive_array_length(oopDesc* archive_array) {
156 return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes());
157 }
158
159 static size_t archive_object_size(oopDesc* archive_object) {
160 Klass* klass = archive_object->klass();
161 int lh = klass->layout_helper();
162
163 if (Klass::layout_helper_is_instance(lh)) {
164 // Instance
165 if (Klass::layout_helper_needs_slow_path(lh)) {
166 return ((size_t*)(archive_object))[-1];
167 } else {
168 return (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize;
169 }
170 } else if (Klass::layout_helper_is_array(lh)) {
171 // Array
172 size_t size_in_bytes;
173 size_t array_length = (size_t)archive_array_length(archive_object);
174 size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh);
175 size_in_bytes += (size_t)Klass::layout_helper_header_size(lh);
176
177 return align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize;
178 } else {
179 // Other
180 return ((size_t*)(archive_object))[-1];
181 }
182 }
183
184 oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) {
185 assert(!archive_object->is_stackChunk(), "no such objects are archived");
186
187 NoJvmtiEventsMark njem;
188 oop heap_object;
189
190 Klass* klass = archive_object->klass();
191 if (klass->is_mirror_instance_klass()) {
192 heap_object = Universe::heap()->class_allocate(klass, size, CHECK_NULL);
193 } else if (klass->is_instance_klass()) {
194 heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL);
195 } else {
196 assert(klass->is_array_klass(), "must be");
197 int length = archive_array_length(archive_object);
198 bool do_zero = klass->is_objArray_klass();
199 heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL);
200 }
201
202 heap_object->set_mark(mark);
203
204 return heap_object;
205 }
206
207 void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) {
208 objArrayOop roots = objArrayOop(_roots.resolve());
209 OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock
210 roots->obj_at_put(root_index, heap_object);
211 }
212
299 // intended linked object.
300 oop obj = linker(heap_p_offset, pointee_object_index);
301 if (obj != nullptr) {
302 heap_object->obj_field_put(heap_p_offset, obj);
303 }
304
305 unfinished_bit++;
306 next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
307 }
308 }
309 }
310
311 template <bool use_coops, typename LinkerT>
312 void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object,
313 oop heap_object,
314 size_t size,
315 LinkerT linker) {
316 if (!_allow_gc) {
317 // Without concurrent GC running, we can copy incorrect object references
318 // and metadata references into the heap object and then fix them up in-place.
319 size_t payload_size = size - 1;
320 HeapWord* archive_start = ((HeapWord*)archive_object) + 1;
321 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object) + 1;
322
323 Copy::disjoint_words(archive_start, heap_start, payload_size);
324
325 // In-place linking fixes up object indices from references of the heap object,
326 // and patches them up to refer to objects. This can be done because we just copied
327 // the payload of the object from the archive to the heap object, including the
328 // reference object indices. However, this is only okay to do before the GC can run.
329 // A concurrent GC thread might racingly read the object payload after GC is enabled.
330 InPlaceLinkingOopClosure cl(heap_object, linker);
331 heap_object->oop_iterate(&cl);
332 HeapShared::remap_loaded_metadata(heap_object);
333 return;
334 }
335
336 // When a concurrent GC may be running, we take care not to copy incorrect oops,
337 // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the
338 // intended runtime linked values only.
339 size_t word_scale = use_coops ? 2 : 1;
340 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
341
342 // Skip the markWord; it is set at allocation time
343 size_t header_size = word_scale;
344
345 size_t buffer_offset = buffer_offset_for_archive_object(archive_object);
346 const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset<use_coops>(buffer_offset);
347 const BitMap::idx_t start_bit = header_bit + header_size;
348 const BitMap::idx_t end_bit = header_bit + size * word_scale;
349
350 BitMap::idx_t curr_bit = start_bit;
351
352 // We are a bit paranoid about GC or other safepointing operations observing
353 // shady metadata fields from the archive that do not point at real metadata.
354 // We deal with this by explicitly reading the requested address from the
355 // archive and fixing it to real Metadata before writing it into the heap object.
356 HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) {
357 BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT);
358 BitMap::idx_t skip = word_scale;
359 assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit,
360 "Metadata field out of bounds");
361
362 // Copy payload before metadata field
363 copy_payload_carefully<use_coops>(archive_object,
364 heap_object,
365 header_bit,
366 curr_bit,
367 metadata_field_idx,
368 linker);
369
370 // Copy metadata field
371 Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset);
|
148
149 int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) {
150 assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string");
151 address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset();
152 return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr;
153 }
154
155 static int archive_array_length(oopDesc* archive_array) {
156 return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes());
157 }
158
159 static size_t archive_object_size(oopDesc* archive_object) {
160 Klass* klass = archive_object->klass();
161 int lh = klass->layout_helper();
162
163 if (Klass::layout_helper_is_instance(lh)) {
164 // Instance
165 if (Klass::layout_helper_needs_slow_path(lh)) {
166 return ((size_t*)(archive_object))[-1];
167 } else {
168 size_t size = (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize;
169 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && klass->expand_for_hash(archive_object, archive_object->mark())) {
170 size = align_object_size(size + 1);
171 }
172 return size;
173 }
174 } else if (Klass::layout_helper_is_array(lh)) {
175 // Array
176 size_t size_in_bytes;
177 size_t array_length = (size_t)archive_array_length(archive_object);
178 size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh);
179 size_in_bytes += (size_t)Klass::layout_helper_header_size(lh);
180
181 size_t size = align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize;
182 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && klass->expand_for_hash(archive_object, archive_object->mark())) {
183 size = align_object_size(size + 1);
184 }
185 return size;
186 } else {
187 // Other
188 return ((size_t*)(archive_object))[-1];
189 }
190 }
191
192 oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) {
193 assert(!archive_object->is_stackChunk(), "no such objects are archived");
194
195 NoJvmtiEventsMark njem;
196 oop heap_object;
197
198 Klass* klass = archive_object->klass();
199 assert(!(UseCompactObjectHeaders && mark.is_hashed_not_expanded()), "Must not be hashed/not-expanded");
200 if (klass->is_mirror_instance_klass()) {
201 size_t base_size = size;
202 assert(!(UseCompactObjectHeaders && mark.is_not_hashed_expanded()), "should not happen");
203 heap_object = Universe::heap()->class_allocate(klass, size, base_size, CHECK_NULL);
204 } else if (klass->is_instance_klass()) {
205 heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL);
206 } else {
207 assert(klass->is_array_klass(), "must be");
208 int length = archive_array_length(archive_object);
209 bool do_zero = klass->is_objArray_klass();
210 heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL);
211 }
212
213 heap_object->set_mark(mark);
214
215 return heap_object;
216 }
217
218 void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) {
219 objArrayOop roots = objArrayOop(_roots.resolve());
220 OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock
221 roots->obj_at_put(root_index, heap_object);
222 }
223
310 // intended linked object.
311 oop obj = linker(heap_p_offset, pointee_object_index);
312 if (obj != nullptr) {
313 heap_object->obj_field_put(heap_p_offset, obj);
314 }
315
316 unfinished_bit++;
317 next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
318 }
319 }
320 }
321
322 template <bool use_coops, typename LinkerT>
323 void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object,
324 oop heap_object,
325 size_t size,
326 LinkerT linker) {
327 if (!_allow_gc) {
328 // Without concurrent GC running, we can copy incorrect object references
329 // and metadata references into the heap object and then fix them up in-place.
330 size_t offset = 1;
331 size_t payload_size = size - offset;
332 HeapWord* archive_start = ((HeapWord*)archive_object);
333 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
334
335 Copy::disjoint_words(archive_start + offset, heap_start + offset, payload_size);
336
337 if (UseCompactObjectHeaders) {
338 // The copying might have missed the first 4 bytes of payload/arraylength, copy that also.
339 *(reinterpret_cast<jint*>(heap_start) + 1) = *(reinterpret_cast<jint*>(archive_start) + 1);
340 }
341
342 // In-place linking fixes up object indices from references of the heap object,
343 // and patches them up to refer to objects. This can be done because we just copied
344 // the payload of the object from the archive to the heap object, including the
345 // reference object indices. However, this is only okay to do before the GC can run.
346 // A concurrent GC thread might racingly read the object payload after GC is enabled.
347 InPlaceLinkingOopClosure cl(heap_object, linker);
348 heap_object->oop_iterate(&cl);
349 HeapShared::remap_loaded_metadata(heap_object);
350 return;
351 }
352
353 // When a concurrent GC may be running, we take care not to copy incorrect oops,
354 // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the
355 // intended runtime linked values only.
356 size_t word_scale = use_coops ? 2 : 1;
357 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
358
359 // Skip the markWord; it is set at allocation time
360 size_t header_size = (UseCompactObjectHeaders && use_coops) ? 1 : word_scale;
361
362 size_t buffer_offset = buffer_offset_for_archive_object(archive_object);
363 const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset<use_coops>(buffer_offset);
364 const BitMap::idx_t start_bit = header_bit + header_size;
365 const BitMap::idx_t end_bit = header_bit + size * word_scale;
366
367 BitMap::idx_t curr_bit = start_bit;
368
369 if (UseCompactObjectHeaders && !use_coops) {
370 // Copy first 4 primitive bytes.
371 jint* archive_start = reinterpret_cast<jint*>(archive_object);
372 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
373 *(reinterpret_cast<jint*>(heap_start) + 1) = *(archive_start + 1);
374 }
375
376 // We are a bit paranoid about GC or other safepointing operations observing
377 // shady metadata fields from the archive that do not point at real metadata.
378 // We deal with this by explicitly reading the requested address from the
379 // archive and fixing it to real Metadata before writing it into the heap object.
380 HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) {
381 BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT);
382 BitMap::idx_t skip = word_scale;
383 assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit,
384 "Metadata field out of bounds");
385
386 // Copy payload before metadata field
387 copy_payload_carefully<use_coops>(archive_object,
388 heap_object,
389 header_bit,
390 curr_bit,
391 metadata_field_idx,
392 linker);
393
394 // Copy metadata field
395 Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset);
|