139 assert(heap_object_for_object_index(object_index) == nullptr, "Should only set once with this API");
140 if (_objects_are_handles) {
141 oop* handle = Universe::vm_global()->allocate();
142 NativeAccess<>::oop_store(handle, heap_object);
143 _object_index_to_heap_object_table[object_index] = (void*)handle;
144 } else {
145 _object_index_to_heap_object_table[object_index] = cast_from_oop<void*>(heap_object);
146 }
147 }
148
149 int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) {
150 assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string");
151 address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset();
152 return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr;
153 }
154
155 static int archive_array_length(oopDesc* archive_array) {
156 return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes());
157 }
158
159 static size_t archive_object_size(oopDesc* archive_object) {
160 Klass* klass = archive_object->klass();
161 int lh = klass->layout_helper();
162
163 if (Klass::layout_helper_is_instance(lh)) {
164 // Instance
165 if (Klass::layout_helper_needs_slow_path(lh)) {
166 return ((size_t*)(archive_object))[-1];
167 } else {
168 return (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize;
169 }
170 } else if (Klass::layout_helper_is_array(lh)) {
171 // Array
172 size_t size_in_bytes;
173 size_t array_length = (size_t)archive_array_length(archive_object);
174 size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh);
175 size_in_bytes += (size_t)Klass::layout_helper_header_size(lh);
176
177 return align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize;
178 } else {
179 // Other
180 return ((size_t*)(archive_object))[-1];
181 }
182 }
183
184 oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) {
185 assert(!archive_object->is_stackChunk(), "no such objects are archived");
186
187 NoJvmtiEventsMark njem;
188 oop heap_object;
189
190 Klass* klass = archive_object->klass();
191 if (klass->is_mirror_instance_klass()) {
192 heap_object = Universe::heap()->class_allocate(klass, size, CHECK_NULL);
193 } else if (klass->is_instance_klass()) {
194 heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL);
195 } else {
196 assert(klass->is_array_klass(), "must be");
197 int length = archive_array_length(archive_object);
198 bool do_zero = klass->is_objArray_klass();
199 heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL);
200 }
201
202 heap_object->set_mark(mark);
203
204 return heap_object;
205 }
206
207 void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) {
208 objArrayOop roots = objArrayOop(_roots.resolve());
209 OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock
210 roots->obj_at_put(root_index, heap_object);
211 }
212
299 // intended linked object.
300 oop obj = linker(heap_p_offset, pointee_object_index);
301 if (obj != nullptr) {
302 heap_object->obj_field_put(heap_p_offset, obj);
303 }
304
305 unfinished_bit++;
306 next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
307 }
308 }
309 }
310
311 template <bool use_coops, typename LinkerT>
312 void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object,
313 oop heap_object,
314 size_t size,
315 LinkerT linker) {
316 if (!_allow_gc) {
317 // Without concurrent GC running, we can copy incorrect object references
318 // and metadata references into the heap object and then fix them up in-place.
319 size_t payload_size = size - 1;
320 HeapWord* archive_start = ((HeapWord*)archive_object) + 1;
321 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object) + 1;
322
323 Copy::disjoint_words(archive_start, heap_start, payload_size);
324
325 // In-place linking fixes up object indices from references of the heap object,
326 // and patches them up to refer to objects. This can be done because we just copied
327 // the payload of the object from the archive to the heap object, including the
328 // reference object indices. However, this is only okay to do before the GC can run.
329 // A concurrent GC thread might racingly read the object payload after GC is enabled.
330 InPlaceLinkingOopClosure cl(heap_object, linker);
331 heap_object->oop_iterate(&cl);
332 HeapShared::remap_loaded_metadata(heap_object);
333 return;
334 }
335
336 // When a concurrent GC may be running, we take care not to copy incorrect oops,
337 // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the
338 // intended runtime linked values only.
339 size_t word_scale = use_coops ? 2 : 1;
340 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
341
342 // Skip the markWord; it is set at allocation time
343 size_t header_size = word_scale;
344
345 size_t buffer_offset = buffer_offset_for_archive_object(archive_object);
346 const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset<use_coops>(buffer_offset);
347 const BitMap::idx_t start_bit = header_bit + header_size;
348 const BitMap::idx_t end_bit = header_bit + size * word_scale;
349
350 BitMap::idx_t curr_bit = start_bit;
351
352 // We are a bit paranoid about GC or other safepointing operations observing
353 // shady metadata fields from the archive that do not point at real metadata.
354 // We deal with this by explicitly reading the requested address from the
355 // archive and fixing it to real Metadata before writing it into the heap object.
356 HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) {
357 BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT);
358 BitMap::idx_t skip = word_scale;
359 assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit,
360 "Metadata field out of bounds");
361
362 // Copy payload before metadata field
363 copy_payload_carefully<use_coops>(archive_object,
364 heap_object,
365 header_bit,
366 curr_bit,
367 metadata_field_idx,
368 linker);
369
370 // Copy metadata field
371 Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset);
|
139 assert(heap_object_for_object_index(object_index) == nullptr, "Should only set once with this API");
140 if (_objects_are_handles) {
141 oop* handle = Universe::vm_global()->allocate();
142 NativeAccess<>::oop_store(handle, heap_object);
143 _object_index_to_heap_object_table[object_index] = (void*)handle;
144 } else {
145 _object_index_to_heap_object_table[object_index] = cast_from_oop<void*>(heap_object);
146 }
147 }
148
149 int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) {
150 assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string");
151 address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset();
152 return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr;
153 }
154
155 static int archive_array_length(oopDesc* archive_array) {
156 return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes());
157 }
158
159 // archive_object lives in CDS mapped memory, not in the GC heap.
160 // Calling expand_for_hash() converts the raw oopDesc* to an oop, which
161 // triggers oop verification. ZGC's verifier rejects non-heap addresses,
162 // so we must suspend the check for that call.
163 #ifdef CHECK_UNHANDLED_OOPS
164 class SuspendCheckOopFunction : public StackObj {
165 CheckOopFunctionPointer _saved;
166 public:
167 SuspendCheckOopFunction() : _saved(check_oop_function) { check_oop_function = nullptr; }
168 ~SuspendCheckOopFunction() { check_oop_function = _saved; }
169 };
170 #endif
171
172 static bool archive_expand_for_hash(Klass* klass, oopDesc* archive_object) {
173 CHECK_UNHANDLED_OOPS_ONLY(SuspendCheckOopFunction suspend;)
174 return klass->expand_for_hash(archive_object, archive_object->mark());
175 }
176
177 static size_t archive_object_size(oopDesc* archive_object) {
178 Klass* klass = archive_object->klass();
179 int lh = klass->layout_helper();
180
181 if (Klass::layout_helper_is_instance(lh)) {
182 // Instance
183 if (Klass::layout_helper_needs_slow_path(lh)) {
184 return ((size_t*)(archive_object))[-1];
185 } else {
186 size_t size = (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize;
187 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && archive_expand_for_hash(klass, archive_object)) {
188 size = align_object_size(size + 1);
189 }
190 return size;
191 }
192 } else if (Klass::layout_helper_is_array(lh)) {
193 // Array
194 size_t size_in_bytes;
195 size_t array_length = (size_t)archive_array_length(archive_object);
196 size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh);
197 size_in_bytes += (size_t)Klass::layout_helper_header_size(lh);
198
199 size_t size = align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize;
200 if (UseCompactObjectHeaders && archive_object->mark().is_expanded() && archive_expand_for_hash(klass, archive_object)) {
201 size = align_object_size(size + 1);
202 }
203 return size;
204 } else {
205 // Other
206 return ((size_t*)(archive_object))[-1];
207 }
208 }
209
210 oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) {
211 assert(!archive_object->is_stackChunk(), "no such objects are archived");
212
213 NoJvmtiEventsMark njem;
214 oop heap_object;
215
216 Klass* klass = archive_object->klass();
217 assert(!(UseCompactObjectHeaders && mark.is_hashed_not_expanded()), "Must not be hashed/not-expanded");
218 if (klass->is_mirror_instance_klass()) {
219 size_t base_size = size;
220 assert(!(UseCompactObjectHeaders && mark.is_not_hashed_expanded()), "should not happen");
221 heap_object = Universe::heap()->class_allocate(klass, size, base_size, CHECK_NULL);
222 } else if (klass->is_instance_klass()) {
223 heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL);
224 } else {
225 assert(klass->is_array_klass(), "must be");
226 int length = archive_array_length(archive_object);
227 bool do_zero = klass->is_objArray_klass();
228 heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL);
229 }
230
231 heap_object->set_mark(mark);
232
233 return heap_object;
234 }
235
236 void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) {
237 objArrayOop roots = objArrayOop(_roots.resolve());
238 OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock
239 roots->obj_at_put(root_index, heap_object);
240 }
241
328 // intended linked object.
329 oop obj = linker(heap_p_offset, pointee_object_index);
330 if (obj != nullptr) {
331 heap_object->obj_field_put(heap_p_offset, obj);
332 }
333
334 unfinished_bit++;
335 next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit);
336 }
337 }
338 }
339
340 template <bool use_coops, typename LinkerT>
341 void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object,
342 oop heap_object,
343 size_t size,
344 LinkerT linker) {
345 if (!_allow_gc) {
346 // Without concurrent GC running, we can copy incorrect object references
347 // and metadata references into the heap object and then fix them up in-place.
348 size_t offset = 1;
349 size_t payload_size = size - offset;
350 HeapWord* archive_start = ((HeapWord*)archive_object);
351 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
352
353 Copy::disjoint_words(archive_start + offset, heap_start + offset, payload_size);
354
355 if (UseCompactObjectHeaders) {
356 // The copying might have missed the first 4 bytes of payload/arraylength, copy that also.
357 *(reinterpret_cast<jint*>(heap_start) + 1) = *(reinterpret_cast<jint*>(archive_start) + 1);
358 }
359
360 // In-place linking fixes up object indices from references of the heap object,
361 // and patches them up to refer to objects. This can be done because we just copied
362 // the payload of the object from the archive to the heap object, including the
363 // reference object indices. However, this is only okay to do before the GC can run.
364 // A concurrent GC thread might racingly read the object payload after GC is enabled.
365 InPlaceLinkingOopClosure cl(heap_object, linker);
366 heap_object->oop_iterate(&cl);
367 HeapShared::remap_loaded_metadata(heap_object);
368 return;
369 }
370
371 // When a concurrent GC may be running, we take care not to copy incorrect oops,
372 // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the
373 // intended runtime linked values only.
374 size_t word_scale = use_coops ? 2 : 1;
375 using RawElementT = std::conditional_t<use_coops, int32_t, int64_t>;
376
377 // Skip the markWord; it is set at allocation time
378 size_t header_size = (UseCompactObjectHeaders && use_coops) ? 1 : word_scale;
379
380 size_t buffer_offset = buffer_offset_for_archive_object(archive_object);
381 const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset<use_coops>(buffer_offset);
382 const BitMap::idx_t start_bit = header_bit + header_size;
383 const BitMap::idx_t end_bit = header_bit + size * word_scale;
384
385 BitMap::idx_t curr_bit = start_bit;
386
387 if (UseCompactObjectHeaders && !use_coops) {
388 // Copy first 4 primitive bytes.
389 jint* archive_start = reinterpret_cast<jint*>(archive_object);
390 HeapWord* heap_start = cast_from_oop<HeapWord*>(heap_object);
391 *(reinterpret_cast<jint*>(heap_start) + 1) = *(archive_start + 1);
392 }
393
394 // We are a bit paranoid about GC or other safepointing operations observing
395 // shady metadata fields from the archive that do not point at real metadata.
396 // We deal with this by explicitly reading the requested address from the
397 // archive and fixing it to real Metadata before writing it into the heap object.
398 HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) {
399 BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT);
400 BitMap::idx_t skip = word_scale;
401 assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit,
402 "Metadata field out of bounds");
403
404 // Copy payload before metadata field
405 copy_payload_carefully<use_coops>(archive_object,
406 heap_object,
407 header_bit,
408 curr_bit,
409 metadata_field_idx,
410 linker);
411
412 // Copy metadata field
413 Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset);
|