229
230 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
231 if (result != nullptr) {
232 return result;
233 }
234 }
235 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
236 }
237
238 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
239 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
240 }
241
242 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
243 // to OLD_GENERATION.
244 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
245 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
246 bool alloc_from_lab = true;
247 bool has_plab = false;
248 HeapWord* copy = nullptr;
249 size_t size = ShenandoahForwarding::size(p);
250 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
251
252 #ifdef ASSERT
253 if (ShenandoahOOMDuringEvacALot &&
254 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
255 copy = nullptr;
256 } else {
257 #endif
258 if (UseTLAB) {
259 switch (TO_GENERATION) {
260 case YOUNG_GENERATION: {
261 copy = allocate_from_gclab(thread, size);
262 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
263 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
264 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
265 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
266 copy = allocate_from_gclab(thread, size);
267 // If we still get nullptr, we'll try a shared allocation below.
268 }
269 break;
323 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
324 old_generation()->handle_failed_promotion(thread, size);
325 return nullptr;
326 } else {
327 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
328 // after the evacuation threads have finished.
329 old_generation()->handle_failed_evacuation();
330 }
331 }
332
333 control_thread()->handle_alloc_failure_evac(size);
334 oom_evac_handler()->handle_out_of_memory_during_evacuation();
335 return ShenandoahBarrierSet::resolve_forwarded(p);
336 }
337
338 if (ShenandoahEvacTracking) {
339 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
340 }
341
342 // Copy the object:
343 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
344 oop copy_val = cast_to_oop(copy);
345
346 // Update the age of the evacuated object
347 if (TO_GENERATION == YOUNG_GENERATION && is_aging_cycle()) {
348 increase_object_age(copy_val, from_region_age + 1);
349 }
350
351 // Try to install the new forwarding pointer.
352 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
353 if (result == copy_val) {
354 // Successfully evacuated. Our copy is now the public one!
355
356 // This is necessary for virtual thread support. This uses the mark word without
357 // considering that it may now be a forwarding pointer (and could therefore crash).
358 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
359 // that lost the evacuation race (and will therefore not become visible). It is
360 // safe to do this on the public copy (this is also done during concurrent mark).
361 ContinuationGCSupport::relativize_stack_chunk(copy_val);
362
363 if (ShenandoahEvacTracking) {
364 // Record that the evacuation succeeded
365 evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
|
229
230 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
231 if (result != nullptr) {
232 return result;
233 }
234 }
235 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
236 }
237
238 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
239 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
240 }
241
242 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
243 // to OLD_GENERATION.
244 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
245 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
246 bool alloc_from_lab = true;
247 bool has_plab = false;
248 HeapWord* copy = nullptr;
249
250 markWord mark = p->mark();
251 if (ShenandoahForwarding::is_forwarded(mark)) {
252 return ShenandoahForwarding::get_forwardee(p);
253 }
254 size_t old_size = ShenandoahForwarding::size(p);
255 size_t size = p->copy_size(old_size, mark);
256
257 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
258
259 #ifdef ASSERT
260 if (ShenandoahOOMDuringEvacALot &&
261 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
262 copy = nullptr;
263 } else {
264 #endif
265 if (UseTLAB) {
266 switch (TO_GENERATION) {
267 case YOUNG_GENERATION: {
268 copy = allocate_from_gclab(thread, size);
269 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
270 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
271 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
272 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
273 copy = allocate_from_gclab(thread, size);
274 // If we still get nullptr, we'll try a shared allocation below.
275 }
276 break;
330 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
331 old_generation()->handle_failed_promotion(thread, size);
332 return nullptr;
333 } else {
334 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
335 // after the evacuation threads have finished.
336 old_generation()->handle_failed_evacuation();
337 }
338 }
339
340 control_thread()->handle_alloc_failure_evac(size);
341 oom_evac_handler()->handle_out_of_memory_during_evacuation();
342 return ShenandoahBarrierSet::resolve_forwarded(p);
343 }
344
345 if (ShenandoahEvacTracking) {
346 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
347 }
348
349 // Copy the object:
350 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
351 oop copy_val = cast_to_oop(copy);
352
353 // Initialize the identity hash on the copy before installing the forwarding
354 // pointer, using the mark word we captured earlier. We must do this before
355 // the CAS so that the copy is fully initialized when it becomes visible to
356 // other threads. Using the captured mark (rather than re-reading the copy's
357 // mark) avoids races with other threads that may have evacuated p and
358 // installed a forwarding pointer in the meantime.
359 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
360 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
361 }
362
363 // Update the age of the evacuated object
364 if (TO_GENERATION == YOUNG_GENERATION && is_aging_cycle()) {
365 increase_object_age(copy_val, from_region_age + 1);
366 }
367
368 // Try to install the new forwarding pointer.
369 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
370 if (result == copy_val) {
371 // Successfully evacuated. Our copy is now the public one!
372
373 // This is necessary for virtual thread support. This uses the mark word without
374 // considering that it may now be a forwarding pointer (and could therefore crash).
375 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
376 // that lost the evacuation race (and will therefore not become visible). It is
377 // safe to do this on the public copy (this is also done during concurrent mark).
378 ContinuationGCSupport::relativize_stack_chunk(copy_val);
379
380 if (ShenandoahEvacTracking) {
381 // Record that the evacuation succeeded
382 evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
|