225 // We don't want to deal with MT here just to ensure we read the right mark word.
226 // Skip the potential promotion attempt for this one.
227 } else if (age_census()->is_tenurable(r->age() + mark.age())) {
228 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
229 if (result != nullptr) {
230 return result;
231 }
232 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
233 }
234 }
235 return try_evacuate_object(p, thread, r, target_gen);
236 }
237
238 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
239 // to OLD_GENERATION.
240 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
241 ShenandoahAffiliation target_gen) {
242 bool alloc_from_lab = true;
243 bool has_plab = false;
244 HeapWord* copy = nullptr;
245 size_t size = ShenandoahForwarding::size(p);
246 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
247
248 #ifdef ASSERT
249 if (ShenandoahOOMDuringEvacALot &&
250 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
251 copy = nullptr;
252 } else {
253 #endif
254 if (UseTLAB) {
255 switch (target_gen) {
256 case YOUNG_GENERATION: {
257 copy = allocate_from_gclab(thread, size);
258 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
259 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
260 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
261 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
262 copy = allocate_from_gclab(thread, size);
263 // If we still get nullptr, we'll try a shared allocation below.
264 }
265 break;
321 return nullptr;
322 } else {
323 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
324 // after the evacuation threads have finished.
325 old_generation()->handle_failed_evacuation();
326 }
327 }
328
329 control_thread()->handle_alloc_failure_evac(size);
330
331 oom_evac_handler()->handle_out_of_memory_during_evacuation();
332
333 return ShenandoahBarrierSet::resolve_forwarded(p);
334 }
335
336 if (ShenandoahEvacTracking) {
337 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
338 }
339
340 // Copy the object:
341 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
342 oop copy_val = cast_to_oop(copy);
343
344 // Update the age of the evacuated object
345 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
346 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
347 }
348
349 // Try to install the new forwarding pointer.
350 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
351 if (result == copy_val) {
352 // Successfully evacuated. Our copy is now the public one!
353
354 // This is necessary for virtual thread support. This uses the mark word without
355 // considering that it may now be a forwarding pointer (and could therefore crash).
356 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
357 // that lost the evacuation race (and will therefore not become visible). It is
358 // safe to do this on the public copy (this is also done during concurrent mark).
359 ContinuationGCSupport::relativize_stack_chunk(copy_val);
360
361 if (ShenandoahEvacTracking) {
362 // Record that the evacuation succeeded
363 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
364 }
365
366 if (target_gen == OLD_GENERATION) {
367 old_generation()->handle_evacuation(copy, size, from_region->is_young());
368 } else {
369 // When copying to the old generation above, we don't care
370 // about recording object age in the census stats.
371 assert(target_gen == YOUNG_GENERATION, "Error");
372 }
373 shenandoah_assert_correct(nullptr, copy_val);
374 return copy_val;
375 } else {
376 // Failed to evacuate. We need to deal with the object that is left behind. Since this
377 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
378 // But if it happens to contain references to evacuated regions, those references would
|
225 // We don't want to deal with MT here just to ensure we read the right mark word.
226 // Skip the potential promotion attempt for this one.
227 } else if (age_census()->is_tenurable(r->age() + mark.age())) {
228 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
229 if (result != nullptr) {
230 return result;
231 }
232 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
233 }
234 }
235 return try_evacuate_object(p, thread, r, target_gen);
236 }
237
238 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
239 // to OLD_GENERATION.
240 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
241 ShenandoahAffiliation target_gen) {
242 bool alloc_from_lab = true;
243 bool has_plab = false;
244 HeapWord* copy = nullptr;
245
246 markWord mark = p->mark();
247 if (ShenandoahForwarding::is_forwarded(mark)) {
248 return ShenandoahForwarding::get_forwardee(p);
249 }
250 size_t old_size = ShenandoahForwarding::size(p);
251 size_t size = p->copy_size(old_size, mark);
252
253 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
254
255 #ifdef ASSERT
256 if (ShenandoahOOMDuringEvacALot &&
257 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
258 copy = nullptr;
259 } else {
260 #endif
261 if (UseTLAB) {
262 switch (target_gen) {
263 case YOUNG_GENERATION: {
264 copy = allocate_from_gclab(thread, size);
265 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
266 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
267 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
268 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
269 copy = allocate_from_gclab(thread, size);
270 // If we still get nullptr, we'll try a shared allocation below.
271 }
272 break;
328 return nullptr;
329 } else {
330 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
331 // after the evacuation threads have finished.
332 old_generation()->handle_failed_evacuation();
333 }
334 }
335
336 control_thread()->handle_alloc_failure_evac(size);
337
338 oom_evac_handler()->handle_out_of_memory_during_evacuation();
339
340 return ShenandoahBarrierSet::resolve_forwarded(p);
341 }
342
343 if (ShenandoahEvacTracking) {
344 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
345 }
346
347 // Copy the object:
348 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
349 oop copy_val = cast_to_oop(copy);
350
351 // Update the age of the evacuated object
352 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
353 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
354 }
355
356 // Try to install the new forwarding pointer.
357 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
358 if (result == copy_val) {
359 // Successfully evacuated. Our copy is now the public one!
360
361 // This is necessary for virtual thread support. This uses the mark word without
362 // considering that it may now be a forwarding pointer (and could therefore crash).
363 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
364 // that lost the evacuation race (and will therefore not become visible). It is
365 // safe to do this on the public copy (this is also done during concurrent mark).
366 copy_val->initialize_hash_if_necessary(p);
367 ContinuationGCSupport::relativize_stack_chunk(copy_val);
368
369 if (ShenandoahEvacTracking) {
370 // Record that the evacuation succeeded
371 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
372 }
373
374 if (target_gen == OLD_GENERATION) {
375 old_generation()->handle_evacuation(copy, size, from_region->is_young());
376 } else {
377 // When copying to the old generation above, we don't care
378 // about recording object age in the census stats.
379 assert(target_gen == YOUNG_GENERATION, "Error");
380 }
381 shenandoah_assert_correct(nullptr, copy_val);
382 return copy_val;
383 } else {
384 // Failed to evacuate. We need to deal with the object that is left behind. Since this
385 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
386 // But if it happens to contain references to evacuated regions, those references would
|