225 // We don't want to deal with MT here just to ensure we read the right mark word.
226 // Skip the potential promotion attempt for this one.
227 } else if (age_census()->is_tenurable(r->age() + mark.age())) {
228 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
229 if (result != nullptr) {
230 return result;
231 }
232 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
233 }
234 }
235 return try_evacuate_object(p, thread, r, target_gen);
236 }
237
238 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
239 // to OLD_GENERATION.
240 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
241 ShenandoahAffiliation target_gen) {
242 bool alloc_from_lab = true;
243 bool has_plab = false;
244 HeapWord* copy = nullptr;
245 size_t size = ShenandoahForwarding::size(p);
246 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
247
248 #ifdef ASSERT
249 if (ShenandoahOOMDuringEvacALot &&
250 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
251 copy = nullptr;
252 } else {
253 #endif
254 if (UseTLAB) {
255 switch (target_gen) {
256 case YOUNG_GENERATION: {
257 copy = allocate_from_gclab(thread, size);
258 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
259 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
260 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
261 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
262 copy = allocate_from_gclab(thread, size);
263 // If we still get nullptr, we'll try a shared allocation below.
264 }
265 break;
321 return nullptr;
322 } else {
323 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
324 // after the evacuation threads have finished.
325 old_generation()->handle_failed_evacuation();
326 }
327 }
328
329 control_thread()->handle_alloc_failure_evac(size);
330
331 oom_evac_handler()->handle_out_of_memory_during_evacuation();
332
333 return ShenandoahBarrierSet::resolve_forwarded(p);
334 }
335
336 if (ShenandoahEvacTracking) {
337 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
338 }
339
340 // Copy the object:
341 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
342 oop copy_val = cast_to_oop(copy);
343
344 // Update the age of the evacuated object
345 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
346 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
347 }
348
349 // Try to install the new forwarding pointer.
350 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
351 if (result == copy_val) {
352 // Successfully evacuated. Our copy is now the public one!
353
354 // This is necessary for virtual thread support. This uses the mark word without
355 // considering that it may now be a forwarding pointer (and could therefore crash).
356 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
357 // that lost the evacuation race (and will therefore not become visible). It is
358 // safe to do this on the public copy (this is also done during concurrent mark).
359 ContinuationGCSupport::relativize_stack_chunk(copy_val);
360
361 if (ShenandoahEvacTracking) {
362 // Record that the evacuation succeeded
363 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
|
225 // We don't want to deal with MT here just to ensure we read the right mark word.
226 // Skip the potential promotion attempt for this one.
227 } else if (age_census()->is_tenurable(r->age() + mark.age())) {
228 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
229 if (result != nullptr) {
230 return result;
231 }
232 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
233 }
234 }
235 return try_evacuate_object(p, thread, r, target_gen);
236 }
237
238 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
239 // to OLD_GENERATION.
240 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
241 ShenandoahAffiliation target_gen) {
242 bool alloc_from_lab = true;
243 bool has_plab = false;
244 HeapWord* copy = nullptr;
245
246 markWord mark = p->mark();
247 if (ShenandoahForwarding::is_forwarded(mark)) {
248 return ShenandoahForwarding::get_forwardee(p);
249 }
250 size_t old_size = ShenandoahForwarding::size(p);
251 size_t size = p->copy_size(old_size, mark);
252
253 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
254
255 #ifdef ASSERT
256 if (ShenandoahOOMDuringEvacALot &&
257 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
258 copy = nullptr;
259 } else {
260 #endif
261 if (UseTLAB) {
262 switch (target_gen) {
263 case YOUNG_GENERATION: {
264 copy = allocate_from_gclab(thread, size);
265 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
266 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
267 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
268 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
269 copy = allocate_from_gclab(thread, size);
270 // If we still get nullptr, we'll try a shared allocation below.
271 }
272 break;
328 return nullptr;
329 } else {
330 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
331 // after the evacuation threads have finished.
332 old_generation()->handle_failed_evacuation();
333 }
334 }
335
336 control_thread()->handle_alloc_failure_evac(size);
337
338 oom_evac_handler()->handle_out_of_memory_during_evacuation();
339
340 return ShenandoahBarrierSet::resolve_forwarded(p);
341 }
342
343 if (ShenandoahEvacTracking) {
344 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
345 }
346
347 // Copy the object:
348 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
349 oop copy_val = cast_to_oop(copy);
350
351 // Initialize the identity hash on the copy before installing the forwarding
352 // pointer, using the mark word we captured earlier. We must do this before
353 // the CAS so that the copy is fully initialized when it becomes visible to
354 // other threads. Using the captured mark (rather than re-reading the copy's
355 // mark) avoids races with other threads that may have evacuated p and
356 // installed a forwarding pointer in the meantime.
357 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
358 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
359 }
360
361 // Update the age of the evacuated object
362 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
363 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
364 }
365
366 // Try to install the new forwarding pointer.
367 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
368 if (result == copy_val) {
369 // Successfully evacuated. Our copy is now the public one!
370
371 // This is necessary for virtual thread support. This uses the mark word without
372 // considering that it may now be a forwarding pointer (and could therefore crash).
373 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
374 // that lost the evacuation race (and will therefore not become visible). It is
375 // safe to do this on the public copy (this is also done during concurrent mark).
376 ContinuationGCSupport::relativize_stack_chunk(copy_val);
377
378 if (ShenandoahEvacTracking) {
379 // Record that the evacuation succeeded
380 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
|