230 // We don't want to deal with MT here just to ensure we read the right mark word.
231 // Skip the potential promotion attempt for this one.
232 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
233 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
234 if (result != nullptr) {
235 return result;
236 }
237 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
238 }
239 }
240 return try_evacuate_object(p, thread, r, target_gen);
241 }
242
243 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
244 // to OLD_GENERATION.
245 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
246 ShenandoahAffiliation target_gen) {
247 bool alloc_from_lab = true;
248 bool has_plab = false;
249 HeapWord* copy = nullptr;
250 size_t size = ShenandoahForwarding::size(p);
251 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
252
253 #ifdef ASSERT
254 if (ShenandoahOOMDuringEvacALot &&
255 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
256 copy = nullptr;
257 } else {
258 #endif
259 if (UseTLAB) {
260 switch (target_gen) {
261 case YOUNG_GENERATION: {
262 copy = allocate_from_gclab(thread, size);
263 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
264 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
265 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
266 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
267 copy = allocate_from_gclab(thread, size);
268 // If we still get nullptr, we'll try a shared allocation below.
269 }
270 break;
322 if (from_region->is_young()) {
323 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
324 old_generation()->handle_failed_promotion(thread, size);
325 return nullptr;
326 } else {
327 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
328 // after the evacuation threads have finished.
329 old_generation()->handle_failed_evacuation();
330 }
331 }
332
333 control_thread()->handle_alloc_failure_evac(size);
334
335 oom_evac_handler()->handle_out_of_memory_during_evacuation();
336
337 return ShenandoahBarrierSet::resolve_forwarded(p);
338 }
339
340 // Copy the object:
341 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
342 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
343 oop copy_val = cast_to_oop(copy);
344
345 // Update the age of the evacuated object
346 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
347 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
348 }
349
350 // Try to install the new forwarding pointer.
351 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
352 if (result == copy_val) {
353 // Successfully evacuated. Our copy is now the public one!
354
355 // This is necessary for virtual thread support. This uses the mark word without
356 // considering that it may now be a forwarding pointer (and could therefore crash).
357 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
358 // that lost the evacuation race (and will therefore not become visible). It is
359 // safe to do this on the public copy (this is also done during concurrent mark).
360 ContinuationGCSupport::relativize_stack_chunk(copy_val);
361
362 // Record that the evacuation succeeded
363 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
364
365 if (target_gen == OLD_GENERATION) {
366 old_generation()->handle_evacuation(copy, size, from_region->is_young());
367 } else {
368 // When copying to the old generation above, we don't care
369 // about recording object age in the census stats.
370 assert(target_gen == YOUNG_GENERATION, "Error");
371 // We record this census only when simulating pre-adaptive tenuring behavior, or
372 // when we have been asked to record the census at evacuation rather than at mark
373 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
374 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
375 }
376 }
377 shenandoah_assert_correct(nullptr, copy_val);
378 return copy_val;
379 } else {
|
230 // We don't want to deal with MT here just to ensure we read the right mark word.
231 // Skip the potential promotion attempt for this one.
232 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
233 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
234 if (result != nullptr) {
235 return result;
236 }
237 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
238 }
239 }
240 return try_evacuate_object(p, thread, r, target_gen);
241 }
242
243 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
244 // to OLD_GENERATION.
245 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
246 ShenandoahAffiliation target_gen) {
247 bool alloc_from_lab = true;
248 bool has_plab = false;
249 HeapWord* copy = nullptr;
250
251 markWord mark = p->mark();
252 if (ShenandoahForwarding::is_forwarded(mark)) {
253 return ShenandoahForwarding::get_forwardee(p);
254 }
255 size_t old_size = ShenandoahForwarding::size(p);
256 size_t size = p->copy_size(old_size, mark);
257
258 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
259
260 #ifdef ASSERT
261 if (ShenandoahOOMDuringEvacALot &&
262 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
263 copy = nullptr;
264 } else {
265 #endif
266 if (UseTLAB) {
267 switch (target_gen) {
268 case YOUNG_GENERATION: {
269 copy = allocate_from_gclab(thread, size);
270 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
271 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
272 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
273 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
274 copy = allocate_from_gclab(thread, size);
275 // If we still get nullptr, we'll try a shared allocation below.
276 }
277 break;
329 if (from_region->is_young()) {
330 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
331 old_generation()->handle_failed_promotion(thread, size);
332 return nullptr;
333 } else {
334 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
335 // after the evacuation threads have finished.
336 old_generation()->handle_failed_evacuation();
337 }
338 }
339
340 control_thread()->handle_alloc_failure_evac(size);
341
342 oom_evac_handler()->handle_out_of_memory_during_evacuation();
343
344 return ShenandoahBarrierSet::resolve_forwarded(p);
345 }
346
347 // Copy the object:
348 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
349 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
350 oop copy_val = cast_to_oop(copy);
351
352 // Update the age of the evacuated object
353 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
354 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
355 }
356
357 // Try to install the new forwarding pointer.
358 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
359 if (result == copy_val) {
360 // Successfully evacuated. Our copy is now the public one!
361
362 // This is necessary for virtual thread support. This uses the mark word without
363 // considering that it may now be a forwarding pointer (and could therefore crash).
364 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
365 // that lost the evacuation race (and will therefore not become visible). It is
366 // safe to do this on the public copy (this is also done during concurrent mark).
367 copy_val->initialize_hash_if_necessary(p);
368 ContinuationGCSupport::relativize_stack_chunk(copy_val);
369
370 // Record that the evacuation succeeded
371 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
372
373 if (target_gen == OLD_GENERATION) {
374 old_generation()->handle_evacuation(copy, size, from_region->is_young());
375 } else {
376 // When copying to the old generation above, we don't care
377 // about recording object age in the census stats.
378 assert(target_gen == YOUNG_GENERATION, "Error");
379 // We record this census only when simulating pre-adaptive tenuring behavior, or
380 // when we have been asked to record the census at evacuation rather than at mark
381 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
382 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
383 }
384 }
385 shenandoah_assert_correct(nullptr, copy_val);
386 return copy_val;
387 } else {
|