223 // We don't want to deal with MT here just to ensure we read the right mark word.
224 // Skip the potential promotion attempt for this one.
225 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
226 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
227 if (result != nullptr) {
228 return result;
229 }
230 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
231 }
232 }
233 return try_evacuate_object(p, thread, r, target_gen);
234 }
235
236 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
237 // to OLD_GENERATION.
238 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
239 ShenandoahAffiliation target_gen) {
240 bool alloc_from_lab = true;
241 bool has_plab = false;
242 HeapWord* copy = nullptr;
243 size_t size = ShenandoahForwarding::size(p);
244 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
245
246 #ifdef ASSERT
247 if (ShenandoahOOMDuringEvacALot &&
248 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
249 copy = nullptr;
250 } else {
251 #endif
252 if (UseTLAB) {
253 switch (target_gen) {
254 case YOUNG_GENERATION: {
255 copy = allocate_from_gclab(thread, size);
256 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
257 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
258 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
259 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
260 copy = allocate_from_gclab(thread, size);
261 // If we still get nullptr, we'll try a shared allocation below.
262 }
263 break;
315 if (from_region->is_young()) {
316 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
317 old_generation()->handle_failed_promotion(thread, size);
318 return nullptr;
319 } else {
320 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
321 // after the evacuation threads have finished.
322 old_generation()->handle_failed_evacuation();
323 }
324 }
325
326 control_thread()->handle_alloc_failure_evac(size);
327
328 oom_evac_handler()->handle_out_of_memory_during_evacuation();
329
330 return ShenandoahBarrierSet::resolve_forwarded(p);
331 }
332
333 // Copy the object:
334 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
335 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
336 oop copy_val = cast_to_oop(copy);
337
338 // Update the age of the evacuated object
339 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
340 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
341 }
342
343 // Try to install the new forwarding pointer.
344 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
345 if (result == copy_val) {
346 // Successfully evacuated. Our copy is now the public one!
347
348 // This is necessary for virtual thread support. This uses the mark word without
349 // considering that it may now be a forwarding pointer (and could therefore crash).
350 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
351 // that lost the evacuation race (and will therefore not become visible). It is
352 // safe to do this on the public copy (this is also done during concurrent mark).
353 ContinuationGCSupport::relativize_stack_chunk(copy_val);
354
355 // Record that the evacuation succeeded
356 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
357
358 if (target_gen == OLD_GENERATION) {
359 old_generation()->handle_evacuation(copy, size, from_region->is_young());
360 } else {
361 // When copying to the old generation above, we don't care
362 // about recording object age in the census stats.
363 assert(target_gen == YOUNG_GENERATION, "Error");
364 // We record this census only when simulating pre-adaptive tenuring behavior, or
365 // when we have been asked to record the census at evacuation rather than at mark
366 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
367 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
368 }
369 }
370 shenandoah_assert_correct(nullptr, copy_val);
371 return copy_val;
372 } else {
|
223 // We don't want to deal with MT here just to ensure we read the right mark word.
224 // Skip the potential promotion attempt for this one.
225 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
226 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
227 if (result != nullptr) {
228 return result;
229 }
230 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
231 }
232 }
233 return try_evacuate_object(p, thread, r, target_gen);
234 }
235
236 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
237 // to OLD_GENERATION.
238 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
239 ShenandoahAffiliation target_gen) {
240 bool alloc_from_lab = true;
241 bool has_plab = false;
242 HeapWord* copy = nullptr;
243
244 markWord mark = p->mark();
245 if (ShenandoahForwarding::is_forwarded(mark)) {
246 return ShenandoahForwarding::get_forwardee(p);
247 }
248 size_t old_size = ShenandoahForwarding::size(p);
249 size_t size = p->copy_size(old_size, mark);
250
251 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
252
253 #ifdef ASSERT
254 if (ShenandoahOOMDuringEvacALot &&
255 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
256 copy = nullptr;
257 } else {
258 #endif
259 if (UseTLAB) {
260 switch (target_gen) {
261 case YOUNG_GENERATION: {
262 copy = allocate_from_gclab(thread, size);
263 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
264 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
265 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
266 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
267 copy = allocate_from_gclab(thread, size);
268 // If we still get nullptr, we'll try a shared allocation below.
269 }
270 break;
322 if (from_region->is_young()) {
323 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
324 old_generation()->handle_failed_promotion(thread, size);
325 return nullptr;
326 } else {
327 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
328 // after the evacuation threads have finished.
329 old_generation()->handle_failed_evacuation();
330 }
331 }
332
333 control_thread()->handle_alloc_failure_evac(size);
334
335 oom_evac_handler()->handle_out_of_memory_during_evacuation();
336
337 return ShenandoahBarrierSet::resolve_forwarded(p);
338 }
339
340 // Copy the object:
341 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
342 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
343 oop copy_val = cast_to_oop(copy);
344
345 // Update the age of the evacuated object
346 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
347 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
348 }
349
350 // Try to install the new forwarding pointer.
351 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
352 if (result == copy_val) {
353 // Successfully evacuated. Our copy is now the public one!
354
355 // This is necessary for virtual thread support. This uses the mark word without
356 // considering that it may now be a forwarding pointer (and could therefore crash).
357 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
358 // that lost the evacuation race (and will therefore not become visible). It is
359 // safe to do this on the public copy (this is also done during concurrent mark).
360 copy_val->initialize_hash_if_necessary(p);
361 ContinuationGCSupport::relativize_stack_chunk(copy_val);
362
363 // Record that the evacuation succeeded
364 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
365
366 if (target_gen == OLD_GENERATION) {
367 old_generation()->handle_evacuation(copy, size, from_region->is_young());
368 } else {
369 // When copying to the old generation above, we don't care
370 // about recording object age in the census stats.
371 assert(target_gen == YOUNG_GENERATION, "Error");
372 // We record this census only when simulating pre-adaptive tenuring behavior, or
373 // when we have been asked to record the census at evacuation rather than at mark
374 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
375 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
376 }
377 }
378 shenandoah_assert_correct(nullptr, copy_val);
379 return copy_val;
380 } else {
|