222 // We don't want to deal with MT here just to ensure we read the right mark word.
223 // Skip the potential promotion attempt for this one.
224 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
225 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
226 if (result != nullptr) {
227 return result;
228 }
229 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
230 }
231 }
232 return try_evacuate_object(p, thread, r, target_gen);
233 }
234
235 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
236 // to OLD_GENERATION.
237 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
238 ShenandoahAffiliation target_gen) {
239 bool alloc_from_lab = true;
240 bool has_plab = false;
241 HeapWord* copy = nullptr;
242 size_t size = ShenandoahForwarding::size(p);
243 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
244
245 #ifdef ASSERT
246 if (ShenandoahOOMDuringEvacALot &&
247 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
248 copy = nullptr;
249 } else {
250 #endif
251 if (UseTLAB) {
252 switch (target_gen) {
253 case YOUNG_GENERATION: {
254 copy = allocate_from_gclab(thread, size);
255 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
256 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
257 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
258 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
259 copy = allocate_from_gclab(thread, size);
260 // If we still get nullptr, we'll try a shared allocation below.
261 }
262 break;
314 if (from_region->is_young()) {
315 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
316 old_generation()->handle_failed_promotion(thread, size);
317 return nullptr;
318 } else {
319 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
320 // after the evacuation threads have finished.
321 old_generation()->handle_failed_evacuation();
322 }
323 }
324
325 control_thread()->handle_alloc_failure_evac(size);
326
327 oom_evac_handler()->handle_out_of_memory_during_evacuation();
328
329 return ShenandoahBarrierSet::resolve_forwarded(p);
330 }
331
332 // Copy the object:
333 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
334 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
335 oop copy_val = cast_to_oop(copy);
336
337 // Update the age of the evacuated object
338 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
339 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
340 }
341
342 // Try to install the new forwarding pointer.
343 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
344 if (result == copy_val) {
345 // Successfully evacuated. Our copy is now the public one!
346
347 // This is necessary for virtual thread support. This uses the mark word without
348 // considering that it may now be a forwarding pointer (and could therefore crash).
349 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
350 // that lost the evacuation race (and will therefore not become visible). It is
351 // safe to do this on the public copy (this is also done during concurrent mark).
352 ContinuationGCSupport::relativize_stack_chunk(copy_val);
353
354 // Record that the evacuation succeeded
355 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
356
357 if (target_gen == OLD_GENERATION) {
358 old_generation()->handle_evacuation(copy, size, from_region->is_young());
359 } else {
360 // When copying to the old generation above, we don't care
361 // about recording object age in the census stats.
362 assert(target_gen == YOUNG_GENERATION, "Error");
363 // We record this census only when simulating pre-adaptive tenuring behavior, or
364 // when we have been asked to record the census at evacuation rather than at mark
365 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
366 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
367 }
368 }
369 shenandoah_assert_correct(nullptr, copy_val);
370 return copy_val;
371 } else {
|
222 // We don't want to deal with MT here just to ensure we read the right mark word.
223 // Skip the potential promotion attempt for this one.
224 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
225 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
226 if (result != nullptr) {
227 return result;
228 }
229 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
230 }
231 }
232 return try_evacuate_object(p, thread, r, target_gen);
233 }
234
235 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
236 // to OLD_GENERATION.
237 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
238 ShenandoahAffiliation target_gen) {
239 bool alloc_from_lab = true;
240 bool has_plab = false;
241 HeapWord* copy = nullptr;
242
243 markWord mark = p->mark();
244 if (ShenandoahForwarding::is_forwarded(mark)) {
245 return ShenandoahForwarding::get_forwardee(p);
246 }
247 size_t old_size = ShenandoahForwarding::size(p);
248 size_t size = p->copy_size(old_size, mark);
249
250 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
251
252 #ifdef ASSERT
253 if (ShenandoahOOMDuringEvacALot &&
254 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
255 copy = nullptr;
256 } else {
257 #endif
258 if (UseTLAB) {
259 switch (target_gen) {
260 case YOUNG_GENERATION: {
261 copy = allocate_from_gclab(thread, size);
262 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
263 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
264 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
265 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
266 copy = allocate_from_gclab(thread, size);
267 // If we still get nullptr, we'll try a shared allocation below.
268 }
269 break;
321 if (from_region->is_young()) {
322 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
323 old_generation()->handle_failed_promotion(thread, size);
324 return nullptr;
325 } else {
326 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
327 // after the evacuation threads have finished.
328 old_generation()->handle_failed_evacuation();
329 }
330 }
331
332 control_thread()->handle_alloc_failure_evac(size);
333
334 oom_evac_handler()->handle_out_of_memory_during_evacuation();
335
336 return ShenandoahBarrierSet::resolve_forwarded(p);
337 }
338
339 // Copy the object:
340 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
341 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
342 oop copy_val = cast_to_oop(copy);
343
344 // Update the age of the evacuated object
345 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
346 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
347 }
348
349 // Try to install the new forwarding pointer.
350 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
351 if (result == copy_val) {
352 // Successfully evacuated. Our copy is now the public one!
353
354 // This is necessary for virtual thread support. This uses the mark word without
355 // considering that it may now be a forwarding pointer (and could therefore crash).
356 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
357 // that lost the evacuation race (and will therefore not become visible). It is
358 // safe to do this on the public copy (this is also done during concurrent mark).
359 copy_val->initialize_hash_if_necessary(p);
360 ContinuationGCSupport::relativize_stack_chunk(copy_val);
361
362 // Record that the evacuation succeeded
363 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
364
365 if (target_gen == OLD_GENERATION) {
366 old_generation()->handle_evacuation(copy, size, from_region->is_young());
367 } else {
368 // When copying to the old generation above, we don't care
369 // about recording object age in the census stats.
370 assert(target_gen == YOUNG_GENERATION, "Error");
371 // We record this census only when simulating pre-adaptive tenuring behavior, or
372 // when we have been asked to record the census at evacuation rather than at mark
373 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
374 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
375 }
376 }
377 shenandoah_assert_correct(nullptr, copy_val);
378 return copy_val;
379 } else {
|