222
223 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
224 if (result != nullptr) {
225 return result;
226 }
227 }
228 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
229 }
230
231 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
232 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
233 }
234
235 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
236 // to OLD_GENERATION.
237 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
238 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
239 bool alloc_from_lab = true;
240 bool has_plab = false;
241 HeapWord* copy = nullptr;
242 size_t size = ShenandoahForwarding::size(p);
243 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
244
245 #ifdef ASSERT
246 if (ShenandoahOOMDuringEvacALot &&
247 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
248 copy = nullptr;
249 } else {
250 #endif
251 if (UseTLAB) {
252 switch (TO_GENERATION) {
253 case YOUNG_GENERATION: {
254 copy = allocate_from_gclab(thread, size);
255 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
256 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
257 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
258 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
259 copy = allocate_from_gclab(thread, size);
260 // If we still get nullptr, we'll try a shared allocation below.
261 }
262 break;
329 // self-forwarded first.
330 markWord old_mark = p->mark();
331 if (old_mark.is_forwarded()) {
332 return ShenandoahForwarding::get_forwardee(p);
333 }
334 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
335 if (winner == nullptr) {
336 // We own the self-forwarding. Flag the from-region so the degen/full
337 // GC entry drain knows to scan it for self_fwd bits to clear.
338 heap_region_containing(p)->set_has_self_forwards();
339 return p;
340 }
341 return winner;
342 }
343
344 if (ShenandoahEvacTracking) {
345 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
346 }
347
348 // Copy the object:
349 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
350 oop copy_val = cast_to_oop(copy);
351
352 // Update the age of the evacuated object
353 if (TO_GENERATION == YOUNG_GENERATION && is_aging_cycle()) {
354 increase_object_age(copy_val, from_region_age + 1);
355 }
356
357 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
358 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
359 // relativize derived pointers and set gc_mode before that happens. Skip if the
360 // copy's mark word is already a forwarding pointer (another thread won the race
361 // and overwrote the original's header before we copied it).
362 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
363 ContinuationGCSupport::relativize_stack_chunk(copy_val);
364 }
365
366 // Try to install the new forwarding pointer.
367 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
368 if (result == copy_val) {
369 // Successfully evacuated. Our copy is now the public one!
370 if (ShenandoahEvacTracking) {
371 // Record that the evacuation succeeded
|
222
223 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
224 if (result != nullptr) {
225 return result;
226 }
227 }
228 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
229 }
230
231 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
232 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
233 }
234
235 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
236 // to OLD_GENERATION.
237 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
238 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
239 bool alloc_from_lab = true;
240 bool has_plab = false;
241 HeapWord* copy = nullptr;
242
243 markWord mark = p->mark();
244 if (ShenandoahForwarding::is_forwarded(mark)) {
245 return ShenandoahForwarding::get_forwardee(p);
246 }
247 size_t old_size = ShenandoahForwarding::size(p);
248 size_t size = p->copy_size(old_size, mark);
249
250 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
251
252 #ifdef ASSERT
253 if (ShenandoahOOMDuringEvacALot &&
254 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
255 copy = nullptr;
256 } else {
257 #endif
258 if (UseTLAB) {
259 switch (TO_GENERATION) {
260 case YOUNG_GENERATION: {
261 copy = allocate_from_gclab(thread, size);
262 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
263 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
264 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
265 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
266 copy = allocate_from_gclab(thread, size);
267 // If we still get nullptr, we'll try a shared allocation below.
268 }
269 break;
336 // self-forwarded first.
337 markWord old_mark = p->mark();
338 if (old_mark.is_forwarded()) {
339 return ShenandoahForwarding::get_forwardee(p);
340 }
341 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
342 if (winner == nullptr) {
343 // We own the self-forwarding. Flag the from-region so the degen/full
344 // GC entry drain knows to scan it for self_fwd bits to clear.
345 heap_region_containing(p)->set_has_self_forwards();
346 return p;
347 }
348 return winner;
349 }
350
351 if (ShenandoahEvacTracking) {
352 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
353 }
354
355 // Copy the object:
356 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
357 oop copy_val = cast_to_oop(copy);
358
359 // Initialize the identity hash on the copy before installing the forwarding
360 // pointer, using the mark word we captured earlier. We must do this before
361 // the CAS so that the copy is fully initialized when it becomes visible to
362 // other threads. Using the captured mark (rather than re-reading the copy's
363 // mark) avoids races with other threads that may have evacuated p and
364 // installed a forwarding pointer in the meantime.
365 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
366 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
367 }
368
369 // Update the age of the evacuated object
370 if (TO_GENERATION == YOUNG_GENERATION && is_aging_cycle()) {
371 increase_object_age(copy_val, from_region_age + 1);
372 }
373
374 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
375 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
376 // relativize derived pointers and set gc_mode before that happens. Skip if the
377 // copy's mark word is already a forwarding pointer (another thread won the race
378 // and overwrote the original's header before we copied it).
379 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
380 ContinuationGCSupport::relativize_stack_chunk(copy_val);
381 }
382
383 // Try to install the new forwarding pointer.
384 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
385 if (result == copy_val) {
386 // Successfully evacuated. Our copy is now the public one!
387 if (ShenandoahEvacTracking) {
388 // Record that the evacuation succeeded
|