219
220 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
221 if (result != nullptr) {
222 return result;
223 }
224 }
225 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
226 }
227
228 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
229 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
230 }
231
232 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
233 // to OLD_GENERATION.
234 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
235 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
236 bool alloc_from_lab = true;
237 bool has_plab = false;
238 HeapWord* copy = nullptr;
239 size_t size = ShenandoahForwarding::size(p);
240 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
241
242 #ifdef ASSERT
243 if (ShenandoahOOMDuringEvacALot &&
244 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
245 copy = nullptr;
246 } else {
247 #endif
248 if (UseTLAB) {
249 switch (TO_GENERATION) {
250 case YOUNG_GENERATION: {
251 copy = allocate_from_gclab(thread, size);
252 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
253 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
254 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
255 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
256 copy = allocate_from_gclab(thread, size);
257 // If we still get nullptr, we'll try a shared allocation below.
258 }
259 break;
326 // self-forwarded first.
327 markWord old_mark = p->mark();
328 if (old_mark.is_forwarded()) {
329 return ShenandoahForwarding::get_forwardee(p);
330 }
331 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
332 if (winner == nullptr) {
333 // We own the self-forwarding. Flag the from-region so the degen/full
334 // GC entry drain knows to scan it for self_fwd bits to clear.
335 heap_region_containing(p)->set_has_self_forwards();
336 return p;
337 }
338 return winner;
339 }
340
341 if (ShenandoahEvacTracking) {
342 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
343 }
344
345 // Copy the object:
346 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
347 oop copy_val = cast_to_oop(copy);
348
349 // Update the age of the evacuated object
350 if (TO_GENERATION == YOUNG_GENERATION) {
351 increase_object_age(copy_val, from_region_age + 1);
352 }
353
354 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
355 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
356 // relativize derived pointers and set gc_mode before that happens. Skip if the
357 // copy's mark word is already a forwarding pointer (another thread won the race
358 // and overwrote the original's header before we copied it).
359 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
360 ContinuationGCSupport::relativize_stack_chunk(copy_val);
361 }
362
363 // Try to install the new forwarding pointer.
364 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
365 if (result == copy_val) {
366 // Successfully evacuated. Our copy is now the public one!
367 if (ShenandoahEvacTracking) {
368 // Record that the evacuation succeeded
|
219
220 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
221 if (result != nullptr) {
222 return result;
223 }
224 }
225 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
226 }
227
228 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
229 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
230 }
231
232 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
233 // to OLD_GENERATION.
234 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
235 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
236 bool alloc_from_lab = true;
237 bool has_plab = false;
238 HeapWord* copy = nullptr;
239
240 markWord mark = p->mark();
241 if (mark.is_forwarded()) {
242 return ShenandoahForwarding::get_forwardee(p);
243 }
244 size_t old_size = ShenandoahForwarding::size(p);
245 size_t size = p->copy_size(old_size, mark);
246
247 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
248
249 #ifdef ASSERT
250 if (ShenandoahOOMDuringEvacALot &&
251 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
252 copy = nullptr;
253 } else {
254 #endif
255 if (UseTLAB) {
256 switch (TO_GENERATION) {
257 case YOUNG_GENERATION: {
258 copy = allocate_from_gclab(thread, size);
259 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
260 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
261 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
262 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
263 copy = allocate_from_gclab(thread, size);
264 // If we still get nullptr, we'll try a shared allocation below.
265 }
266 break;
333 // self-forwarded first.
334 markWord old_mark = p->mark();
335 if (old_mark.is_forwarded()) {
336 return ShenandoahForwarding::get_forwardee(p);
337 }
338 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
339 if (winner == nullptr) {
340 // We own the self-forwarding. Flag the from-region so the degen/full
341 // GC entry drain knows to scan it for self_fwd bits to clear.
342 heap_region_containing(p)->set_has_self_forwards();
343 return p;
344 }
345 return winner;
346 }
347
348 if (ShenandoahEvacTracking) {
349 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
350 }
351
352 // Copy the object:
353 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
354 oop copy_val = cast_to_oop(copy);
355
356 // Initialize the identity hash on the copy before installing the forwarding
357 // pointer, using the mark word we captured earlier. We must do this before
358 // the CAS so that the copy is fully initialized when it becomes visible to
359 // other threads. Using the captured mark (rather than re-reading the copy's
360 // mark) avoids races with other threads that may have evacuated p and
361 // installed a forwarding pointer in the meantime.
362 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
363 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
364 }
365
366 // Update the age of the evacuated object
367 if (TO_GENERATION == YOUNG_GENERATION) {
368 increase_object_age(copy_val, from_region_age + 1);
369 }
370
371 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
372 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
373 // relativize derived pointers and set gc_mode before that happens. Skip if the
374 // copy's mark word is already a forwarding pointer (another thread won the race
375 // and overwrote the original's header before we copied it).
376 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
377 ContinuationGCSupport::relativize_stack_chunk(copy_val);
378 }
379
380 // Try to install the new forwarding pointer.
381 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
382 if (result == copy_val) {
383 // Successfully evacuated. Our copy is now the public one!
384 if (ShenandoahEvacTracking) {
385 // Record that the evacuation succeeded
|