267 // No GCLABs in this thread, fallback to shared allocation
268 return nullptr;
269 }
270 HeapWord* obj = gclab->allocate(size);
271 if (obj != nullptr) {
272 return obj;
273 }
274 // Otherwise...
275 return allocate_from_gclab_slow(thread, size);
276 }
277
278 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
279 if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
280 // This thread went through the OOM during evac protocol and it is safe to return
281 // the forward pointer. It must not attempt to evacuate any more.
282 return ShenandoahBarrierSet::resolve_forwarded(p);
283 }
284
285 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
286
287 size_t size = p->size();
288
289 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
290
291 bool alloc_from_gclab = true;
292 HeapWord* copy = nullptr;
293
294 #ifdef ASSERT
295 if (ShenandoahOOMDuringEvacALot &&
296 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
297 copy = nullptr;
298 } else {
299 #endif
300 if (UseTLAB) {
301 copy = allocate_from_gclab(thread, size);
302 }
303 if (copy == nullptr) {
304 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
305 copy = allocate_memory(req);
306 alloc_from_gclab = false;
307 }
308 #ifdef ASSERT
309 }
310 #endif
311
312 if (copy == nullptr) {
313 control_thread()->handle_alloc_failure_evac(size);
314
315 _oom_evac_handler.handle_out_of_memory_during_evacuation();
316
317 return ShenandoahBarrierSet::resolve_forwarded(p);
318 }
319
320 // Copy the object:
321 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
322
323 // Try to install the new forwarding pointer.
324 oop copy_val = cast_to_oop(copy);
325 ContinuationGCSupport::relativize_stack_chunk(copy_val);
326
327 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
328 if (result == copy_val) {
329 // Successfully evacuated. Our copy is now the public one!
330 shenandoah_assert_correct(nullptr, copy_val);
331 return copy_val;
332 } else {
333 // Failed to evacuate. We need to deal with the object that is left behind. Since this
334 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
335 // But if it happens to contain references to evacuated regions, those references would
336 // not get updated for this stale copy during this cycle, and we will crash while scanning
337 // it the next cycle.
338 //
339 // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
340 // object will overwrite this stale copy, or the filler object on LAB retirement will
341 // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
342 // have to explicitly overwrite the copy with the filler object. With that overwrite,
343 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
344 if (alloc_from_gclab) {
345 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
346 } else {
486 assert(oopDesc::is_oop(obj), "sanity");
487 assert(ctx->is_marked(obj), "object expected to be marked");
488 cl->do_object(obj);
489 cb += skip_bitmap_delta;
490 if (cb < limit_bitmap) {
491 cb = ctx->get_next_marked_addr(cb, limit_bitmap);
492 }
493 }
494 }
495
496 // Step 2. Accurate size-based traversal, happens past the TAMS.
497 // This restarts the scan at TAMS, which makes sure we traverse all objects,
498 // regardless of what happened at Step 1.
499 HeapWord* cs = tams;
500 while (cs < limit) {
501 assert (cs >= tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
502 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
503 oop obj = cast_to_oop(cs);
504 assert(oopDesc::is_oop(obj), "sanity");
505 assert(ctx->is_marked(obj), "object expected to be marked");
506 size_t size = obj->size();
507 cl->do_object(obj);
508 cs += size;
509 }
510 }
511
512 template <class T>
513 class ShenandoahObjectToOopClosure : public ObjectClosure {
514 T* _cl;
515 public:
516 ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {}
517
518 void do_object(oop obj) {
519 obj->oop_iterate(_cl);
520 }
521 };
522
523 template <class T>
524 class ShenandoahObjectToOopBoundedClosure : public ObjectClosure {
525 T* _cl;
526 MemRegion _bounds;
|
267 // No GCLABs in this thread, fallback to shared allocation
268 return nullptr;
269 }
270 HeapWord* obj = gclab->allocate(size);
271 if (obj != nullptr) {
272 return obj;
273 }
274 // Otherwise...
275 return allocate_from_gclab_slow(thread, size);
276 }
277
278 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
279 if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
280 // This thread went through the OOM during evac protocol and it is safe to return
281 // the forward pointer. It must not attempt to evacuate any more.
282 return ShenandoahBarrierSet::resolve_forwarded(p);
283 }
284
285 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
286
287 size_t size = p->forward_safe_size();
288
289 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
290
291 bool alloc_from_gclab = true;
292 HeapWord* copy = nullptr;
293
294 #ifdef ASSERT
295 if (ShenandoahOOMDuringEvacALot &&
296 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
297 copy = nullptr;
298 } else {
299 #endif
300 if (UseTLAB) {
301 copy = allocate_from_gclab(thread, size);
302 }
303 if (copy == nullptr) {
304 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
305 copy = allocate_memory(req);
306 alloc_from_gclab = false;
307 }
308 #ifdef ASSERT
309 }
310 #endif
311
312 if (copy == nullptr) {
313 control_thread()->handle_alloc_failure_evac(size);
314
315 _oom_evac_handler.handle_out_of_memory_during_evacuation();
316
317 return ShenandoahBarrierSet::resolve_forwarded(p);
318 }
319
320 // Copy the object:
321 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
322 oop copy_val = cast_to_oop(copy);
323
324 if (UseCompactObjectHeaders) {
325 // The copy above is not atomic. Make sure we have seen the proper mark
326 // and re-install it into the copy, so that Klass* is guaranteed to be correct.
327 markWord mark = copy_val->mark();
328 if (!mark.is_marked()) {
329 copy_val->set_mark(mark);
330 ContinuationGCSupport::relativize_stack_chunk(copy_val);
331 } else {
332 // If we copied a mark-word that indicates 'forwarded' state, the object
333 // installation would not succeed. We cannot access Klass* anymore either.
334 // Skip the transformation.
335 }
336 } else {
337 ContinuationGCSupport::relativize_stack_chunk(copy_val);
338 }
339
340 // Try to install the new forwarding pointer.
341 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
342 if (result == copy_val) {
343 // Successfully evacuated. Our copy is now the public one!
344 shenandoah_assert_correct(nullptr, copy_val);
345 return copy_val;
346 } else {
347 // Failed to evacuate. We need to deal with the object that is left behind. Since this
348 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
349 // But if it happens to contain references to evacuated regions, those references would
350 // not get updated for this stale copy during this cycle, and we will crash while scanning
351 // it the next cycle.
352 //
353 // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
354 // object will overwrite this stale copy, or the filler object on LAB retirement will
355 // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
356 // have to explicitly overwrite the copy with the filler object. With that overwrite,
357 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
358 if (alloc_from_gclab) {
359 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
360 } else {
500 assert(oopDesc::is_oop(obj), "sanity");
501 assert(ctx->is_marked(obj), "object expected to be marked");
502 cl->do_object(obj);
503 cb += skip_bitmap_delta;
504 if (cb < limit_bitmap) {
505 cb = ctx->get_next_marked_addr(cb, limit_bitmap);
506 }
507 }
508 }
509
510 // Step 2. Accurate size-based traversal, happens past the TAMS.
511 // This restarts the scan at TAMS, which makes sure we traverse all objects,
512 // regardless of what happened at Step 1.
513 HeapWord* cs = tams;
514 while (cs < limit) {
515 assert (cs >= tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
516 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
517 oop obj = cast_to_oop(cs);
518 assert(oopDesc::is_oop(obj), "sanity");
519 assert(ctx->is_marked(obj), "object expected to be marked");
520 size_t size = obj->forward_safe_size();
521 cl->do_object(obj);
522 cs += size;
523 }
524 }
525
526 template <class T>
527 class ShenandoahObjectToOopClosure : public ObjectClosure {
528 T* _cl;
529 public:
530 ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {}
531
532 void do_object(oop obj) {
533 obj->oop_iterate(_cl);
534 }
535 };
536
537 template <class T>
538 class ShenandoahObjectToOopBoundedClosure : public ObjectClosure {
539 T* _cl;
540 MemRegion _bounds;
|