463 // Private inline function, for direct internal use and providing the
464 // implementation of the public not-inline function.
465 MAYBE_INLINE_EVACUATION
466 oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr,
467 oop const old,
468 markWord const old_mark) {
469 assert(region_attr.is_in_cset(),
470 "Unexpected region attr type: %s", region_attr.get_type_str());
471
472 // NOTE: With compact headers, it is not safe to load the Klass* from old, because
473 // that would access the mark-word, that might change at any time by concurrent
474 // workers.
475 // This mark word would refer to a forwardee, which may not yet have completed
476 // copying. Therefore we must load the Klass* from the mark-word that we already
477 // loaded. This is safe, because we only enter here if not yet forwarded.
478 assert(!old_mark.is_forwarded(), "precondition");
479 Klass* klass = UseCompactObjectHeaders
480 ? old_mark.klass()
481 : old->klass();
482
483 const size_t word_sz = old->size_given_klass(klass);
484
485 // JNI only allows pinning of typeArrays, so we only need to keep those in place.
486 if (region_attr.is_pinned() && klass->is_typeArray_klass()) {
487 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, true /* cause_pinned */);
488 }
489
490 uint age = 0;
491 G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
492 G1HeapRegion* const from_region = _g1h->heap_region_containing(old);
493 uint node_index = from_region->node_index();
494
495 HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_attr, word_sz, node_index);
496
497 // PLAB allocations should succeed most of the time, so we'll
498 // normally check against null once and that's it.
499 if (obj_ptr == nullptr) {
500 obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index);
501 if (obj_ptr == nullptr) {
502 // This will either forward-to-self, or detect that someone else has
503 // installed a forwarding pointer.
504 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, false /* cause_pinned */);
505 }
506 }
507
508 assert(obj_ptr != nullptr, "when we get here, allocation should have succeeded");
509 assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
510
511 // Should this evacuation fail?
512 if (inject_allocation_failure(from_region->hrm_index())) {
513 // Doing this after all the allocation attempts also tests the
514 // undo_allocation() method too.
515 undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
516 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, false /* cause_pinned */);
517 }
518
519 // We're going to allocate linearly, so might as well prefetch ahead.
520 Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
521 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), obj_ptr, word_sz);
522
523 const oop obj = cast_to_oop(obj_ptr);
524 // Because the forwarding is done with memory_order_relaxed there is no
525 // ordering with the above copy. Clients that get the forwardee must not
526 // examine its contents without other synchronization, since the contents
527 // may not be up to date for them.
528 const oop forward_ptr = old->forward_to_atomic(obj, old_mark, memory_order_relaxed);
529 if (forward_ptr == nullptr) {
530
531 {
532 const uint young_index = from_region->young_index_in_cset();
533 assert((from_region->is_young() && young_index > 0) ||
534 (!from_region->is_young() && young_index == 0), "invariant" );
535 _surviving_young_words[young_index] += word_sz;
536 }
537
538 if (dest_attr.is_young()) {
539 if (age < markWord::max_age) {
540 age++;
541 obj->incr_age();
542 }
543 _age_table.add(age, word_sz);
544 } else {
545 update_bot_after_copying(obj, word_sz);
546 }
547
548 {
549 // Skip the card enqueue iff the object (obj) is in survivor region.
550 // However, G1HeapRegion::is_survivor() is too expensive here.
551 // Instead, we use dest_attr.is_young() because the two values are always
552 // equal: successfully allocated young regions must be survivor regions.
553 assert(dest_attr.is_young() == _g1h->heap_region_containing(obj)->is_survivor(), "must be");
554 G1SkipCardMarkSetter x(&_scanner, dest_attr.is_young());
555 do_iterate_object(obj, old, klass, region_attr, dest_attr, age);
556 }
557
|
463 // Private inline function, for direct internal use and providing the
464 // implementation of the public not-inline function.
465 MAYBE_INLINE_EVACUATION
466 oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr,
467 oop const old,
468 markWord const old_mark) {
469 assert(region_attr.is_in_cset(),
470 "Unexpected region attr type: %s", region_attr.get_type_str());
471
472 // NOTE: With compact headers, it is not safe to load the Klass* from old, because
473 // that would access the mark-word, that might change at any time by concurrent
474 // workers.
475 // This mark word would refer to a forwardee, which may not yet have completed
476 // copying. Therefore we must load the Klass* from the mark-word that we already
477 // loaded. This is safe, because we only enter here if not yet forwarded.
478 assert(!old_mark.is_forwarded(), "precondition");
479 Klass* klass = UseCompactObjectHeaders
480 ? old_mark.klass()
481 : old->klass();
482
483 const size_t old_size = old->size_given_mark_and_klass(old_mark, klass);
484 const size_t word_sz = old->copy_size(old_size, old_mark);
485
486 // JNI only allows pinning of typeArrays, so we only need to keep those in place.
487 if (region_attr.is_pinned() && klass->is_typeArray_klass()) {
488 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, true /* cause_pinned */);
489 }
490
491 uint age = 0;
492 G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
493 G1HeapRegion* const from_region = _g1h->heap_region_containing(old);
494 uint node_index = from_region->node_index();
495
496 HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_attr, word_sz, node_index);
497
498 // PLAB allocations should succeed most of the time, so we'll
499 // normally check against null once and that's it.
500 if (obj_ptr == nullptr) {
501 obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index);
502 if (obj_ptr == nullptr) {
503 // This will either forward-to-self, or detect that someone else has
504 // installed a forwarding pointer.
505 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, false /* cause_pinned */);
506 }
507 }
508
509 assert(obj_ptr != nullptr, "when we get here, allocation should have succeeded");
510 assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
511
512 // Should this evacuation fail?
513 if (inject_allocation_failure(from_region->hrm_index())) {
514 // Doing this after all the allocation attempts also tests the
515 // undo_allocation() method too.
516 undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
517 return handle_evacuation_failure_par(old, old_mark, klass, region_attr, word_sz, false /* cause_pinned */);
518 }
519
520 // We're going to allocate linearly, so might as well prefetch ahead.
521 Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
522 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), obj_ptr, old_size);
523
524 const oop obj = cast_to_oop(obj_ptr);
525 // Because the forwarding is done with memory_order_relaxed there is no
526 // ordering with the above copy. Clients that get the forwardee must not
527 // examine its contents without other synchronization, since the contents
528 // may not be up to date for them.
529 const oop forward_ptr = old->forward_to_atomic(obj, old_mark, memory_order_relaxed);
530 if (forward_ptr == nullptr) {
531
532 {
533 const uint young_index = from_region->young_index_in_cset();
534 assert((from_region->is_young() && young_index > 0) ||
535 (!from_region->is_young() && young_index == 0), "invariant" );
536 _surviving_young_words[young_index] += word_sz;
537 }
538
539 obj->initialize_hash_if_necessary(old);
540
541 if (dest_attr.is_young()) {
542 if (age < markWord::max_age) {
543 age++;
544 obj->incr_age();
545 }
546 _age_table.add(age, word_sz);
547 } else {
548 update_bot_after_copying(obj, word_sz);
549 }
550
551 {
552 // Skip the card enqueue iff the object (obj) is in survivor region.
553 // However, G1HeapRegion::is_survivor() is too expensive here.
554 // Instead, we use dest_attr.is_young() because the two values are always
555 // equal: successfully allocated young regions must be survivor regions.
556 assert(dest_attr.is_young() == _g1h->heap_region_containing(obj)->is_survivor(), "must be");
557 G1SkipCardMarkSetter x(&_scanner, dest_attr.is_young());
558 do_iterate_object(obj, old, klass, region_attr, dest_attr, age);
559 }
560
|