1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/vmClasses.hpp" 27 #include "classfile/vmSymbols.hpp" 28 #include "gc/shared/collectedHeap.inline.hpp" 29 #include "gc/shared/genCollectedHeap.hpp" 30 #include "gc/shared/slidingForwarding.inline.hpp" 31 #include "gc/shared/space.hpp" 32 #include "gc/shared/space.inline.hpp" 33 #include "gc/shared/spaceDecorator.inline.hpp" 34 #include "memory/iterator.inline.hpp" 35 #include "memory/universe.hpp" 36 #include "oops/oop.inline.hpp" 37 #include "runtime/atomic.hpp" 38 #include "runtime/java.hpp" 39 #include "runtime/prefetch.inline.hpp" 40 #include "runtime/safepoint.hpp" 41 #include "utilities/align.hpp" 42 #include "utilities/copy.hpp" 43 #include "utilities/globalDefinitions.hpp" 44 #include "utilities/macros.hpp" 45 #if INCLUDE_SERIALGC 46 #include "gc/serial/serialBlockOffsetTable.inline.hpp" 47 #include "gc/serial/defNewGeneration.hpp" 48 #endif 49 50 HeapWord* DirtyCardToOopClosure::get_actual_top(HeapWord* top, 51 HeapWord* top_obj) { 52 if (top_obj != nullptr && top_obj < (_sp->toContiguousSpace())->top()) { 53 if (cast_to_oop(top_obj)->is_objArray() || cast_to_oop(top_obj)->is_typeArray()) { 54 // An arrayOop is starting on the dirty card - since we do exact 55 // store checks for objArrays we are done. 56 } else { 57 // Otherwise, it is possible that the object starting on the dirty 58 // card spans the entire card, and that the store happened on a 59 // later card. Figure out where the object ends. 60 assert(_sp->block_size(top_obj) == cast_to_oop(top_obj)->size(), 61 "Block size and object size mismatch"); 62 top = top_obj + cast_to_oop(top_obj)->size(); 63 } 64 } else { 65 top = (_sp->toContiguousSpace())->top(); 66 } 67 return top; 68 } 69 70 void DirtyCardToOopClosure::walk_mem_region(MemRegion mr, 71 HeapWord* bottom, 72 HeapWord* top) { 73 // Note that this assumption won't hold if we have a concurrent 74 // collector in this space, which may have freed up objects after 75 // they were dirtied and before the stop-the-world GC that is 76 // examining cards here. 77 assert(bottom < top, "ought to be at least one obj on a dirty card."); 78 79 walk_mem_region_with_cl(mr, bottom, top, _cl); 80 } 81 82 // We get called with "mr" representing the dirty region 83 // that we want to process. Because of imprecise marking, 84 // we may need to extend the incoming "mr" to the right, 85 // and scan more. However, because we may already have 86 // scanned some of that extended region, we may need to 87 // trim its right-end back some so we do not scan what 88 // we (or another worker thread) may already have scanned 89 // or planning to scan. 90 void DirtyCardToOopClosure::do_MemRegion(MemRegion mr) { 91 HeapWord* bottom = mr.start(); 92 HeapWord* last = mr.last(); 93 HeapWord* top = mr.end(); 94 HeapWord* bottom_obj; 95 HeapWord* top_obj; 96 97 assert(_last_bottom == nullptr || top <= _last_bottom, 98 "Not decreasing"); 99 NOT_PRODUCT(_last_bottom = mr.start()); 100 101 bottom_obj = _sp->block_start(bottom); 102 top_obj = _sp->block_start(last); 103 104 assert(bottom_obj <= bottom, "just checking"); 105 assert(top_obj <= top, "just checking"); 106 107 // Given what we think is the top of the memory region and 108 // the start of the object at the top, get the actual 109 // value of the top. 110 top = get_actual_top(top, top_obj); 111 112 // If the previous call did some part of this region, don't redo. 113 if (_min_done != nullptr && _min_done < top) { 114 top = _min_done; 115 } 116 117 // Top may have been reset, and in fact may be below bottom, 118 // e.g. the dirty card region is entirely in a now free object 119 // -- something that could happen with a concurrent sweeper. 120 bottom = MIN2(bottom, top); 121 MemRegion extended_mr = MemRegion(bottom, top); 122 assert(bottom <= top && 123 (_min_done == nullptr || top <= _min_done), 124 "overlap!"); 125 126 // Walk the region if it is not empty; otherwise there is nothing to do. 127 if (!extended_mr.is_empty()) { 128 walk_mem_region(extended_mr, bottom_obj, top); 129 } 130 131 _min_done = bottom; 132 } 133 134 void DirtyCardToOopClosure::walk_mem_region_with_cl(MemRegion mr, 135 HeapWord* bottom, 136 HeapWord* top, 137 OopIterateClosure* cl) { 138 bottom += cast_to_oop(bottom)->oop_iterate_size(cl, mr); 139 if (bottom < top) { 140 HeapWord* next_obj = bottom + cast_to_oop(bottom)->size(); 141 while (next_obj < top) { 142 /* Bottom lies entirely below top, so we can call the */ 143 /* non-memRegion version of oop_iterate below. */ 144 cast_to_oop(bottom)->oop_iterate(cl); 145 bottom = next_obj; 146 next_obj = bottom + cast_to_oop(bottom)->size(); 147 } 148 /* Last object. */ 149 cast_to_oop(bottom)->oop_iterate(cl, mr); 150 } 151 } 152 153 void Space::initialize(MemRegion mr, 154 bool clear_space, 155 bool mangle_space) { 156 HeapWord* bottom = mr.start(); 157 HeapWord* end = mr.end(); 158 assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), 159 "invalid space boundaries"); 160 set_bottom(bottom); 161 set_end(end); 162 if (clear_space) clear(mangle_space); 163 } 164 165 void Space::clear(bool mangle_space) { 166 if (ZapUnusedHeapArea && mangle_space) { 167 mangle_unused_area(); 168 } 169 } 170 171 ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(nullptr) { 172 _mangler = new GenSpaceMangler(this); 173 } 174 175 ContiguousSpace::~ContiguousSpace() { 176 delete _mangler; 177 } 178 179 void ContiguousSpace::initialize(MemRegion mr, 180 bool clear_space, 181 bool mangle_space) 182 { 183 CompactibleSpace::initialize(mr, clear_space, mangle_space); 184 } 185 186 void ContiguousSpace::clear(bool mangle_space) { 187 set_top(bottom()); 188 set_saved_mark(); 189 CompactibleSpace::clear(mangle_space); 190 } 191 192 bool ContiguousSpace::is_free_block(const HeapWord* p) const { 193 return p >= _top; 194 } 195 196 #if INCLUDE_SERIALGC 197 void TenuredSpace::clear(bool mangle_space) { 198 ContiguousSpace::clear(mangle_space); 199 _offsets.initialize_threshold(); 200 } 201 202 void TenuredSpace::set_bottom(HeapWord* new_bottom) { 203 Space::set_bottom(new_bottom); 204 _offsets.set_bottom(new_bottom); 205 } 206 207 void TenuredSpace::set_end(HeapWord* new_end) { 208 // Space should not advertise an increase in size 209 // until after the underlying offset table has been enlarged. 210 _offsets.resize(pointer_delta(new_end, bottom())); 211 Space::set_end(new_end); 212 } 213 #endif // INCLUDE_SERIALGC 214 215 #ifndef PRODUCT 216 217 void ContiguousSpace::set_top_for_allocations(HeapWord* v) { 218 mangler()->set_top_for_allocations(v); 219 } 220 void ContiguousSpace::set_top_for_allocations() { 221 mangler()->set_top_for_allocations(top()); 222 } 223 void ContiguousSpace::check_mangled_unused_area(HeapWord* limit) { 224 mangler()->check_mangled_unused_area(limit); 225 } 226 227 void ContiguousSpace::check_mangled_unused_area_complete() { 228 mangler()->check_mangled_unused_area_complete(); 229 } 230 231 // Mangled only the unused space that has not previously 232 // been mangled and that has not been allocated since being 233 // mangled. 234 void ContiguousSpace::mangle_unused_area() { 235 mangler()->mangle_unused_area(); 236 } 237 void ContiguousSpace::mangle_unused_area_complete() { 238 mangler()->mangle_unused_area_complete(); 239 } 240 #endif // NOT_PRODUCT 241 242 void CompactibleSpace::initialize(MemRegion mr, 243 bool clear_space, 244 bool mangle_space) { 245 Space::initialize(mr, clear_space, mangle_space); 246 set_compaction_top(bottom()); 247 _next_compaction_space = nullptr; 248 } 249 250 void CompactibleSpace::clear(bool mangle_space) { 251 Space::clear(mangle_space); 252 _compaction_top = bottom(); 253 } 254 255 HeapWord* CompactibleSpace::forward(oop q, size_t size, 256 CompactPoint* cp, HeapWord* compact_top, SlidingForwarding* const forwarding) { 257 // q is alive 258 // First check if we should switch compaction space 259 assert(this == cp->space, "'this' should be current compaction space."); 260 size_t compaction_max_size = pointer_delta(end(), compact_top); 261 while (size > compaction_max_size) { 262 // switch to next compaction space 263 cp->space->set_compaction_top(compact_top); 264 cp->space = cp->space->next_compaction_space(); 265 if (cp->space == nullptr) { 266 cp->gen = GenCollectedHeap::heap()->young_gen(); 267 assert(cp->gen != nullptr, "compaction must succeed"); 268 cp->space = cp->gen->first_compaction_space(); 269 assert(cp->space != nullptr, "generation must have a first compaction space"); 270 } 271 compact_top = cp->space->bottom(); 272 cp->space->set_compaction_top(compact_top); 273 cp->space->initialize_threshold(); 274 compaction_max_size = pointer_delta(cp->space->end(), compact_top); 275 } 276 277 // store the forwarding pointer into the mark word 278 if (cast_from_oop<HeapWord*>(q) != compact_top) { 279 forwarding->forward_to(q, cast_to_oop(compact_top)); 280 assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); 281 } else { 282 // if the object isn't moving we can just set the mark to the default 283 // mark and handle it specially later on. 284 q->init_mark(); 285 assert(!q->is_forwarded(), "should not be forwarded"); 286 } 287 288 compact_top += size; 289 290 // We need to update the offset table so that the beginnings of objects can be 291 // found during scavenge. Note that we are updating the offset table based on 292 // where the object will be once the compaction phase finishes. 293 cp->space->alloc_block(compact_top - size, compact_top); 294 return compact_top; 295 } 296 297 #if INCLUDE_SERIALGC 298 299 void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) { 300 // Compute the new addresses for the live objects and store it in the mark 301 // Used by universe::mark_sweep_phase2() 302 303 // We're sure to be here before any objects are compacted into this 304 // space, so this is a good time to initialize this: 305 set_compaction_top(bottom()); 306 307 if (cp->space == nullptr) { 308 assert(cp->gen != nullptr, "need a generation"); 309 assert(cp->gen->first_compaction_space() == this, "just checking"); 310 cp->space = cp->gen->first_compaction_space(); 311 cp->space->initialize_threshold(); 312 cp->space->set_compaction_top(cp->space->bottom()); 313 } 314 315 HeapWord* compact_top = cp->space->compaction_top(); // This is where we are currently compacting to. 316 317 DeadSpacer dead_spacer(this); 318 319 HeapWord* end_of_live = bottom(); // One byte beyond the last byte of the last live object. 320 HeapWord* first_dead = nullptr; // The first dead object. 321 322 const intx interval = PrefetchScanIntervalInBytes; 323 324 HeapWord* cur_obj = bottom(); 325 HeapWord* scan_limit = top(); 326 327 SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding(); 328 while (cur_obj < scan_limit) { 329 if (cast_to_oop(cur_obj)->is_gc_marked()) { 330 // prefetch beyond cur_obj 331 Prefetch::write(cur_obj, interval); 332 size_t size = cast_to_oop(cur_obj)->size(); 333 compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top, forwarding); 334 cur_obj += size; 335 end_of_live = cur_obj; 336 } else { 337 // run over all the contiguous dead objects 338 HeapWord* end = cur_obj; 339 do { 340 // prefetch beyond end 341 Prefetch::write(end, interval); 342 end += cast_to_oop(end)->size(); 343 } while (end < scan_limit && !cast_to_oop(end)->is_gc_marked()); 344 345 // see if we might want to pretend this object is alive so that 346 // we don't have to compact quite as often. 347 if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) { 348 oop obj = cast_to_oop(cur_obj); 349 compact_top = cp->space->forward(obj, obj->size(), cp, compact_top, forwarding); 350 end_of_live = end; 351 } else { 352 // otherwise, it really is a free region. 353 354 // cur_obj is a pointer to a dead object. Use this dead memory to store a pointer to the next live object. 355 *(HeapWord**)cur_obj = end; 356 357 // see if this is the first dead region. 358 if (first_dead == nullptr) { 359 first_dead = cur_obj; 360 } 361 } 362 363 // move on to the next object 364 cur_obj = end; 365 } 366 } 367 368 assert(cur_obj == scan_limit, "just checking"); 369 _end_of_live = end_of_live; 370 if (first_dead != nullptr) { 371 _first_dead = first_dead; 372 } else { 373 _first_dead = end_of_live; 374 } 375 376 // save the compaction_top of the compaction space. 377 cp->space->set_compaction_top(compact_top); 378 } 379 380 void CompactibleSpace::adjust_pointers() { 381 // Check first is there is any work to do. 382 if (used() == 0) { 383 return; // Nothing to do. 384 } 385 386 // adjust all the interior pointers to point at the new locations of objects 387 // Used by MarkSweep::mark_sweep_phase3() 388 389 HeapWord* cur_obj = bottom(); 390 HeapWord* const end_of_live = _end_of_live; // Established by prepare_for_compaction(). 391 HeapWord* const first_dead = _first_dead; // Established by prepare_for_compaction(). 392 const SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding(); 393 394 assert(first_dead <= end_of_live, "Stands to reason, no?"); 395 396 const intx interval = PrefetchScanIntervalInBytes; 397 398 debug_only(HeapWord* prev_obj = nullptr); 399 while (cur_obj < end_of_live) { 400 Prefetch::write(cur_obj, interval); 401 if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) { 402 // cur_obj is alive 403 // point all the oops to the new location 404 size_t size = MarkSweep::adjust_pointers(forwarding, cast_to_oop(cur_obj)); 405 debug_only(prev_obj = cur_obj); 406 cur_obj += size; 407 } else { 408 debug_only(prev_obj = cur_obj); 409 // cur_obj is not a live object, instead it points at the next live object 410 cur_obj = *(HeapWord**)cur_obj; 411 assert(cur_obj > prev_obj, "we should be moving forward through memory, cur_obj: " PTR_FORMAT ", prev_obj: " PTR_FORMAT, p2i(cur_obj), p2i(prev_obj)); 412 } 413 } 414 415 assert(cur_obj == end_of_live, "just checking"); 416 } 417 418 void CompactibleSpace::compact() { 419 // Copy all live objects to their new location 420 // Used by MarkSweep::mark_sweep_phase4() 421 422 verify_up_to_first_dead(this); 423 424 HeapWord* const start = bottom(); 425 HeapWord* const end_of_live = _end_of_live; 426 427 assert(_first_dead <= end_of_live, "Invariant. _first_dead: " PTR_FORMAT " <= end_of_live: " PTR_FORMAT, p2i(_first_dead), p2i(end_of_live)); 428 if (_first_dead == end_of_live && (start == end_of_live || !cast_to_oop(start)->is_gc_marked())) { 429 // Nothing to compact. The space is either empty or all live object should be left in place. 430 clear_empty_region(this); 431 return; 432 } 433 434 const intx scan_interval = PrefetchScanIntervalInBytes; 435 const intx copy_interval = PrefetchCopyIntervalInBytes; 436 437 assert(start < end_of_live, "bottom: " PTR_FORMAT " should be < end_of_live: " PTR_FORMAT, p2i(start), p2i(end_of_live)); 438 HeapWord* cur_obj = start; 439 if (_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) { 440 // All object before _first_dead can be skipped. They should not be moved. 441 // A pointer to the first live object is stored at the memory location for _first_dead. 442 cur_obj = *(HeapWord**)(_first_dead); 443 } 444 445 const SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding(); 446 447 debug_only(HeapWord* prev_obj = nullptr); 448 while (cur_obj < end_of_live) { 449 if (!cast_to_oop(cur_obj)->is_forwarded()) { 450 debug_only(prev_obj = cur_obj); 451 // The first word of the dead object contains a pointer to the next live object or end of space. 452 cur_obj = *(HeapWord**)cur_obj; 453 assert(cur_obj > prev_obj, "we should be moving forward through memory"); 454 } else { 455 // prefetch beyond q 456 Prefetch::read(cur_obj, scan_interval); 457 458 // size and destination 459 size_t size = cast_to_oop(cur_obj)->size(); 460 HeapWord* compaction_top = cast_from_oop<HeapWord*>(forwarding->forwardee(cast_to_oop(cur_obj))); 461 462 // prefetch beyond compaction_top 463 Prefetch::write(compaction_top, copy_interval); 464 465 // copy object and reinit its mark 466 assert(cur_obj != compaction_top, "everything in this pass should be moving"); 467 Copy::aligned_conjoint_words(cur_obj, compaction_top, size); 468 oop new_obj = cast_to_oop(compaction_top); 469 470 ContinuationGCSupport::transform_stack_chunk(new_obj); 471 472 new_obj->init_mark(); 473 assert(new_obj->klass() != nullptr, "should have a class"); 474 475 debug_only(prev_obj = cur_obj); 476 cur_obj += size; 477 } 478 } 479 480 clear_empty_region(this); 481 } 482 483 #endif // INCLUDE_SERIALGC 484 485 void Space::print_short() const { print_short_on(tty); } 486 487 void Space::print_short_on(outputStream* st) const { 488 st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K, 489 (int) ((double) used() * 100 / capacity())); 490 } 491 492 void Space::print() const { print_on(tty); } 493 494 void Space::print_on(outputStream* st) const { 495 print_short_on(st); 496 st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ")", 497 p2i(bottom()), p2i(end())); 498 } 499 500 void ContiguousSpace::print_on(outputStream* st) const { 501 print_short_on(st); 502 st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")", 503 p2i(bottom()), p2i(top()), p2i(end())); 504 } 505 506 #if INCLUDE_SERIALGC 507 void TenuredSpace::print_on(outputStream* st) const { 508 print_short_on(st); 509 st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " 510 PTR_FORMAT ", " PTR_FORMAT ")", 511 p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end())); 512 } 513 #endif 514 515 void ContiguousSpace::verify() const { 516 HeapWord* p = bottom(); 517 HeapWord* t = top(); 518 HeapWord* prev_p = nullptr; 519 while (p < t) { 520 oopDesc::verify(cast_to_oop(p)); 521 prev_p = p; 522 p += cast_to_oop(p)->size(); 523 } 524 guarantee(p == top(), "end of last object must match end of space"); 525 if (top() != end()) { 526 guarantee(top() == block_start_const(end()-1) && 527 top() == block_start_const(top()), 528 "top should be start of unallocated block, if it exists"); 529 } 530 } 531 532 void Space::oop_iterate(OopIterateClosure* blk) { 533 ObjectToOopClosure blk2(blk); 534 object_iterate(&blk2); 535 } 536 537 bool Space::obj_is_alive(const HeapWord* p) const { 538 assert (block_is_obj(p), "The address should point to an object"); 539 return true; 540 } 541 542 void ContiguousSpace::oop_iterate(OopIterateClosure* blk) { 543 if (is_empty()) return; 544 HeapWord* obj_addr = bottom(); 545 HeapWord* t = top(); 546 // Could call objects iterate, but this is easier. 547 while (obj_addr < t) { 548 obj_addr += cast_to_oop(obj_addr)->oop_iterate_size(blk); 549 } 550 } 551 552 void ContiguousSpace::object_iterate(ObjectClosure* blk) { 553 if (is_empty()) return; 554 object_iterate_from(bottom(), blk); 555 } 556 557 void ContiguousSpace::object_iterate_from(HeapWord* mark, ObjectClosure* blk) { 558 while (mark < top()) { 559 blk->do_object(cast_to_oop(mark)); 560 mark += cast_to_oop(mark)->size(); 561 } 562 } 563 564 // Very general, slow implementation. 565 HeapWord* ContiguousSpace::block_start_const(const void* p) const { 566 assert(MemRegion(bottom(), end()).contains(p), 567 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", 568 p2i(p), p2i(bottom()), p2i(end())); 569 if (p >= top()) { 570 return top(); 571 } else { 572 HeapWord* last = bottom(); 573 HeapWord* cur = last; 574 while (cur <= p) { 575 last = cur; 576 cur += cast_to_oop(cur)->size(); 577 } 578 assert(oopDesc::is_oop(cast_to_oop(last)), PTR_FORMAT " should be an object start", p2i(last)); 579 return last; 580 } 581 } 582 583 size_t ContiguousSpace::block_size(const HeapWord* p) const { 584 assert(MemRegion(bottom(), end()).contains(p), 585 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", 586 p2i(p), p2i(bottom()), p2i(end())); 587 HeapWord* current_top = top(); 588 assert(p <= current_top, 589 "p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT, 590 p2i(p), p2i(current_top)); 591 assert(p == current_top || oopDesc::is_oop(cast_to_oop(p)), 592 "p (" PTR_FORMAT ") is not a block start - " 593 "current_top: " PTR_FORMAT ", is_oop: %s", 594 p2i(p), p2i(current_top), BOOL_TO_STR(oopDesc::is_oop(cast_to_oop(p)))); 595 if (p < current_top) { 596 return cast_to_oop(p)->size(); 597 } else { 598 assert(p == current_top, "just checking"); 599 return pointer_delta(end(), (HeapWord*) p); 600 } 601 } 602 603 // This version requires locking. 604 inline HeapWord* ContiguousSpace::allocate_impl(size_t size) { 605 assert(Heap_lock->owned_by_self() || 606 (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), 607 "not locked"); 608 HeapWord* obj = top(); 609 if (pointer_delta(end(), obj) >= size) { 610 HeapWord* new_top = obj + size; 611 set_top(new_top); 612 assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); 613 return obj; 614 } else { 615 return nullptr; 616 } 617 } 618 619 // This version is lock-free. 620 inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) { 621 do { 622 HeapWord* obj = top(); 623 if (pointer_delta(end(), obj) >= size) { 624 HeapWord* new_top = obj + size; 625 HeapWord* result = Atomic::cmpxchg(top_addr(), obj, new_top); 626 // result can be one of two: 627 // the old top value: the exchange succeeded 628 // otherwise: the new value of the top is returned. 629 if (result == obj) { 630 assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); 631 return obj; 632 } 633 } else { 634 return nullptr; 635 } 636 } while (true); 637 } 638 639 // Requires locking. 640 HeapWord* ContiguousSpace::allocate(size_t size) { 641 return allocate_impl(size); 642 } 643 644 // Lock-free. 645 HeapWord* ContiguousSpace::par_allocate(size_t size) { 646 return par_allocate_impl(size); 647 } 648 649 #if INCLUDE_SERIALGC 650 void TenuredSpace::initialize_threshold() { 651 _offsets.initialize_threshold(); 652 } 653 654 void TenuredSpace::alloc_block(HeapWord* start, HeapWord* end) { 655 _offsets.alloc_block(start, end); 656 } 657 658 TenuredSpace::TenuredSpace(BlockOffsetSharedArray* sharedOffsetArray, 659 MemRegion mr) : 660 _offsets(sharedOffsetArray, mr), 661 _par_alloc_lock(Mutex::safepoint, "TenuredSpaceParAlloc_lock", true) 662 { 663 _offsets.set_contig_space(this); 664 initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); 665 } 666 667 #define OBJ_SAMPLE_INTERVAL 0 668 #define BLOCK_SAMPLE_INTERVAL 100 669 670 void TenuredSpace::verify() const { 671 HeapWord* p = bottom(); 672 HeapWord* prev_p = nullptr; 673 int objs = 0; 674 int blocks = 0; 675 676 if (VerifyObjectStartArray) { 677 _offsets.verify(); 678 } 679 680 while (p < top()) { 681 size_t size = cast_to_oop(p)->size(); 682 // For a sampling of objects in the space, find it using the 683 // block offset table. 684 if (blocks == BLOCK_SAMPLE_INTERVAL) { 685 guarantee(p == block_start_const(p + (size/2)), 686 "check offset computation"); 687 blocks = 0; 688 } else { 689 blocks++; 690 } 691 692 if (objs == OBJ_SAMPLE_INTERVAL) { 693 oopDesc::verify(cast_to_oop(p)); 694 objs = 0; 695 } else { 696 objs++; 697 } 698 prev_p = p; 699 p += size; 700 } 701 guarantee(p == top(), "end of last object must match end of space"); 702 } 703 704 705 size_t TenuredSpace::allowed_dead_ratio() const { 706 return MarkSweepDeadRatio; 707 } 708 #endif // INCLUDE_SERIALGC