1 /* 2 * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "classfile/classLoaderDataGraph.hpp" 27 #include "gc/epsilon/epsilonHeap.hpp" 28 #include "gc/epsilon/epsilonInitLogger.hpp" 29 #include "gc/epsilon/epsilonMemoryPool.hpp" 30 #include "gc/epsilon/epsilonThreadLocalData.hpp" 31 #include "gc/shared/fullGCForwarding.inline.hpp" 32 #include "gc/shared/gcArguments.hpp" 33 #include "gc/shared/gcLocker.inline.hpp" 34 #include "gc/shared/gcTraceTime.inline.hpp" 35 #include "gc/shared/locationPrinter.inline.hpp" 36 #include "gc/shared/markBitMap.inline.hpp" 37 #include "gc/shared/oopStorageSet.inline.hpp" 38 #include "gc/shared/preservedMarks.inline.hpp" 39 #include "logging/log.hpp" 40 #include "memory/allocation.hpp" 41 #include "memory/iterator.inline.hpp" 42 #include "memory/memoryReserver.hpp" 43 #include "memory/metaspaceUtils.hpp" 44 #include "memory/resourceArea.hpp" 45 #include "memory/universe.hpp" 46 #include "nmt/memTracker.hpp" 47 #include "oops/compressedOops.inline.hpp" 48 #include "runtime/atomicAccess.hpp" 49 #include "runtime/globals.hpp" 50 #include "runtime/thread.hpp" 51 #include "runtime/threads.hpp" 52 #include "runtime/vmOperations.hpp" 53 #include "runtime/vmThread.hpp" 54 #include "services/management.hpp" 55 #include "utilities/macros.hpp" 56 #include "utilities/ostream.hpp" 57 #include "utilities/stack.inline.hpp" 58 59 jint EpsilonHeap::initialize() { 60 size_t align = HeapAlignment; 61 size_t init_byte_size = align_up(InitialHeapSize, align); 62 size_t max_byte_size = align_up(MaxHeapSize, align); 63 64 // Initialize backing storage 65 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, align); 66 _virtual_space.initialize(heap_rs, init_byte_size); 67 68 MemRegion committed_region((HeapWord*)_virtual_space.low(), (HeapWord*)_virtual_space.high()); 69 70 initialize_reserved_region(heap_rs); 71 72 _space = new ContiguousSpace(); 73 _space->initialize(committed_region, /* clear_space = */ true, /* mangle_space = */ true); 74 75 // Precompute hot fields 76 _max_tlab_size = MIN2(CollectedHeap::max_tlab_size(), align_object_size(EpsilonMaxTLABSize / HeapWordSize)); 77 _step_counter_update = MIN2<size_t>(max_byte_size / 16, EpsilonUpdateCountersStep); 78 _step_heap_print = (EpsilonPrintHeapSteps == 0) ? SIZE_MAX : (max_byte_size / EpsilonPrintHeapSteps); 79 _decay_time_ns = (int64_t) EpsilonTLABDecayTime * NANOSECS_PER_MILLISEC; 80 81 // Enable monitoring 82 _monitoring_support = new EpsilonMonitoringSupport(this); 83 _last_counter_update = 0; 84 _last_heap_print = 0; 85 86 // Install barrier set 87 BarrierSet::set_barrier_set(new EpsilonBarrierSet()); 88 89 if (EpsilonSlidingGC) { 90 // Initialize marking bitmap, but not commit it yet 91 size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); 92 size_t alignment = MAX2(os::vm_page_size(), os::vm_allocation_granularity()); 93 94 size_t _bitmap_size = MarkBitMap::compute_size(heap_rs.size()); 95 _bitmap_size = align_up(_bitmap_size, bitmap_page_size); 96 _bitmap_size = align_up(_bitmap_size, alignment); 97 98 const ReservedSpace bitmap = MemoryReserver::reserve(_bitmap_size, alignment, bitmap_page_size, mtGC); 99 if (!bitmap.is_reserved()) { 100 vm_exit_during_initialization("Could not reserve space for bitmap"); 101 } 102 _bitmap_region = MemRegion((HeapWord *) bitmap.base(), bitmap.size() / HeapWordSize); 103 MemRegion heap_region = MemRegion((HeapWord *) heap_rs.base(), heap_rs.size() / HeapWordSize); 104 _bitmap.initialize(heap_region, _bitmap_region); 105 106 // Initialize full GC forwarding for compact object headers 107 FullGCForwarding::initialize(_reserved); 108 109 // Initialize GC Locker 110 GCLocker::initialize(); 111 } 112 113 // All done, print out the configuration 114 EpsilonInitLogger::print(); 115 116 return JNI_OK; 117 } 118 119 void EpsilonHeap::initialize_serviceability() { 120 _pool = new EpsilonMemoryPool(this); 121 _memory_manager.add_pool(_pool); 122 } 123 124 GrowableArray<GCMemoryManager*> EpsilonHeap::memory_managers() { 125 GrowableArray<GCMemoryManager*> memory_managers(1); 126 memory_managers.append(&_memory_manager); 127 return memory_managers; 128 } 129 130 GrowableArray<MemoryPool*> EpsilonHeap::memory_pools() { 131 GrowableArray<MemoryPool*> memory_pools(1); 132 memory_pools.append(_pool); 133 return memory_pools; 134 } 135 136 size_t EpsilonHeap::unsafe_max_tlab_alloc(Thread* thr) const { 137 // Return max allocatable TLAB size, and let allocation path figure out 138 // the actual allocation size. Note: result should be in bytes. 139 return _max_tlab_size * HeapWordSize; 140 } 141 142 EpsilonHeap* EpsilonHeap::heap() { 143 return named_heap<EpsilonHeap>(CollectedHeap::Epsilon); 144 } 145 146 HeapWord* EpsilonHeap::allocate_work(size_t size, bool verbose) { 147 assert(is_object_aligned(size), "Allocation size should be aligned: %zu", size); 148 149 HeapWord* res = nullptr; 150 while (true) { 151 // Try to allocate, assume space is available 152 res = _space->par_allocate(size); 153 if (res != nullptr) { 154 break; 155 } 156 157 // Allocation failed, attempt expansion, and retry: 158 { 159 MutexLocker ml(Heap_lock); 160 161 // Try to allocate under the lock, assume another thread was able to expand 162 res = _space->par_allocate(size); 163 if (res != nullptr) { 164 break; 165 } 166 167 // Expand and loop back if space is available 168 size_t size_in_bytes = size * HeapWordSize; 169 size_t uncommitted_space = max_capacity() - capacity(); 170 size_t unused_space = max_capacity() - used(); 171 size_t want_space = MAX2(size_in_bytes, EpsilonMinHeapExpand); 172 assert(unused_space >= uncommitted_space, 173 "Unused (%zu) >= uncommitted (%zu)", 174 unused_space, uncommitted_space); 175 176 if (want_space < uncommitted_space) { 177 // Enough space to expand in bulk: 178 bool expand = _virtual_space.expand_by(want_space); 179 assert(expand, "Should be able to expand"); 180 } else if (size_in_bytes < unused_space) { 181 // No space to expand in bulk, and this allocation is still possible, 182 // take all the remaining space: 183 bool expand = _virtual_space.expand_by(uncommitted_space); 184 assert(expand, "Should be able to expand"); 185 } else { 186 // No space left: 187 return nullptr; 188 } 189 190 _space->set_end((HeapWord *) _virtual_space.high()); 191 } 192 } 193 194 size_t used = _space->used(); 195 196 // Allocation successful, update counters 197 if (verbose) { 198 size_t last = _last_counter_update; 199 if ((used - last >= _step_counter_update) && AtomicAccess::cmpxchg(&_last_counter_update, last, used) == last) { 200 _monitoring_support->update_counters(); 201 } 202 } 203 204 // ...and print the occupancy line, if needed 205 if (verbose) { 206 size_t last = _last_heap_print; 207 if ((used - last >= _step_heap_print) && AtomicAccess::cmpxchg(&_last_heap_print, last, used) == last) { 208 print_heap_info(used); 209 print_metaspace_info(); 210 } 211 } 212 213 assert(is_object_aligned(res), "Object should be aligned: " PTR_FORMAT, p2i(res)); 214 return res; 215 } 216 217 HeapWord* EpsilonHeap::allocate_new_tlab(size_t min_size, 218 size_t requested_size, 219 size_t* actual_size) { 220 Thread* thread = Thread::current(); 221 222 // Defaults in case elastic paths are not taken 223 bool fits = true; 224 size_t size = requested_size; 225 size_t ergo_tlab = requested_size; 226 int64_t time = 0; 227 228 if (EpsilonElasticTLAB) { 229 ergo_tlab = EpsilonThreadLocalData::ergo_tlab_size(thread); 230 231 if (EpsilonElasticTLABDecay) { 232 int64_t last_time = EpsilonThreadLocalData::last_tlab_time(thread); 233 time = (int64_t) os::javaTimeNanos(); 234 235 assert(last_time <= time, "time should be monotonic"); 236 237 // If the thread had not allocated recently, retract the ergonomic size. 238 // This conserves memory when the thread had initial burst of allocations, 239 // and then started allocating only sporadically. 240 if (last_time != 0 && (time - last_time > _decay_time_ns)) { 241 ergo_tlab = 0; 242 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); 243 } 244 } 245 246 // If we can fit the allocation under current TLAB size, do so. 247 // Otherwise, we want to elastically increase the TLAB size. 248 fits = (requested_size <= ergo_tlab); 249 if (!fits) { 250 size = (size_t) (ergo_tlab * EpsilonTLABElasticity); 251 } 252 } 253 254 // Always honor boundaries 255 size = clamp(size, min_size, _max_tlab_size); 256 257 // Always honor alignment 258 size = align_up(size, MinObjAlignment); 259 260 // Check that adjustments did not break local and global invariants 261 assert(is_object_aligned(size), 262 "Size honors object alignment: %zu", size); 263 assert(min_size <= size, 264 "Size honors min size: %zu <= %zu", min_size, size); 265 assert(size <= _max_tlab_size, 266 "Size honors max size: %zu <= %zu", size, _max_tlab_size); 267 assert(size <= CollectedHeap::max_tlab_size(), 268 "Size honors global max size: %zu <= %zu", size, CollectedHeap::max_tlab_size()); 269 270 if (log_is_enabled(Trace, gc)) { 271 ResourceMark rm; 272 log_trace(gc)("TLAB size for \"%s\" (Requested: %zuK, Min: %zu" 273 "K, Max: %zuK, Ergo: %zuK) -> %zuK", 274 thread->name(), 275 requested_size * HeapWordSize / K, 276 min_size * HeapWordSize / K, 277 _max_tlab_size * HeapWordSize / K, 278 ergo_tlab * HeapWordSize / K, 279 size * HeapWordSize / K); 280 } 281 282 // All prepared, let's do it! 283 HeapWord* res = allocate_or_collect_work(size); 284 285 if (res != nullptr) { 286 // Allocation successful 287 *actual_size = size; 288 if (EpsilonElasticTLABDecay) { 289 EpsilonThreadLocalData::set_last_tlab_time(thread, time); 290 } 291 if (EpsilonElasticTLAB && !fits) { 292 // If we requested expansion, this is our new ergonomic TLAB size 293 EpsilonThreadLocalData::set_ergo_tlab_size(thread, size); 294 } 295 } else { 296 // Allocation failed, reset ergonomics to try and fit smaller TLABs 297 if (EpsilonElasticTLAB) { 298 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); 299 } 300 } 301 302 return res; 303 } 304 305 HeapWord* EpsilonHeap::mem_allocate(size_t size) { 306 return allocate_or_collect_work(size); 307 } 308 309 HeapWord* EpsilonHeap::allocate_loaded_archive_space(size_t size) { 310 // Cannot use verbose=true because Metaspace is not initialized 311 return allocate_work(size, /* verbose = */false); 312 } 313 314 void EpsilonHeap::collect(GCCause::Cause cause) { 315 switch (cause) { 316 case GCCause::_metadata_GC_threshold: 317 case GCCause::_metadata_GC_clear_soft_refs: 318 // Receiving these causes means the VM itself entered the safepoint for metadata collection. 319 // While Epsilon does not do GC, it has to perform sizing adjustments, otherwise we would 320 // re-enter the safepoint again very soon. 321 322 assert(SafepointSynchronize::is_at_safepoint(), "Expected at safepoint"); 323 log_info(gc)("GC request for \"%s\" is handled", GCCause::to_string(cause)); 324 MetaspaceGC::compute_new_size(); 325 print_metaspace_info(); 326 break; 327 default: 328 if (EpsilonSlidingGC) { 329 if (SafepointSynchronize::is_at_safepoint()) { 330 entry_collect(cause); 331 } else { 332 vmentry_collect(cause); 333 } 334 } else { 335 log_info(gc)("GC request for \"%s\" is ignored", GCCause::to_string(cause)); 336 } 337 } 338 _monitoring_support->update_counters(); 339 } 340 341 void EpsilonHeap::do_full_collection(bool clear_all_soft_refs) { 342 collect(gc_cause()); 343 } 344 345 void EpsilonHeap::object_iterate(ObjectClosure *cl) { 346 _space->object_iterate(cl); 347 } 348 349 void EpsilonHeap::print_heap_on(outputStream *st) const { 350 st->print_cr("Epsilon Heap"); 351 352 StreamIndentor si(st, 1); 353 354 _virtual_space.print_on(st); 355 356 if (_space != nullptr) { 357 st->print_cr("Allocation space:"); 358 359 StreamIndentor si(st, 1); 360 _space->print_on(st, ""); 361 } 362 } 363 364 bool EpsilonHeap::print_location(outputStream* st, void* addr) const { 365 return BlockLocationPrinter<EpsilonHeap>::print_location(st, addr); 366 } 367 368 void EpsilonHeap::print_tracing_info() const { 369 print_heap_info(used()); 370 print_metaspace_info(); 371 } 372 373 void EpsilonHeap::print_heap_info(size_t used) const { 374 size_t reserved = max_capacity(); 375 size_t committed = capacity(); 376 377 if (reserved != 0) { 378 log_info(gc)("Heap: %zu%s reserved, %zu%s (%.2f%%) committed, " 379 "%zu%s (%.2f%%) used", 380 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved), 381 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed), 382 percent_of(committed, reserved), 383 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), 384 percent_of(used, reserved) 385 ); 386 } else { 387 log_info(gc)("Heap: no reliable data"); 388 } 389 } 390 391 void EpsilonHeap::print_metaspace_info() const { 392 MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics(); 393 size_t reserved = stats.reserved(); 394 size_t committed = stats.committed(); 395 size_t used = stats.used(); 396 397 if (reserved != 0) { 398 log_info(gc, metaspace)("Metaspace: %zu%s reserved, %zu%s (%.2f%%) committed, " 399 "%zu%s (%.2f%%) used", 400 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved), 401 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed), 402 percent_of(committed, reserved), 403 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), 404 percent_of(used, reserved) 405 ); 406 } else { 407 log_info(gc, metaspace)("Metaspace: no reliable data"); 408 } 409 } 410 411 // ------------------ EXPERIMENTAL MARK-COMPACT ------------------------------- 412 // 413 // This implements a trivial Lisp2-style sliding collector: 414 // https://en.wikipedia.org/wiki/Mark-compact_algorithm#LISP2_algorithm 415 // 416 // The goal for this implementation is to be as simple as possible, ignoring 417 // non-trivial performance optimizations. This collector does not implement 418 // reference processing: no soft/weak/phantom/finalizeable references are ever 419 // cleared. It also does not implement class unloading and other runtime 420 // cleanups. 421 // 422 423 // VM operation that executes collection cycle under safepoint 424 class VM_EpsilonCollect: public VM_Operation { 425 private: 426 const GCCause::Cause _cause; 427 EpsilonHeap* const _heap; 428 static size_t _req_id; 429 public: 430 VM_EpsilonCollect(GCCause::Cause cause) : VM_Operation(), 431 _cause(cause), 432 _heap(EpsilonHeap::heap()) {}; 433 434 VM_Operation::VMOp_Type type() const { return VMOp_EpsilonCollect; } 435 const char* name() const { return "Epsilon Collection"; } 436 437 virtual bool doit_prologue() { 438 size_t id = AtomicAccess::load_acquire(&_req_id); 439 440 // Need to take the Heap lock before managing backing storage. 441 Heap_lock->lock(); 442 443 // Heap lock also naturally serializes GC requests, and allows us to coalesce 444 // back-to-back GC requests from many threads. Avoid the consecutive GCs 445 // if we started waiting when other GC request was being handled. 446 if (id < AtomicAccess::load_acquire(&_req_id)) { 447 Heap_lock->unlock(); 448 return false; 449 } 450 451 // No contenders. Start handling a new GC request. 452 AtomicAccess::inc(&_req_id); 453 return true; 454 } 455 456 virtual void doit() { 457 _heap->entry_collect(_cause); 458 } 459 460 virtual void doit_epilogue() { 461 Heap_lock->unlock(); 462 } 463 }; 464 465 size_t VM_EpsilonCollect::_req_id = 0; 466 467 void EpsilonHeap::vmentry_collect(GCCause::Cause cause) { 468 VM_EpsilonCollect vmop(cause); 469 VMThread::execute(&vmop); 470 } 471 472 HeapWord* EpsilonHeap::allocate_or_collect_work(size_t size, bool verbose) { 473 HeapWord* res = allocate_work(size); 474 if (res == nullptr && EpsilonSlidingGC && EpsilonImplicitGC) { 475 vmentry_collect(GCCause::_allocation_failure); 476 GCLocker::block(); 477 res = allocate_work(size, verbose); 478 GCLocker::unblock(); 479 } 480 return res; 481 } 482 483 typedef Stack<oop, mtGC> EpsilonMarkStack; 484 485 void EpsilonHeap::process_roots(OopClosure* cl, bool update) { 486 // Need to tell runtime we are about to walk the roots with 1 thread 487 ThreadsClaimTokenScope scope; 488 489 // Need to adapt oop closure for some special root types. 490 CLDToOopClosure clds(cl, ClassLoaderData::_claim_none); 491 NMethodToOopClosure code_roots(cl, update); 492 493 // Strong roots: always reachable roots 494 495 // General strong roots that are registered in OopStorages 496 OopStorageSet::strong_oops_do(cl); 497 498 // Subsystems that still have their own root handling 499 ClassLoaderDataGraph::cld_do(&clds); 500 Threads::possibly_parallel_oops_do(false, cl, nullptr); 501 502 { 503 MutexLocker lock(CodeCache_lock, Mutex::_no_safepoint_check_flag); 504 CodeCache::nmethods_do(&code_roots); 505 } 506 507 // Weak roots: in an advanced GC these roots would be skipped during 508 // the initial scan, and walked again after the marking is complete. 509 // Then, we could discover which roots are not actually pointing 510 // to surviving Java objects, and either clean the roots, or mark them. 511 // Current simple implementation does not handle weak roots specially, 512 // and therefore, we mark through them as if they are strong roots. 513 for (auto id : EnumRange<OopStorageSet::WeakId>()) { 514 OopStorageSet::storage(id)->oops_do(cl); 515 } 516 } 517 518 // Walk the marking bitmap and call object closure on every marked object. 519 // This is much faster that walking a (very sparse) parsable heap, but it 520 // takes up to 1/64-th of heap size for the bitmap. 521 void EpsilonHeap::walk_bitmap(ObjectClosure* cl) { 522 HeapWord* limit = _space->top(); 523 HeapWord* addr = _bitmap.get_next_marked_addr(_space->bottom(), limit); 524 while (addr < limit) { 525 oop obj = cast_to_oop(addr); 526 assert(_bitmap.is_marked(obj), "sanity"); 527 cl->do_object(obj); 528 addr += 1; 529 if (addr < limit) { 530 addr = _bitmap.get_next_marked_addr(addr, limit); 531 } 532 } 533 } 534 535 class EpsilonScanOopClosure : public BasicOopIterateClosure { 536 private: 537 EpsilonMarkStack* const _stack; 538 MarkBitMap* const _bitmap; 539 540 template <class T> 541 void do_oop_work(T* p) { 542 // p is the pointer to memory location where oop is, load the value 543 // from it, unpack the compressed reference, if needed: 544 T o = RawAccess<>::oop_load(p); 545 if (!CompressedOops::is_null(o)) { 546 oop obj = CompressedOops::decode_not_null(o); 547 548 // Object is discovered. See if it is marked already. If not, 549 // mark and push it on mark stack for further traversal. Non-atomic 550 // check and set would do, as this closure is called by single thread. 551 if (!_bitmap->is_marked(obj)) { 552 // Support Virtual Threads: transform the stack chunks as we visit them. 553 ContinuationGCSupport::transform_stack_chunk(obj); 554 555 _bitmap->mark(obj); 556 _stack->push(obj); 557 } 558 } 559 } 560 561 public: 562 EpsilonScanOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) : 563 _stack(stack), _bitmap(bitmap) {} 564 virtual void do_oop(oop* p) { do_oop_work(p); } 565 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 566 }; 567 568 class EpsilonCalcNewLocationObjectClosure : public ObjectClosure { 569 private: 570 HeapWord* _compact_point; 571 PreservedMarks* const _preserved_marks; 572 573 public: 574 EpsilonCalcNewLocationObjectClosure(HeapWord* start, PreservedMarks* pm) : 575 _compact_point(start), 576 _preserved_marks(pm) {} 577 578 void do_object(oop obj) { 579 // Record the new location of the object: it is current compaction point. 580 // If object stays at the same location (which is true for objects in 581 // dense prefix, that we would normally get), do not bother recording the 582 // move, letting downstream code ignore it. 583 if (obj != cast_to_oop(_compact_point)) { 584 markWord mark = obj->mark(); 585 _preserved_marks->push_if_necessary(obj, mark); 586 FullGCForwarding::forward_to(obj, cast_to_oop(_compact_point)); 587 } 588 _compact_point += obj->size(); 589 } 590 591 HeapWord* compact_point() { 592 return _compact_point; 593 } 594 }; 595 596 class EpsilonAdjustPointersOopClosure : public BasicOopIterateClosure { 597 private: 598 template <class T> 599 void do_oop_work(T* p) { 600 // p is the pointer to memory location where oop is, load the value 601 // from it, unpack the compressed reference, if needed: 602 T o = RawAccess<>::oop_load(p); 603 if (!CompressedOops::is_null(o)) { 604 oop obj = CompressedOops::decode_not_null(o); 605 606 // Rewrite the current pointer to the object with its forwardee. 607 // Skip the write if update is not needed. 608 if (FullGCForwarding::is_forwarded(obj)) { 609 oop fwd = FullGCForwarding::forwardee(obj); 610 assert(fwd != nullptr, "just checking"); 611 RawAccess<>::oop_store(p, fwd); 612 } 613 } 614 } 615 616 public: 617 virtual void do_oop(oop* p) { do_oop_work(p); } 618 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 619 }; 620 621 class EpsilonAdjustPointersObjectClosure : public ObjectClosure { 622 private: 623 EpsilonAdjustPointersOopClosure _cl; 624 public: 625 void do_object(oop obj) { 626 // Apply the updates to all references reachable from current object: 627 obj->oop_iterate(&_cl); 628 } 629 }; 630 631 class EpsilonMoveObjectsObjectClosure : public ObjectClosure { 632 private: 633 size_t _moved; 634 public: 635 EpsilonMoveObjectsObjectClosure() : ObjectClosure(), _moved(0) {} 636 637 void do_object(oop obj) { 638 // Copy the object to its new location, if needed. This is final step, 639 // so we have to re-initialize its new mark word, dropping the forwardee 640 // data from it. 641 if (FullGCForwarding::is_forwarded(obj)) { 642 oop fwd = FullGCForwarding::forwardee(obj); 643 assert(fwd != nullptr, "just checking"); 644 Copy::aligned_conjoint_words(cast_from_oop<HeapWord*>(obj), cast_from_oop<HeapWord*>(fwd), obj->size()); 645 fwd->init_mark(); 646 _moved++; 647 } 648 } 649 650 size_t moved() { 651 return _moved; 652 } 653 }; 654 655 class EpsilonVerifyOopClosure : public BasicOopIterateClosure { 656 private: 657 EpsilonHeap* const _heap; 658 EpsilonMarkStack* const _stack; 659 MarkBitMap* const _bitmap; 660 661 template <class T> 662 void do_oop_work(T* p) { 663 T o = RawAccess<>::oop_load(p); 664 if (!CompressedOops::is_null(o)) { 665 oop obj = CompressedOops::decode_not_null(o); 666 if (!_bitmap->is_marked(obj)) { 667 _bitmap->mark(obj); 668 669 guarantee(_heap->is_in(obj), "Is in heap: " PTR_FORMAT, p2i(obj)); 670 guarantee(oopDesc::is_oop(obj), "Is an object: " PTR_FORMAT, p2i(obj)); 671 guarantee(!obj->mark().is_marked(), "Mark is gone: " PTR_FORMAT, p2i(obj)); 672 673 _stack->push(obj); 674 } 675 } 676 } 677 678 public: 679 EpsilonVerifyOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) : 680 _heap(EpsilonHeap::heap()), _stack(stack), _bitmap(bitmap) {} 681 virtual void do_oop(oop* p) { do_oop_work(p); } 682 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 683 }; 684 685 void EpsilonHeap::entry_collect(GCCause::Cause cause) { 686 if (GCLocker::is_active()) { 687 return; 688 } 689 690 GCIdMark mark; 691 GCTraceTime(Info, gc) time("Lisp2-style Mark-Compact", nullptr, cause, true); 692 693 // Some statistics, for fun and profit: 694 size_t stat_reachable_roots = 0; 695 size_t stat_reachable_heap = 0; 696 size_t stat_moved = 0; 697 size_t stat_preserved_marks = 0; 698 699 { 700 GCTraceTime(Info, gc) time("Step 0: Prologue", nullptr); 701 702 // Commit marking bitmap memory. There are several upsides of doing this 703 // before the cycle: no memory is taken if GC is not happening, the memory 704 // is "cleared" on first touch, and untouched parts of bitmap are mapped 705 // to zero page, boosting performance on sparse heaps. 706 if (!os::commit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size(), false)) { 707 log_warning(gc)("Could not commit native memory for marking bitmap, GC failed"); 708 return; 709 } 710 711 // We do not need parsable heap for this algorithm to work, but we want 712 // threads to give up their TLABs. 713 ensure_parsability(true); 714 } 715 716 { 717 GCTraceTime(Info, gc) time("Step 1: Mark", nullptr); 718 719 #if COMPILER2_OR_JVMCI 720 // Derived pointers would be re-discovered during the mark. 721 // Clear and activate the table for them. 722 DerivedPointerTable::clear(); 723 #endif 724 725 // TODO: Do we need this if we do not do class unloading? 726 CodeCache::on_gc_marking_cycle_start(); 727 728 // Marking stack and the closure that does most of the work. The closure 729 // would scan the outgoing references, mark them, and push newly-marked 730 // objects to stack for further processing. 731 EpsilonMarkStack stack; 732 EpsilonScanOopClosure cl(&stack, &_bitmap); 733 734 // Seed the marking with roots. 735 process_roots(&cl, false); 736 stat_reachable_roots = stack.size(); 737 738 // Scan the rest of the heap until we run out of objects. Termination is 739 // guaranteed, because all reachable objects would be marked eventually. 740 while (!stack.is_empty()) { 741 oop obj = stack.pop(); 742 obj->oop_iterate(&cl); 743 stat_reachable_heap++; 744 } 745 746 // TODO: Do we need this if we do not do class unloading? 747 CodeCache::on_gc_marking_cycle_finish(); 748 CodeCache::arm_all_nmethods(); 749 750 #if COMPILER2_OR_JVMCI 751 // No more derived pointers discovered after marking is done. 752 DerivedPointerTable::set_active(false); 753 #endif 754 } 755 756 // We are going to store forwarding information (where the new copy resides) 757 // in mark words. Some of those mark words need to be carefully preserved. 758 // This is an utility that maintains the list of those special mark words. 759 PreservedMarks preserved_marks; 760 761 // New top of the allocated space. 762 HeapWord* new_top; 763 764 { 765 GCTraceTime(Info, gc) time("Step 2: Calculate new locations", nullptr); 766 767 // Walk all alive objects, compute their new addresses and store those 768 // addresses in mark words. Optionally preserve some marks. 769 EpsilonCalcNewLocationObjectClosure cl(_space->bottom(), &preserved_marks); 770 walk_bitmap(&cl); 771 772 // After addresses are calculated, we know the new top for the allocated 773 // space. We cannot set it just yet, because some asserts check that objects 774 // are "in heap" based on current "top". 775 new_top = cl.compact_point(); 776 777 stat_preserved_marks = preserved_marks.size(); 778 } 779 780 { 781 GCTraceTime(Info, gc) time("Step 3: Adjust pointers", nullptr); 782 783 // Walk all alive objects _and their reference fields_, and put "new 784 // addresses" there. We know the new addresses from the forwarding data 785 // in mark words. Take care of the heap objects first. 786 EpsilonAdjustPointersObjectClosure cl; 787 walk_bitmap(&cl); 788 789 // Now do the same, but for all VM roots, which reference the objects on 790 // their own: their references should also be updated. 791 EpsilonAdjustPointersOopClosure cli; 792 process_roots(&cli, true); 793 794 // Finally, make sure preserved marks know the objects are about to move. 795 preserved_marks.adjust_during_full_gc(); 796 } 797 798 { 799 GCTraceTime(Info, gc) time("Step 4: Move objects", nullptr); 800 801 // Move all alive objects to their new locations. All the references are 802 // already adjusted at previous step. 803 EpsilonMoveObjectsObjectClosure cl; 804 walk_bitmap(&cl); 805 stat_moved = cl.moved(); 806 807 // Now we moved all objects to their relevant locations, we can retract 808 // the "top" of the allocation space to the end of the compacted prefix. 809 _space->set_top(new_top); 810 } 811 812 { 813 GCTraceTime(Info, gc) time("Step 5: Epilogue", nullptr); 814 815 // Restore all special mark words. 816 preserved_marks.restore(); 817 818 #if COMPILER2_OR_JVMCI 819 // Tell the rest of runtime we have finished the GC. 820 DerivedPointerTable::update_pointers(); 821 #endif 822 823 // Verification code walks entire heap and verifies nothing is broken. 824 if (EpsilonVerify) { 825 // The basic implementation turns heap into entirely parsable one with 826 // only alive objects, which mean we could just walked the heap object 827 // by object and verify it. But, it would be inconvenient for verification 828 // to assume heap has only alive objects. Any future change that leaves 829 // at least one dead object with dead outgoing references would fail the 830 // verification. Therefore, it makes more sense to mark through the heap 831 // again, not assuming objects are all alive. 832 EpsilonMarkStack stack; 833 EpsilonVerifyOopClosure cl(&stack, &_bitmap); 834 835 _bitmap.clear(); 836 837 // Verify all roots are correct, and that we have the same number of 838 // object reachable from roots. 839 process_roots(&cl, false); 840 841 size_t verified_roots = stack.size(); 842 guarantee(verified_roots == stat_reachable_roots, 843 "Verification discovered %zu roots out of %zu", 844 verified_roots, stat_reachable_roots); 845 846 // Verify the rest of the heap is correct, and that we have the same 847 // number of objects reachable from heap. 848 size_t verified_heap = 0; 849 while (!stack.is_empty()) { 850 oop obj = stack.pop(); 851 obj->oop_iterate(&cl); 852 verified_heap++; 853 } 854 855 guarantee(verified_heap == stat_reachable_heap, 856 "Verification discovered %zu heap objects out of %zu", 857 verified_heap, stat_reachable_heap); 858 859 // Ask parts of runtime to verify themselves too 860 Universe::verify("Epsilon"); 861 } 862 863 // Marking bitmap is not needed anymore 864 if (!os::uncommit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size())) { 865 log_warning(gc)("Could not uncommit native memory for marking bitmap"); 866 } 867 868 // Return all memory back if so requested. On large heaps, this would 869 // take a while. 870 if (EpsilonUncommit) { 871 _virtual_space.shrink_by((_space->end() - new_top) * HeapWordSize); 872 _space->set_end((HeapWord*)_virtual_space.high()); 873 } 874 } 875 876 size_t stat_reachable = stat_reachable_roots + stat_reachable_heap; 877 log_info(gc)("GC Stats: %zu (%.2f%%) reachable from roots, %zu (%.2f%%) reachable from heap, " 878 "%zu (%.2f%%) moved, %zu (%.2f%%) markwords preserved", 879 stat_reachable_roots, percent_of(stat_reachable_roots, stat_reachable), 880 stat_reachable_heap, percent_of(stat_reachable_heap, stat_reachable), 881 stat_moved, percent_of(stat_moved, stat_reachable), 882 stat_preserved_marks, percent_of(stat_preserved_marks, stat_reachable) 883 ); 884 885 print_heap_info(used()); 886 print_metaspace_info(); 887 } 888 889 void EpsilonHeap::pin_object(JavaThread* thread, oop obj) { 890 if (EpsilonSlidingGC) { 891 GCLocker::enter(thread); 892 } 893 } 894 895 void EpsilonHeap::unpin_object(JavaThread* thread, oop obj) { 896 if (EpsilonSlidingGC) { 897 GCLocker::exit(thread); 898 } 899 } --- EOF ---