6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/epsilon/epsilonHeap.hpp"
27 #include "gc/epsilon/epsilonInitLogger.hpp"
28 #include "gc/epsilon/epsilonMemoryPool.hpp"
29 #include "gc/epsilon/epsilonThreadLocalData.hpp"
30 #include "gc/shared/gcArguments.hpp"
31 #include "gc/shared/locationPrinter.inline.hpp"
32 #include "logging/log.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/metaspaceUtils.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "memory/universe.hpp"
37 #include "runtime/atomicAccess.hpp"
38 #include "runtime/globals.hpp"
39 #include "utilities/ostream.hpp"
40
41 jint EpsilonHeap::initialize() {
42 size_t align = HeapAlignment;
43 size_t init_byte_size = align_up(InitialHeapSize, align);
44 size_t max_byte_size = align_up(MaxHeapSize, align);
45
46 // Initialize backing storage
47 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, align);
48 _virtual_space.initialize(heap_rs, init_byte_size);
49
50 MemRegion committed_region((HeapWord*)_virtual_space.low(), (HeapWord*)_virtual_space.high());
51
52 initialize_reserved_region(heap_rs);
53
54 _space = new ContiguousSpace();
55 _space->initialize(committed_region, /* clear_space = */ true, /* mangle_space = */ true);
56
57 // Precompute hot fields
58 _max_tlab_size = MIN2(CollectedHeap::max_tlab_size(), align_object_size(EpsilonMaxTLABSize / HeapWordSize));
59 _step_counter_update = MIN2<size_t>(max_byte_size / 16, EpsilonUpdateCountersStep);
60 _step_heap_print = (EpsilonPrintHeapSteps == 0) ? SIZE_MAX : (max_byte_size / EpsilonPrintHeapSteps);
61 _decay_time_ns = (int64_t) EpsilonTLABDecayTime * NANOSECS_PER_MILLISEC;
62
63 // Enable monitoring
64 _monitoring_support = new EpsilonMonitoringSupport(this);
65 _last_counter_update = 0;
66 _last_heap_print = 0;
67
68 // Install barrier set
69 BarrierSet::set_barrier_set(new EpsilonBarrierSet());
70
71 // All done, print out the configuration
72 EpsilonInitLogger::print();
73
74 return JNI_OK;
75 }
76
77 void EpsilonHeap::initialize_serviceability() {
78 _pool = new EpsilonMemoryPool(this);
79 _memory_manager.add_pool(_pool);
80 }
81
82 GrowableArray<GCMemoryManager*> EpsilonHeap::memory_managers() {
83 GrowableArray<GCMemoryManager*> memory_managers(1);
84 memory_managers.append(&_memory_manager);
85 return memory_managers;
86 }
87
88 GrowableArray<MemoryPool*> EpsilonHeap::memory_pools() {
89 GrowableArray<MemoryPool*> memory_pools(1);
90 memory_pools.append(_pool);
221 assert(min_size <= size,
222 "Size honors min size: %zu <= %zu", min_size, size);
223 assert(size <= _max_tlab_size,
224 "Size honors max size: %zu <= %zu", size, _max_tlab_size);
225 assert(size <= CollectedHeap::max_tlab_size(),
226 "Size honors global max size: %zu <= %zu", size, CollectedHeap::max_tlab_size());
227
228 if (log_is_enabled(Trace, gc)) {
229 ResourceMark rm;
230 log_trace(gc)("TLAB size for \"%s\" (Requested: %zuK, Min: %zu"
231 "K, Max: %zuK, Ergo: %zuK) -> %zuK",
232 thread->name(),
233 requested_size * HeapWordSize / K,
234 min_size * HeapWordSize / K,
235 _max_tlab_size * HeapWordSize / K,
236 ergo_tlab * HeapWordSize / K,
237 size * HeapWordSize / K);
238 }
239
240 // All prepared, let's do it!
241 HeapWord* res = allocate_work(size);
242
243 if (res != nullptr) {
244 // Allocation successful
245 *actual_size = size;
246 if (EpsilonElasticTLABDecay) {
247 EpsilonThreadLocalData::set_last_tlab_time(thread, time);
248 }
249 if (EpsilonElasticTLAB && !fits) {
250 // If we requested expansion, this is our new ergonomic TLAB size
251 EpsilonThreadLocalData::set_ergo_tlab_size(thread, size);
252 }
253 } else {
254 // Allocation failed, reset ergonomics to try and fit smaller TLABs
255 if (EpsilonElasticTLAB) {
256 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0);
257 }
258 }
259
260 return res;
261 }
262
263 HeapWord* EpsilonHeap::mem_allocate(size_t size) {
264 return allocate_work(size);
265 }
266
267 HeapWord* EpsilonHeap::allocate_loaded_archive_space(size_t size) {
268 // Cannot use verbose=true because Metaspace is not initialized
269 return allocate_work(size, /* verbose = */false);
270 }
271
272 void EpsilonHeap::collect(GCCause::Cause cause) {
273 switch (cause) {
274 case GCCause::_metadata_GC_threshold:
275 case GCCause::_metadata_GC_clear_soft_refs:
276 // Receiving these causes means the VM itself entered the safepoint for metadata collection.
277 // While Epsilon does not do GC, it has to perform sizing adjustments, otherwise we would
278 // re-enter the safepoint again very soon.
279
280 assert(SafepointSynchronize::is_at_safepoint(), "Expected at safepoint");
281 log_info(gc)("GC request for \"%s\" is handled", GCCause::to_string(cause));
282 MetaspaceGC::compute_new_size();
283 print_metaspace_info();
284 break;
285 default:
286 log_info(gc)("GC request for \"%s\" is ignored", GCCause::to_string(cause));
287 }
288 _monitoring_support->update_counters();
289 }
290
291 void EpsilonHeap::do_full_collection(bool clear_all_soft_refs) {
292 collect(gc_cause());
293 }
294
295 void EpsilonHeap::object_iterate(ObjectClosure *cl) {
296 _space->object_iterate(cl);
297 }
298
299 void EpsilonHeap::print_heap_on(outputStream *st) const {
300 st->print_cr("Epsilon Heap");
301
302 StreamIndentor si(st, 1);
303
304 _virtual_space.print_on(st);
305
306 if (_space != nullptr) {
312 }
313
314 bool EpsilonHeap::print_location(outputStream* st, void* addr) const {
315 return BlockLocationPrinter<EpsilonHeap>::print_location(st, addr);
316 }
317
318 void EpsilonHeap::print_tracing_info() const {
319 print_heap_info(used());
320 print_metaspace_info();
321 }
322
323 void EpsilonHeap::print_heap_info(size_t used) const {
324 size_t reserved = max_capacity();
325 size_t committed = capacity();
326
327 if (reserved != 0) {
328 log_info(gc)("Heap: %zu%s reserved, %zu%s (%.2f%%) committed, "
329 "%zu%s (%.2f%%) used",
330 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved),
331 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed),
332 committed * 100.0 / reserved,
333 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used),
334 used * 100.0 / reserved);
335 } else {
336 log_info(gc)("Heap: no reliable data");
337 }
338 }
339
340 void EpsilonHeap::print_metaspace_info() const {
341 MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics();
342 size_t reserved = stats.reserved();
343 size_t committed = stats.committed();
344 size_t used = stats.used();
345
346 if (reserved != 0) {
347 log_info(gc, metaspace)("Metaspace: %zu%s reserved, %zu%s (%.2f%%) committed, "
348 "%zu%s (%.2f%%) used",
349 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved),
350 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed),
351 committed * 100.0 / reserved,
352 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used),
353 used * 100.0 / reserved);
354 } else {
355 log_info(gc, metaspace)("Metaspace: no reliable data");
356 }
357 }
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/classLoaderDataGraph.hpp"
27 #include "gc/epsilon/epsilonHeap.hpp"
28 #include "gc/epsilon/epsilonInitLogger.hpp"
29 #include "gc/epsilon/epsilonMemoryPool.hpp"
30 #include "gc/epsilon/epsilonThreadLocalData.hpp"
31 #include "gc/shared/fullGCForwarding.inline.hpp"
32 #include "gc/shared/gcArguments.hpp"
33 #include "gc/shared/gcLocker.inline.hpp"
34 #include "gc/shared/gcTraceTime.inline.hpp"
35 #include "gc/shared/locationPrinter.inline.hpp"
36 #include "gc/shared/markBitMap.inline.hpp"
37 #include "gc/shared/oopStorageSet.inline.hpp"
38 #include "gc/shared/preservedMarks.inline.hpp"
39 #include "logging/log.hpp"
40 #include "memory/allocation.hpp"
41 #include "memory/iterator.inline.hpp"
42 #include "memory/memoryReserver.hpp"
43 #include "memory/metaspaceUtils.hpp"
44 #include "memory/resourceArea.hpp"
45 #include "memory/universe.hpp"
46 #include "nmt/memTracker.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "runtime/atomicAccess.hpp"
49 #include "runtime/globals.hpp"
50 #include "runtime/thread.hpp"
51 #include "runtime/threads.hpp"
52 #include "runtime/vmOperations.hpp"
53 #include "runtime/vmThread.hpp"
54 #include "services/management.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/ostream.hpp"
57 #include "utilities/stack.inline.hpp"
58
59 jint EpsilonHeap::initialize() {
60 size_t align = HeapAlignment;
61 size_t init_byte_size = align_up(InitialHeapSize, align);
62 size_t max_byte_size = align_up(MaxHeapSize, align);
63
64 // Initialize backing storage
65 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, align);
66 _virtual_space.initialize(heap_rs, init_byte_size);
67
68 MemRegion committed_region((HeapWord*)_virtual_space.low(), (HeapWord*)_virtual_space.high());
69
70 initialize_reserved_region(heap_rs);
71
72 _space = new ContiguousSpace();
73 _space->initialize(committed_region, /* clear_space = */ true, /* mangle_space = */ true);
74
75 // Precompute hot fields
76 _max_tlab_size = MIN2(CollectedHeap::max_tlab_size(), align_object_size(EpsilonMaxTLABSize / HeapWordSize));
77 _step_counter_update = MIN2<size_t>(max_byte_size / 16, EpsilonUpdateCountersStep);
78 _step_heap_print = (EpsilonPrintHeapSteps == 0) ? SIZE_MAX : (max_byte_size / EpsilonPrintHeapSteps);
79 _decay_time_ns = (int64_t) EpsilonTLABDecayTime * NANOSECS_PER_MILLISEC;
80
81 // Enable monitoring
82 _monitoring_support = new EpsilonMonitoringSupport(this);
83 _last_counter_update = 0;
84 _last_heap_print = 0;
85
86 // Install barrier set
87 BarrierSet::set_barrier_set(new EpsilonBarrierSet());
88
89 if (EpsilonSlidingGC) {
90 // Initialize marking bitmap, but not commit it yet
91 size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
92 size_t alignment = MAX2(os::vm_page_size(), os::vm_allocation_granularity());
93
94 size_t _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
95 _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
96 _bitmap_size = align_up(_bitmap_size, alignment);
97
98 const ReservedSpace bitmap = MemoryReserver::reserve(_bitmap_size, alignment, bitmap_page_size, mtGC);
99 if (!bitmap.is_reserved()) {
100 vm_exit_during_initialization("Could not reserve space for bitmap");
101 }
102 _bitmap_region = MemRegion((HeapWord *) bitmap.base(), bitmap.size() / HeapWordSize);
103 MemRegion heap_region = MemRegion((HeapWord *) heap_rs.base(), heap_rs.size() / HeapWordSize);
104 _bitmap.initialize(heap_region, _bitmap_region);
105
106 // Initialize full GC forwarding for compact object headers
107 FullGCForwarding::initialize(_reserved);
108
109 // Initialize GC Locker
110 GCLocker::initialize();
111 }
112
113 // All done, print out the configuration
114 EpsilonInitLogger::print();
115
116 return JNI_OK;
117 }
118
119 void EpsilonHeap::initialize_serviceability() {
120 _pool = new EpsilonMemoryPool(this);
121 _memory_manager.add_pool(_pool);
122 }
123
124 GrowableArray<GCMemoryManager*> EpsilonHeap::memory_managers() {
125 GrowableArray<GCMemoryManager*> memory_managers(1);
126 memory_managers.append(&_memory_manager);
127 return memory_managers;
128 }
129
130 GrowableArray<MemoryPool*> EpsilonHeap::memory_pools() {
131 GrowableArray<MemoryPool*> memory_pools(1);
132 memory_pools.append(_pool);
263 assert(min_size <= size,
264 "Size honors min size: %zu <= %zu", min_size, size);
265 assert(size <= _max_tlab_size,
266 "Size honors max size: %zu <= %zu", size, _max_tlab_size);
267 assert(size <= CollectedHeap::max_tlab_size(),
268 "Size honors global max size: %zu <= %zu", size, CollectedHeap::max_tlab_size());
269
270 if (log_is_enabled(Trace, gc)) {
271 ResourceMark rm;
272 log_trace(gc)("TLAB size for \"%s\" (Requested: %zuK, Min: %zu"
273 "K, Max: %zuK, Ergo: %zuK) -> %zuK",
274 thread->name(),
275 requested_size * HeapWordSize / K,
276 min_size * HeapWordSize / K,
277 _max_tlab_size * HeapWordSize / K,
278 ergo_tlab * HeapWordSize / K,
279 size * HeapWordSize / K);
280 }
281
282 // All prepared, let's do it!
283 HeapWord* res = allocate_or_collect_work(size);
284
285 if (res != nullptr) {
286 // Allocation successful
287 *actual_size = size;
288 if (EpsilonElasticTLABDecay) {
289 EpsilonThreadLocalData::set_last_tlab_time(thread, time);
290 }
291 if (EpsilonElasticTLAB && !fits) {
292 // If we requested expansion, this is our new ergonomic TLAB size
293 EpsilonThreadLocalData::set_ergo_tlab_size(thread, size);
294 }
295 } else {
296 // Allocation failed, reset ergonomics to try and fit smaller TLABs
297 if (EpsilonElasticTLAB) {
298 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0);
299 }
300 }
301
302 return res;
303 }
304
305 HeapWord* EpsilonHeap::mem_allocate(size_t size) {
306 return allocate_or_collect_work(size);
307 }
308
309 HeapWord* EpsilonHeap::allocate_loaded_archive_space(size_t size) {
310 // Cannot use verbose=true because Metaspace is not initialized
311 return allocate_work(size, /* verbose = */false);
312 }
313
314 void EpsilonHeap::collect(GCCause::Cause cause) {
315 switch (cause) {
316 case GCCause::_metadata_GC_threshold:
317 case GCCause::_metadata_GC_clear_soft_refs:
318 // Receiving these causes means the VM itself entered the safepoint for metadata collection.
319 // While Epsilon does not do GC, it has to perform sizing adjustments, otherwise we would
320 // re-enter the safepoint again very soon.
321
322 assert(SafepointSynchronize::is_at_safepoint(), "Expected at safepoint");
323 log_info(gc)("GC request for \"%s\" is handled", GCCause::to_string(cause));
324 MetaspaceGC::compute_new_size();
325 print_metaspace_info();
326 break;
327 default:
328 if (EpsilonSlidingGC) {
329 if (SafepointSynchronize::is_at_safepoint()) {
330 entry_collect(cause);
331 } else {
332 vmentry_collect(cause);
333 }
334 } else {
335 log_info(gc)("GC request for \"%s\" is ignored", GCCause::to_string(cause));
336 }
337 }
338 _monitoring_support->update_counters();
339 }
340
341 void EpsilonHeap::do_full_collection(bool clear_all_soft_refs) {
342 collect(gc_cause());
343 }
344
345 void EpsilonHeap::object_iterate(ObjectClosure *cl) {
346 _space->object_iterate(cl);
347 }
348
349 void EpsilonHeap::print_heap_on(outputStream *st) const {
350 st->print_cr("Epsilon Heap");
351
352 StreamIndentor si(st, 1);
353
354 _virtual_space.print_on(st);
355
356 if (_space != nullptr) {
362 }
363
364 bool EpsilonHeap::print_location(outputStream* st, void* addr) const {
365 return BlockLocationPrinter<EpsilonHeap>::print_location(st, addr);
366 }
367
368 void EpsilonHeap::print_tracing_info() const {
369 print_heap_info(used());
370 print_metaspace_info();
371 }
372
373 void EpsilonHeap::print_heap_info(size_t used) const {
374 size_t reserved = max_capacity();
375 size_t committed = capacity();
376
377 if (reserved != 0) {
378 log_info(gc)("Heap: %zu%s reserved, %zu%s (%.2f%%) committed, "
379 "%zu%s (%.2f%%) used",
380 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved),
381 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed),
382 percent_of(committed, reserved),
383 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used),
384 percent_of(used, reserved)
385 );
386 } else {
387 log_info(gc)("Heap: no reliable data");
388 }
389 }
390
391 void EpsilonHeap::print_metaspace_info() const {
392 MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics();
393 size_t reserved = stats.reserved();
394 size_t committed = stats.committed();
395 size_t used = stats.used();
396
397 if (reserved != 0) {
398 log_info(gc, metaspace)("Metaspace: %zu%s reserved, %zu%s (%.2f%%) committed, "
399 "%zu%s (%.2f%%) used",
400 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved),
401 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed),
402 percent_of(committed, reserved),
403 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used),
404 percent_of(used, reserved)
405 );
406 } else {
407 log_info(gc, metaspace)("Metaspace: no reliable data");
408 }
409 }
410
411 // ------------------ EXPERIMENTAL MARK-COMPACT -------------------------------
412 //
413 // This implements a trivial Lisp2-style sliding collector:
414 // https://en.wikipedia.org/wiki/Mark-compact_algorithm#LISP2_algorithm
415 //
416 // The goal for this implementation is to be as simple as possible, ignoring
417 // non-trivial performance optimizations. This collector does not implement
418 // reference processing: no soft/weak/phantom/finalizeable references are ever
419 // cleared. It also does not implement class unloading and other runtime
420 // cleanups.
421 //
422
423 // VM operation that executes collection cycle under safepoint
424 class VM_EpsilonCollect: public VM_Operation {
425 private:
426 const GCCause::Cause _cause;
427 EpsilonHeap* const _heap;
428 static size_t _req_id;
429 public:
430 VM_EpsilonCollect(GCCause::Cause cause) : VM_Operation(),
431 _cause(cause),
432 _heap(EpsilonHeap::heap()) {};
433
434 VM_Operation::VMOp_Type type() const { return VMOp_EpsilonCollect; }
435 const char* name() const { return "Epsilon Collection"; }
436
437 virtual bool doit_prologue() {
438 size_t id = AtomicAccess::load_acquire(&_req_id);
439
440 // Need to take the Heap lock before managing backing storage.
441 Heap_lock->lock();
442
443 // Heap lock also naturally serializes GC requests, and allows us to coalesce
444 // back-to-back GC requests from many threads. Avoid the consecutive GCs
445 // if we started waiting when other GC request was being handled.
446 if (id < AtomicAccess::load_acquire(&_req_id)) {
447 Heap_lock->unlock();
448 return false;
449 }
450
451 // No contenders. Start handling a new GC request.
452 AtomicAccess::inc(&_req_id);
453 return true;
454 }
455
456 virtual void doit() {
457 _heap->entry_collect(_cause);
458 }
459
460 virtual void doit_epilogue() {
461 Heap_lock->unlock();
462 }
463 };
464
465 size_t VM_EpsilonCollect::_req_id = 0;
466
467 void EpsilonHeap::vmentry_collect(GCCause::Cause cause) {
468 VM_EpsilonCollect vmop(cause);
469 VMThread::execute(&vmop);
470 }
471
472 HeapWord* EpsilonHeap::allocate_or_collect_work(size_t size, bool verbose) {
473 HeapWord* res = allocate_work(size);
474 if (res == nullptr && EpsilonSlidingGC && EpsilonImplicitGC) {
475 vmentry_collect(GCCause::_allocation_failure);
476 GCLocker::block();
477 res = allocate_work(size, verbose);
478 GCLocker::unblock();
479 }
480 return res;
481 }
482
483 typedef Stack<oop, mtGC> EpsilonMarkStack;
484
485 void EpsilonHeap::process_roots(OopClosure* cl, bool update) {
486 // Need to tell runtime we are about to walk the roots with 1 thread
487 ThreadsClaimTokenScope scope;
488
489 // Need to adapt oop closure for some special root types.
490 CLDToOopClosure clds(cl, ClassLoaderData::_claim_none);
491 NMethodToOopClosure code_roots(cl, update);
492
493 // Strong roots: always reachable roots
494
495 // General strong roots that are registered in OopStorages
496 OopStorageSet::strong_oops_do(cl);
497
498 // Subsystems that still have their own root handling
499 ClassLoaderDataGraph::cld_do(&clds);
500 Threads::possibly_parallel_oops_do(false, cl, nullptr);
501
502 {
503 MutexLocker lock(CodeCache_lock, Mutex::_no_safepoint_check_flag);
504 CodeCache::nmethods_do(&code_roots);
505 }
506
507 // Weak roots: in an advanced GC these roots would be skipped during
508 // the initial scan, and walked again after the marking is complete.
509 // Then, we could discover which roots are not actually pointing
510 // to surviving Java objects, and either clean the roots, or mark them.
511 // Current simple implementation does not handle weak roots specially,
512 // and therefore, we mark through them as if they are strong roots.
513 for (auto id : EnumRange<OopStorageSet::WeakId>()) {
514 OopStorageSet::storage(id)->oops_do(cl);
515 }
516 }
517
518 // Walk the marking bitmap and call object closure on every marked object.
519 // This is much faster that walking a (very sparse) parsable heap, but it
520 // takes up to 1/64-th of heap size for the bitmap.
521 void EpsilonHeap::walk_bitmap(ObjectClosure* cl) {
522 HeapWord* limit = _space->top();
523 HeapWord* addr = _bitmap.get_next_marked_addr(_space->bottom(), limit);
524 while (addr < limit) {
525 oop obj = cast_to_oop(addr);
526 assert(_bitmap.is_marked(obj), "sanity");
527 cl->do_object(obj);
528 addr += 1;
529 if (addr < limit) {
530 addr = _bitmap.get_next_marked_addr(addr, limit);
531 }
532 }
533 }
534
535 class EpsilonScanOopClosure : public BasicOopIterateClosure {
536 private:
537 EpsilonMarkStack* const _stack;
538 MarkBitMap* const _bitmap;
539
540 template <class T>
541 void do_oop_work(T* p) {
542 // p is the pointer to memory location where oop is, load the value
543 // from it, unpack the compressed reference, if needed:
544 T o = RawAccess<>::oop_load(p);
545 if (!CompressedOops::is_null(o)) {
546 oop obj = CompressedOops::decode_not_null(o);
547
548 // Object is discovered. See if it is marked already. If not,
549 // mark and push it on mark stack for further traversal. Non-atomic
550 // check and set would do, as this closure is called by single thread.
551 if (!_bitmap->is_marked(obj)) {
552 // Support Virtual Threads: transform the stack chunks as we visit them.
553 ContinuationGCSupport::transform_stack_chunk(obj);
554
555 _bitmap->mark(obj);
556 _stack->push(obj);
557 }
558 }
559 }
560
561 public:
562 EpsilonScanOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) :
563 _stack(stack), _bitmap(bitmap) {}
564 virtual void do_oop(oop* p) { do_oop_work(p); }
565 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
566 };
567
568 class EpsilonCalcNewLocationObjectClosure : public ObjectClosure {
569 private:
570 HeapWord* _compact_point;
571 PreservedMarks* const _preserved_marks;
572
573 public:
574 EpsilonCalcNewLocationObjectClosure(HeapWord* start, PreservedMarks* pm) :
575 _compact_point(start),
576 _preserved_marks(pm) {}
577
578 void do_object(oop obj) {
579 // Record the new location of the object: it is current compaction point.
580 // If object stays at the same location (which is true for objects in
581 // dense prefix, that we would normally get), do not bother recording the
582 // move, letting downstream code ignore it.
583 if (obj != cast_to_oop(_compact_point)) {
584 markWord mark = obj->mark();
585 _preserved_marks->push_if_necessary(obj, mark);
586 FullGCForwarding::forward_to(obj, cast_to_oop(_compact_point));
587 }
588 _compact_point += obj->size();
589 }
590
591 HeapWord* compact_point() {
592 return _compact_point;
593 }
594 };
595
596 class EpsilonAdjustPointersOopClosure : public BasicOopIterateClosure {
597 private:
598 template <class T>
599 void do_oop_work(T* p) {
600 // p is the pointer to memory location where oop is, load the value
601 // from it, unpack the compressed reference, if needed:
602 T o = RawAccess<>::oop_load(p);
603 if (!CompressedOops::is_null(o)) {
604 oop obj = CompressedOops::decode_not_null(o);
605
606 // Rewrite the current pointer to the object with its forwardee.
607 // Skip the write if update is not needed.
608 if (FullGCForwarding::is_forwarded(obj)) {
609 oop fwd = FullGCForwarding::forwardee(obj);
610 assert(fwd != nullptr, "just checking");
611 RawAccess<>::oop_store(p, fwd);
612 }
613 }
614 }
615
616 public:
617 virtual void do_oop(oop* p) { do_oop_work(p); }
618 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
619 };
620
621 class EpsilonAdjustPointersObjectClosure : public ObjectClosure {
622 private:
623 EpsilonAdjustPointersOopClosure _cl;
624 public:
625 void do_object(oop obj) {
626 // Apply the updates to all references reachable from current object:
627 obj->oop_iterate(&_cl);
628 }
629 };
630
631 class EpsilonMoveObjectsObjectClosure : public ObjectClosure {
632 private:
633 size_t _moved;
634 public:
635 EpsilonMoveObjectsObjectClosure() : ObjectClosure(), _moved(0) {}
636
637 void do_object(oop obj) {
638 // Copy the object to its new location, if needed. This is final step,
639 // so we have to re-initialize its new mark word, dropping the forwardee
640 // data from it.
641 if (FullGCForwarding::is_forwarded(obj)) {
642 oop fwd = FullGCForwarding::forwardee(obj);
643 assert(fwd != nullptr, "just checking");
644 Copy::aligned_conjoint_words(cast_from_oop<HeapWord*>(obj), cast_from_oop<HeapWord*>(fwd), obj->size());
645 fwd->init_mark();
646 _moved++;
647 }
648 }
649
650 size_t moved() {
651 return _moved;
652 }
653 };
654
655 class EpsilonVerifyOopClosure : public BasicOopIterateClosure {
656 private:
657 EpsilonHeap* const _heap;
658 EpsilonMarkStack* const _stack;
659 MarkBitMap* const _bitmap;
660
661 template <class T>
662 void do_oop_work(T* p) {
663 T o = RawAccess<>::oop_load(p);
664 if (!CompressedOops::is_null(o)) {
665 oop obj = CompressedOops::decode_not_null(o);
666 if (!_bitmap->is_marked(obj)) {
667 _bitmap->mark(obj);
668
669 guarantee(_heap->is_in(obj), "Is in heap: " PTR_FORMAT, p2i(obj));
670 guarantee(oopDesc::is_oop(obj), "Is an object: " PTR_FORMAT, p2i(obj));
671 guarantee(!obj->mark().is_marked(), "Mark is gone: " PTR_FORMAT, p2i(obj));
672
673 _stack->push(obj);
674 }
675 }
676 }
677
678 public:
679 EpsilonVerifyOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) :
680 _heap(EpsilonHeap::heap()), _stack(stack), _bitmap(bitmap) {}
681 virtual void do_oop(oop* p) { do_oop_work(p); }
682 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
683 };
684
685 void EpsilonHeap::entry_collect(GCCause::Cause cause) {
686 if (GCLocker::is_active()) {
687 return;
688 }
689
690 GCIdMark mark;
691 GCTraceTime(Info, gc) time("Lisp2-style Mark-Compact", nullptr, cause, true);
692
693 // Some statistics, for fun and profit:
694 size_t stat_reachable_roots = 0;
695 size_t stat_reachable_heap = 0;
696 size_t stat_moved = 0;
697 size_t stat_preserved_marks = 0;
698
699 {
700 GCTraceTime(Info, gc) time("Step 0: Prologue", nullptr);
701
702 // Commit marking bitmap memory. There are several upsides of doing this
703 // before the cycle: no memory is taken if GC is not happening, the memory
704 // is "cleared" on first touch, and untouched parts of bitmap are mapped
705 // to zero page, boosting performance on sparse heaps.
706 if (!os::commit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size(), false)) {
707 log_warning(gc)("Could not commit native memory for marking bitmap, GC failed");
708 return;
709 }
710
711 // We do not need parsable heap for this algorithm to work, but we want
712 // threads to give up their TLABs.
713 ensure_parsability(true);
714 }
715
716 {
717 GCTraceTime(Info, gc) time("Step 1: Mark", nullptr);
718
719 #if COMPILER2_OR_JVMCI
720 // Derived pointers would be re-discovered during the mark.
721 // Clear and activate the table for them.
722 DerivedPointerTable::clear();
723 #endif
724
725 // TODO: Do we need this if we do not do class unloading?
726 CodeCache::on_gc_marking_cycle_start();
727
728 // Marking stack and the closure that does most of the work. The closure
729 // would scan the outgoing references, mark them, and push newly-marked
730 // objects to stack for further processing.
731 EpsilonMarkStack stack;
732 EpsilonScanOopClosure cl(&stack, &_bitmap);
733
734 // Seed the marking with roots.
735 process_roots(&cl, false);
736 stat_reachable_roots = stack.size();
737
738 // Scan the rest of the heap until we run out of objects. Termination is
739 // guaranteed, because all reachable objects would be marked eventually.
740 while (!stack.is_empty()) {
741 oop obj = stack.pop();
742 obj->oop_iterate(&cl);
743 stat_reachable_heap++;
744 }
745
746 // TODO: Do we need this if we do not do class unloading?
747 CodeCache::on_gc_marking_cycle_finish();
748 CodeCache::arm_all_nmethods();
749
750 #if COMPILER2_OR_JVMCI
751 // No more derived pointers discovered after marking is done.
752 DerivedPointerTable::set_active(false);
753 #endif
754 }
755
756 // We are going to store forwarding information (where the new copy resides)
757 // in mark words. Some of those mark words need to be carefully preserved.
758 // This is an utility that maintains the list of those special mark words.
759 PreservedMarks preserved_marks;
760
761 // New top of the allocated space.
762 HeapWord* new_top;
763
764 {
765 GCTraceTime(Info, gc) time("Step 2: Calculate new locations", nullptr);
766
767 // Walk all alive objects, compute their new addresses and store those
768 // addresses in mark words. Optionally preserve some marks.
769 EpsilonCalcNewLocationObjectClosure cl(_space->bottom(), &preserved_marks);
770 walk_bitmap(&cl);
771
772 // After addresses are calculated, we know the new top for the allocated
773 // space. We cannot set it just yet, because some asserts check that objects
774 // are "in heap" based on current "top".
775 new_top = cl.compact_point();
776
777 stat_preserved_marks = preserved_marks.size();
778 }
779
780 {
781 GCTraceTime(Info, gc) time("Step 3: Adjust pointers", nullptr);
782
783 // Walk all alive objects _and their reference fields_, and put "new
784 // addresses" there. We know the new addresses from the forwarding data
785 // in mark words. Take care of the heap objects first.
786 EpsilonAdjustPointersObjectClosure cl;
787 walk_bitmap(&cl);
788
789 // Now do the same, but for all VM roots, which reference the objects on
790 // their own: their references should also be updated.
791 EpsilonAdjustPointersOopClosure cli;
792 process_roots(&cli, true);
793
794 // Finally, make sure preserved marks know the objects are about to move.
795 preserved_marks.adjust_during_full_gc();
796 }
797
798 {
799 GCTraceTime(Info, gc) time("Step 4: Move objects", nullptr);
800
801 // Move all alive objects to their new locations. All the references are
802 // already adjusted at previous step.
803 EpsilonMoveObjectsObjectClosure cl;
804 walk_bitmap(&cl);
805 stat_moved = cl.moved();
806
807 // Now we moved all objects to their relevant locations, we can retract
808 // the "top" of the allocation space to the end of the compacted prefix.
809 _space->set_top(new_top);
810 }
811
812 {
813 GCTraceTime(Info, gc) time("Step 5: Epilogue", nullptr);
814
815 // Restore all special mark words.
816 preserved_marks.restore();
817
818 #if COMPILER2_OR_JVMCI
819 // Tell the rest of runtime we have finished the GC.
820 DerivedPointerTable::update_pointers();
821 #endif
822
823 // Verification code walks entire heap and verifies nothing is broken.
824 if (EpsilonVerify) {
825 // The basic implementation turns heap into entirely parsable one with
826 // only alive objects, which mean we could just walked the heap object
827 // by object and verify it. But, it would be inconvenient for verification
828 // to assume heap has only alive objects. Any future change that leaves
829 // at least one dead object with dead outgoing references would fail the
830 // verification. Therefore, it makes more sense to mark through the heap
831 // again, not assuming objects are all alive.
832 EpsilonMarkStack stack;
833 EpsilonVerifyOopClosure cl(&stack, &_bitmap);
834
835 _bitmap.clear();
836
837 // Verify all roots are correct, and that we have the same number of
838 // object reachable from roots.
839 process_roots(&cl, false);
840
841 size_t verified_roots = stack.size();
842 guarantee(verified_roots == stat_reachable_roots,
843 "Verification discovered %zu roots out of %zu",
844 verified_roots, stat_reachable_roots);
845
846 // Verify the rest of the heap is correct, and that we have the same
847 // number of objects reachable from heap.
848 size_t verified_heap = 0;
849 while (!stack.is_empty()) {
850 oop obj = stack.pop();
851 obj->oop_iterate(&cl);
852 verified_heap++;
853 }
854
855 guarantee(verified_heap == stat_reachable_heap,
856 "Verification discovered %zu heap objects out of %zu",
857 verified_heap, stat_reachable_heap);
858
859 // Ask parts of runtime to verify themselves too
860 Universe::verify("Epsilon");
861 }
862
863 // Marking bitmap is not needed anymore
864 if (!os::uncommit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size())) {
865 log_warning(gc)("Could not uncommit native memory for marking bitmap");
866 }
867
868 // Return all memory back if so requested. On large heaps, this would
869 // take a while.
870 if (EpsilonUncommit) {
871 _virtual_space.shrink_by((_space->end() - new_top) * HeapWordSize);
872 _space->set_end((HeapWord*)_virtual_space.high());
873 }
874 }
875
876 size_t stat_reachable = stat_reachable_roots + stat_reachable_heap;
877 log_info(gc)("GC Stats: %zu (%.2f%%) reachable from roots, %zu (%.2f%%) reachable from heap, "
878 "%zu (%.2f%%) moved, %zu (%.2f%%) markwords preserved",
879 stat_reachable_roots, percent_of(stat_reachable_roots, stat_reachable),
880 stat_reachable_heap, percent_of(stat_reachable_heap, stat_reachable),
881 stat_moved, percent_of(stat_moved, stat_reachable),
882 stat_preserved_marks, percent_of(stat_preserved_marks, stat_reachable)
883 );
884
885 print_heap_info(used());
886 print_metaspace_info();
887 }
888
889 void EpsilonHeap::pin_object(JavaThread* thread, oop obj) {
890 if (EpsilonSlidingGC) {
891 GCLocker::enter(thread);
892 }
893 }
894
895 void EpsilonHeap::unpin_object(JavaThread* thread, oop obj) {
896 if (EpsilonSlidingGC) {
897 GCLocker::exit(thread);
898 }
899 }
|