< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page

        

*** 1,7 **** /* ! * Copyright (c) 2013, 2018, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * --- 1,7 ---- /* ! * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. *
*** 33,43 **** #include "gc/shenandoah/shenandoahAllocTracker.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahBrooksPointer.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" - #include "gc/shenandoah/shenandoahConcurrentMark.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" --- 33,42 ----
*** 47,57 **** #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahMemoryPool.hpp" #include "gc/shenandoah/shenandoahMetrics.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" - #include "gc/shenandoah/shenandoahPacer.hpp" #include "gc/shenandoah/shenandoahPacer.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" --- 46,55 ----
*** 65,74 **** --- 63,74 ---- #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp" #include "memory/metaspace.hpp" + #include "runtime/interfaceSupport.inline.hpp" + #include "runtime/safepointMechanism.hpp" #include "runtime/vmThread.hpp" #include "services/mallocTracker.hpp" ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
*** 84,112 **** void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); } void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_work(p); } #endif ! class ShenandoahPretouchTask : public AbstractGangTask { private: ShenandoahRegionIterator _regions; - const size_t _bitmap_size; const size_t _page_size; - char* _bitmap_base; public: ! ShenandoahPretouchTask(char* bitmap_base, size_t bitmap_size, size_t page_size) : ! AbstractGangTask("Shenandoah PreTouch"), ! _bitmap_size(bitmap_size), ! _page_size(page_size), ! _bitmap_base(bitmap_base) { ! } virtual void work(uint worker_id) { ShenandoahHeapRegion* r = _regions.next(); while (r != NULL) { os::pretouch_memory(r->bottom(), r->end(), _page_size); size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor(); size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor(); assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size); os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size); --- 84,127 ---- void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); } void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_work(p); } #endif ! class ShenandoahPretouchHeapTask : public AbstractGangTask { private: ShenandoahRegionIterator _regions; const size_t _page_size; public: ! ShenandoahPretouchHeapTask(size_t page_size) : ! AbstractGangTask("Shenandoah Pretouch Heap"), ! _page_size(page_size) {} virtual void work(uint worker_id) { ShenandoahHeapRegion* r = _regions.next(); while (r != NULL) { os::pretouch_memory(r->bottom(), r->end(), _page_size); + r = _regions.next(); + } + } + }; + + class ShenandoahPretouchBitmapTask : public AbstractGangTask { + private: + ShenandoahRegionIterator _regions; + char* _bitmap_base; + const size_t _bitmap_size; + const size_t _page_size; + public: + ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) : + AbstractGangTask("Shenandoah Pretouch Bitmap"), + _bitmap_base(bitmap_base), + _bitmap_size(bitmap_size), + _page_size(page_size) {} + virtual void work(uint worker_id) { + ShenandoahHeapRegion* r = _regions.next(); + while (r != NULL) { size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor(); size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor(); assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size); os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
*** 119,196 **** jint ShenandoahHeap::initialize() { ShenandoahBrooksPointer::initial_checks(); initialize_heuristics(); size_t init_byte_size = collector_policy()->initial_heap_byte_size(); ! size_t max_byte_size = collector_policy()->max_heap_byte_size(); size_t heap_alignment = collector_policy()->heap_alignment(); if (ShenandoahAlwaysPreTouch) { // Enabled pre-touch means the entire heap is committed right away. init_byte_size = max_byte_size; } ! Universe::check_alignment(max_byte_size, ! ShenandoahHeapRegion::region_size_bytes(), ! "shenandoah heap"); ! Universe::check_alignment(init_byte_size, ! ShenandoahHeapRegion::region_size_bytes(), ! "shenandoah heap"); ! ! ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, ! heap_alignment); ! initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size())); ! ! ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size); _num_regions = ShenandoahHeapRegion::region_count(); ! size_t num_committed_regions = init_byte_size / ShenandoahHeapRegion::region_size_bytes(); num_committed_regions = MIN2(num_committed_regions, _num_regions); assert(num_committed_regions <= _num_regions, "sanity"); ! _initial_size = num_committed_regions * ShenandoahHeapRegion::region_size_bytes(); _committed = _initial_size; ! log_info(gc, heap)("Initialize Shenandoah heap with initial size " SIZE_FORMAT "%s", ! byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size)); ! if (!os::commit_memory(pgc_rs.base(), _initial_size, false)) { ! vm_exit_out_of_memory(_initial_size, OOM_MMAP_ERROR, "Shenandoah failed to initialize heap"); ! } ! size_t reg_size_words = ShenandoahHeapRegion::region_size_words(); ! size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes(); ! _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC); ! _free_set = new ShenandoahFreeSet(this, _num_regions); ! _collection_set = new ShenandoahCollectionSet(this, (HeapWord*)pgc_rs.base()); ! if (ShenandoahPacing) { ! _pacer = new ShenandoahPacer(this); ! _pacer->setup_for_idle(); ! } else { ! _pacer = NULL; } ! assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0, ! "misaligned heap: " PTR_FORMAT, p2i(base())); - // The call below uses stuff (the SATB* things) that are in G1, but probably - // belong into a shared location. - ShenandoahBarrierSet::satb_mark_queue_set().initialize(this, - SATB_Q_CBL_mon, - 20 /*G1SATBProcessCompletedThreshold */, - 60 /* G1SATBBufferEnqueueingThresholdPercent */, - Shared_SATB_Q_lock); - - // Reserve space for prev and next bitmap. - size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); _bitmap_size = MarkBitMap::compute_size(heap_rs.size()); _bitmap_size = align_up(_bitmap_size, bitmap_page_size); - _heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize); size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor(); guarantee(bitmap_bytes_per_region != 0, "Bitmap bytes per region should not be zero"); --- 134,197 ---- jint ShenandoahHeap::initialize() { ShenandoahBrooksPointer::initial_checks(); initialize_heuristics(); + // + // Figure out heap sizing + // + size_t init_byte_size = collector_policy()->initial_heap_byte_size(); ! size_t max_byte_size = collector_policy()->max_heap_byte_size(); size_t heap_alignment = collector_policy()->heap_alignment(); + size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + if (ShenandoahAlwaysPreTouch) { // Enabled pre-touch means the entire heap is committed right away. init_byte_size = max_byte_size; } ! Universe::check_alignment(max_byte_size, reg_size_bytes, "Shenandoah heap"); ! Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap"); _num_regions = ShenandoahHeapRegion::region_count(); ! size_t num_committed_regions = init_byte_size / reg_size_bytes; num_committed_regions = MIN2(num_committed_regions, _num_regions); assert(num_committed_regions <= _num_regions, "sanity"); ! _initial_size = num_committed_regions * reg_size_bytes; _committed = _initial_size; ! size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); ! size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); ! // ! // Reserve and commit memory for heap ! // ! ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment); ! initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size())); ! _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize); ! _heap_region_special = heap_rs.special(); ! assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0, ! "Misaligned heap: " PTR_FORMAT, p2i(base())); ! ReservedSpace sh_rs = heap_rs.first_part(max_byte_size); ! if (!_heap_region_special) { ! os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false, ! "Cannot commit heap memory"); } ! // ! // Reserve and commit memory for bitmap(s) ! // _bitmap_size = MarkBitMap::compute_size(heap_rs.size()); _bitmap_size = align_up(_bitmap_size, bitmap_page_size); size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor(); guarantee(bitmap_bytes_per_region != 0, "Bitmap bytes per region should not be zero");
*** 211,252 **** guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0, "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT, _bitmap_bytes_per_slice, bitmap_page_size); ! ReservedSpace bitmap0(_bitmap_size, bitmap_page_size); ! MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC); ! _bitmap_region = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize); size_t bitmap_init_commit = _bitmap_bytes_per_slice * align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice; bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit); ! os::commit_memory_or_exit((char *) (_bitmap_region.start()), bitmap_init_commit, false, ! "couldn't allocate initial bitmap"); ! size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); if (ShenandoahVerify) { ! ReservedSpace verify_bitmap(_bitmap_size, page_size); ! os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), false, ! "couldn't allocate verification bitmap"); MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC); MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize); _verification_bit_map.initialize(_heap_region, verify_bitmap_region); _verifier = new ShenandoahVerifier(this, &_verification_bit_map); } ! _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions); { ShenandoahHeapLocker locker(lock()); for (size_t i = 0; i < _num_regions; i++) { ! ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this, ! (HeapWord*) pgc_rs.base() + reg_size_words * i, ! reg_size_words, ! i, ! i < num_committed_regions); _marking_context->initialize_top_at_mark_start(r); _regions[i] = r; assert(!collection_set()->is_in(i), "New region should not be in collection set"); } --- 212,272 ---- guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0, "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT, _bitmap_bytes_per_slice, bitmap_page_size); ! ReservedSpace bitmap(_bitmap_size, bitmap_page_size); ! MemTracker::record_virtual_memory_type(bitmap.base(), mtGC); ! _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize); ! _bitmap_region_special = bitmap.special(); size_t bitmap_init_commit = _bitmap_bytes_per_slice * align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice; bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit); ! if (!_bitmap_region_special) { ! os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false, ! "Cannot commit bitmap memory"); ! } ! _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions); if (ShenandoahVerify) { ! ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size); ! if (!verify_bitmap.special()) { ! os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false, ! "Cannot commit verification bitmap memory"); ! } MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC); MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize); _verification_bit_map.initialize(_heap_region, verify_bitmap_region); _verifier = new ShenandoahVerifier(this, &_verification_bit_map); } ! // Reserve aux bitmap for use in object_iterate(). We don't commit it here. ! ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size); ! MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC); ! _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize); ! _aux_bitmap_region_special = aux_bitmap.special(); ! _aux_bit_map.initialize(_heap_region, _aux_bitmap_region); ! ! // ! // Create regions and region sets ! // ! ! _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC); ! _free_set = new ShenandoahFreeSet(this, _num_regions); ! _collection_set = new ShenandoahCollectionSet(this, (HeapWord*)sh_rs.base()); { ShenandoahHeapLocker locker(lock()); + + size_t size_words = ShenandoahHeapRegion::region_size_words(); + for (size_t i = 0; i < _num_regions; i++) { ! HeapWord* start = (HeapWord*)sh_rs.base() + size_words * i; ! bool is_committed = i < num_committed_regions; ! ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this, start, size_words, i, is_committed); _marking_context->initialize_top_at_mark_start(r); _regions[i] = r; assert(!collection_set()->is_in(i), "New region should not be in collection set"); }
*** 256,312 **** _free_set->rebuild(); } if (ShenandoahAlwaysPreTouch) { ! assert (!AlwaysPreTouch, "Should have been overridden"); // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads, // before initialize() below zeroes it with initializing thread. For any given region, // we touch the region and the corresponding bitmaps from the same thread. ShenandoahPushWorkerScope scope(workers(), _max_workers, false); ! log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages", ! _num_regions, page_size); ! ShenandoahPretouchTask cl(bitmap0.base(), _bitmap_size, page_size); ! _workers->run_task(&cl); ! } ! // Reserve aux bitmap for use in object_iterate(). We don't commit it here. ! ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size); ! MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC); ! _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize); ! _aux_bit_map.initialize(_heap_region, _aux_bitmap_region); ! _traversal_gc = heuristics()->can_do_traversal_gc() ? ! new ShenandoahTraversalGC(this, _num_regions) : ! NULL; ! _monitoring_support = new ShenandoahMonitoringSupport(this); _phase_timings = new ShenandoahPhaseTimings(); if (ShenandoahAllocationTrace) { _alloc_tracker = new ShenandoahAllocTracker(); } ! ShenandoahStringDedup::initialize(); _control_thread = new ShenandoahControlThread(); ! ShenandoahCodeRoots::initialize(); log_info(gc, init)("Safepointing mechanism: %s", SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" : (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown")); - _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC); - for (uint worker = 0; worker < _max_workers; worker++) { - _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC); - Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort)); - } - return JNI_OK; } void ShenandoahHeap::initialize_heuristics() { if (ShenandoahGCHeuristics != NULL) { --- 276,365 ---- _free_set->rebuild(); } if (ShenandoahAlwaysPreTouch) { ! assert(!AlwaysPreTouch, "Should have been overridden"); // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads, // before initialize() below zeroes it with initializing thread. For any given region, // we touch the region and the corresponding bitmaps from the same thread. ShenandoahPushWorkerScope scope(workers(), _max_workers, false); ! size_t pretouch_heap_page_size = heap_page_size; ! size_t pretouch_bitmap_page_size = bitmap_page_size; ! #ifdef LINUX ! // UseTransparentHugePages would madvise that backing memory can be coalesced into huge ! // pages. But, the kernel needs to know that every small page is used, in order to coalesce ! // them into huge one. Therefore, we need to pretouch with smaller pages. ! if (UseTransparentHugePages) { ! pretouch_heap_page_size = (size_t)os::vm_page_size(); ! pretouch_bitmap_page_size = (size_t)os::vm_page_size(); ! } ! #endif ! // OS memory managers may want to coalesce back-to-back pages. Make their jobs ! // simpler by pre-touching continuous spaces (heap and bitmap) separately. ! log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page", ! _num_regions, pretouch_bitmap_page_size); ! ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size); ! _workers->run_task(&bcl); ! ! log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page", ! _num_regions, pretouch_heap_page_size); ! ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size); ! _workers->run_task(&hcl); ! } ! ! // ! // Initialize the rest of GC subsystems ! // ! ! _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC); ! for (uint worker = 0; worker < _max_workers; worker++) { ! _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC); ! Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort)); ! } ! ! // The call below uses stuff (the SATB* things) that are in G1, but probably ! // belong into a shared location. ! ShenandoahBarrierSet::satb_mark_queue_set().initialize(this, ! SATB_Q_CBL_mon, ! 20 /* G1SATBProcessCompletedThreshold */, ! 60 /* G1SATBBufferEnqueueingThresholdPercent */); + _monitoring_support = new ShenandoahMonitoringSupport(this); _phase_timings = new ShenandoahPhaseTimings(); + ShenandoahStringDedup::initialize(); + ShenandoahCodeRoots::initialize(); if (ShenandoahAllocationTrace) { _alloc_tracker = new ShenandoahAllocTracker(); } ! if (ShenandoahPacing) { ! _pacer = new ShenandoahPacer(this); ! _pacer->setup_for_idle(); ! } else { ! _pacer = NULL; ! } ! ! _traversal_gc = heuristics()->can_do_traversal_gc() ? ! new ShenandoahTraversalGC(this, _num_regions) : ! NULL; _control_thread = new ShenandoahControlThread(); ! log_info(gc, init)("Initialize Shenandoah heap with initial size " SIZE_FORMAT "%s", ! byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size)); log_info(gc, init)("Safepointing mechanism: %s", SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" : (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown")); return JNI_OK; } void ShenandoahHeap::initialize_heuristics() { if (ShenandoahGCHeuristics != NULL) {
*** 334,347 **** if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) { vm_exit_during_initialization( err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", _heuristics->name())); } - - if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) { - vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier"); - } log_info(gc, init)("Shenandoah heuristics: %s", _heuristics->name()); } else { ShouldNotReachHere(); } --- 387,396 ----
*** 360,369 **** --- 409,419 ---- _committed(0), _bytes_allocated_since_gc_start(0), _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)), _workers(NULL), _safepoint_workers(NULL), + _heap_region_special(false), _num_regions(0), _regions(NULL), _update_refs_iterator(this), _control_thread(NULL), _shenandoah_policy(policy),
*** 382,391 **** --- 432,447 ---- _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"), _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), _soft_ref_policy(), _ref_processor(NULL), _marking_context(NULL), + _bitmap_size(0), + _bitmap_regions_per_slice(0), + _bitmap_bytes_per_slice(0), + _bitmap_region_special(false), + _aux_bitmap_region_special(false), + _liveness_cache(NULL), _collection_set(NULL) { log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads); log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
*** 477,503 **** if (Verbose) { print_heap_regions_on(st); } } ! class ShenandoahInitGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! if (thread != NULL && (thread->is_Java_thread() || thread->is_Worker_thread())) { ! ShenandoahThreadLocalData::initialize_gclab(thread); ! } } }; void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); MutexLocker ml(Threads_lock); ! ShenandoahInitGCLABClosure init_gclabs; ! Threads::threads_do(&init_gclabs); _workers->threads_do(&init_gclabs); - _safepoint_workers->threads_do(&init_gclabs); // gclab can not be initialized early during VM startup, as it can not determinate its max_size. // Now, we will let WorkGang to initialize gclab when new worker is created. _workers->set_initialize_gclab(); --- 533,557 ---- if (Verbose) { print_heap_regions_on(st); } } ! class ShenandoahInitWorkerGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! assert(thread != NULL, "Sanity"); ! assert(thread->is_Worker_thread(), "Only worker thread expected"); ! ShenandoahThreadLocalData::initialize_gclab(thread); } }; void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); MutexLocker ml(Threads_lock); ! ShenandoahInitWorkerGCLABClosure init_gclabs; _workers->threads_do(&init_gclabs); // gclab can not be initialized early during VM startup, as it can not determinate its max_size. // Now, we will let WorkGang to initialize gclab when new worker is created. _workers->set_initialize_gclab();
*** 1025,1035 **** ShenandoahRetireGCLABClosure cl; for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { cl.do_thread(t); } workers()->threads_do(&cl); - _safepoint_workers->threads_do(&cl); } void ShenandoahHeap::resize_tlabs() { CollectedHeap::resize_all_tlabs(); } --- 1079,1088 ----
*** 1045,1055 **** void work(uint worker_id) { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahEvacOOMScope oom_evac_scope; ShenandoahEvacuateUpdateRootsClosure cl; - MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); _rp->process_evacuate_roots(&cl, &blobsCl, worker_id); } }; --- 1098,1107 ----
*** 1111,1121 **** ShenandoahRetireAndResetGCLABClosure cl; for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { cl.do_thread(t); } workers()->threads_do(&cl); - _safepoint_workers->threads_do(&cl); } void ShenandoahHeap::collect(GCCause::Cause cause) { control_thread()->request_gc(cause); } --- 1163,1172 ----
*** 1134,1149 **** return sp->block_start(addr); } return NULL; } - size_t ShenandoahHeap::block_size(const HeapWord* addr) const { - Space* sp = heap_region_containing(addr); - assert(sp != NULL, "block_size of address outside of heap"); - return sp->block_size(addr); - } - bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const { Space* sp = heap_region_containing(addr); return sp->block_is_obj(addr); } --- 1185,1194 ----
*** 1270,1280 **** * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap * is allowed to report dead objects, but is not required to do so. */ void ShenandoahHeap::object_iterate(ObjectClosure* cl) { assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints"); ! if (!os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) { log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration"); return; } // Reset bitmap --- 1315,1325 ---- * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap * is allowed to report dead objects, but is not required to do so. */ void ShenandoahHeap::object_iterate(ObjectClosure* cl) { assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints"); ! if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) { log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration"); return; } // Reset bitmap
*** 1297,1307 **** obj->oop_iterate(&oops); } assert(oop_stack.is_empty(), "should be empty"); ! if (!os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) { log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration"); } } void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) { --- 1342,1352 ---- obj->oop_iterate(&oops); } assert(oop_stack.is_empty(), "should be empty"); ! if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) { log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration"); } } void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
*** 1863,1872 **** --- 1908,1933 ---- size_t ShenandoahHeap::tlab_used(Thread* thread) const { return _free_set->used(); } + bool ShenandoahHeap::try_cancel_gc() { + while (true) { + jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE); + if (prev == CANCELLABLE) return true; + else if (prev == CANCELLED) return false; + assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers"); + assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED"); + { + // We need to provide a safepoint here, otherwise we might + // spin forever if a SP is pending. + ThreadBlockInVM sp(JavaThread::current()); + SpinPause(); + } + } + } + void ShenandoahHeap::cancel_gc(GCCause::Cause cause) { if (try_cancel_gc()) { FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause)); log_info(gc)("%s", msg.buffer()); Events::log(Thread::current(), "%s", msg.buffer());
*** 1923,1944 **** { ShenandoahGCPhase phase(full_gc ? ShenandoahPhaseTimings::full_gc_purge_par : ShenandoahPhaseTimings::purge_par); uint active = _workers->active_workers(); ! StringDedupUnlinkOrOopsDoClosure dedup_cl(is_alive, NULL); ! ParallelCleaningTask unlink_task(is_alive, &dedup_cl, active, purged_class); _workers->run_task(&unlink_task); } - if (ShenandoahStringDedup::is_enabled()) { - ShenandoahGCPhase phase(full_gc ? - ShenandoahPhaseTimings::full_gc_purge_string_dedup : - ShenandoahPhaseTimings::purge_string_dedup); - ShenandoahStringDedup::parallel_cleanup(); - } - { ShenandoahGCPhase phase(full_gc ? ShenandoahPhaseTimings::full_gc_purge_cldg : ShenandoahPhaseTimings::purge_cldg); ClassLoaderDataGraph::purge(); --- 1984,1997 ---- { ShenandoahGCPhase phase(full_gc ? ShenandoahPhaseTimings::full_gc_purge_par : ShenandoahPhaseTimings::purge_par); uint active = _workers->active_workers(); ! ParallelCleaningTask unlink_task(is_alive, active, purged_class, true); _workers->run_task(&unlink_task); } { ShenandoahGCPhase phase(full_gc ? ShenandoahPhaseTimings::full_gc_purge_cldg : ShenandoahPhaseTimings::purge_cldg); ClassLoaderDataGraph::purge();
*** 2011,2028 **** void ShenandoahHeap::unregister_nmethod(nmethod* nm) { ShenandoahCodeRoots::remove_nmethod(nm); } oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->write_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_pinned(); return o; } void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->read_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_unpinned(); } GCTimer* ShenandoahHeap::gc_timer() const { --- 2064,2079 ----
*** 2216,2225 **** --- 2267,2281 ---- } bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) { assert_heaplock_owned_by_current_thread(); + // Bitmaps in special regions do not need commits + if (_bitmap_region_special) { + return true; + } + if (is_bitmap_slice_committed(r, true)) { // Some other region from the group is already committed, meaning the bitmap // slice is already committed, we exit right away. return true; }
*** 2235,2244 **** --- 2291,2305 ---- } bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) { assert_heaplock_owned_by_current_thread(); + // Bitmaps in special regions do not need uncommits + if (_bitmap_region_special) { + return true; + } + if (is_bitmap_slice_committed(r, true)) { // Some other region from the group is still committed, meaning the bitmap // slice is should stay committed, exit right away. return true; }
*** 2629,2638 **** --- 2690,2703 ---- GrowableArray<MemoryPool*> memory_pools(1); memory_pools.append(_memory_pool); return memory_pools; } + MemoryUsage ShenandoahHeap::memory_usage() { + return _memory_pool->get_memory_usage(); + } + void ShenandoahHeap::enter_evacuation() { _oom_evac_handler.enter_evacuation(); } void ShenandoahHeap::leave_evacuation() {
*** 2759,2778 **** --- 2824,2845 ---- } } jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) { #ifdef ASSERT + assert(_liveness_cache != NULL, "sanity"); assert(worker_id < _max_workers, "sanity"); for (uint i = 0; i < num_regions(); i++) { assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty"); } #endif return _liveness_cache[worker_id]; } void ShenandoahHeap::flush_liveness_cache(uint worker_id) { assert(worker_id < _max_workers, "sanity"); + assert(_liveness_cache != NULL, "sanity"); jushort* ld = _liveness_cache[worker_id]; for (uint i = 0; i < num_regions(); i++) { ShenandoahHeapRegion* r = get_region(i); jushort live = ld[i]; if (live > 0) {
< prev index next >