1 /*
2 * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "memory/allocation.hpp"
27 #include "memory/universe.hpp"
28
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shared/gcTimer.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/locationPrinter.inline.hpp"
33 #include "gc/shared/memAllocator.hpp"
34 #include "gc/shared/plab.hpp"
35 #include "gc/shared/slidingForwarding.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37
38 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
39 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
41 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
42 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
43 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
44 #include "gc/shenandoah/shenandoahControlThread.hpp"
45 #include "gc/shenandoah/shenandoahFreeSet.hpp"
46 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
47 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
48 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
49 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
50 #include "gc/shenandoah/shenandoahInitLogger.hpp"
51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
53 #include "gc/shenandoah/shenandoahMetrics.hpp"
54 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
55 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
56 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
57 #include "gc/shenandoah/shenandoahPadding.hpp"
58 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
59 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
60 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
61 #include "gc/shenandoah/shenandoahStringDedup.hpp"
62 #include "gc/shenandoah/shenandoahSTWMark.hpp"
63 #include "gc/shenandoah/shenandoahUtils.hpp"
64 #include "gc/shenandoah/shenandoahVerifier.hpp"
65 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
66 #include "gc/shenandoah/shenandoahVMOperations.hpp"
67 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
68 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
69 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
70 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
71 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
72 #if INCLUDE_JFR
73 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
74 #endif
75
76 #include "classfile/systemDictionary.hpp"
77 #include "code/codeCache.hpp"
78 #include "memory/classLoaderMetaspace.hpp"
79 #include "memory/metaspaceUtils.hpp"
80 #include "oops/compressedOops.inline.hpp"
81 #include "prims/jvmtiTagMap.hpp"
82 #include "runtime/atomic.hpp"
83 #include "runtime/globals.hpp"
84 #include "runtime/interfaceSupport.inline.hpp"
85 #include "runtime/java.hpp"
86 #include "runtime/orderAccess.hpp"
87 #include "runtime/safepointMechanism.hpp"
88 #include "runtime/vmThread.hpp"
89 #include "services/mallocTracker.hpp"
90 #include "services/memTracker.hpp"
91 #include "utilities/events.hpp"
92 #include "utilities/powerOfTwo.hpp"
93
94 class ShenandoahPretouchHeapTask : public WorkerTask {
95 private:
96 ShenandoahRegionIterator _regions;
97 const size_t _page_size;
98 public:
99 ShenandoahPretouchHeapTask(size_t page_size) :
100 WorkerTask("Shenandoah Pretouch Heap"),
101 _page_size(page_size) {}
102
103 virtual void work(uint worker_id) {
104 ShenandoahHeapRegion* r = _regions.next();
105 while (r != NULL) {
106 if (r->is_committed()) {
107 os::pretouch_memory(r->bottom(), r->end(), _page_size);
108 }
109 r = _regions.next();
110 }
111 }
112 };
113
114 class ShenandoahPretouchBitmapTask : public WorkerTask {
115 private:
116 ShenandoahRegionIterator _regions;
117 char* _bitmap_base;
118 const size_t _bitmap_size;
119 const size_t _page_size;
120 public:
121 ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
122 WorkerTask("Shenandoah Pretouch Bitmap"),
123 _bitmap_base(bitmap_base),
124 _bitmap_size(bitmap_size),
125 _page_size(page_size) {}
126
127 virtual void work(uint worker_id) {
128 ShenandoahHeapRegion* r = _regions.next();
129 while (r != NULL) {
130 size_t start = r->index() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
131 size_t end = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
132 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
133
134 if (r->is_committed()) {
135 os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
136 }
137
138 r = _regions.next();
139 }
140 }
141 };
142
143 jint ShenandoahHeap::initialize() {
144 //
145 // Figure out heap sizing
146 //
147
148 size_t init_byte_size = InitialHeapSize;
149 size_t min_byte_size = MinHeapSize;
150 size_t max_byte_size = MaxHeapSize;
151 size_t heap_alignment = HeapAlignment;
152
153 size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
154
155 Universe::check_alignment(max_byte_size, reg_size_bytes, "Shenandoah heap");
156 Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
157
158 _num_regions = ShenandoahHeapRegion::region_count();
159 assert(_num_regions == (max_byte_size / reg_size_bytes),
160 "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
161 _num_regions, max_byte_size, reg_size_bytes);
162
163 // Now we know the number of regions, initialize the heuristics.
164 initialize_heuristics();
165
166 size_t num_committed_regions = init_byte_size / reg_size_bytes;
167 num_committed_regions = MIN2(num_committed_regions, _num_regions);
168 assert(num_committed_regions <= _num_regions, "sanity");
169 _initial_size = num_committed_regions * reg_size_bytes;
170
171 size_t num_min_regions = min_byte_size / reg_size_bytes;
172 num_min_regions = MIN2(num_min_regions, _num_regions);
173 assert(num_min_regions <= _num_regions, "sanity");
174 _minimum_size = num_min_regions * reg_size_bytes;
175
176 // Default to max heap size.
177 _soft_max_size = _num_regions * reg_size_bytes;
178
179 _committed = _initial_size;
180
181 size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
182 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
183 size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
184
185 //
186 // Reserve and commit memory for heap
187 //
188
189 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
190 initialize_reserved_region(heap_rs);
191 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
192 _heap_region_special = heap_rs.special();
193
194 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
195 "Misaligned heap: " PTR_FORMAT, p2i(base()));
196
197 _forwarding = new SlidingForwarding(_heap_region, ShenandoahHeapRegion::region_size_words_shift());
198
199 #if SHENANDOAH_OPTIMIZED_MARKTASK
200 // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
201 // Fail if we ever attempt to address more than we can.
202 if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
203 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
204 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
205 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
206 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
207 vm_exit_during_initialization("Fatal Error", buf);
208 }
209 #endif
210
211 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
212 if (!_heap_region_special) {
213 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
214 "Cannot commit heap memory");
215 }
216
217 //
218 // Reserve and commit memory for bitmap(s)
219 //
220
221 _bitmap_size = ShenandoahMarkBitMap::compute_size(heap_rs.size());
222 _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
223
224 size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
225
226 guarantee(bitmap_bytes_per_region != 0,
227 "Bitmap bytes per region should not be zero");
228 guarantee(is_power_of_2(bitmap_bytes_per_region),
229 "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
230
231 if (bitmap_page_size > bitmap_bytes_per_region) {
232 _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
233 _bitmap_bytes_per_slice = bitmap_page_size;
234 } else {
235 _bitmap_regions_per_slice = 1;
236 _bitmap_bytes_per_slice = bitmap_bytes_per_region;
237 }
238
239 guarantee(_bitmap_regions_per_slice >= 1,
240 "Should have at least one region per slice: " SIZE_FORMAT,
241 _bitmap_regions_per_slice);
242
243 guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
244 "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
245 _bitmap_bytes_per_slice, bitmap_page_size);
246
247 ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
248 MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
249 _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
250 _bitmap_region_special = bitmap.special();
251
252 size_t bitmap_init_commit = _bitmap_bytes_per_slice *
253 align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
254 bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
255 if (!_bitmap_region_special) {
256 os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
257 "Cannot commit bitmap memory");
258 }
259
260 _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
261
262 if (ShenandoahVerify) {
263 ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
264 if (!verify_bitmap.special()) {
265 os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
266 "Cannot commit verification bitmap memory");
267 }
268 MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
269 MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
270 _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
271 _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
272 }
273
274 // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
275 ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
276 MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
277 _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
278 _aux_bitmap_region_special = aux_bitmap.special();
279 _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
280
281 //
282 // Create regions and region sets
283 //
284 size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
285 size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
286 region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
287
288 ReservedSpace region_storage(region_storage_size, region_page_size);
289 MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
290 if (!region_storage.special()) {
291 os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
292 "Cannot commit region memory");
293 }
294
295 // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
296 // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
297 // If not successful, bite a bullet and allocate at whatever address.
298 {
299 size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
300 size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
301
302 uintptr_t min = round_up_power_of_2(cset_align);
303 uintptr_t max = (1u << 30u);
304
305 for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
306 char* req_addr = (char*)addr;
307 assert(is_aligned(req_addr, cset_align), "Should be aligned");
308 ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size(), req_addr);
309 if (cset_rs.is_reserved()) {
310 assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
311 _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
312 break;
313 }
314 }
315
316 if (_collection_set == NULL) {
317 ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size());
318 _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
319 }
320 }
321
322 _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
323 _free_set = new ShenandoahFreeSet(this, _num_regions);
324
325 {
326 ShenandoahHeapLocker locker(lock());
327
328 for (size_t i = 0; i < _num_regions; i++) {
329 HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
330 bool is_committed = i < num_committed_regions;
331 void* loc = region_storage.base() + i * region_align;
332
333 ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
334 assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
335
336 _marking_context->initialize_top_at_mark_start(r);
337 _regions[i] = r;
338 assert(!collection_set()->is_in(i), "New region should not be in collection set");
339 }
340
341 // Initialize to complete
342 _marking_context->mark_complete();
343
344 _free_set->rebuild();
345 }
346
347 if (AlwaysPreTouch) {
348 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
349 // before initialize() below zeroes it with initializing thread. For any given region,
350 // we touch the region and the corresponding bitmaps from the same thread.
351 ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
352
353 _pretouch_heap_page_size = heap_page_size;
354 _pretouch_bitmap_page_size = bitmap_page_size;
355
356 #ifdef LINUX
357 // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
358 // pages. But, the kernel needs to know that every small page is used, in order to coalesce
359 // them into huge one. Therefore, we need to pretouch with smaller pages.
360 if (UseTransparentHugePages) {
361 _pretouch_heap_page_size = (size_t)os::vm_page_size();
362 _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
363 }
364 #endif
365
366 // OS memory managers may want to coalesce back-to-back pages. Make their jobs
367 // simpler by pre-touching continuous spaces (heap and bitmap) separately.
368
369 ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
370 _workers->run_task(&bcl);
371
372 ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
373 _workers->run_task(&hcl);
374 }
375
376 //
377 // Initialize the rest of GC subsystems
378 //
379
380 _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
381 for (uint worker = 0; worker < _max_workers; worker++) {
382 _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
383 Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
384 }
385
386 // There should probably be Shenandoah-specific options for these,
387 // just as there are G1-specific options.
388 {
389 ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
390 satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
391 satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
392 }
393
394 _monitoring_support = new ShenandoahMonitoringSupport(this);
395 _phase_timings = new ShenandoahPhaseTimings(max_workers());
396 ShenandoahCodeRoots::initialize();
397
398 if (ShenandoahPacing) {
399 _pacer = new ShenandoahPacer(this);
400 _pacer->setup_for_idle();
401 } else {
402 _pacer = NULL;
403 }
404
405 _control_thread = new ShenandoahControlThread();
406
407 ShenandoahInitLogger::print();
408
409 return JNI_OK;
410 }
411
412 void ShenandoahHeap::initialize_mode() {
413 if (ShenandoahGCMode != NULL) {
414 if (strcmp(ShenandoahGCMode, "satb") == 0) {
415 _gc_mode = new ShenandoahSATBMode();
416 } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
417 _gc_mode = new ShenandoahIUMode();
418 } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
419 _gc_mode = new ShenandoahPassiveMode();
420 } else {
421 vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
422 }
423 } else {
424 vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
425 }
426 _gc_mode->initialize_flags();
427 if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
428 vm_exit_during_initialization(
429 err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
430 _gc_mode->name()));
431 }
432 if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
433 vm_exit_during_initialization(
434 err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
435 _gc_mode->name()));
436 }
437 }
438
439 void ShenandoahHeap::initialize_heuristics() {
440 assert(_gc_mode != NULL, "Must be initialized");
441 _heuristics = _gc_mode->initialize_heuristics();
442
443 if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
444 vm_exit_during_initialization(
445 err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
446 _heuristics->name()));
447 }
448 if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
449 vm_exit_during_initialization(
450 err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
451 _heuristics->name()));
452 }
453 }
454
455 #ifdef _MSC_VER
456 #pragma warning( push )
457 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
458 #endif
459
460 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
461 CollectedHeap(),
462 _initial_size(0),
463 _used(0),
464 _committed(0),
465 _bytes_allocated_since_gc_start(0),
466 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
467 _workers(NULL),
468 _safepoint_workers(NULL),
469 _heap_region_special(false),
470 _num_regions(0),
471 _regions(NULL),
472 _update_refs_iterator(this),
473 _control_thread(NULL),
474 _shenandoah_policy(policy),
475 _gc_mode(NULL),
476 _heuristics(NULL),
477 _free_set(NULL),
478 _pacer(NULL),
479 _verifier(NULL),
480 _phase_timings(NULL),
481 _monitoring_support(NULL),
482 _memory_pool(NULL),
483 _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
484 _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
485 _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
486 _soft_ref_policy(),
487 _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
488 _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
489 _marking_context(NULL),
490 _bitmap_size(0),
491 _bitmap_regions_per_slice(0),
492 _bitmap_bytes_per_slice(0),
493 _bitmap_region_special(false),
494 _aux_bitmap_region_special(false),
495 _liveness_cache(NULL),
496 _collection_set(NULL)
497 {
498 // Initialize GC mode early, so we can adjust barrier support
499 initialize_mode();
500 BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
501
502 _max_workers = MAX2(_max_workers, 1U);
503 _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
504 if (_workers == NULL) {
505 vm_exit_during_initialization("Failed necessary allocation.");
506 } else {
507 _workers->initialize_workers();
508 }
509
510 if (ParallelGCThreads > 1) {
511 _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
512 ParallelGCThreads);
513 _safepoint_workers->initialize_workers();
514 }
515 }
516
517 #ifdef _MSC_VER
518 #pragma warning( pop )
519 #endif
520
521 class ShenandoahResetBitmapTask : public WorkerTask {
522 private:
523 ShenandoahRegionIterator _regions;
524
525 public:
526 ShenandoahResetBitmapTask() :
527 WorkerTask("Shenandoah Reset Bitmap") {}
528
529 void work(uint worker_id) {
530 ShenandoahHeapRegion* region = _regions.next();
531 ShenandoahHeap* heap = ShenandoahHeap::heap();
532 ShenandoahMarkingContext* const ctx = heap->marking_context();
533 while (region != NULL) {
534 if (heap->is_bitmap_slice_committed(region)) {
535 ctx->clear_bitmap(region);
536 }
537 region = _regions.next();
538 }
539 }
540 };
541
542 void ShenandoahHeap::reset_mark_bitmap() {
543 assert_gc_workers(_workers->active_workers());
544 mark_incomplete_marking_context();
545
546 ShenandoahResetBitmapTask task;
547 _workers->run_task(&task);
548 }
549
550 void ShenandoahHeap::print_on(outputStream* st) const {
551 st->print_cr("Shenandoah Heap");
552 st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
553 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
554 byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
555 byte_size_in_proper_unit(committed()), proper_unit_for_byte_size(committed()),
556 byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
557 st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
558 num_regions(),
559 byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
560 proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
561
562 st->print("Status: ");
563 if (has_forwarded_objects()) st->print("has forwarded objects, ");
564 if (is_concurrent_mark_in_progress()) st->print("marking, ");
565 if (is_evacuation_in_progress()) st->print("evacuating, ");
566 if (is_update_refs_in_progress()) st->print("updating refs, ");
567 if (is_degenerated_gc_in_progress()) st->print("degenerated gc, ");
568 if (is_full_gc_in_progress()) st->print("full gc, ");
569 if (is_full_gc_move_in_progress()) st->print("full gc move, ");
570 if (is_concurrent_weak_root_in_progress()) st->print("concurrent weak roots, ");
571 if (is_concurrent_strong_root_in_progress() &&
572 !is_concurrent_weak_root_in_progress()) st->print("concurrent strong roots, ");
573
574 if (cancelled_gc()) {
575 st->print("cancelled");
576 } else {
577 st->print("not cancelled");
578 }
579 st->cr();
580
581 st->print_cr("Reserved region:");
582 st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
583 p2i(reserved_region().start()),
584 p2i(reserved_region().end()));
585
586 ShenandoahCollectionSet* cset = collection_set();
587 st->print_cr("Collection set:");
588 if (cset != NULL) {
589 st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
590 st->print_cr(" - map (biased): " PTR_FORMAT, p2i(cset->biased_map_address()));
591 } else {
592 st->print_cr(" (NULL)");
593 }
594
595 st->cr();
596 MetaspaceUtils::print_on(st);
597
598 if (Verbose) {
599 print_heap_regions_on(st);
600 }
601 }
602
603 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
604 public:
605 void do_thread(Thread* thread) {
606 assert(thread != NULL, "Sanity");
607 assert(thread->is_Worker_thread(), "Only worker thread expected");
608 ShenandoahThreadLocalData::initialize_gclab(thread);
609 }
610 };
611
612 void ShenandoahHeap::post_initialize() {
613 CollectedHeap::post_initialize();
614 MutexLocker ml(Threads_lock);
615
616 ShenandoahInitWorkerGCLABClosure init_gclabs;
617 _workers->threads_do(&init_gclabs);
618
619 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
620 // Now, we will let WorkerThreads to initialize gclab when new worker is created.
621 _workers->set_initialize_gclab();
622 if (_safepoint_workers != NULL) {
623 _safepoint_workers->threads_do(&init_gclabs);
624 _safepoint_workers->set_initialize_gclab();
625 }
626
627 _heuristics->initialize();
628
629 JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
630 }
631
632 size_t ShenandoahHeap::used() const {
633 return Atomic::load(&_used);
634 }
635
636 size_t ShenandoahHeap::committed() const {
637 return Atomic::load(&_committed);
638 }
639
640 void ShenandoahHeap::increase_committed(size_t bytes) {
641 shenandoah_assert_heaplocked_or_safepoint();
642 _committed += bytes;
643 }
644
645 void ShenandoahHeap::decrease_committed(size_t bytes) {
646 shenandoah_assert_heaplocked_or_safepoint();
647 _committed -= bytes;
648 }
649
650 void ShenandoahHeap::increase_used(size_t bytes) {
651 Atomic::add(&_used, bytes, memory_order_relaxed);
652 }
653
654 void ShenandoahHeap::set_used(size_t bytes) {
655 Atomic::store(&_used, bytes);
656 }
657
658 void ShenandoahHeap::decrease_used(size_t bytes) {
659 assert(used() >= bytes, "never decrease heap size by more than we've left");
660 Atomic::sub(&_used, bytes, memory_order_relaxed);
661 }
662
663 void ShenandoahHeap::increase_allocated(size_t bytes) {
664 Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
665 }
666
667 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
668 size_t bytes = words * HeapWordSize;
669 if (!waste) {
670 increase_used(bytes);
671 }
672 increase_allocated(bytes);
673 if (ShenandoahPacing) {
674 control_thread()->pacing_notify_alloc(words);
675 if (waste) {
676 pacer()->claim_for_alloc(words, true);
677 }
678 }
679 }
680
681 size_t ShenandoahHeap::capacity() const {
682 return committed();
683 }
684
685 size_t ShenandoahHeap::max_capacity() const {
686 return _num_regions * ShenandoahHeapRegion::region_size_bytes();
687 }
688
689 size_t ShenandoahHeap::soft_max_capacity() const {
690 size_t v = Atomic::load(&_soft_max_size);
691 assert(min_capacity() <= v && v <= max_capacity(),
692 "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
693 min_capacity(), v, max_capacity());
694 return v;
695 }
696
697 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
698 assert(min_capacity() <= v && v <= max_capacity(),
699 "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
700 min_capacity(), v, max_capacity());
701 Atomic::store(&_soft_max_size, v);
702 }
703
704 size_t ShenandoahHeap::min_capacity() const {
705 return _minimum_size;
706 }
707
708 size_t ShenandoahHeap::initial_capacity() const {
709 return _initial_size;
710 }
711
712 bool ShenandoahHeap::is_in(const void* p) const {
713 HeapWord* heap_base = (HeapWord*) base();
714 HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
715 return p >= heap_base && p < last_region_end;
716 }
717
718 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
719 assert (ShenandoahUncommit, "should be enabled");
720
721 // Application allocates from the beginning of the heap, and GC allocates at
722 // the end of it. It is more efficient to uncommit from the end, so that applications
723 // could enjoy the near committed regions. GC allocations are much less frequent,
724 // and therefore can accept the committing costs.
725
726 size_t count = 0;
727 for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
728 ShenandoahHeapRegion* r = get_region(i - 1);
729 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
730 ShenandoahHeapLocker locker(lock());
731 if (r->is_empty_committed()) {
732 if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
733 break;
734 }
735
736 r->make_uncommitted();
737 count++;
738 }
739 }
740 SpinPause(); // allow allocators to take the lock
741 }
742
743 if (count > 0) {
744 control_thread()->notify_heap_changed();
745 }
746 }
747
748 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
749 // New object should fit the GCLAB size
750 size_t min_size = MAX2(size, PLAB::min_size());
751
752 // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
753 size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
754 new_size = MIN2(new_size, PLAB::max_size());
755 new_size = MAX2(new_size, PLAB::min_size());
756
757 // Record new heuristic value even if we take any shortcut. This captures
758 // the case when moderately-sized objects always take a shortcut. At some point,
759 // heuristics should catch up with them.
760 ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
761
762 if (new_size < size) {
763 // New size still does not fit the object. Fall back to shared allocation.
764 // This avoids retiring perfectly good GCLABs, when we encounter a large object.
765 return NULL;
766 }
767
768 // Retire current GCLAB, and allocate a new one.
769 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
770 gclab->retire();
771
772 size_t actual_size = 0;
773 HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
774 if (gclab_buf == NULL) {
775 return NULL;
776 }
777
778 assert (size <= actual_size, "allocation should fit");
779
780 if (ZeroTLAB) {
781 // ..and clear it.
782 Copy::zero_to_words(gclab_buf, actual_size);
783 } else {
784 // ...and zap just allocated object.
785 #ifdef ASSERT
786 // Skip mangling the space corresponding to the object header to
787 // ensure that the returned space is not considered parsable by
788 // any concurrent GC thread.
789 size_t hdr_size = oopDesc::header_size();
790 Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
791 #endif // ASSERT
792 }
793 gclab->set_buf(gclab_buf, actual_size);
794 return gclab->allocate(size);
795 }
796
797 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
798 size_t requested_size,
799 size_t* actual_size) {
800 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
801 HeapWord* res = allocate_memory(req);
802 if (res != NULL) {
803 *actual_size = req.actual_size();
804 } else {
805 *actual_size = 0;
806 }
807 return res;
808 }
809
810 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
811 size_t word_size,
812 size_t* actual_size) {
813 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
814 HeapWord* res = allocate_memory(req);
815 if (res != NULL) {
816 *actual_size = req.actual_size();
817 } else {
818 *actual_size = 0;
819 }
820 return res;
821 }
822
823 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
824 intptr_t pacer_epoch = 0;
825 bool in_new_region = false;
826 HeapWord* result = NULL;
827
828 if (req.is_mutator_alloc()) {
829 if (ShenandoahPacing) {
830 pacer()->pace_for_alloc(req.size());
831 pacer_epoch = pacer()->epoch();
832 }
833
834 if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
835 result = allocate_memory_under_lock(req, in_new_region);
836 }
837
838 // Allocation failed, block until control thread reacted, then retry allocation.
839 //
840 // It might happen that one of the threads requesting allocation would unblock
841 // way later after GC happened, only to fail the second allocation, because
842 // other threads have already depleted the free storage. In this case, a better
843 // strategy is to try again, as long as GC makes progress.
844 //
845 // Then, we need to make sure the allocation was retried after at least one
846 // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
847
848 size_t tries = 0;
849
850 while (result == NULL && _progress_last_gc.is_set()) {
851 tries++;
852 control_thread()->handle_alloc_failure(req);
853 result = allocate_memory_under_lock(req, in_new_region);
854 }
855
856 while (result == NULL && tries <= ShenandoahFullGCThreshold) {
857 tries++;
858 control_thread()->handle_alloc_failure(req);
859 result = allocate_memory_under_lock(req, in_new_region);
860 }
861
862 } else {
863 assert(req.is_gc_alloc(), "Can only accept GC allocs here");
864 result = allocate_memory_under_lock(req, in_new_region);
865 // Do not call handle_alloc_failure() here, because we cannot block.
866 // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
867 }
868
869 if (in_new_region) {
870 control_thread()->notify_heap_changed();
871 }
872
873 if (result != NULL) {
874 size_t requested = req.size();
875 size_t actual = req.actual_size();
876
877 assert (req.is_lab_alloc() || (requested == actual),
878 "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
879 ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
880
881 if (req.is_mutator_alloc()) {
882 notify_mutator_alloc_words(actual, false);
883
884 // If we requested more than we were granted, give the rest back to pacer.
885 // This only matters if we are in the same pacing epoch: do not try to unpace
886 // over the budget for the other phase.
887 if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
888 pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
889 }
890 } else {
891 increase_used(actual*HeapWordSize);
892 }
893 }
894
895 return result;
896 }
897
898 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
899 ShenandoahHeapLocker locker(lock());
900 return _free_set->allocate(req, in_new_region);
901 }
902
903 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
904 bool* gc_overhead_limit_was_exceeded) {
905 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
906 return allocate_memory(req);
907 }
908
909 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
910 size_t size,
911 Metaspace::MetadataType mdtype) {
912 MetaWord* result;
913
914 // Inform metaspace OOM to GC heuristics if class unloading is possible.
915 if (heuristics()->can_unload_classes()) {
916 ShenandoahHeuristics* h = heuristics();
917 h->record_metaspace_oom();
918 }
919
920 // Expand and retry allocation
921 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
922 if (result != NULL) {
923 return result;
924 }
925
926 // Start full GC
927 collect(GCCause::_metadata_GC_clear_soft_refs);
928
929 // Retry allocation
930 result = loader_data->metaspace_non_null()->allocate(size, mdtype);
931 if (result != NULL) {
932 return result;
933 }
934
935 // Expand and retry allocation
936 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
937 if (result != NULL) {
938 return result;
939 }
940
941 // Out of memory
942 return NULL;
943 }
944
945 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
946 private:
947 ShenandoahHeap* const _heap;
948 Thread* const _thread;
949 public:
950 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
951 _heap(heap), _thread(Thread::current()) {}
952
953 void do_object(oop p) {
954 shenandoah_assert_marked(NULL, p);
955 if (!ShenandoahForwarding::is_forwarded(p)) {
956 _heap->evacuate_object(p, _thread);
957 }
958 }
959 };
960
961 class ShenandoahEvacuationTask : public WorkerTask {
962 private:
963 ShenandoahHeap* const _sh;
964 ShenandoahCollectionSet* const _cs;
965 bool _concurrent;
966 public:
967 ShenandoahEvacuationTask(ShenandoahHeap* sh,
968 ShenandoahCollectionSet* cs,
969 bool concurrent) :
970 WorkerTask("Shenandoah Evacuation"),
971 _sh(sh),
972 _cs(cs),
973 _concurrent(concurrent)
974 {}
975
976 void work(uint worker_id) {
977 if (_concurrent) {
978 ShenandoahConcurrentWorkerSession worker_session(worker_id);
979 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
980 ShenandoahEvacOOMScope oom_evac_scope;
981 do_work();
982 } else {
983 ShenandoahParallelWorkerSession worker_session(worker_id);
984 ShenandoahEvacOOMScope oom_evac_scope;
985 do_work();
986 }
987 }
988
989 private:
990 void do_work() {
991 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
992 ShenandoahHeapRegion* r;
993 while ((r =_cs->claim_next()) != NULL) {
994 assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
995 _sh->marked_object_iterate(r, &cl);
996
997 if (ShenandoahPacing) {
998 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
999 }
1000
1001 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1002 break;
1003 }
1004 }
1005 }
1006 };
1007
1008 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1009 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1010 workers()->run_task(&task);
1011 }
1012
1013 void ShenandoahHeap::trash_cset_regions() {
1014 ShenandoahHeapLocker locker(lock());
1015
1016 ShenandoahCollectionSet* set = collection_set();
1017 ShenandoahHeapRegion* r;
1018 set->clear_current_index();
1019 while ((r = set->next()) != NULL) {
1020 r->make_trash();
1021 }
1022 collection_set()->clear();
1023 }
1024
1025 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1026 st->print_cr("Heap Regions:");
1027 st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1028 st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1029 st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1030 st->print_cr("SN=alloc sequence number");
1031
1032 for (size_t i = 0; i < num_regions(); i++) {
1033 get_region(i)->print_on(st);
1034 }
1035 }
1036
1037 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1038 assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1039
1040 oop humongous_obj = cast_to_oop(start->bottom());
1041 size_t size = humongous_obj->size();
1042 size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1043 size_t index = start->index() + required_regions - 1;
1044
1045 assert(!start->has_live(), "liveness must be zero");
1046
1047 for(size_t i = 0; i < required_regions; i++) {
1048 // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1049 // as it expects that every region belongs to a humongous region starting with a humongous start region.
1050 ShenandoahHeapRegion* region = get_region(index --);
1051
1052 assert(region->is_humongous(), "expect correct humongous start or continuation");
1053 assert(!region->is_cset(), "Humongous region should not be in collection set");
1054
1055 region->make_trash_immediate();
1056 }
1057 }
1058
1059 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1060 public:
1061 ShenandoahCheckCleanGCLABClosure() {}
1062 void do_thread(Thread* thread) {
1063 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1064 assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1065 assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1066 }
1067 };
1068
1069 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1070 private:
1071 bool const _resize;
1072 public:
1073 ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1074 void do_thread(Thread* thread) {
1075 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1076 assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1077 gclab->retire();
1078 if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1079 ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1080 }
1081 }
1082 };
1083
1084 void ShenandoahHeap::labs_make_parsable() {
1085 assert(UseTLAB, "Only call with UseTLAB");
1086
1087 ShenandoahRetireGCLABClosure cl(false);
1088
1089 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1090 ThreadLocalAllocBuffer& tlab = t->tlab();
1091 tlab.make_parsable();
1092 cl.do_thread(t);
1093 }
1094
1095 workers()->threads_do(&cl);
1096 }
1097
1098 void ShenandoahHeap::tlabs_retire(bool resize) {
1099 assert(UseTLAB, "Only call with UseTLAB");
1100 assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1101
1102 ThreadLocalAllocStats stats;
1103
1104 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1105 ThreadLocalAllocBuffer& tlab = t->tlab();
1106 tlab.retire(&stats);
1107 if (resize) {
1108 tlab.resize();
1109 }
1110 }
1111
1112 stats.publish();
1113
1114 #ifdef ASSERT
1115 ShenandoahCheckCleanGCLABClosure cl;
1116 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1117 cl.do_thread(t);
1118 }
1119 workers()->threads_do(&cl);
1120 #endif
1121 }
1122
1123 void ShenandoahHeap::gclabs_retire(bool resize) {
1124 assert(UseTLAB, "Only call with UseTLAB");
1125 assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1126
1127 ShenandoahRetireGCLABClosure cl(resize);
1128 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1129 cl.do_thread(t);
1130 }
1131 workers()->threads_do(&cl);
1132
1133 if (safepoint_workers() != NULL) {
1134 safepoint_workers()->threads_do(&cl);
1135 }
1136 }
1137
1138 // Returns size in bytes
1139 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1140 if (ShenandoahElasticTLAB) {
1141 // With Elastic TLABs, return the max allowed size, and let the allocation path
1142 // figure out the safe size for current allocation.
1143 return ShenandoahHeapRegion::max_tlab_size_bytes();
1144 } else {
1145 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1146 }
1147 }
1148
1149 size_t ShenandoahHeap::max_tlab_size() const {
1150 // Returns size in words
1151 return ShenandoahHeapRegion::max_tlab_size_words();
1152 }
1153
1154 void ShenandoahHeap::collect(GCCause::Cause cause) {
1155 control_thread()->request_gc(cause);
1156 }
1157
1158 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1159 //assert(false, "Shouldn't need to do full collections");
1160 }
1161
1162 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1163 ShenandoahHeapRegion* r = heap_region_containing(addr);
1164 if (r != NULL) {
1165 return r->block_start(addr);
1166 }
1167 return NULL;
1168 }
1169
1170 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1171 ShenandoahHeapRegion* r = heap_region_containing(addr);
1172 return r->block_is_obj(addr);
1173 }
1174
1175 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1176 return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1177 }
1178
1179 void ShenandoahHeap::prepare_for_verify() {
1180 if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1181 labs_make_parsable();
1182 }
1183 }
1184
1185 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1186 tcl->do_thread(_control_thread);
1187 workers()->threads_do(tcl);
1188 if (_safepoint_workers != NULL) {
1189 _safepoint_workers->threads_do(tcl);
1190 }
1191 if (ShenandoahStringDedup::is_enabled()) {
1192 ShenandoahStringDedup::threads_do(tcl);
1193 }
1194 }
1195
1196 void ShenandoahHeap::print_tracing_info() const {
1197 LogTarget(Info, gc, stats) lt;
1198 if (lt.is_enabled()) {
1199 ResourceMark rm;
1200 LogStream ls(lt);
1201
1202 phase_timings()->print_global_on(&ls);
1203
1204 ls.cr();
1205 ls.cr();
1206
1207 shenandoah_policy()->print_gc_stats(&ls);
1208
1209 ls.cr();
1210 ls.cr();
1211 }
1212 }
1213
1214 void ShenandoahHeap::verify(VerifyOption vo) {
1215 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1216 if (ShenandoahVerify) {
1217 verifier()->verify_generic(vo);
1218 } else {
1219 // TODO: Consider allocating verification bitmaps on demand,
1220 // and turn this on unconditionally.
1221 }
1222 }
1223 }
1224 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1225 return _free_set->capacity();
1226 }
1227
1228 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1229 private:
1230 MarkBitMap* _bitmap;
1231 ShenandoahScanObjectStack* _oop_stack;
1232 ShenandoahHeap* const _heap;
1233 ShenandoahMarkingContext* const _marking_context;
1234
1235 template <class T>
1236 void do_oop_work(T* p) {
1237 T o = RawAccess<>::oop_load(p);
1238 if (!CompressedOops::is_null(o)) {
1239 oop obj = CompressedOops::decode_not_null(o);
1240 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1241 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1242 return;
1243 }
1244 obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1245
1246 assert(oopDesc::is_oop(obj), "must be a valid oop");
1247 if (!_bitmap->is_marked(obj)) {
1248 _bitmap->mark(obj);
1249 _oop_stack->push(obj);
1250 }
1251 }
1252 }
1253 public:
1254 ObjectIterateScanRootClosure(MarkBitMap* bitmap, ShenandoahScanObjectStack* oop_stack) :
1255 _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1256 _marking_context(_heap->marking_context()) {}
1257 void do_oop(oop* p) { do_oop_work(p); }
1258 void do_oop(narrowOop* p) { do_oop_work(p); }
1259 };
1260
1261 /*
1262 * This is public API, used in preparation of object_iterate().
1263 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1264 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1265 * control, we call SH::tlabs_retire, SH::gclabs_retire.
1266 */
1267 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1268 // No-op.
1269 }
1270
1271 /*
1272 * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1273 *
1274 * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1275 * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1276 * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1277 * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1278 * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1279 * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1280 * wiped the bitmap in preparation for next marking).
1281 *
1282 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1283 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1284 * is allowed to report dead objects, but is not required to do so.
1285 */
1286 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1287 // Reset bitmap
1288 if (!prepare_aux_bitmap_for_iteration())
1289 return;
1290
1291 ShenandoahScanObjectStack oop_stack;
1292 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1293 // Seed the stack with root scan
1294 scan_roots_for_iteration(&oop_stack, &oops);
1295
1296 // Work through the oop stack to traverse heap
1297 while (! oop_stack.is_empty()) {
1298 oop obj = oop_stack.pop();
1299 assert(oopDesc::is_oop(obj), "must be a valid oop");
1300 shenandoah_assert_not_in_cset_except(NULL, obj, cancelled_gc());
1301 cl->do_object(obj);
1302 obj->oop_iterate(&oops);
1303 }
1304
1305 assert(oop_stack.is_empty(), "should be empty");
1306 // Reclaim bitmap
1307 reclaim_aux_bitmap_for_iteration();
1308 }
1309
1310 bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
1311 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1312
1313 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1314 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1315 return false;
1316 }
1317 // Reset bitmap
1318 _aux_bit_map.clear();
1319 return true;
1320 }
1321
1322 void ShenandoahHeap::scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops) {
1323 // Process GC roots according to current GC cycle
1324 // This populates the work stack with initial objects
1325 // It is important to relinquish the associated locks before diving
1326 // into heap dumper
1327 uint n_workers = safepoint_workers() != NULL ? safepoint_workers()->active_workers() : 1;
1328 ShenandoahHeapIterationRootScanner rp(n_workers);
1329 rp.roots_do(oops);
1330 }
1331
1332 void ShenandoahHeap::reclaim_aux_bitmap_for_iteration() {
1333 if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1334 log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1335 }
1336 }
1337
1338 // Closure for parallelly iterate objects
1339 class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure {
1340 private:
1341 MarkBitMap* _bitmap;
1342 ShenandoahObjToScanQueue* _queue;
1343 ShenandoahHeap* const _heap;
1344 ShenandoahMarkingContext* const _marking_context;
1345
1346 template <class T>
1347 void do_oop_work(T* p) {
1348 T o = RawAccess<>::oop_load(p);
1349 if (!CompressedOops::is_null(o)) {
1350 oop obj = CompressedOops::decode_not_null(o);
1351 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1352 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1353 return;
1354 }
1355 obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1356
1357 assert(oopDesc::is_oop(obj), "Must be a valid oop");
1358 if (_bitmap->par_mark(obj)) {
1359 _queue->push(ShenandoahMarkTask(obj));
1360 }
1361 }
1362 }
1363 public:
1364 ShenandoahObjectIterateParScanClosure(MarkBitMap* bitmap, ShenandoahObjToScanQueue* q) :
1365 _bitmap(bitmap), _queue(q), _heap(ShenandoahHeap::heap()),
1366 _marking_context(_heap->marking_context()) {}
1367 void do_oop(oop* p) { do_oop_work(p); }
1368 void do_oop(narrowOop* p) { do_oop_work(p); }
1369 };
1370
1371 // Object iterator for parallel heap iteraion.
1372 // The root scanning phase happenes in construction as a preparation of
1373 // parallel marking queues.
1374 // Every worker processes it's own marking queue. work-stealing is used
1375 // to balance workload.
1376 class ShenandoahParallelObjectIterator : public ParallelObjectIteratorImpl {
1377 private:
1378 uint _num_workers;
1379 bool _init_ready;
1380 MarkBitMap* _aux_bit_map;
1381 ShenandoahHeap* _heap;
1382 ShenandoahScanObjectStack _roots_stack; // global roots stack
1383 ShenandoahObjToScanQueueSet* _task_queues;
1384 public:
1385 ShenandoahParallelObjectIterator(uint num_workers, MarkBitMap* bitmap) :
1386 _num_workers(num_workers),
1387 _init_ready(false),
1388 _aux_bit_map(bitmap),
1389 _heap(ShenandoahHeap::heap()) {
1390 // Initialize bitmap
1391 _init_ready = _heap->prepare_aux_bitmap_for_iteration();
1392 if (!_init_ready) {
1393 return;
1394 }
1395
1396 ObjectIterateScanRootClosure oops(_aux_bit_map, &_roots_stack);
1397 _heap->scan_roots_for_iteration(&_roots_stack, &oops);
1398
1399 _init_ready = prepare_worker_queues();
1400 }
1401
1402 ~ShenandoahParallelObjectIterator() {
1403 // Reclaim bitmap
1404 _heap->reclaim_aux_bitmap_for_iteration();
1405 // Reclaim queue for workers
1406 if (_task_queues!= NULL) {
1407 for (uint i = 0; i < _num_workers; ++i) {
1408 ShenandoahObjToScanQueue* q = _task_queues->queue(i);
1409 if (q != NULL) {
1410 delete q;
1411 _task_queues->register_queue(i, NULL);
1412 }
1413 }
1414 delete _task_queues;
1415 _task_queues = NULL;
1416 }
1417 }
1418
1419 virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
1420 if (_init_ready) {
1421 object_iterate_parallel(cl, worker_id, _task_queues);
1422 }
1423 }
1424
1425 private:
1426 // Divide global root_stack into worker queues
1427 bool prepare_worker_queues() {
1428 _task_queues = new ShenandoahObjToScanQueueSet((int) _num_workers);
1429 // Initialize queues for every workers
1430 for (uint i = 0; i < _num_workers; ++i) {
1431 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
1432 _task_queues->register_queue(i, task_queue);
1433 }
1434 // Divide roots among the workers. Assume that object referencing distribution
1435 // is related with root kind, use round-robin to make every worker have same chance
1436 // to process every kind of roots
1437 size_t roots_num = _roots_stack.size();
1438 if (roots_num == 0) {
1439 // No work to do
1440 return false;
1441 }
1442
1443 for (uint j = 0; j < roots_num; j++) {
1444 uint stack_id = j % _num_workers;
1445 oop obj = _roots_stack.pop();
1446 _task_queues->queue(stack_id)->push(ShenandoahMarkTask(obj));
1447 }
1448 return true;
1449 }
1450
1451 void object_iterate_parallel(ObjectClosure* cl,
1452 uint worker_id,
1453 ShenandoahObjToScanQueueSet* queue_set) {
1454 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1455 assert(queue_set != NULL, "task queue must not be NULL");
1456
1457 ShenandoahObjToScanQueue* q = queue_set->queue(worker_id);
1458 assert(q != NULL, "object iterate queue must not be NULL");
1459
1460 ShenandoahMarkTask t;
1461 ShenandoahObjectIterateParScanClosure oops(_aux_bit_map, q);
1462
1463 // Work through the queue to traverse heap.
1464 // Steal when there is no task in queue.
1465 while (q->pop(t) || queue_set->steal(worker_id, t)) {
1466 oop obj = t.obj();
1467 assert(oopDesc::is_oop(obj), "must be a valid oop");
1468 cl->do_object(obj);
1469 obj->oop_iterate(&oops);
1470 }
1471 assert(q->is_empty(), "should be empty");
1472 }
1473 };
1474
1475 ParallelObjectIteratorImpl* ShenandoahHeap::parallel_object_iterator(uint workers) {
1476 return new ShenandoahParallelObjectIterator(workers, &_aux_bit_map);
1477 }
1478
1479 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1480 void ShenandoahHeap::keep_alive(oop obj) {
1481 if (is_concurrent_mark_in_progress() && (obj != NULL)) {
1482 ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1483 }
1484 }
1485
1486 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1487 for (size_t i = 0; i < num_regions(); i++) {
1488 ShenandoahHeapRegion* current = get_region(i);
1489 blk->heap_region_do(current);
1490 }
1491 }
1492
1493 class ShenandoahParallelHeapRegionTask : public WorkerTask {
1494 private:
1495 ShenandoahHeap* const _heap;
1496 ShenandoahHeapRegionClosure* const _blk;
1497
1498 shenandoah_padding(0);
1499 volatile size_t _index;
1500 shenandoah_padding(1);
1501
1502 public:
1503 ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1504 WorkerTask("Shenandoah Parallel Region Operation"),
1505 _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1506
1507 void work(uint worker_id) {
1508 ShenandoahParallelWorkerSession worker_session(worker_id);
1509 size_t stride = ShenandoahParallelRegionStride;
1510
1511 size_t max = _heap->num_regions();
1512 while (Atomic::load(&_index) < max) {
1513 size_t cur = Atomic::fetch_and_add(&_index, stride, memory_order_relaxed);
1514 size_t start = cur;
1515 size_t end = MIN2(cur + stride, max);
1516 if (start >= max) break;
1517
1518 for (size_t i = cur; i < end; i++) {
1519 ShenandoahHeapRegion* current = _heap->get_region(i);
1520 _blk->heap_region_do(current);
1521 }
1522 }
1523 }
1524 };
1525
1526 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1527 assert(blk->is_thread_safe(), "Only thread-safe closures here");
1528 if (num_regions() > ShenandoahParallelRegionStride) {
1529 ShenandoahParallelHeapRegionTask task(blk);
1530 workers()->run_task(&task);
1531 } else {
1532 heap_region_iterate(blk);
1533 }
1534 }
1535
1536 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1537 private:
1538 ShenandoahMarkingContext* const _ctx;
1539 public:
1540 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1541
1542 void heap_region_do(ShenandoahHeapRegion* r) {
1543 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1544 if (r->is_active()) {
1545 // Check if region needs updating its TAMS. We have updated it already during concurrent
1546 // reset, so it is very likely we don't need to do another write here.
1547 if (_ctx->top_at_mark_start(r) != r->top()) {
1548 _ctx->capture_top_at_mark_start(r);
1549 }
1550 } else {
1551 assert(_ctx->top_at_mark_start(r) == r->top(),
1552 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1553 }
1554 }
1555
1556 bool is_thread_safe() { return true; }
1557 };
1558
1559 class ShenandoahRendezvousClosure : public HandshakeClosure {
1560 public:
1561 inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1562 inline void do_thread(Thread* thread) {}
1563 };
1564
1565 void ShenandoahHeap::rendezvous_threads() {
1566 ShenandoahRendezvousClosure cl;
1567 Handshake::execute(&cl);
1568 }
1569
1570 void ShenandoahHeap::recycle_trash() {
1571 free_set()->recycle_trash();
1572 }
1573
1574 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1575 private:
1576 ShenandoahMarkingContext* const _ctx;
1577 public:
1578 ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1579
1580 void heap_region_do(ShenandoahHeapRegion* r) {
1581 if (r->is_active()) {
1582 // Reset live data and set TAMS optimistically. We would recheck these under the pause
1583 // anyway to capture any updates that happened since now.
1584 r->clear_live_data();
1585 _ctx->capture_top_at_mark_start(r);
1586 }
1587 }
1588
1589 bool is_thread_safe() { return true; }
1590 };
1591
1592 void ShenandoahHeap::prepare_gc() {
1593 reset_mark_bitmap();
1594
1595 ShenandoahResetUpdateRegionStateClosure cl;
1596 parallel_heap_region_iterate(&cl);
1597 }
1598
1599 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1600 private:
1601 ShenandoahMarkingContext* const _ctx;
1602 ShenandoahHeapLock* const _lock;
1603
1604 public:
1605 ShenandoahFinalMarkUpdateRegionStateClosure() :
1606 _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1607
1608 void heap_region_do(ShenandoahHeapRegion* r) {
1609 if (r->is_active()) {
1610 // All allocations past TAMS are implicitly live, adjust the region data.
1611 // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1612 HeapWord *tams = _ctx->top_at_mark_start(r);
1613 HeapWord *top = r->top();
1614 if (top > tams) {
1615 r->increase_live_data_alloc_words(pointer_delta(top, tams));
1616 }
1617
1618 // We are about to select the collection set, make sure it knows about
1619 // current pinning status. Also, this allows trashing more regions that
1620 // now have their pinning status dropped.
1621 if (r->is_pinned()) {
1622 if (r->pin_count() == 0) {
1623 ShenandoahHeapLocker locker(_lock);
1624 r->make_unpinned();
1625 }
1626 } else {
1627 if (r->pin_count() > 0) {
1628 ShenandoahHeapLocker locker(_lock);
1629 r->make_pinned();
1630 }
1631 }
1632
1633 // Remember limit for updating refs. It's guaranteed that we get no
1634 // from-space-refs written from here on.
1635 r->set_update_watermark_at_safepoint(r->top());
1636 } else {
1637 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1638 assert(_ctx->top_at_mark_start(r) == r->top(),
1639 "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1640 }
1641 }
1642
1643 bool is_thread_safe() { return true; }
1644 };
1645
1646 void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
1647 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1648 {
1649 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
1650 ShenandoahPhaseTimings::degen_gc_final_update_region_states);
1651 ShenandoahFinalMarkUpdateRegionStateClosure cl;
1652 parallel_heap_region_iterate(&cl);
1653
1654 assert_pinned_region_status();
1655 }
1656
1657 {
1658 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
1659 ShenandoahPhaseTimings::degen_gc_choose_cset);
1660 ShenandoahHeapLocker locker(lock());
1661 _collection_set->clear();
1662 heuristics()->choose_collection_set(_collection_set);
1663 }
1664
1665 {
1666 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
1667 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
1668 ShenandoahHeapLocker locker(lock());
1669 _free_set->rebuild();
1670 }
1671 }
1672
1673 void ShenandoahHeap::do_class_unloading() {
1674 _unloader.unload();
1675 }
1676
1677 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1678 // Weak refs processing
1679 ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1680 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1681 ShenandoahTimingsTracker t(phase);
1682 ShenandoahGCWorkerPhase worker_phase(phase);
1683 ref_processor()->process_references(phase, workers(), false /* concurrent */);
1684 }
1685
1686 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1687 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1688
1689 // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1690 // make them parsable for update code to work correctly. Plus, we can compute new sizes
1691 // for future GCLABs here.
1692 if (UseTLAB) {
1693 ShenandoahGCPhase phase(concurrent ?
1694 ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1695 ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1696 gclabs_retire(ResizeTLAB);
1697 }
1698
1699 _update_refs_iterator.reset();
1700 }
1701
1702 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1703 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1704 ShenandoahThreadLocalData::set_gc_state(t, state);
1705 }
1706 }
1707
1708 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1709 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1710 _gc_state.set_cond(mask, value);
1711 set_gc_state_all_threads(_gc_state.raw_value());
1712 }
1713
1714 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1715 assert(!has_forwarded_objects(), "Not expected before/after mark phase");
1716 set_gc_state_mask(MARKING, in_progress);
1717 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1718 }
1719
1720 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1721 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1722 set_gc_state_mask(EVACUATION, in_progress);
1723 }
1724
1725 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
1726 if (in_progress) {
1727 _concurrent_strong_root_in_progress.set();
1728 } else {
1729 _concurrent_strong_root_in_progress.unset();
1730 }
1731 }
1732
1733 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
1734 set_gc_state_mask(WEAK_ROOTS, cond);
1735 }
1736
1737 GCTracer* ShenandoahHeap::tracer() {
1738 return shenandoah_policy()->tracer();
1739 }
1740
1741 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1742 return _free_set->used();
1743 }
1744
1745 bool ShenandoahHeap::try_cancel_gc() {
1746 while (true) {
1747 jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1748 if (prev == CANCELLABLE) return true;
1749 else if (prev == CANCELLED) return false;
1750 assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
1751 assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
1752 Thread* thread = Thread::current();
1753 if (thread->is_Java_thread()) {
1754 // We need to provide a safepoint here, otherwise we might
1755 // spin forever if a SP is pending.
1756 ThreadBlockInVM sp(JavaThread::cast(thread));
1757 SpinPause();
1758 }
1759 }
1760 }
1761
1762 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1763 if (try_cancel_gc()) {
1764 FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1765 log_info(gc)("%s", msg.buffer());
1766 Events::log(Thread::current(), "%s", msg.buffer());
1767 }
1768 }
1769
1770 uint ShenandoahHeap::max_workers() {
1771 return _max_workers;
1772 }
1773
1774 void ShenandoahHeap::stop() {
1775 // The shutdown sequence should be able to terminate when GC is running.
1776
1777 // Step 0. Notify policy to disable event recording.
1778 _shenandoah_policy->record_shutdown();
1779
1780 // Step 1. Notify control thread that we are in shutdown.
1781 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1782 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1783 control_thread()->prepare_for_graceful_shutdown();
1784
1785 // Step 2. Notify GC workers that we are cancelling GC.
1786 cancel_gc(GCCause::_shenandoah_stop_vm);
1787
1788 // Step 3. Wait until GC worker exits normally.
1789 control_thread()->stop();
1790 }
1791
1792 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
1793 if (!unload_classes()) return;
1794 // Unload classes and purge SystemDictionary.
1795 {
1796 ShenandoahPhaseTimings::Phase phase = full_gc ?
1797 ShenandoahPhaseTimings::full_gc_purge_class_unload :
1798 ShenandoahPhaseTimings::degen_gc_purge_class_unload;
1799 ShenandoahIsAliveSelector is_alive;
1800 CodeCache::UnloadingScope scope(is_alive.is_alive_closure());
1801 ShenandoahGCPhase gc_phase(phase);
1802 ShenandoahGCWorkerPhase worker_phase(phase);
1803 bool purged_class = SystemDictionary::do_unloading(gc_timer());
1804
1805 uint num_workers = _workers->active_workers();
1806 ShenandoahClassUnloadingTask unlink_task(phase, num_workers, purged_class);
1807 _workers->run_task(&unlink_task);
1808 }
1809
1810 {
1811 ShenandoahGCPhase phase(full_gc ?
1812 ShenandoahPhaseTimings::full_gc_purge_cldg :
1813 ShenandoahPhaseTimings::degen_gc_purge_cldg);
1814 ClassLoaderDataGraph::purge(/*at_safepoint*/true);
1815 }
1816 // Resize and verify metaspace
1817 MetaspaceGC::compute_new_size();
1818 DEBUG_ONLY(MetaspaceUtils::verify();)
1819 }
1820
1821 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
1822 // so they should not have forwarded oops.
1823 // However, we do need to "null" dead oops in the roots, if can not be done
1824 // in concurrent cycles.
1825 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
1826 uint num_workers = _workers->active_workers();
1827 ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
1828 ShenandoahPhaseTimings::full_gc_purge_weak_par :
1829 ShenandoahPhaseTimings::degen_gc_purge_weak_par;
1830 ShenandoahGCPhase phase(timing_phase);
1831 ShenandoahGCWorkerPhase worker_phase(timing_phase);
1832 // Cleanup weak roots
1833 if (has_forwarded_objects()) {
1834 ShenandoahForwardedIsAliveClosure is_alive;
1835 ShenandoahUpdateRefsClosure keep_alive;
1836 ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
1837 cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers);
1838 _workers->run_task(&cleaning_task);
1839 } else {
1840 ShenandoahIsAliveClosure is_alive;
1841 #ifdef ASSERT
1842 ShenandoahAssertNotForwardedClosure verify_cl;
1843 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
1844 cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers);
1845 #else
1846 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
1847 cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers);
1848 #endif
1849 _workers->run_task(&cleaning_task);
1850 }
1851 }
1852
1853 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
1854 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1855 assert(is_stw_gc_in_progress(), "Only for Degenerated and Full GC");
1856 ShenandoahGCPhase phase(full_gc ?
1857 ShenandoahPhaseTimings::full_gc_purge :
1858 ShenandoahPhaseTimings::degen_gc_purge);
1859 stw_weak_refs(full_gc);
1860 stw_process_weak_roots(full_gc);
1861 stw_unload_classes(full_gc);
1862 }
1863
1864 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1865 set_gc_state_mask(HAS_FORWARDED, cond);
1866 }
1867
1868 void ShenandoahHeap::set_unload_classes(bool uc) {
1869 _unload_classes.set_cond(uc);
1870 }
1871
1872 bool ShenandoahHeap::unload_classes() const {
1873 return _unload_classes.is_set();
1874 }
1875
1876 address ShenandoahHeap::in_cset_fast_test_addr() {
1877 ShenandoahHeap* heap = ShenandoahHeap::heap();
1878 assert(heap->collection_set() != NULL, "Sanity");
1879 return (address) heap->collection_set()->biased_map_address();
1880 }
1881
1882 address ShenandoahHeap::cancelled_gc_addr() {
1883 return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
1884 }
1885
1886 address ShenandoahHeap::gc_state_addr() {
1887 return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
1888 }
1889
1890 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
1891 return Atomic::load(&_bytes_allocated_since_gc_start);
1892 }
1893
1894 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
1895 Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
1896 }
1897
1898 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
1899 _degenerated_gc_in_progress.set_cond(in_progress);
1900 }
1901
1902 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
1903 _full_gc_in_progress.set_cond(in_progress);
1904 }
1905
1906 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
1907 assert (is_full_gc_in_progress(), "should be");
1908 _full_gc_move_in_progress.set_cond(in_progress);
1909 }
1910
1911 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
1912 set_gc_state_mask(UPDATEREFS, in_progress);
1913 }
1914
1915 void ShenandoahHeap::register_nmethod(nmethod* nm) {
1916 ShenandoahCodeRoots::register_nmethod(nm);
1917 }
1918
1919 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
1920 ShenandoahCodeRoots::unregister_nmethod(nm);
1921 }
1922
1923 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
1924 heap_region_containing(o)->record_pin();
1925 return o;
1926 }
1927
1928 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
1929 ShenandoahHeapRegion* r = heap_region_containing(o);
1930 assert(r != NULL, "Sanity");
1931 assert(r->pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", r->index());
1932 r->record_unpin();
1933 }
1934
1935 void ShenandoahHeap::sync_pinned_region_status() {
1936 ShenandoahHeapLocker locker(lock());
1937
1938 for (size_t i = 0; i < num_regions(); i++) {
1939 ShenandoahHeapRegion *r = get_region(i);
1940 if (r->is_active()) {
1941 if (r->is_pinned()) {
1942 if (r->pin_count() == 0) {
1943 r->make_unpinned();
1944 }
1945 } else {
1946 if (r->pin_count() > 0) {
1947 r->make_pinned();
1948 }
1949 }
1950 }
1951 }
1952
1953 assert_pinned_region_status();
1954 }
1955
1956 #ifdef ASSERT
1957 void ShenandoahHeap::assert_pinned_region_status() {
1958 for (size_t i = 0; i < num_regions(); i++) {
1959 ShenandoahHeapRegion* r = get_region(i);
1960 assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
1961 "Region " SIZE_FORMAT " pinning status is inconsistent", i);
1962 }
1963 }
1964 #endif
1965
1966 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
1967 return _gc_timer;
1968 }
1969
1970 void ShenandoahHeap::prepare_concurrent_roots() {
1971 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1972 assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1973 set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
1974 set_concurrent_weak_root_in_progress(true);
1975 if (unload_classes()) {
1976 _unloader.prepare();
1977 }
1978 }
1979
1980 void ShenandoahHeap::finish_concurrent_roots() {
1981 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1982 assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1983 if (unload_classes()) {
1984 _unloader.finish();
1985 }
1986 }
1987
1988 #ifdef ASSERT
1989 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
1990 assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
1991
1992 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1993 if (UseDynamicNumberOfGCThreads) {
1994 assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
1995 } else {
1996 // Use ParallelGCThreads inside safepoints
1997 assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
1998 }
1999 } else {
2000 if (UseDynamicNumberOfGCThreads) {
2001 assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2002 } else {
2003 // Use ConcGCThreads outside safepoints
2004 assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2005 }
2006 }
2007 }
2008 #endif
2009
2010 ShenandoahVerifier* ShenandoahHeap::verifier() {
2011 guarantee(ShenandoahVerify, "Should be enabled");
2012 assert (_verifier != NULL, "sanity");
2013 return _verifier;
2014 }
2015
2016 template<bool CONCURRENT>
2017 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2018 private:
2019 ShenandoahHeap* _heap;
2020 ShenandoahRegionIterator* _regions;
2021 public:
2022 ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2023 WorkerTask("Shenandoah Update References"),
2024 _heap(ShenandoahHeap::heap()),
2025 _regions(regions) {
2026 }
2027
2028 void work(uint worker_id) {
2029 if (CONCURRENT) {
2030 ShenandoahConcurrentWorkerSession worker_session(worker_id);
2031 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2032 do_work<ShenandoahConcUpdateRefsClosure>();
2033 } else {
2034 ShenandoahParallelWorkerSession worker_session(worker_id);
2035 do_work<ShenandoahSTWUpdateRefsClosure>();
2036 }
2037 }
2038
2039 private:
2040 template<class T>
2041 void do_work() {
2042 T cl;
2043 ShenandoahHeapRegion* r = _regions->next();
2044 ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2045 while (r != NULL) {
2046 HeapWord* update_watermark = r->get_update_watermark();
2047 assert (update_watermark >= r->bottom(), "sanity");
2048 if (r->is_active() && !r->is_cset()) {
2049 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2050 }
2051 if (ShenandoahPacing) {
2052 _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2053 }
2054 if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2055 return;
2056 }
2057 r = _regions->next();
2058 }
2059 }
2060 };
2061
2062 void ShenandoahHeap::update_heap_references(bool concurrent) {
2063 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2064
2065 if (concurrent) {
2066 ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2067 workers()->run_task(&task);
2068 } else {
2069 ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2070 workers()->run_task(&task);
2071 }
2072 }
2073
2074
2075 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2076 private:
2077 ShenandoahHeapLock* const _lock;
2078
2079 public:
2080 ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2081
2082 void heap_region_do(ShenandoahHeapRegion* r) {
2083 // Drop unnecessary "pinned" state from regions that does not have CP marks
2084 // anymore, as this would allow trashing them.
2085
2086 if (r->is_active()) {
2087 if (r->is_pinned()) {
2088 if (r->pin_count() == 0) {
2089 ShenandoahHeapLocker locker(_lock);
2090 r->make_unpinned();
2091 }
2092 } else {
2093 if (r->pin_count() > 0) {
2094 ShenandoahHeapLocker locker(_lock);
2095 r->make_pinned();
2096 }
2097 }
2098 }
2099 }
2100
2101 bool is_thread_safe() { return true; }
2102 };
2103
2104 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2105 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2106 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2107
2108 {
2109 ShenandoahGCPhase phase(concurrent ?
2110 ShenandoahPhaseTimings::final_update_refs_update_region_states :
2111 ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2112 ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2113 parallel_heap_region_iterate(&cl);
2114
2115 assert_pinned_region_status();
2116 }
2117
2118 {
2119 ShenandoahGCPhase phase(concurrent ?
2120 ShenandoahPhaseTimings::final_update_refs_trash_cset :
2121 ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2122 trash_cset_regions();
2123 }
2124 }
2125
2126 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2127 {
2128 ShenandoahGCPhase phase(concurrent ?
2129 ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2130 ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2131 ShenandoahHeapLocker locker(lock());
2132 _free_set->rebuild();
2133 }
2134 }
2135
2136 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2137 print_on(st);
2138 print_heap_regions_on(st);
2139 }
2140
2141 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2142 size_t slice = r->index() / _bitmap_regions_per_slice;
2143
2144 size_t regions_from = _bitmap_regions_per_slice * slice;
2145 size_t regions_to = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2146 for (size_t g = regions_from; g < regions_to; g++) {
2147 assert (g / _bitmap_regions_per_slice == slice, "same slice");
2148 if (skip_self && g == r->index()) continue;
2149 if (get_region(g)->is_committed()) {
2150 return true;
2151 }
2152 }
2153 return false;
2154 }
2155
2156 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2157 shenandoah_assert_heaplocked();
2158
2159 // Bitmaps in special regions do not need commits
2160 if (_bitmap_region_special) {
2161 return true;
2162 }
2163
2164 if (is_bitmap_slice_committed(r, true)) {
2165 // Some other region from the group is already committed, meaning the bitmap
2166 // slice is already committed, we exit right away.
2167 return true;
2168 }
2169
2170 // Commit the bitmap slice:
2171 size_t slice = r->index() / _bitmap_regions_per_slice;
2172 size_t off = _bitmap_bytes_per_slice * slice;
2173 size_t len = _bitmap_bytes_per_slice;
2174 char* start = (char*) _bitmap_region.start() + off;
2175
2176 if (!os::commit_memory(start, len, false)) {
2177 return false;
2178 }
2179
2180 if (AlwaysPreTouch) {
2181 os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2182 }
2183
2184 return true;
2185 }
2186
2187 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2188 shenandoah_assert_heaplocked();
2189
2190 // Bitmaps in special regions do not need uncommits
2191 if (_bitmap_region_special) {
2192 return true;
2193 }
2194
2195 if (is_bitmap_slice_committed(r, true)) {
2196 // Some other region from the group is still committed, meaning the bitmap
2197 // slice is should stay committed, exit right away.
2198 return true;
2199 }
2200
2201 // Uncommit the bitmap slice:
2202 size_t slice = r->index() / _bitmap_regions_per_slice;
2203 size_t off = _bitmap_bytes_per_slice * slice;
2204 size_t len = _bitmap_bytes_per_slice;
2205 if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2206 return false;
2207 }
2208 return true;
2209 }
2210
2211 void ShenandoahHeap::safepoint_synchronize_begin() {
2212 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2213 SuspendibleThreadSet::synchronize();
2214 }
2215 }
2216
2217 void ShenandoahHeap::safepoint_synchronize_end() {
2218 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2219 SuspendibleThreadSet::desynchronize();
2220 }
2221 }
2222
2223 void ShenandoahHeap::entry_uncommit(double shrink_before, size_t shrink_until) {
2224 static const char *msg = "Concurrent uncommit";
2225 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2226 EventMark em("%s", msg);
2227
2228 op_uncommit(shrink_before, shrink_until);
2229 }
2230
2231 void ShenandoahHeap::try_inject_alloc_failure() {
2232 if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2233 _inject_alloc_failure.set();
2234 os::naked_short_sleep(1);
2235 if (cancelled_gc()) {
2236 log_info(gc)("Allocation failure was successfully injected");
2237 }
2238 }
2239 }
2240
2241 bool ShenandoahHeap::should_inject_alloc_failure() {
2242 return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2243 }
2244
2245 void ShenandoahHeap::initialize_serviceability() {
2246 _memory_pool = new ShenandoahMemoryPool(this);
2247 _cycle_memory_manager.add_pool(_memory_pool);
2248 _stw_memory_manager.add_pool(_memory_pool);
2249 }
2250
2251 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2252 GrowableArray<GCMemoryManager*> memory_managers(2);
2253 memory_managers.append(&_cycle_memory_manager);
2254 memory_managers.append(&_stw_memory_manager);
2255 return memory_managers;
2256 }
2257
2258 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2259 GrowableArray<MemoryPool*> memory_pools(1);
2260 memory_pools.append(_memory_pool);
2261 return memory_pools;
2262 }
2263
2264 MemoryUsage ShenandoahHeap::memory_usage() {
2265 return _memory_pool->get_memory_usage();
2266 }
2267
2268 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2269 _heap(ShenandoahHeap::heap()),
2270 _index(0) {}
2271
2272 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2273 _heap(heap),
2274 _index(0) {}
2275
2276 void ShenandoahRegionIterator::reset() {
2277 _index = 0;
2278 }
2279
2280 bool ShenandoahRegionIterator::has_next() const {
2281 return _index < _heap->num_regions();
2282 }
2283
2284 char ShenandoahHeap::gc_state() const {
2285 return _gc_state.raw_value();
2286 }
2287
2288 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2289 #ifdef ASSERT
2290 assert(_liveness_cache != NULL, "sanity");
2291 assert(worker_id < _max_workers, "sanity");
2292 for (uint i = 0; i < num_regions(); i++) {
2293 assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2294 }
2295 #endif
2296 return _liveness_cache[worker_id];
2297 }
2298
2299 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2300 assert(worker_id < _max_workers, "sanity");
2301 assert(_liveness_cache != NULL, "sanity");
2302 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2303 for (uint i = 0; i < num_regions(); i++) {
2304 ShenandoahLiveData live = ld[i];
2305 if (live > 0) {
2306 ShenandoahHeapRegion* r = get_region(i);
2307 r->increase_live_data_gc_words(live);
2308 ld[i] = 0;
2309 }
2310 }
2311 }
2312
2313 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2314 if (is_idle()) return false;
2315
2316 // Objects allocated after marking start are implicitly alive, don't need any barriers during
2317 // marking phase.
2318 if (is_concurrent_mark_in_progress() &&
2319 !marking_context()->allocated_after_mark_start(obj)) {
2320 return true;
2321 }
2322
2323 // Can not guarantee obj is deeply good.
2324 if (has_forwarded_objects()) {
2325 return true;
2326 }
2327
2328 return false;
2329 }