1 /*
2 * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "memory/allocation.hpp"
28 #include "memory/universe.hpp"
29
30 #include "gc/shared/classUnloadingContext.hpp"
31 #include "gc/shared/gcArguments.hpp"
32 #include "gc/shared/gcTimer.hpp"
33 #include "gc/shared/gcTraceTime.inline.hpp"
34 #include "gc/shared/locationPrinter.inline.hpp"
35 #include "gc/shared/memAllocator.hpp"
36 #include "gc/shared/plab.hpp"
37 #include "gc/shared/slidingForwarding.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39
40 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
41 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
42 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
43 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
44 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
45 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
46 #include "gc/shenandoah/shenandoahControlThread.hpp"
47 #include "gc/shenandoah/shenandoahFreeSet.hpp"
48 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
49 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
50 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
51 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
52 #include "gc/shenandoah/shenandoahInitLogger.hpp"
53 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
54 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
55 #include "gc/shenandoah/shenandoahMetrics.hpp"
56 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
57 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
58 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
59 #include "gc/shenandoah/shenandoahPadding.hpp"
60 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
61 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
62 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
63 #include "gc/shenandoah/shenandoahSTWMark.hpp"
64 #include "gc/shenandoah/shenandoahUtils.hpp"
65 #include "gc/shenandoah/shenandoahVerifier.hpp"
66 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
67 #include "gc/shenandoah/shenandoahVMOperations.hpp"
68 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
69 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
70 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
71 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
72 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
73 #if INCLUDE_JFR
74 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
75 #endif
76
77 #include "classfile/systemDictionary.hpp"
78 #include "code/codeCache.hpp"
79 #include "memory/classLoaderMetaspace.hpp"
80 #include "memory/metaspaceUtils.hpp"
81 #include "nmt/mallocTracker.hpp"
82 #include "nmt/memTracker.hpp"
83 #include "oops/compressedOops.inline.hpp"
84 #include "prims/jvmtiTagMap.hpp"
85 #include "runtime/atomic.hpp"
86 #include "runtime/globals.hpp"
87 #include "runtime/interfaceSupport.inline.hpp"
88 #include "runtime/java.hpp"
89 #include "runtime/orderAccess.hpp"
90 #include "runtime/safepointMechanism.hpp"
91 #include "runtime/vmThread.hpp"
92 #include "utilities/events.hpp"
93 #include "utilities/powerOfTwo.hpp"
94
95 class ShenandoahPretouchHeapTask : public WorkerTask {
96 private:
97 ShenandoahRegionIterator _regions;
98 const size_t _page_size;
99 public:
100 ShenandoahPretouchHeapTask(size_t page_size) :
101 WorkerTask("Shenandoah Pretouch Heap"),
102 _page_size(page_size) {}
103
104 virtual void work(uint worker_id) {
105 ShenandoahHeapRegion* r = _regions.next();
106 while (r != nullptr) {
107 if (r->is_committed()) {
108 os::pretouch_memory(r->bottom(), r->end(), _page_size);
109 }
110 r = _regions.next();
111 }
112 }
113 };
114
115 class ShenandoahPretouchBitmapTask : public WorkerTask {
116 private:
117 ShenandoahRegionIterator _regions;
118 char* _bitmap_base;
119 const size_t _bitmap_size;
120 const size_t _page_size;
121 public:
122 ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
123 WorkerTask("Shenandoah Pretouch Bitmap"),
124 _bitmap_base(bitmap_base),
125 _bitmap_size(bitmap_size),
126 _page_size(page_size) {}
127
128 virtual void work(uint worker_id) {
129 ShenandoahHeapRegion* r = _regions.next();
130 while (r != nullptr) {
131 size_t start = r->index() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
132 size_t end = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
133 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
134
135 if (r->is_committed()) {
136 os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
137 }
138
139 r = _regions.next();
140 }
141 }
142 };
143
144 jint ShenandoahHeap::initialize() {
145 //
146 // Figure out heap sizing
147 //
148
149 size_t init_byte_size = InitialHeapSize;
150 size_t min_byte_size = MinHeapSize;
151 size_t max_byte_size = MaxHeapSize;
152 size_t heap_alignment = HeapAlignment;
153
154 size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
155
156 Universe::check_alignment(max_byte_size, reg_size_bytes, "Shenandoah heap");
157 Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
158
159 _num_regions = ShenandoahHeapRegion::region_count();
160 assert(_num_regions == (max_byte_size / reg_size_bytes),
161 "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
162 _num_regions, max_byte_size, reg_size_bytes);
163
164 // Now we know the number of regions, initialize the heuristics.
165 initialize_heuristics();
166
167 size_t num_committed_regions = init_byte_size / reg_size_bytes;
168 num_committed_regions = MIN2(num_committed_regions, _num_regions);
169 assert(num_committed_regions <= _num_regions, "sanity");
170 _initial_size = num_committed_regions * reg_size_bytes;
171
172 size_t num_min_regions = min_byte_size / reg_size_bytes;
173 num_min_regions = MIN2(num_min_regions, _num_regions);
174 assert(num_min_regions <= _num_regions, "sanity");
175 _minimum_size = num_min_regions * reg_size_bytes;
176
177 // Default to max heap size.
178 _soft_max_size = _num_regions * reg_size_bytes;
179
180 _committed = _initial_size;
181
182 size_t heap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
183 size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
184 size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
185
186 //
187 // Reserve and commit memory for heap
188 //
189
190 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
191 initialize_reserved_region(heap_rs);
192 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
193 _heap_region_special = heap_rs.special();
194
195 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
196 "Misaligned heap: " PTR_FORMAT, p2i(base()));
197 os::trace_page_sizes_for_requested_size("Heap",
198 max_byte_size, heap_alignment,
199 heap_rs.base(),
200 heap_rs.size(), heap_rs.page_size());
201
202 #if SHENANDOAH_OPTIMIZED_MARKTASK
203 // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
204 // Fail if we ever attempt to address more than we can.
205 if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
206 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
207 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
208 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
209 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
210 vm_exit_during_initialization("Fatal Error", buf);
211 }
212 #endif
213
214 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
215 if (!_heap_region_special) {
216 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
217 "Cannot commit heap memory");
218 }
219
220 //
221 // Reserve and commit memory for bitmap(s)
222 //
223
224 size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
225 _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
226
227 size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
228
229 guarantee(bitmap_bytes_per_region != 0,
230 "Bitmap bytes per region should not be zero");
231 guarantee(is_power_of_2(bitmap_bytes_per_region),
232 "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
233
234 if (bitmap_page_size > bitmap_bytes_per_region) {
235 _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
236 _bitmap_bytes_per_slice = bitmap_page_size;
237 } else {
238 _bitmap_regions_per_slice = 1;
239 _bitmap_bytes_per_slice = bitmap_bytes_per_region;
240 }
241
242 guarantee(_bitmap_regions_per_slice >= 1,
243 "Should have at least one region per slice: " SIZE_FORMAT,
244 _bitmap_regions_per_slice);
245
246 guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
247 "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
248 _bitmap_bytes_per_slice, bitmap_page_size);
249
250 ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
251 os::trace_page_sizes_for_requested_size("Mark Bitmap",
252 bitmap_size_orig, bitmap_page_size,
253 bitmap.base(),
254 bitmap.size(), bitmap.page_size());
255 MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
256 _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
257 _bitmap_region_special = bitmap.special();
258
259 size_t bitmap_init_commit = _bitmap_bytes_per_slice *
260 align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
261 bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
262 if (!_bitmap_region_special) {
263 os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
264 "Cannot commit bitmap memory");
265 }
266
267 _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
268
269 if (ShenandoahVerify) {
270 ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
271 os::trace_page_sizes_for_requested_size("Verify Bitmap",
272 bitmap_size_orig, bitmap_page_size,
273 verify_bitmap.base(),
274 verify_bitmap.size(), verify_bitmap.page_size());
275 if (!verify_bitmap.special()) {
276 os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
277 "Cannot commit verification bitmap memory");
278 }
279 MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
280 MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
281 _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
282 _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
283 }
284
285 // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
286 size_t aux_bitmap_page_size = bitmap_page_size;
287 #ifdef LINUX
288 // In THP "advise" mode, we refrain from advising the system to use large pages
289 // since we know these commits will be short lived, and there is no reason to trash
290 // the THP area with this bitmap.
291 if (UseTransparentHugePages) {
292 aux_bitmap_page_size = os::vm_page_size();
293 }
294 #endif
295 ReservedSpace aux_bitmap(_bitmap_size, aux_bitmap_page_size);
296 os::trace_page_sizes_for_requested_size("Aux Bitmap",
297 bitmap_size_orig, aux_bitmap_page_size,
298 aux_bitmap.base(),
299 aux_bitmap.size(), aux_bitmap.page_size());
300 MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
301 _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
302 _aux_bitmap_region_special = aux_bitmap.special();
303 _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
304
305 //
306 // Create regions and region sets
307 //
308 size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
309 size_t region_storage_size_orig = region_align * _num_regions;
310 size_t region_storage_size = align_up(region_storage_size_orig,
311 MAX2(region_page_size, os::vm_allocation_granularity()));
312
313 ReservedSpace region_storage(region_storage_size, region_page_size);
314 os::trace_page_sizes_for_requested_size("Region Storage",
315 region_storage_size_orig, region_page_size,
316 region_storage.base(),
317 region_storage.size(), region_storage.page_size());
318 MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
319 if (!region_storage.special()) {
320 os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
321 "Cannot commit region memory");
322 }
323
324 // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
325 // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
326 // If not successful, bite a bullet and allocate at whatever address.
327 {
328 const size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
329 const size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
330 const size_t cset_page_size = os::vm_page_size();
331
332 uintptr_t min = round_up_power_of_2(cset_align);
333 uintptr_t max = (1u << 30u);
334 ReservedSpace cset_rs;
335
336 for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
337 char* req_addr = (char*)addr;
338 assert(is_aligned(req_addr, cset_align), "Should be aligned");
339 cset_rs = ReservedSpace(cset_size, cset_align, cset_page_size, req_addr);
340 if (cset_rs.is_reserved()) {
341 assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
342 _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
343 break;
344 }
345 }
346
347 if (_collection_set == nullptr) {
348 cset_rs = ReservedSpace(cset_size, cset_align, os::vm_page_size());
349 _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
350 }
351 os::trace_page_sizes_for_requested_size("Collection Set",
352 cset_size, cset_page_size,
353 cset_rs.base(),
354 cset_rs.size(), cset_rs.page_size());
355 }
356
357 _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
358 _free_set = new ShenandoahFreeSet(this, _num_regions);
359
360 {
361 ShenandoahHeapLocker locker(lock());
362
363 for (size_t i = 0; i < _num_regions; i++) {
364 HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
365 bool is_committed = i < num_committed_regions;
366 void* loc = region_storage.base() + i * region_align;
367
368 ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
369 assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
370
371 _marking_context->initialize_top_at_mark_start(r);
372 _regions[i] = r;
373 assert(!collection_set()->is_in(i), "New region should not be in collection set");
374 }
375
376 // Initialize to complete
377 _marking_context->mark_complete();
378
379 _free_set->rebuild();
380 }
381
382 if (AlwaysPreTouch) {
383 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
384 // before initialize() below zeroes it with initializing thread. For any given region,
385 // we touch the region and the corresponding bitmaps from the same thread.
386 ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
387
388 _pretouch_heap_page_size = heap_page_size;
389 _pretouch_bitmap_page_size = bitmap_page_size;
390
391 #ifdef LINUX
392 // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
393 // pages. But, the kernel needs to know that every small page is used, in order to coalesce
394 // them into huge one. Therefore, we need to pretouch with smaller pages.
395 if (UseTransparentHugePages) {
396 _pretouch_heap_page_size = (size_t)os::vm_page_size();
397 _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
398 }
399 #endif
400
401 // OS memory managers may want to coalesce back-to-back pages. Make their jobs
402 // simpler by pre-touching continuous spaces (heap and bitmap) separately.
403
404 ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
405 _workers->run_task(&bcl);
406
407 ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
408 _workers->run_task(&hcl);
409 }
410
411 //
412 // Initialize the rest of GC subsystems
413 //
414
415 _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
416 for (uint worker = 0; worker < _max_workers; worker++) {
417 _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
418 Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
419 }
420
421 // There should probably be Shenandoah-specific options for these,
422 // just as there are G1-specific options.
423 {
424 ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
425 satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
426 satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
427 }
428
429 _monitoring_support = new ShenandoahMonitoringSupport(this);
430 _phase_timings = new ShenandoahPhaseTimings(max_workers());
431 ShenandoahCodeRoots::initialize();
432
433 if (ShenandoahPacing) {
434 _pacer = new ShenandoahPacer(this);
435 _pacer->setup_for_idle();
436 }
437
438 _control_thread = new ShenandoahControlThread();
439
440 ShenandoahInitLogger::print();
441
442 SlidingForwarding::initialize(_heap_region, ShenandoahHeapRegion::region_size_words());
443
444 return JNI_OK;
445 }
446
447 void ShenandoahHeap::initialize_mode() {
448 if (ShenandoahGCMode != nullptr) {
449 if (strcmp(ShenandoahGCMode, "satb") == 0) {
450 _gc_mode = new ShenandoahSATBMode();
451 } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
452 _gc_mode = new ShenandoahIUMode();
453 } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
454 _gc_mode = new ShenandoahPassiveMode();
455 } else {
456 vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
457 }
458 } else {
459 vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
460 }
461 _gc_mode->initialize_flags();
462 if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
463 vm_exit_during_initialization(
464 err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
465 _gc_mode->name()));
466 }
467 if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
468 vm_exit_during_initialization(
469 err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
470 _gc_mode->name()));
471 }
472 }
473
474 void ShenandoahHeap::initialize_heuristics() {
475 assert(_gc_mode != nullptr, "Must be initialized");
476 _heuristics = _gc_mode->initialize_heuristics();
477
478 if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
479 vm_exit_during_initialization(
480 err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
481 _heuristics->name()));
482 }
483 if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
484 vm_exit_during_initialization(
485 err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
486 _heuristics->name()));
487 }
488 }
489
490 #ifdef _MSC_VER
491 #pragma warning( push )
492 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
493 #endif
494
495 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
496 CollectedHeap(),
497 _initial_size(0),
498 _used(0),
499 _committed(0),
500 _bytes_allocated_since_gc_start(0),
501 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
502 _workers(nullptr),
503 _safepoint_workers(nullptr),
504 _heap_region_special(false),
505 _num_regions(0),
506 _regions(nullptr),
507 _update_refs_iterator(this),
508 _gc_state_changed(false),
509 _gc_no_progress_count(0),
510 _control_thread(nullptr),
511 _shenandoah_policy(policy),
512 _gc_mode(nullptr),
513 _heuristics(nullptr),
514 _free_set(nullptr),
515 _pacer(nullptr),
516 _verifier(nullptr),
517 _phase_timings(nullptr),
518 _monitoring_support(nullptr),
519 _memory_pool(nullptr),
520 _stw_memory_manager("Shenandoah Pauses"),
521 _cycle_memory_manager("Shenandoah Cycles"),
522 _gc_timer(new ConcurrentGCTimer()),
523 _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
524 _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
525 _marking_context(nullptr),
526 _bitmap_size(0),
527 _bitmap_regions_per_slice(0),
528 _bitmap_bytes_per_slice(0),
529 _bitmap_region_special(false),
530 _aux_bitmap_region_special(false),
531 _liveness_cache(nullptr),
532 _collection_set(nullptr)
533 {
534 // Initialize GC mode early, so we can adjust barrier support
535 initialize_mode();
536 BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
537
538 _max_workers = MAX2(_max_workers, 1U);
539 _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
540 if (_workers == nullptr) {
541 vm_exit_during_initialization("Failed necessary allocation.");
542 } else {
543 _workers->initialize_workers();
544 }
545
546 if (ParallelGCThreads > 1) {
547 _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
548 ParallelGCThreads);
549 _safepoint_workers->initialize_workers();
550 }
551 }
552
553 #ifdef _MSC_VER
554 #pragma warning( pop )
555 #endif
556
557 class ShenandoahResetBitmapTask : public WorkerTask {
558 private:
559 ShenandoahRegionIterator _regions;
560
561 public:
562 ShenandoahResetBitmapTask() :
563 WorkerTask("Shenandoah Reset Bitmap") {}
564
565 void work(uint worker_id) {
566 ShenandoahHeapRegion* region = _regions.next();
567 ShenandoahHeap* heap = ShenandoahHeap::heap();
568 ShenandoahMarkingContext* const ctx = heap->marking_context();
569 while (region != nullptr) {
570 if (heap->is_bitmap_slice_committed(region)) {
571 ctx->clear_bitmap(region);
572 }
573 region = _regions.next();
574 }
575 }
576 };
577
578 void ShenandoahHeap::reset_mark_bitmap() {
579 assert_gc_workers(_workers->active_workers());
580 mark_incomplete_marking_context();
581
582 ShenandoahResetBitmapTask task;
583 _workers->run_task(&task);
584 }
585
586 void ShenandoahHeap::print_on(outputStream* st) const {
587 st->print_cr("Shenandoah Heap");
588 st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
589 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
590 byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
591 byte_size_in_proper_unit(committed()), proper_unit_for_byte_size(committed()),
592 byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
593 st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
594 num_regions(),
595 byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
596 proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
597
598 st->print("Status: ");
599 if (has_forwarded_objects()) st->print("has forwarded objects, ");
600 if (is_concurrent_mark_in_progress()) st->print("marking, ");
601 if (is_evacuation_in_progress()) st->print("evacuating, ");
602 if (is_update_refs_in_progress()) st->print("updating refs, ");
603 if (is_degenerated_gc_in_progress()) st->print("degenerated gc, ");
604 if (is_full_gc_in_progress()) st->print("full gc, ");
605 if (is_full_gc_move_in_progress()) st->print("full gc move, ");
606 if (is_concurrent_weak_root_in_progress()) st->print("concurrent weak roots, ");
607 if (is_concurrent_strong_root_in_progress() &&
608 !is_concurrent_weak_root_in_progress()) st->print("concurrent strong roots, ");
609
610 if (cancelled_gc()) {
611 st->print("cancelled");
612 } else {
613 st->print("not cancelled");
614 }
615 st->cr();
616
617 st->print_cr("Reserved region:");
618 st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
619 p2i(reserved_region().start()),
620 p2i(reserved_region().end()));
621
622 ShenandoahCollectionSet* cset = collection_set();
623 st->print_cr("Collection set:");
624 if (cset != nullptr) {
625 st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
626 st->print_cr(" - map (biased): " PTR_FORMAT, p2i(cset->biased_map_address()));
627 } else {
628 st->print_cr(" (null)");
629 }
630
631 st->cr();
632 MetaspaceUtils::print_on(st);
633
634 if (Verbose) {
635 st->cr();
636 print_heap_regions_on(st);
637 }
638 }
639
640 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
641 public:
642 void do_thread(Thread* thread) {
643 assert(thread != nullptr, "Sanity");
644 assert(thread->is_Worker_thread(), "Only worker thread expected");
645 ShenandoahThreadLocalData::initialize_gclab(thread);
646 }
647 };
648
649 void ShenandoahHeap::post_initialize() {
650 CollectedHeap::post_initialize();
651 MutexLocker ml(Threads_lock);
652
653 ShenandoahInitWorkerGCLABClosure init_gclabs;
654 _workers->threads_do(&init_gclabs);
655
656 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
657 // Now, we will let WorkerThreads to initialize gclab when new worker is created.
658 _workers->set_initialize_gclab();
659 if (_safepoint_workers != nullptr) {
660 _safepoint_workers->threads_do(&init_gclabs);
661 _safepoint_workers->set_initialize_gclab();
662 }
663
664 _heuristics->initialize();
665
666 JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
667 }
668
669 size_t ShenandoahHeap::used() const {
670 return Atomic::load(&_used);
671 }
672
673 size_t ShenandoahHeap::committed() const {
674 return Atomic::load(&_committed);
675 }
676
677 size_t ShenandoahHeap::available() const {
678 return free_set()->available();
679 }
680
681 void ShenandoahHeap::increase_committed(size_t bytes) {
682 shenandoah_assert_heaplocked_or_safepoint();
683 _committed += bytes;
684 }
685
686 void ShenandoahHeap::decrease_committed(size_t bytes) {
687 shenandoah_assert_heaplocked_or_safepoint();
688 _committed -= bytes;
689 }
690
691 void ShenandoahHeap::increase_used(size_t bytes) {
692 Atomic::add(&_used, bytes, memory_order_relaxed);
693 }
694
695 void ShenandoahHeap::set_used(size_t bytes) {
696 Atomic::store(&_used, bytes);
697 }
698
699 void ShenandoahHeap::decrease_used(size_t bytes) {
700 assert(used() >= bytes, "never decrease heap size by more than we've left");
701 Atomic::sub(&_used, bytes, memory_order_relaxed);
702 }
703
704 void ShenandoahHeap::increase_allocated(size_t bytes) {
705 Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
706 }
707
708 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
709 size_t bytes = words * HeapWordSize;
710 if (!waste) {
711 increase_used(bytes);
712 }
713 increase_allocated(bytes);
714 if (ShenandoahPacing) {
715 control_thread()->pacing_notify_alloc(words);
716 if (waste) {
717 pacer()->claim_for_alloc(words, true);
718 }
719 }
720 }
721
722 size_t ShenandoahHeap::capacity() const {
723 return committed();
724 }
725
726 size_t ShenandoahHeap::max_capacity() const {
727 return _num_regions * ShenandoahHeapRegion::region_size_bytes();
728 }
729
730 size_t ShenandoahHeap::soft_max_capacity() const {
731 size_t v = Atomic::load(&_soft_max_size);
732 assert(min_capacity() <= v && v <= max_capacity(),
733 "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
734 min_capacity(), v, max_capacity());
735 return v;
736 }
737
738 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
739 assert(min_capacity() <= v && v <= max_capacity(),
740 "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
741 min_capacity(), v, max_capacity());
742 Atomic::store(&_soft_max_size, v);
743 }
744
745 size_t ShenandoahHeap::min_capacity() const {
746 return _minimum_size;
747 }
748
749 size_t ShenandoahHeap::initial_capacity() const {
750 return _initial_size;
751 }
752
753 bool ShenandoahHeap::is_in(const void* p) const {
754 HeapWord* heap_base = (HeapWord*) base();
755 HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
756 return p >= heap_base && p < last_region_end;
757 }
758
759 void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
760 assert (ShenandoahUncommit, "should be enabled");
761
762 // Determine if there is work to do. This avoids taking heap lock if there is
763 // no work available, avoids spamming logs with superfluous logging messages,
764 // and minimises the amount of work while locks are taken.
765
766 if (committed() <= shrink_until) return;
767
768 bool has_work = false;
769 for (size_t i = 0; i < num_regions(); i++) {
770 ShenandoahHeapRegion* r = get_region(i);
771 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
772 has_work = true;
773 break;
774 }
775 }
776
777 if (has_work) {
778 static const char* msg = "Concurrent uncommit";
779 ShenandoahConcurrentPhase gcPhase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
780 EventMark em("%s", msg);
781
782 op_uncommit(shrink_before, shrink_until);
783 }
784 }
785
786 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
787 assert (ShenandoahUncommit, "should be enabled");
788
789 // Application allocates from the beginning of the heap, and GC allocates at
790 // the end of it. It is more efficient to uncommit from the end, so that applications
791 // could enjoy the near committed regions. GC allocations are much less frequent,
792 // and therefore can accept the committing costs.
793
794 size_t count = 0;
795 for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
796 ShenandoahHeapRegion* r = get_region(i - 1);
797 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
798 ShenandoahHeapLocker locker(lock());
799 if (r->is_empty_committed()) {
800 if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
801 break;
802 }
803
804 r->make_uncommitted();
805 count++;
806 }
807 }
808 SpinPause(); // allow allocators to take the lock
809 }
810
811 if (count > 0) {
812 notify_heap_changed();
813 }
814 }
815
816 bool ShenandoahHeap::check_soft_max_changed() {
817 size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
818 size_t old_soft_max = soft_max_capacity();
819 if (new_soft_max != old_soft_max) {
820 new_soft_max = MAX2(min_capacity(), new_soft_max);
821 new_soft_max = MIN2(max_capacity(), new_soft_max);
822 if (new_soft_max != old_soft_max) {
823 log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
824 byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
825 byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
826 );
827 set_soft_max_capacity(new_soft_max);
828 return true;
829 }
830 }
831 return false;
832 }
833
834 void ShenandoahHeap::notify_heap_changed() {
835 // Update monitoring counters when we took a new region. This amortizes the
836 // update costs on slow path.
837 monitoring_support()->notify_heap_changed();
838
839 // This is called from allocation path, and thus should be fast.
840 _heap_changed.try_set();
841 }
842
843 void ShenandoahHeap::set_forced_counters_update(bool value) {
844 monitoring_support()->set_forced_counters_update(value);
845 }
846
847 void ShenandoahHeap::handle_force_counters_update() {
848 monitoring_support()->handle_force_counters_update();
849 }
850
851 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
852 // New object should fit the GCLAB size
853 size_t min_size = MAX2(size, PLAB::min_size());
854
855 // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
856 size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
857 new_size = MIN2(new_size, PLAB::max_size());
858 new_size = MAX2(new_size, PLAB::min_size());
859
860 // Record new heuristic value even if we take any shortcut. This captures
861 // the case when moderately-sized objects always take a shortcut. At some point,
862 // heuristics should catch up with them.
863 ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
864
865 if (new_size < size) {
866 // New size still does not fit the object. Fall back to shared allocation.
867 // This avoids retiring perfectly good GCLABs, when we encounter a large object.
868 return nullptr;
869 }
870
871 // Retire current GCLAB, and allocate a new one.
872 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
873 gclab->retire();
874
875 size_t actual_size = 0;
876 HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
877 if (gclab_buf == nullptr) {
878 return nullptr;
879 }
880
881 assert (size <= actual_size, "allocation should fit");
882
883 if (ZeroTLAB) {
884 // ..and clear it.
885 Copy::zero_to_words(gclab_buf, actual_size);
886 } else {
887 // ...and zap just allocated object.
888 #ifdef ASSERT
889 // Skip mangling the space corresponding to the object header to
890 // ensure that the returned space is not considered parsable by
891 // any concurrent GC thread.
892 size_t hdr_size = oopDesc::header_size();
893 Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
894 #endif // ASSERT
895 }
896 gclab->set_buf(gclab_buf, actual_size);
897 return gclab->allocate(size);
898 }
899
900 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
901 size_t requested_size,
902 size_t* actual_size) {
903 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
904 HeapWord* res = allocate_memory(req);
905 if (res != nullptr) {
906 *actual_size = req.actual_size();
907 } else {
908 *actual_size = 0;
909 }
910 return res;
911 }
912
913 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
914 size_t word_size,
915 size_t* actual_size) {
916 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
917 HeapWord* res = allocate_memory(req);
918 if (res != nullptr) {
919 *actual_size = req.actual_size();
920 } else {
921 *actual_size = 0;
922 }
923 return res;
924 }
925
926 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
927 intptr_t pacer_epoch = 0;
928 bool in_new_region = false;
929 HeapWord* result = nullptr;
930
931 if (req.is_mutator_alloc()) {
932 if (ShenandoahPacing) {
933 pacer()->pace_for_alloc(req.size());
934 pacer_epoch = pacer()->epoch();
935 }
936
937 if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
938 result = allocate_memory_under_lock(req, in_new_region);
939 }
940
941 // Check that gc overhead is not exceeded.
942 //
943 // Shenandoah will grind along for quite a while allocating one
944 // object at a time using shared (non-tlab) allocations. This check
945 // is testing that the GC overhead limit has not been exceeded.
946 // This will notify the collector to start a cycle, but will raise
947 // an OOME to the mutator if the last Full GCs have not made progress.
948 if (result == nullptr && !req.is_lab_alloc() && get_gc_no_progress_count() > ShenandoahNoProgressThreshold) {
949 control_thread()->handle_alloc_failure(req, false);
950 return nullptr;
951 }
952
953 // Block until control thread reacted, then retry allocation.
954 //
955 // It might happen that one of the threads requesting allocation would unblock
956 // way later after GC happened, only to fail the second allocation, because
957 // other threads have already depleted the free storage. In this case, a better
958 // strategy is to try again, as long as GC makes progress (or until at least
959 // one full GC has completed).
960 size_t original_count = shenandoah_policy()->full_gc_count();
961 while (result == nullptr
962 && (get_gc_no_progress_count() == 0 || original_count == shenandoah_policy()->full_gc_count())) {
963 control_thread()->handle_alloc_failure(req);
964 result = allocate_memory_under_lock(req, in_new_region);
965 }
966
967 if (log_is_enabled(Debug, gc, alloc)) {
968 ResourceMark rm;
969 log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT ", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
970 Thread::current()->name(), p2i(result), req.type_string(), req.size(), original_count, get_gc_no_progress_count());
971 }
972 } else {
973 assert(req.is_gc_alloc(), "Can only accept GC allocs here");
974 result = allocate_memory_under_lock(req, in_new_region);
975 // Do not call handle_alloc_failure() here, because we cannot block.
976 // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
977 }
978
979 if (in_new_region) {
980 notify_heap_changed();
981 }
982
983 if (result != nullptr) {
984 size_t requested = req.size();
985 size_t actual = req.actual_size();
986
987 assert (req.is_lab_alloc() || (requested == actual),
988 "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
989 ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
990
991 if (req.is_mutator_alloc()) {
992 notify_mutator_alloc_words(actual, false);
993
994 // If we requested more than we were granted, give the rest back to pacer.
995 // This only matters if we are in the same pacing epoch: do not try to unpace
996 // over the budget for the other phase.
997 if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
998 pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
999 }
1000 } else {
1001 increase_used(actual*HeapWordSize);
1002 }
1003 }
1004
1005 return result;
1006 }
1007
1008 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1009 // If we are dealing with mutator allocation, then we may need to block for safepoint.
1010 // We cannot block for safepoint for GC allocations, because there is a high chance
1011 // we are already running at safepoint or from stack watermark machinery, and we cannot
1012 // block again.
1013 ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1014 return _free_set->allocate(req, in_new_region);
1015 }
1016
1017 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1018 bool* gc_overhead_limit_was_exceeded) {
1019 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1020 return allocate_memory(req);
1021 }
1022
1023 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1024 size_t size,
1025 Metaspace::MetadataType mdtype) {
1026 MetaWord* result;
1027
1028 // Inform metaspace OOM to GC heuristics if class unloading is possible.
1029 if (heuristics()->can_unload_classes()) {
1030 ShenandoahHeuristics* h = heuristics();
1031 h->record_metaspace_oom();
1032 }
1033
1034 // Expand and retry allocation
1035 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1036 if (result != nullptr) {
1037 return result;
1038 }
1039
1040 // Start full GC
1041 collect(GCCause::_metadata_GC_clear_soft_refs);
1042
1043 // Retry allocation
1044 result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1045 if (result != nullptr) {
1046 return result;
1047 }
1048
1049 // Expand and retry allocation
1050 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1051 if (result != nullptr) {
1052 return result;
1053 }
1054
1055 // Out of memory
1056 return nullptr;
1057 }
1058
1059 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
1060 private:
1061 ShenandoahHeap* const _heap;
1062 Thread* const _thread;
1063 public:
1064 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
1065 _heap(heap), _thread(Thread::current()) {}
1066
1067 void do_object(oop p) {
1068 shenandoah_assert_marked(nullptr, p);
1069 if (!p->is_forwarded()) {
1070 _heap->evacuate_object(p, _thread);
1071 }
1072 }
1073 };
1074
1075 class ShenandoahEvacuationTask : public WorkerTask {
1076 private:
1077 ShenandoahHeap* const _sh;
1078 ShenandoahCollectionSet* const _cs;
1079 bool _concurrent;
1080 public:
1081 ShenandoahEvacuationTask(ShenandoahHeap* sh,
1082 ShenandoahCollectionSet* cs,
1083 bool concurrent) :
1084 WorkerTask("Shenandoah Evacuation"),
1085 _sh(sh),
1086 _cs(cs),
1087 _concurrent(concurrent)
1088 {}
1089
1090 void work(uint worker_id) {
1091 if (_concurrent) {
1092 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1093 ShenandoahSuspendibleThreadSetJoiner stsj;
1094 ShenandoahEvacOOMScope oom_evac_scope;
1095 do_work();
1096 } else {
1097 ShenandoahParallelWorkerSession worker_session(worker_id);
1098 ShenandoahEvacOOMScope oom_evac_scope;
1099 do_work();
1100 }
1101 }
1102
1103 private:
1104 void do_work() {
1105 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1106 ShenandoahHeapRegion* r;
1107 while ((r =_cs->claim_next()) != nullptr) {
1108 assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
1109 _sh->marked_object_iterate(r, &cl);
1110
1111 if (ShenandoahPacing) {
1112 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1113 }
1114
1115 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1116 break;
1117 }
1118 }
1119 }
1120 };
1121
1122 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1123 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1124 workers()->run_task(&task);
1125 }
1126
1127 void ShenandoahHeap::trash_cset_regions() {
1128 ShenandoahHeapLocker locker(lock());
1129
1130 ShenandoahCollectionSet* set = collection_set();
1131 ShenandoahHeapRegion* r;
1132 set->clear_current_index();
1133 while ((r = set->next()) != nullptr) {
1134 r->make_trash();
1135 }
1136 collection_set()->clear();
1137 }
1138
1139 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1140 st->print_cr("Heap Regions:");
1141 st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1142 st->print_cr(" HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1143 st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1144 st->print_cr("UWM=update watermark, U=used");
1145 st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1146 st->print_cr("S=shared allocs, L=live data");
1147 st->print_cr("CP=critical pins");
1148
1149 for (size_t i = 0; i < num_regions(); i++) {
1150 get_region(i)->print_on(st);
1151 }
1152 }
1153
1154 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1155 assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1156
1157 oop humongous_obj = cast_to_oop(start->bottom());
1158 size_t size = humongous_obj->size();
1159 size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1160 size_t index = start->index() + required_regions - 1;
1161
1162 assert(!start->has_live(), "liveness must be zero");
1163
1164 for(size_t i = 0; i < required_regions; i++) {
1165 // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1166 // as it expects that every region belongs to a humongous region starting with a humongous start region.
1167 ShenandoahHeapRegion* region = get_region(index --);
1168
1169 assert(region->is_humongous(), "expect correct humongous start or continuation");
1170 assert(!region->is_cset(), "Humongous region should not be in collection set");
1171
1172 region->make_trash_immediate();
1173 }
1174 }
1175
1176 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1177 public:
1178 ShenandoahCheckCleanGCLABClosure() {}
1179 void do_thread(Thread* thread) {
1180 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1181 assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1182 assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1183 }
1184 };
1185
1186 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1187 private:
1188 bool const _resize;
1189 public:
1190 ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1191 void do_thread(Thread* thread) {
1192 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1193 assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1194 gclab->retire();
1195 if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1196 ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1197 }
1198 }
1199 };
1200
1201 void ShenandoahHeap::labs_make_parsable() {
1202 assert(UseTLAB, "Only call with UseTLAB");
1203
1204 ShenandoahRetireGCLABClosure cl(false);
1205
1206 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1207 ThreadLocalAllocBuffer& tlab = t->tlab();
1208 tlab.make_parsable();
1209 cl.do_thread(t);
1210 }
1211
1212 workers()->threads_do(&cl);
1213 }
1214
1215 void ShenandoahHeap::tlabs_retire(bool resize) {
1216 assert(UseTLAB, "Only call with UseTLAB");
1217 assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1218
1219 ThreadLocalAllocStats stats;
1220
1221 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1222 ThreadLocalAllocBuffer& tlab = t->tlab();
1223 tlab.retire(&stats);
1224 if (resize) {
1225 tlab.resize();
1226 }
1227 }
1228
1229 stats.publish();
1230
1231 #ifdef ASSERT
1232 ShenandoahCheckCleanGCLABClosure cl;
1233 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1234 cl.do_thread(t);
1235 }
1236 workers()->threads_do(&cl);
1237 #endif
1238 }
1239
1240 void ShenandoahHeap::gclabs_retire(bool resize) {
1241 assert(UseTLAB, "Only call with UseTLAB");
1242 assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1243
1244 ShenandoahRetireGCLABClosure cl(resize);
1245 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1246 cl.do_thread(t);
1247 }
1248 workers()->threads_do(&cl);
1249
1250 if (safepoint_workers() != nullptr) {
1251 safepoint_workers()->threads_do(&cl);
1252 }
1253 }
1254
1255 // Returns size in bytes
1256 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1257 // Return the max allowed size, and let the allocation path
1258 // figure out the safe size for current allocation.
1259 return ShenandoahHeapRegion::max_tlab_size_bytes();
1260 }
1261
1262 size_t ShenandoahHeap::max_tlab_size() const {
1263 // Returns size in words
1264 return ShenandoahHeapRegion::max_tlab_size_words();
1265 }
1266
1267 void ShenandoahHeap::collect(GCCause::Cause cause) {
1268 control_thread()->request_gc(cause);
1269 }
1270
1271 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1272 //assert(false, "Shouldn't need to do full collections");
1273 }
1274
1275 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1276 ShenandoahHeapRegion* r = heap_region_containing(addr);
1277 if (r != nullptr) {
1278 return r->block_start(addr);
1279 }
1280 return nullptr;
1281 }
1282
1283 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1284 ShenandoahHeapRegion* r = heap_region_containing(addr);
1285 return r->block_is_obj(addr);
1286 }
1287
1288 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1289 return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1290 }
1291
1292 void ShenandoahHeap::prepare_for_verify() {
1293 if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1294 labs_make_parsable();
1295 }
1296 }
1297
1298 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1299 tcl->do_thread(_control_thread);
1300 workers()->threads_do(tcl);
1301 if (_safepoint_workers != nullptr) {
1302 _safepoint_workers->threads_do(tcl);
1303 }
1304 }
1305
1306 void ShenandoahHeap::print_tracing_info() const {
1307 LogTarget(Info, gc, stats) lt;
1308 if (lt.is_enabled()) {
1309 ResourceMark rm;
1310 LogStream ls(lt);
1311
1312 phase_timings()->print_global_on(&ls);
1313
1314 ls.cr();
1315 ls.cr();
1316
1317 shenandoah_policy()->print_gc_stats(&ls);
1318
1319 ls.cr();
1320 ls.cr();
1321 }
1322 }
1323
1324 void ShenandoahHeap::verify(VerifyOption vo) {
1325 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1326 if (ShenandoahVerify) {
1327 verifier()->verify_generic(vo);
1328 } else {
1329 // TODO: Consider allocating verification bitmaps on demand,
1330 // and turn this on unconditionally.
1331 }
1332 }
1333 }
1334 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1335 return _free_set->capacity();
1336 }
1337
1338 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1339 private:
1340 MarkBitMap* _bitmap;
1341 ShenandoahScanObjectStack* _oop_stack;
1342 ShenandoahHeap* const _heap;
1343 ShenandoahMarkingContext* const _marking_context;
1344
1345 template <class T>
1346 void do_oop_work(T* p) {
1347 T o = RawAccess<>::oop_load(p);
1348 if (!CompressedOops::is_null(o)) {
1349 oop obj = CompressedOops::decode_not_null(o);
1350 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1351 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1352 return;
1353 }
1354 obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1355
1356 assert(oopDesc::is_oop(obj), "must be a valid oop");
1357 if (!_bitmap->is_marked(obj)) {
1358 _bitmap->mark(obj);
1359 _oop_stack->push(obj);
1360 }
1361 }
1362 }
1363 public:
1364 ObjectIterateScanRootClosure(MarkBitMap* bitmap, ShenandoahScanObjectStack* oop_stack) :
1365 _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1366 _marking_context(_heap->marking_context()) {}
1367 void do_oop(oop* p) { do_oop_work(p); }
1368 void do_oop(narrowOop* p) { do_oop_work(p); }
1369 };
1370
1371 /*
1372 * This is public API, used in preparation of object_iterate().
1373 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1374 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1375 * control, we call SH::tlabs_retire, SH::gclabs_retire.
1376 */
1377 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1378 // No-op.
1379 }
1380
1381 /*
1382 * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1383 *
1384 * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1385 * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1386 * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1387 * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1388 * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1389 * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1390 * wiped the bitmap in preparation for next marking).
1391 *
1392 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1393 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1394 * is allowed to report dead objects, but is not required to do so.
1395 */
1396 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1397 // Reset bitmap
1398 if (!prepare_aux_bitmap_for_iteration())
1399 return;
1400
1401 ShenandoahScanObjectStack oop_stack;
1402 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1403 // Seed the stack with root scan
1404 scan_roots_for_iteration(&oop_stack, &oops);
1405
1406 // Work through the oop stack to traverse heap
1407 while (! oop_stack.is_empty()) {
1408 oop obj = oop_stack.pop();
1409 assert(oopDesc::is_oop(obj), "must be a valid oop");
1410 cl->do_object(obj);
1411 obj->oop_iterate(&oops);
1412 }
1413
1414 assert(oop_stack.is_empty(), "should be empty");
1415 // Reclaim bitmap
1416 reclaim_aux_bitmap_for_iteration();
1417 }
1418
1419 bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
1420 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1421
1422 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1423 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1424 return false;
1425 }
1426 // Reset bitmap
1427 _aux_bit_map.clear();
1428 return true;
1429 }
1430
1431 void ShenandoahHeap::scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops) {
1432 // Process GC roots according to current GC cycle
1433 // This populates the work stack with initial objects
1434 // It is important to relinquish the associated locks before diving
1435 // into heap dumper
1436 uint n_workers = safepoint_workers() != nullptr ? safepoint_workers()->active_workers() : 1;
1437 ShenandoahHeapIterationRootScanner rp(n_workers);
1438 rp.roots_do(oops);
1439 }
1440
1441 void ShenandoahHeap::reclaim_aux_bitmap_for_iteration() {
1442 if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1443 log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1444 }
1445 }
1446
1447 // Closure for parallelly iterate objects
1448 class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure {
1449 private:
1450 MarkBitMap* _bitmap;
1451 ShenandoahObjToScanQueue* _queue;
1452 ShenandoahHeap* const _heap;
1453 ShenandoahMarkingContext* const _marking_context;
1454
1455 template <class T>
1456 void do_oop_work(T* p) {
1457 T o = RawAccess<>::oop_load(p);
1458 if (!CompressedOops::is_null(o)) {
1459 oop obj = CompressedOops::decode_not_null(o);
1460 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1461 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1462 return;
1463 }
1464 obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1465
1466 assert(oopDesc::is_oop(obj), "Must be a valid oop");
1467 if (_bitmap->par_mark(obj)) {
1468 _queue->push(ShenandoahMarkTask(obj));
1469 }
1470 }
1471 }
1472 public:
1473 ShenandoahObjectIterateParScanClosure(MarkBitMap* bitmap, ShenandoahObjToScanQueue* q) :
1474 _bitmap(bitmap), _queue(q), _heap(ShenandoahHeap::heap()),
1475 _marking_context(_heap->marking_context()) {}
1476 void do_oop(oop* p) { do_oop_work(p); }
1477 void do_oop(narrowOop* p) { do_oop_work(p); }
1478 };
1479
1480 // Object iterator for parallel heap iteraion.
1481 // The root scanning phase happenes in construction as a preparation of
1482 // parallel marking queues.
1483 // Every worker processes it's own marking queue. work-stealing is used
1484 // to balance workload.
1485 class ShenandoahParallelObjectIterator : public ParallelObjectIteratorImpl {
1486 private:
1487 uint _num_workers;
1488 bool _init_ready;
1489 MarkBitMap* _aux_bit_map;
1490 ShenandoahHeap* _heap;
1491 ShenandoahScanObjectStack _roots_stack; // global roots stack
1492 ShenandoahObjToScanQueueSet* _task_queues;
1493 public:
1494 ShenandoahParallelObjectIterator(uint num_workers, MarkBitMap* bitmap) :
1495 _num_workers(num_workers),
1496 _init_ready(false),
1497 _aux_bit_map(bitmap),
1498 _heap(ShenandoahHeap::heap()) {
1499 // Initialize bitmap
1500 _init_ready = _heap->prepare_aux_bitmap_for_iteration();
1501 if (!_init_ready) {
1502 return;
1503 }
1504
1505 ObjectIterateScanRootClosure oops(_aux_bit_map, &_roots_stack);
1506 _heap->scan_roots_for_iteration(&_roots_stack, &oops);
1507
1508 _init_ready = prepare_worker_queues();
1509 }
1510
1511 ~ShenandoahParallelObjectIterator() {
1512 // Reclaim bitmap
1513 _heap->reclaim_aux_bitmap_for_iteration();
1514 // Reclaim queue for workers
1515 if (_task_queues!= nullptr) {
1516 for (uint i = 0; i < _num_workers; ++i) {
1517 ShenandoahObjToScanQueue* q = _task_queues->queue(i);
1518 if (q != nullptr) {
1519 delete q;
1520 _task_queues->register_queue(i, nullptr);
1521 }
1522 }
1523 delete _task_queues;
1524 _task_queues = nullptr;
1525 }
1526 }
1527
1528 virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
1529 if (_init_ready) {
1530 object_iterate_parallel(cl, worker_id, _task_queues);
1531 }
1532 }
1533
1534 private:
1535 // Divide global root_stack into worker queues
1536 bool prepare_worker_queues() {
1537 _task_queues = new ShenandoahObjToScanQueueSet((int) _num_workers);
1538 // Initialize queues for every workers
1539 for (uint i = 0; i < _num_workers; ++i) {
1540 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
1541 _task_queues->register_queue(i, task_queue);
1542 }
1543 // Divide roots among the workers. Assume that object referencing distribution
1544 // is related with root kind, use round-robin to make every worker have same chance
1545 // to process every kind of roots
1546 size_t roots_num = _roots_stack.size();
1547 if (roots_num == 0) {
1548 // No work to do
1549 return false;
1550 }
1551
1552 for (uint j = 0; j < roots_num; j++) {
1553 uint stack_id = j % _num_workers;
1554 oop obj = _roots_stack.pop();
1555 _task_queues->queue(stack_id)->push(ShenandoahMarkTask(obj));
1556 }
1557 return true;
1558 }
1559
1560 void object_iterate_parallel(ObjectClosure* cl,
1561 uint worker_id,
1562 ShenandoahObjToScanQueueSet* queue_set) {
1563 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1564 assert(queue_set != nullptr, "task queue must not be null");
1565
1566 ShenandoahObjToScanQueue* q = queue_set->queue(worker_id);
1567 assert(q != nullptr, "object iterate queue must not be null");
1568
1569 ShenandoahMarkTask t;
1570 ShenandoahObjectIterateParScanClosure oops(_aux_bit_map, q);
1571
1572 // Work through the queue to traverse heap.
1573 // Steal when there is no task in queue.
1574 while (q->pop(t) || queue_set->steal(worker_id, t)) {
1575 oop obj = t.obj();
1576 assert(oopDesc::is_oop(obj), "must be a valid oop");
1577 cl->do_object(obj);
1578 obj->oop_iterate(&oops);
1579 }
1580 assert(q->is_empty(), "should be empty");
1581 }
1582 };
1583
1584 ParallelObjectIteratorImpl* ShenandoahHeap::parallel_object_iterator(uint workers) {
1585 return new ShenandoahParallelObjectIterator(workers, &_aux_bit_map);
1586 }
1587
1588 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1589 void ShenandoahHeap::keep_alive(oop obj) {
1590 if (is_concurrent_mark_in_progress() && (obj != nullptr)) {
1591 ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1592 }
1593 }
1594
1595 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1596 for (size_t i = 0; i < num_regions(); i++) {
1597 ShenandoahHeapRegion* current = get_region(i);
1598 blk->heap_region_do(current);
1599 }
1600 }
1601
1602 class ShenandoahParallelHeapRegionTask : public WorkerTask {
1603 private:
1604 ShenandoahHeap* const _heap;
1605 ShenandoahHeapRegionClosure* const _blk;
1606
1607 shenandoah_padding(0);
1608 volatile size_t _index;
1609 shenandoah_padding(1);
1610
1611 public:
1612 ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1613 WorkerTask("Shenandoah Parallel Region Operation"),
1614 _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1615
1616 void work(uint worker_id) {
1617 ShenandoahParallelWorkerSession worker_session(worker_id);
1618 size_t stride = ShenandoahParallelRegionStride;
1619
1620 size_t max = _heap->num_regions();
1621 while (Atomic::load(&_index) < max) {
1622 size_t cur = Atomic::fetch_then_add(&_index, stride, memory_order_relaxed);
1623 size_t start = cur;
1624 size_t end = MIN2(cur + stride, max);
1625 if (start >= max) break;
1626
1627 for (size_t i = cur; i < end; i++) {
1628 ShenandoahHeapRegion* current = _heap->get_region(i);
1629 _blk->heap_region_do(current);
1630 }
1631 }
1632 }
1633 };
1634
1635 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1636 assert(blk->is_thread_safe(), "Only thread-safe closures here");
1637 if (num_regions() > ShenandoahParallelRegionStride) {
1638 ShenandoahParallelHeapRegionTask task(blk);
1639 workers()->run_task(&task);
1640 } else {
1641 heap_region_iterate(blk);
1642 }
1643 }
1644
1645 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1646 private:
1647 ShenandoahMarkingContext* const _ctx;
1648 public:
1649 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1650
1651 void heap_region_do(ShenandoahHeapRegion* r) {
1652 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1653 if (r->is_active()) {
1654 // Check if region needs updating its TAMS. We have updated it already during concurrent
1655 // reset, so it is very likely we don't need to do another write here.
1656 if (_ctx->top_at_mark_start(r) != r->top()) {
1657 _ctx->capture_top_at_mark_start(r);
1658 }
1659 } else {
1660 assert(_ctx->top_at_mark_start(r) == r->top(),
1661 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1662 }
1663 }
1664
1665 bool is_thread_safe() { return true; }
1666 };
1667
1668 class ShenandoahRendezvousClosure : public HandshakeClosure {
1669 public:
1670 inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1671 inline void do_thread(Thread* thread) {}
1672 };
1673
1674 void ShenandoahHeap::rendezvous_threads() {
1675 ShenandoahRendezvousClosure cl;
1676 Handshake::execute(&cl);
1677 }
1678
1679 void ShenandoahHeap::recycle_trash() {
1680 free_set()->recycle_trash();
1681 }
1682
1683 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1684 private:
1685 ShenandoahMarkingContext* const _ctx;
1686 public:
1687 ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1688
1689 void heap_region_do(ShenandoahHeapRegion* r) {
1690 if (r->is_active()) {
1691 // Reset live data and set TAMS optimistically. We would recheck these under the pause
1692 // anyway to capture any updates that happened since now.
1693 r->clear_live_data();
1694 _ctx->capture_top_at_mark_start(r);
1695 }
1696 }
1697
1698 bool is_thread_safe() { return true; }
1699 };
1700
1701 void ShenandoahHeap::prepare_gc() {
1702 reset_mark_bitmap();
1703
1704 ShenandoahResetUpdateRegionStateClosure cl;
1705 parallel_heap_region_iterate(&cl);
1706 }
1707
1708 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1709 private:
1710 ShenandoahMarkingContext* const _ctx;
1711 ShenandoahHeapLock* const _lock;
1712
1713 public:
1714 ShenandoahFinalMarkUpdateRegionStateClosure() :
1715 _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1716
1717 void heap_region_do(ShenandoahHeapRegion* r) {
1718 if (r->is_active()) {
1719 // All allocations past TAMS are implicitly live, adjust the region data.
1720 // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1721 HeapWord *tams = _ctx->top_at_mark_start(r);
1722 HeapWord *top = r->top();
1723 if (top > tams) {
1724 r->increase_live_data_alloc_words(pointer_delta(top, tams));
1725 }
1726
1727 // We are about to select the collection set, make sure it knows about
1728 // current pinning status. Also, this allows trashing more regions that
1729 // now have their pinning status dropped.
1730 if (r->is_pinned()) {
1731 if (r->pin_count() == 0) {
1732 ShenandoahHeapLocker locker(_lock);
1733 r->make_unpinned();
1734 }
1735 } else {
1736 if (r->pin_count() > 0) {
1737 ShenandoahHeapLocker locker(_lock);
1738 r->make_pinned();
1739 }
1740 }
1741
1742 // Remember limit for updating refs. It's guaranteed that we get no
1743 // from-space-refs written from here on.
1744 r->set_update_watermark_at_safepoint(r->top());
1745 } else {
1746 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1747 assert(_ctx->top_at_mark_start(r) == r->top(),
1748 "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1749 }
1750 }
1751
1752 bool is_thread_safe() { return true; }
1753 };
1754
1755 void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
1756 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1757 {
1758 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
1759 ShenandoahPhaseTimings::degen_gc_final_update_region_states);
1760 ShenandoahFinalMarkUpdateRegionStateClosure cl;
1761 parallel_heap_region_iterate(&cl);
1762
1763 assert_pinned_region_status();
1764 }
1765
1766 {
1767 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
1768 ShenandoahPhaseTimings::degen_gc_choose_cset);
1769 ShenandoahHeapLocker locker(lock());
1770 _collection_set->clear();
1771 heuristics()->choose_collection_set(_collection_set);
1772 }
1773
1774 {
1775 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
1776 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
1777 ShenandoahHeapLocker locker(lock());
1778 _free_set->rebuild();
1779 }
1780 }
1781
1782 void ShenandoahHeap::do_class_unloading() {
1783 _unloader.unload();
1784 }
1785
1786 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1787 // Weak refs processing
1788 ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1789 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1790 ShenandoahTimingsTracker t(phase);
1791 ShenandoahGCWorkerPhase worker_phase(phase);
1792 ref_processor()->process_references(phase, workers(), false /* concurrent */);
1793 }
1794
1795 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1796 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1797
1798 // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1799 // make them parsable for update code to work correctly. Plus, we can compute new sizes
1800 // for future GCLABs here.
1801 if (UseTLAB) {
1802 ShenandoahGCPhase phase(concurrent ?
1803 ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1804 ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1805 gclabs_retire(ResizeTLAB);
1806 }
1807
1808 _update_refs_iterator.reset();
1809 }
1810
1811 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1812 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1813 if (_gc_state_changed) {
1814 _gc_state_changed = false;
1815 char state = gc_state();
1816 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1817 ShenandoahThreadLocalData::set_gc_state(t, state);
1818 }
1819 }
1820 }
1821
1822 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1823 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1824 _gc_state.set_cond(mask, value);
1825 _gc_state_changed = true;
1826 }
1827
1828 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1829 assert(!has_forwarded_objects(), "Not expected before/after mark phase");
1830 set_gc_state(MARKING, in_progress);
1831 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1832 }
1833
1834 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1835 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1836 set_gc_state(EVACUATION, in_progress);
1837 }
1838
1839 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
1840 if (in_progress) {
1841 _concurrent_strong_root_in_progress.set();
1842 } else {
1843 _concurrent_strong_root_in_progress.unset();
1844 }
1845 }
1846
1847 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
1848 set_gc_state(WEAK_ROOTS, cond);
1849 }
1850
1851 GCTracer* ShenandoahHeap::tracer() {
1852 return shenandoah_policy()->tracer();
1853 }
1854
1855 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1856 return _free_set->used();
1857 }
1858
1859 bool ShenandoahHeap::try_cancel_gc() {
1860 jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1861 return prev == CANCELLABLE;
1862 }
1863
1864 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1865 if (try_cancel_gc()) {
1866 FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1867 log_info(gc)("%s", msg.buffer());
1868 Events::log(Thread::current(), "%s", msg.buffer());
1869 }
1870 }
1871
1872 uint ShenandoahHeap::max_workers() {
1873 return _max_workers;
1874 }
1875
1876 void ShenandoahHeap::stop() {
1877 // The shutdown sequence should be able to terminate when GC is running.
1878
1879 // Step 0. Notify policy to disable event recording.
1880 _shenandoah_policy->record_shutdown();
1881
1882 // Step 1. Notify control thread that we are in shutdown.
1883 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1884 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1885 control_thread()->prepare_for_graceful_shutdown();
1886
1887 // Step 2. Notify GC workers that we are cancelling GC.
1888 cancel_gc(GCCause::_shenandoah_stop_vm);
1889
1890 // Step 3. Wait until GC worker exits normally.
1891 control_thread()->stop();
1892 }
1893
1894 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
1895 if (!unload_classes()) return;
1896 ClassUnloadingContext ctx(_workers->active_workers(),
1897 true /* unregister_nmethods_during_purge */,
1898 false /* lock_codeblob_free_separately */);
1899
1900 // Unload classes and purge SystemDictionary.
1901 {
1902 ShenandoahPhaseTimings::Phase phase = full_gc ?
1903 ShenandoahPhaseTimings::full_gc_purge_class_unload :
1904 ShenandoahPhaseTimings::degen_gc_purge_class_unload;
1905 ShenandoahIsAliveSelector is_alive;
1906 {
1907 CodeCache::UnlinkingScope scope(is_alive.is_alive_closure());
1908 ShenandoahGCPhase gc_phase(phase);
1909 ShenandoahGCWorkerPhase worker_phase(phase);
1910 bool unloading_occurred = SystemDictionary::do_unloading(gc_timer());
1911
1912 uint num_workers = _workers->active_workers();
1913 ShenandoahClassUnloadingTask unlink_task(phase, num_workers, unloading_occurred);
1914 _workers->run_task(&unlink_task);
1915 }
1916 // Release unloaded nmethods's memory.
1917 ClassUnloadingContext::context()->purge_and_free_nmethods();
1918 }
1919
1920 {
1921 ShenandoahGCPhase phase(full_gc ?
1922 ShenandoahPhaseTimings::full_gc_purge_cldg :
1923 ShenandoahPhaseTimings::degen_gc_purge_cldg);
1924 ClassLoaderDataGraph::purge(true /* at_safepoint */);
1925 }
1926 // Resize and verify metaspace
1927 MetaspaceGC::compute_new_size();
1928 DEBUG_ONLY(MetaspaceUtils::verify();)
1929 }
1930
1931 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
1932 // so they should not have forwarded oops.
1933 // However, we do need to "null" dead oops in the roots, if can not be done
1934 // in concurrent cycles.
1935 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
1936 uint num_workers = _workers->active_workers();
1937 ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
1938 ShenandoahPhaseTimings::full_gc_purge_weak_par :
1939 ShenandoahPhaseTimings::degen_gc_purge_weak_par;
1940 ShenandoahGCPhase phase(timing_phase);
1941 ShenandoahGCWorkerPhase worker_phase(timing_phase);
1942 // Cleanup weak roots
1943 if (has_forwarded_objects()) {
1944 ShenandoahForwardedIsAliveClosure is_alive;
1945 ShenandoahUpdateRefsClosure keep_alive;
1946 ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
1947 cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers);
1948 _workers->run_task(&cleaning_task);
1949 } else {
1950 ShenandoahIsAliveClosure is_alive;
1951 #ifdef ASSERT
1952 ShenandoahAssertNotForwardedClosure verify_cl;
1953 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
1954 cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers);
1955 #else
1956 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
1957 cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers);
1958 #endif
1959 _workers->run_task(&cleaning_task);
1960 }
1961 }
1962
1963 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
1964 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1965 assert(is_stw_gc_in_progress(), "Only for Degenerated and Full GC");
1966 ShenandoahGCPhase phase(full_gc ?
1967 ShenandoahPhaseTimings::full_gc_purge :
1968 ShenandoahPhaseTimings::degen_gc_purge);
1969 stw_weak_refs(full_gc);
1970 stw_process_weak_roots(full_gc);
1971 stw_unload_classes(full_gc);
1972 }
1973
1974 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1975 set_gc_state(HAS_FORWARDED, cond);
1976 }
1977
1978 void ShenandoahHeap::set_unload_classes(bool uc) {
1979 _unload_classes.set_cond(uc);
1980 }
1981
1982 bool ShenandoahHeap::unload_classes() const {
1983 return _unload_classes.is_set();
1984 }
1985
1986 address ShenandoahHeap::in_cset_fast_test_addr() {
1987 ShenandoahHeap* heap = ShenandoahHeap::heap();
1988 assert(heap->collection_set() != nullptr, "Sanity");
1989 return (address) heap->collection_set()->biased_map_address();
1990 }
1991
1992 size_t ShenandoahHeap::bytes_allocated_since_gc_start() const {
1993 return Atomic::load(&_bytes_allocated_since_gc_start);
1994 }
1995
1996 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
1997 Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
1998 }
1999
2000 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2001 _degenerated_gc_in_progress.set_cond(in_progress);
2002 }
2003
2004 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2005 _full_gc_in_progress.set_cond(in_progress);
2006 }
2007
2008 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2009 assert (is_full_gc_in_progress(), "should be");
2010 _full_gc_move_in_progress.set_cond(in_progress);
2011 }
2012
2013 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2014 set_gc_state(UPDATEREFS, in_progress);
2015 }
2016
2017 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2018 ShenandoahCodeRoots::register_nmethod(nm);
2019 }
2020
2021 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2022 ShenandoahCodeRoots::unregister_nmethod(nm);
2023 }
2024
2025 void ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2026 heap_region_containing(o)->record_pin();
2027 }
2028
2029 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2030 ShenandoahHeapRegion* r = heap_region_containing(o);
2031 assert(r != nullptr, "Sanity");
2032 assert(r->pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", r->index());
2033 r->record_unpin();
2034 }
2035
2036 void ShenandoahHeap::sync_pinned_region_status() {
2037 ShenandoahHeapLocker locker(lock());
2038
2039 for (size_t i = 0; i < num_regions(); i++) {
2040 ShenandoahHeapRegion *r = get_region(i);
2041 if (r->is_active()) {
2042 if (r->is_pinned()) {
2043 if (r->pin_count() == 0) {
2044 r->make_unpinned();
2045 }
2046 } else {
2047 if (r->pin_count() > 0) {
2048 r->make_pinned();
2049 }
2050 }
2051 }
2052 }
2053
2054 assert_pinned_region_status();
2055 }
2056
2057 #ifdef ASSERT
2058 void ShenandoahHeap::assert_pinned_region_status() {
2059 for (size_t i = 0; i < num_regions(); i++) {
2060 ShenandoahHeapRegion* r = get_region(i);
2061 assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2062 "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2063 }
2064 }
2065 #endif
2066
2067 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2068 return _gc_timer;
2069 }
2070
2071 void ShenandoahHeap::prepare_concurrent_roots() {
2072 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2073 assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2074 set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2075 set_concurrent_weak_root_in_progress(true);
2076 if (unload_classes()) {
2077 _unloader.prepare();
2078 }
2079 }
2080
2081 void ShenandoahHeap::finish_concurrent_roots() {
2082 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2083 assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2084 if (unload_classes()) {
2085 _unloader.finish();
2086 }
2087 }
2088
2089 #ifdef ASSERT
2090 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2091 assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2092
2093 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2094 // Use ParallelGCThreads inside safepoints
2095 assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads (%u) within safepoint, not %u",
2096 ParallelGCThreads, nworkers);
2097 } else {
2098 // Use ConcGCThreads outside safepoints
2099 assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2100 ConcGCThreads, nworkers);
2101 }
2102 }
2103 #endif
2104
2105 ShenandoahVerifier* ShenandoahHeap::verifier() {
2106 guarantee(ShenandoahVerify, "Should be enabled");
2107 assert (_verifier != nullptr, "sanity");
2108 return _verifier;
2109 }
2110
2111 template<bool CONCURRENT>
2112 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2113 private:
2114 ShenandoahHeap* _heap;
2115 ShenandoahRegionIterator* _regions;
2116 public:
2117 ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2118 WorkerTask("Shenandoah Update References"),
2119 _heap(ShenandoahHeap::heap()),
2120 _regions(regions) {
2121 }
2122
2123 void work(uint worker_id) {
2124 if (CONCURRENT) {
2125 ShenandoahConcurrentWorkerSession worker_session(worker_id);
2126 ShenandoahSuspendibleThreadSetJoiner stsj;
2127 do_work<ShenandoahConcUpdateRefsClosure>();
2128 } else {
2129 ShenandoahParallelWorkerSession worker_session(worker_id);
2130 do_work<ShenandoahSTWUpdateRefsClosure>();
2131 }
2132 }
2133
2134 private:
2135 template<class T>
2136 void do_work() {
2137 T cl;
2138 ShenandoahHeapRegion* r = _regions->next();
2139 ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2140 while (r != nullptr) {
2141 HeapWord* update_watermark = r->get_update_watermark();
2142 assert (update_watermark >= r->bottom(), "sanity");
2143 if (r->is_active() && !r->is_cset()) {
2144 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2145 }
2146 if (ShenandoahPacing) {
2147 _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2148 }
2149 if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2150 return;
2151 }
2152 r = _regions->next();
2153 }
2154 }
2155 };
2156
2157 void ShenandoahHeap::update_heap_references(bool concurrent) {
2158 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2159
2160 if (concurrent) {
2161 ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2162 workers()->run_task(&task);
2163 } else {
2164 ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2165 workers()->run_task(&task);
2166 }
2167 }
2168
2169
2170 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2171 private:
2172 ShenandoahHeapLock* const _lock;
2173
2174 public:
2175 ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2176
2177 void heap_region_do(ShenandoahHeapRegion* r) {
2178 // Drop unnecessary "pinned" state from regions that does not have CP marks
2179 // anymore, as this would allow trashing them.
2180
2181 if (r->is_active()) {
2182 if (r->is_pinned()) {
2183 if (r->pin_count() == 0) {
2184 ShenandoahHeapLocker locker(_lock);
2185 r->make_unpinned();
2186 }
2187 } else {
2188 if (r->pin_count() > 0) {
2189 ShenandoahHeapLocker locker(_lock);
2190 r->make_pinned();
2191 }
2192 }
2193 }
2194 }
2195
2196 bool is_thread_safe() { return true; }
2197 };
2198
2199 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2200 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2201 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2202
2203 {
2204 ShenandoahGCPhase phase(concurrent ?
2205 ShenandoahPhaseTimings::final_update_refs_update_region_states :
2206 ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2207 ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2208 parallel_heap_region_iterate(&cl);
2209
2210 assert_pinned_region_status();
2211 }
2212
2213 {
2214 ShenandoahGCPhase phase(concurrent ?
2215 ShenandoahPhaseTimings::final_update_refs_trash_cset :
2216 ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2217 trash_cset_regions();
2218 }
2219 }
2220
2221 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2222 {
2223 ShenandoahGCPhase phase(concurrent ?
2224 ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2225 ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2226 ShenandoahHeapLocker locker(lock());
2227 _free_set->rebuild();
2228 }
2229 }
2230
2231 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2232 print_on(st);
2233 st->cr();
2234 print_heap_regions_on(st);
2235 }
2236
2237 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2238 size_t slice = r->index() / _bitmap_regions_per_slice;
2239
2240 size_t regions_from = _bitmap_regions_per_slice * slice;
2241 size_t regions_to = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2242 for (size_t g = regions_from; g < regions_to; g++) {
2243 assert (g / _bitmap_regions_per_slice == slice, "same slice");
2244 if (skip_self && g == r->index()) continue;
2245 if (get_region(g)->is_committed()) {
2246 return true;
2247 }
2248 }
2249 return false;
2250 }
2251
2252 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2253 shenandoah_assert_heaplocked();
2254
2255 // Bitmaps in special regions do not need commits
2256 if (_bitmap_region_special) {
2257 return true;
2258 }
2259
2260 if (is_bitmap_slice_committed(r, true)) {
2261 // Some other region from the group is already committed, meaning the bitmap
2262 // slice is already committed, we exit right away.
2263 return true;
2264 }
2265
2266 // Commit the bitmap slice:
2267 size_t slice = r->index() / _bitmap_regions_per_slice;
2268 size_t off = _bitmap_bytes_per_slice * slice;
2269 size_t len = _bitmap_bytes_per_slice;
2270 char* start = (char*) _bitmap_region.start() + off;
2271
2272 if (!os::commit_memory(start, len, false)) {
2273 return false;
2274 }
2275
2276 if (AlwaysPreTouch) {
2277 os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2278 }
2279
2280 return true;
2281 }
2282
2283 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2284 shenandoah_assert_heaplocked();
2285
2286 // Bitmaps in special regions do not need uncommits
2287 if (_bitmap_region_special) {
2288 return true;
2289 }
2290
2291 if (is_bitmap_slice_committed(r, true)) {
2292 // Some other region from the group is still committed, meaning the bitmap
2293 // slice is should stay committed, exit right away.
2294 return true;
2295 }
2296
2297 // Uncommit the bitmap slice:
2298 size_t slice = r->index() / _bitmap_regions_per_slice;
2299 size_t off = _bitmap_bytes_per_slice * slice;
2300 size_t len = _bitmap_bytes_per_slice;
2301 if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2302 return false;
2303 }
2304 return true;
2305 }
2306
2307 void ShenandoahHeap::safepoint_synchronize_begin() {
2308 SuspendibleThreadSet::synchronize();
2309 }
2310
2311 void ShenandoahHeap::safepoint_synchronize_end() {
2312 SuspendibleThreadSet::desynchronize();
2313 }
2314
2315 void ShenandoahHeap::try_inject_alloc_failure() {
2316 if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2317 _inject_alloc_failure.set();
2318 os::naked_short_sleep(1);
2319 if (cancelled_gc()) {
2320 log_info(gc)("Allocation failure was successfully injected");
2321 }
2322 }
2323 }
2324
2325 bool ShenandoahHeap::should_inject_alloc_failure() {
2326 return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2327 }
2328
2329 void ShenandoahHeap::initialize_serviceability() {
2330 _memory_pool = new ShenandoahMemoryPool(this);
2331 _cycle_memory_manager.add_pool(_memory_pool);
2332 _stw_memory_manager.add_pool(_memory_pool);
2333 }
2334
2335 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2336 GrowableArray<GCMemoryManager*> memory_managers(2);
2337 memory_managers.append(&_cycle_memory_manager);
2338 memory_managers.append(&_stw_memory_manager);
2339 return memory_managers;
2340 }
2341
2342 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2343 GrowableArray<MemoryPool*> memory_pools(1);
2344 memory_pools.append(_memory_pool);
2345 return memory_pools;
2346 }
2347
2348 MemoryUsage ShenandoahHeap::memory_usage() {
2349 return _memory_pool->get_memory_usage();
2350 }
2351
2352 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2353 _heap(ShenandoahHeap::heap()),
2354 _index(0) {}
2355
2356 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2357 _heap(heap),
2358 _index(0) {}
2359
2360 void ShenandoahRegionIterator::reset() {
2361 _index = 0;
2362 }
2363
2364 bool ShenandoahRegionIterator::has_next() const {
2365 return _index < _heap->num_regions();
2366 }
2367
2368 char ShenandoahHeap::gc_state() const {
2369 return _gc_state.raw_value();
2370 }
2371
2372 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2373 #ifdef ASSERT
2374 assert(_liveness_cache != nullptr, "sanity");
2375 assert(worker_id < _max_workers, "sanity");
2376 for (uint i = 0; i < num_regions(); i++) {
2377 assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2378 }
2379 #endif
2380 return _liveness_cache[worker_id];
2381 }
2382
2383 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2384 assert(worker_id < _max_workers, "sanity");
2385 assert(_liveness_cache != nullptr, "sanity");
2386 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2387 for (uint i = 0; i < num_regions(); i++) {
2388 ShenandoahLiveData live = ld[i];
2389 if (live > 0) {
2390 ShenandoahHeapRegion* r = get_region(i);
2391 r->increase_live_data_gc_words(live);
2392 ld[i] = 0;
2393 }
2394 }
2395 }
2396
2397 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2398 if (is_idle()) return false;
2399
2400 // Objects allocated after marking start are implicitly alive, don't need any barriers during
2401 // marking phase.
2402 if (is_concurrent_mark_in_progress() &&
2403 !marking_context()->allocated_after_mark_start(obj)) {
2404 return true;
2405 }
2406
2407 // Can not guarantee obj is deeply good.
2408 if (has_forwarded_objects()) {
2409 return true;
2410 }
2411
2412 return false;
2413 }