56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahSTWMark.hpp"
77 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
78 #include "gc/shenandoah/shenandoahUtils.hpp"
79 #include "gc/shenandoah/shenandoahVerifier.hpp"
80 #include "gc/shenandoah/shenandoahVMOperations.hpp"
81 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
82 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
83 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
84 #include "memory/allocation.hpp"
85 #include "memory/classLoaderMetaspace.hpp"
86 #include "memory/memoryReserver.hpp"
87 #include "memory/metaspaceUtils.hpp"
88 #include "memory/universe.hpp"
89 #include "nmt/mallocTracker.hpp"
90 #include "nmt/memTracker.hpp"
91 #include "oops/compressedOops.inline.hpp"
92 #include "prims/jvmtiTagMap.hpp"
93 #include "runtime/atomic.hpp"
94 #include "runtime/atomicAccess.hpp"
95 #include "runtime/globals.hpp"
1214 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1215 _retire(ResizeTLAB), _propagator(gc_state) {}
1216
1217 void do_thread(Thread* thread) override {
1218 _propagator.do_thread(thread);
1219 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1220 _retire.do_thread(thread);
1221 }
1222 }
1223 private:
1224 ShenandoahRetireGCLABClosure _retire;
1225 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1226 };
1227
1228 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1229 assert(generation->is_global(), "Only global generation expected here");
1230 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1231 workers()->run_task(&task);
1232 }
1233
1234 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1235 {
1236 // Java threads take this lock while they are being attached and added to the list of threads.
1237 // If another thread holds this lock before we update the gc state, it will receive a stale
1238 // gc state, but they will have been added to the list of java threads and so will be corrected
1239 // by the following handshake.
1240 MutexLocker lock(Threads_lock);
1241
1242 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1243 set_gc_state_concurrent(EVACUATION, false);
1244 set_gc_state_concurrent(UPDATE_REFS, true);
1245 }
1246
1247 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1248 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1249
1250 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1251 Threads::non_java_threads_do(&prepare_for_update_refs);
1252
1253 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1254 Handshake::execute(&prepare_for_update_refs);
1255
1256 _update_refs_iterator.reset();
1257 }
1258
1259 void ShenandoahHeap::concurrent_final_roots() {
1260 {
1261 MutexLocker lock(Threads_lock);
1262
1263 #ifdef ASSERT
1264 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1265 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1266 assert(sw == nullptr || sw->processing_completed(),
1267 "Cannot turn off weak roots before stack watermark processing is complete");
1268 }
1269 #endif
1270
1271 set_gc_state_concurrent(WEAK_ROOTS, false);
1272 }
1273
1274 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1275 Threads::non_java_threads_do(&propagator);
1276 Handshake::execute(&propagator);
1277 }
1278
1279 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1280 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1281
1282 ShenandoahHeapRegion* r = heap_region_containing(p);
1283 assert(!r->is_humongous(), "never evacuate humongous objects");
1284
1285 ShenandoahAffiliation target_gen = r->affiliation();
1286 return try_evacuate_object(p, thread, r, target_gen);
1287 }
1288
1289 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1290 ShenandoahAffiliation target_gen) {
1291 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1292 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1293 bool alloc_from_lab = true;
1294 HeapWord* copy = nullptr;
1295 size_t size = ShenandoahForwarding::size(p);
1296
2877 }
2878 #endif
2879 return _liveness_cache[worker_id];
2880 }
2881
2882 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2883 assert(worker_id < _max_workers, "sanity");
2884 assert(_liveness_cache != nullptr, "sanity");
2885 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2886 for (uint i = 0; i < num_regions(); i++) {
2887 ShenandoahLiveData live = ld[i];
2888 if (live > 0) {
2889 ShenandoahHeapRegion* r = get_region(i);
2890 r->increase_live_data_gc_words(live);
2891 ld[i] = 0;
2892 }
2893 }
2894 }
2895
2896 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2897 if (is_idle()) return false;
2898
2899 // Objects allocated after marking start are implicitly alive, don't need any barriers during
2900 // marking phase.
2901 if (is_concurrent_mark_in_progress() &&
2902 !marking_context()->allocated_after_mark_start(obj)) {
2903 return true;
2904 }
2905
2906 // Can not guarantee obj is deeply good.
2907 if (has_forwarded_objects()) {
2908 return true;
2909 }
2910
2911 return false;
2912 }
2913
2914 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
2915 #if INCLUDE_CDS_JAVA_HEAP
2916 // CDS wants a raw continuous memory range to load a bunch of objects itself.
2917 // This is an unusual request, since all requested regions should be regular, not humongous.
2918 //
2919 // CDS would guarantee no objects straddle multiple regions, as long as regions are as large
2920 // as MIN_GC_REGION_ALIGNMENT.
2921 guarantee(ShenandoahHeapRegion::region_size_bytes() >= AOTMappedHeapWriter::MIN_GC_REGION_ALIGNMENT, "Must be");
2922
2923 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_cds(size);
2924 return allocate_memory(req);
2925 #else
2926 assert(false, "Archive heap loader should not be available, should not be here");
2927 return nullptr;
|
56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahStackChunkGCData.inline.hpp"
77 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
78 #include "gc/shenandoah/shenandoahSTWMark.hpp"
79 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
80 #include "gc/shenandoah/shenandoahUtils.hpp"
81 #include "gc/shenandoah/shenandoahVerifier.hpp"
82 #include "gc/shenandoah/shenandoahVMOperations.hpp"
83 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
84 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
85 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
86 #include "memory/allocation.hpp"
87 #include "memory/classLoaderMetaspace.hpp"
88 #include "memory/memoryReserver.hpp"
89 #include "memory/metaspaceUtils.hpp"
90 #include "memory/universe.hpp"
91 #include "nmt/mallocTracker.hpp"
92 #include "nmt/memTracker.hpp"
93 #include "oops/compressedOops.inline.hpp"
94 #include "prims/jvmtiTagMap.hpp"
95 #include "runtime/atomic.hpp"
96 #include "runtime/atomicAccess.hpp"
97 #include "runtime/globals.hpp"
1216 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1217 _retire(ResizeTLAB), _propagator(gc_state) {}
1218
1219 void do_thread(Thread* thread) override {
1220 _propagator.do_thread(thread);
1221 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1222 _retire.do_thread(thread);
1223 }
1224 }
1225 private:
1226 ShenandoahRetireGCLABClosure _retire;
1227 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1228 };
1229
1230 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1231 assert(generation->is_global(), "Only global generation expected here");
1232 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1233 workers()->run_task(&task);
1234 }
1235
1236 class ShenandoahCompleteStackWatermarkHandshakeClosure : public HandshakeClosure {
1237 public:
1238 ShenandoahCompleteStackWatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
1239 void do_thread(Thread* thread) override {
1240 if (thread->is_Java_thread()) {
1241 JavaThread* jt = JavaThread::cast(thread);
1242 StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
1243 }
1244 }
1245 };
1246
1247 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1248 // The stack watermarks are active since op_final_roots().
1249 // Make sure the current stack watermark machinery has completed before we drop evac flags.
1250 // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
1251 ShenandoahCompleteStackWatermarkHandshakeClosure cl;
1252 Handshake::execute(&cl);
1253
1254 {
1255 // Java threads take this lock while they are being attached and added to the list of threads.
1256 // If another thread holds this lock before we update the gc state, it will receive a stale
1257 // gc state, but they will have been added to the list of java threads and so will be corrected
1258 // by the following handshake.
1259 MutexLocker lock(Threads_lock);
1260
1261 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1262 set_gc_state_concurrent(EVACUATION, false);
1263 set_gc_state_concurrent(UPDATE_REFS, true);
1264 }
1265
1266 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1267 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1268
1269 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1270 Threads::non_java_threads_do(&prepare_for_update_refs);
1271
1272 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1273 Handshake::execute(&prepare_for_update_refs);
1274
1275 _update_refs_iterator.reset();
1276 }
1277
1278 void ShenandoahHeap::op_final_roots() {
1279 #ifdef ASSERT
1280 // Check if stack watermark machinery is in safe state:
1281 // 1. With evac-in-progress, we are about to supersede evac processing with new epoch.
1282 // op_thread_roots() should have completed the stack watermark processing before
1283 // we reach here.
1284 // 2. Without evac-in-progress, we are in abbreviated cycle, and we are switching
1285 // to a new epoch that *also* does not change any oops, which is safe.
1286 if (is_evacuation_in_progress()) {
1287 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1288 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1289 assert(sw == nullptr || sw->processing_completed(),
1290 "Cannot turn off weak roots before stack watermark processing is complete");
1291 }
1292 }
1293 #endif
1294
1295 set_gc_state_at_safepoint(WEAK_ROOTS, false);
1296
1297 // Arm the nmethods to change the barriers.
1298 CodeCache::arm_all_nmethods();
1299
1300 {
1301 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
1302 propagate_gc_state_to_all_threads();
1303 }
1304 }
1305
1306 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1307 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1308
1309 ShenandoahHeapRegion* r = heap_region_containing(p);
1310 assert(!r->is_humongous(), "never evacuate humongous objects");
1311
1312 ShenandoahAffiliation target_gen = r->affiliation();
1313 return try_evacuate_object(p, thread, r, target_gen);
1314 }
1315
1316 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1317 ShenandoahAffiliation target_gen) {
1318 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1319 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1320 bool alloc_from_lab = true;
1321 HeapWord* copy = nullptr;
1322 size_t size = ShenandoahForwarding::size(p);
1323
2904 }
2905 #endif
2906 return _liveness_cache[worker_id];
2907 }
2908
2909 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2910 assert(worker_id < _max_workers, "sanity");
2911 assert(_liveness_cache != nullptr, "sanity");
2912 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2913 for (uint i = 0; i < num_regions(); i++) {
2914 ShenandoahLiveData live = ld[i];
2915 if (live > 0) {
2916 ShenandoahHeapRegion* r = get_region(i);
2917 r->increase_live_data_gc_words(live);
2918 ld[i] = 0;
2919 }
2920 }
2921 }
2922
2923 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2924 // Can not guarantee obj is deeply good.
2925 if (has_forwarded_objects()) {
2926 return true;
2927 }
2928
2929 // Objects allocated after marking start are implicitly alive.
2930 if (is_concurrent_mark_in_progress() &&
2931 !marking_context()->allocated_after_mark_start(obj)) {
2932 return true;
2933 }
2934
2935 // Nmethods referenced by stack chunk may need the barrier fixups.
2936 if (ShenandoahStackChunkGCData::is_different_epoch(obj)) {
2937 return true;
2938 }
2939
2940 return false;
2941 }
2942
2943 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
2944 #if INCLUDE_CDS_JAVA_HEAP
2945 // CDS wants a raw continuous memory range to load a bunch of objects itself.
2946 // This is an unusual request, since all requested regions should be regular, not humongous.
2947 //
2948 // CDS would guarantee no objects straddle multiple regions, as long as regions are as large
2949 // as MIN_GC_REGION_ALIGNMENT.
2950 guarantee(ShenandoahHeapRegion::region_size_bytes() >= AOTMappedHeapWriter::MIN_GC_REGION_ALIGNMENT, "Must be");
2951
2952 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_cds(size);
2953 return allocate_memory(req);
2954 #else
2955 assert(false, "Archive heap loader should not be available, should not be here");
2956 return nullptr;
|