56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahSTWMark.hpp"
77 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
78 #include "gc/shenandoah/shenandoahUtils.hpp"
79 #include "gc/shenandoah/shenandoahVerifier.hpp"
80 #include "gc/shenandoah/shenandoahVMOperations.hpp"
81 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
82 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
83 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
84 #include "memory/allocation.hpp"
85 #include "memory/classLoaderMetaspace.hpp"
86 #include "memory/memoryReserver.hpp"
87 #include "memory/metaspaceUtils.hpp"
88 #include "memory/universe.hpp"
89 #include "nmt/mallocTracker.hpp"
90 #include "nmt/memTracker.hpp"
91 #include "oops/compressedOops.inline.hpp"
92 #include "prims/jvmtiTagMap.hpp"
93 #include "runtime/atomic.hpp"
94 #include "runtime/atomicAccess.hpp"
95 #include "runtime/globals.hpp"
1210 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1211 _retire(ResizeTLAB), _propagator(gc_state) {}
1212
1213 void do_thread(Thread* thread) override {
1214 _propagator.do_thread(thread);
1215 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1216 _retire.do_thread(thread);
1217 }
1218 }
1219 private:
1220 ShenandoahRetireGCLABClosure _retire;
1221 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1222 };
1223
1224 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1225 assert(generation->is_global(), "Only global generation expected here");
1226 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1227 workers()->run_task(&task);
1228 }
1229
1230 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1231 {
1232 // Java threads take this lock while they are being attached and added to the list of threads.
1233 // If another thread holds this lock before we update the gc state, it will receive a stale
1234 // gc state, but they will have been added to the list of java threads and so will be corrected
1235 // by the following handshake.
1236 MutexLocker lock(Threads_lock);
1237
1238 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1239 set_gc_state_concurrent(EVACUATION, false);
1240 set_gc_state_concurrent(UPDATE_REFS, true);
1241 }
1242
1243 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1244 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1245
1246 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1247 Threads::non_java_threads_do(&prepare_for_update_refs);
1248
1249 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1250 Handshake::execute(&prepare_for_update_refs);
1251
1252 _update_refs_iterator.reset();
1253 }
1254
1255 void ShenandoahHeap::concurrent_final_roots() {
1256 {
1257 MutexLocker lock(Threads_lock);
1258
1259 #ifdef ASSERT
1260 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1261 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1262 assert(sw == nullptr || sw->processing_completed(),
1263 "Cannot turn off weak roots before stack watermark processing is complete");
1264 }
1265 #endif
1266
1267 set_gc_state_concurrent(WEAK_ROOTS, false);
1268 }
1269
1270 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1271 Threads::non_java_threads_do(&propagator);
1272 Handshake::execute(&propagator);
1273 }
1274
1275 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1276 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1277
1278 ShenandoahHeapRegion* r = heap_region_containing(p);
1279 assert(!r->is_humongous(), "never evacuate humongous objects");
1280
1281 ShenandoahAffiliation target_gen = r->affiliation();
1282 return try_evacuate_object(p, thread, r, target_gen);
1283 }
1284
1285 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1286 ShenandoahAffiliation target_gen) {
1287 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1288 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1289 bool alloc_from_lab = true;
1290 HeapWord* copy = nullptr;
1291 size_t size = ShenandoahForwarding::size(p);
1292
|
56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
77 #include "gc/shenandoah/shenandoahSTWMark.hpp"
78 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
79 #include "gc/shenandoah/shenandoahUtils.hpp"
80 #include "gc/shenandoah/shenandoahVerifier.hpp"
81 #include "gc/shenandoah/shenandoahVMOperations.hpp"
82 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
83 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
84 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
85 #include "memory/allocation.hpp"
86 #include "memory/classLoaderMetaspace.hpp"
87 #include "memory/memoryReserver.hpp"
88 #include "memory/metaspaceUtils.hpp"
89 #include "memory/universe.hpp"
90 #include "nmt/mallocTracker.hpp"
91 #include "nmt/memTracker.hpp"
92 #include "oops/compressedOops.inline.hpp"
93 #include "prims/jvmtiTagMap.hpp"
94 #include "runtime/atomic.hpp"
95 #include "runtime/atomicAccess.hpp"
96 #include "runtime/globals.hpp"
1211 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1212 _retire(ResizeTLAB), _propagator(gc_state) {}
1213
1214 void do_thread(Thread* thread) override {
1215 _propagator.do_thread(thread);
1216 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1217 _retire.do_thread(thread);
1218 }
1219 }
1220 private:
1221 ShenandoahRetireGCLABClosure _retire;
1222 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1223 };
1224
1225 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1226 assert(generation->is_global(), "Only global generation expected here");
1227 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1228 workers()->run_task(&task);
1229 }
1230
1231 class ShenandoahCompleteStackWatermarkHandshakeClosure : public HandshakeClosure {
1232 public:
1233 ShenandoahCompleteStackWatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
1234 void do_thread(Thread* thread) override {
1235 if (thread->is_Java_thread()) {
1236 JavaThread* jt = JavaThread::cast(thread);
1237 StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
1238 }
1239 }
1240 };
1241
1242 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1243 // The stack watermarks are active since op_final_roots().
1244 // Make sure the current stack watermark machinery has completed before we drop evac flags.
1245 // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
1246 ShenandoahCompleteStackWatermarkHandshakeClosure cl;
1247 Handshake::execute(&cl);
1248
1249 {
1250 // Java threads take this lock while they are being attached and added to the list of threads.
1251 // If another thread holds this lock before we update the gc state, it will receive a stale
1252 // gc state, but they will have been added to the list of java threads and so will be corrected
1253 // by the following handshake.
1254 MutexLocker lock(Threads_lock);
1255
1256 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1257 set_gc_state_concurrent(EVACUATION, false);
1258 set_gc_state_concurrent(UPDATE_REFS, true);
1259 }
1260
1261 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1262 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1263
1264 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1265 Threads::non_java_threads_do(&prepare_for_update_refs);
1266
1267 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1268 Handshake::execute(&prepare_for_update_refs);
1269
1270 _update_refs_iterator.reset();
1271 }
1272
1273 void ShenandoahHeap::op_final_roots() {
1274 #ifdef ASSERT
1275 // Check if stack watermark machinery is in safe state:
1276 // 1. With evac-in-progress, we are about to supersede evac processing with new epoch.
1277 // op_thread_roots() should have completed the stack watermark processing before
1278 // we reach here.
1279 // 2. Without evac-in-progress, we are in abbreviated cycle, and we are switching
1280 // to a new epoch that *also* does not change any oops, which is safe.
1281 if (is_evacuation_in_progress()) {
1282 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1283 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1284 assert(sw == nullptr || sw->processing_completed(),
1285 "Cannot turn off weak roots before stack watermark processing is complete");
1286 }
1287 }
1288 #endif
1289
1290 set_gc_state_at_safepoint(WEAK_ROOTS, false);
1291
1292 // Arm the nmethods to change the barriers.
1293 ShenandoahCodeRoots::arm_nmethods();
1294
1295 {
1296 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
1297 propagate_gc_state_to_all_threads();
1298 }
1299 }
1300
1301 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1302 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1303
1304 ShenandoahHeapRegion* r = heap_region_containing(p);
1305 assert(!r->is_humongous(), "never evacuate humongous objects");
1306
1307 ShenandoahAffiliation target_gen = r->affiliation();
1308 return try_evacuate_object(p, thread, r, target_gen);
1309 }
1310
1311 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1312 ShenandoahAffiliation target_gen) {
1313 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1314 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1315 bool alloc_from_lab = true;
1316 HeapWord* copy = nullptr;
1317 size_t size = ShenandoahForwarding::size(p);
1318
|