56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahSTWMark.hpp"
77 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
78 #include "gc/shenandoah/shenandoahUtils.hpp"
79 #include "gc/shenandoah/shenandoahVerifier.hpp"
80 #include "gc/shenandoah/shenandoahVMOperations.hpp"
81 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
82 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
83 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
84 #include "memory/allocation.hpp"
85 #include "memory/classLoaderMetaspace.hpp"
86 #include "memory/memoryReserver.hpp"
87 #include "memory/metaspaceUtils.hpp"
88 #include "memory/universe.hpp"
89 #include "nmt/mallocTracker.hpp"
90 #include "nmt/memTracker.hpp"
91 #include "oops/compressedOops.inline.hpp"
92 #include "prims/jvmtiTagMap.hpp"
93 #include "runtime/atomic.hpp"
94 #include "runtime/atomicAccess.hpp"
95 #include "runtime/globals.hpp"
1210 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1211 _retire(ResizeTLAB), _propagator(gc_state) {}
1212
1213 void do_thread(Thread* thread) override {
1214 _propagator.do_thread(thread);
1215 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1216 _retire.do_thread(thread);
1217 }
1218 }
1219 private:
1220 ShenandoahRetireGCLABClosure _retire;
1221 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1222 };
1223
1224 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1225 assert(generation->is_global(), "Only global generation expected here");
1226 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1227 workers()->run_task(&task);
1228 }
1229
1230 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1231 {
1232 // Java threads take this lock while they are being attached and added to the list of threads.
1233 // If another thread holds this lock before we update the gc state, it will receive a stale
1234 // gc state, but they will have been added to the list of java threads and so will be corrected
1235 // by the following handshake.
1236 MutexLocker lock(Threads_lock);
1237
1238 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1239 set_gc_state_concurrent(EVACUATION, false);
1240 set_gc_state_concurrent(UPDATE_REFS, true);
1241 }
1242
1243 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1244 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1245
1246 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1247 Threads::non_java_threads_do(&prepare_for_update_refs);
1248
1249 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1250 Handshake::execute(&prepare_for_update_refs);
1251
1252 _update_refs_iterator.reset();
1253 }
1254
1255 void ShenandoahHeap::concurrent_final_roots() {
1256 {
1257 MutexLocker lock(Threads_lock);
1258
1259 #ifdef ASSERT
1260 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1261 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1262 assert(sw == nullptr || sw->processing_completed(),
1263 "Cannot turn off weak roots before stack watermark processing is complete");
1264 }
1265 #endif
1266
1267 set_gc_state_concurrent(WEAK_ROOTS, false);
1268 }
1269
1270 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1271 Threads::non_java_threads_do(&propagator);
1272 Handshake::execute(&propagator);
1273 }
1274
1275 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1276 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1277
1278 ShenandoahHeapRegion* r = heap_region_containing(p);
1279 assert(!r->is_humongous(), "never evacuate humongous objects");
1280
1281 ShenandoahAffiliation target_gen = r->affiliation();
1282 return try_evacuate_object(p, thread, r, target_gen);
1283 }
1284
1285 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1286 ShenandoahAffiliation target_gen) {
1287 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1288 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1289 bool alloc_from_lab = true;
1290 HeapWord* copy = nullptr;
1291 size_t size = ShenandoahForwarding::size(p);
1292
1293 #ifdef ASSERT
1294 if (ShenandoahOOMDuringEvacALot &&
|
56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
77 #include "gc/shenandoah/shenandoahSTWMark.hpp"
78 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
79 #include "gc/shenandoah/shenandoahUtils.hpp"
80 #include "gc/shenandoah/shenandoahVerifier.hpp"
81 #include "gc/shenandoah/shenandoahVMOperations.hpp"
82 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
83 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
84 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
85 #include "memory/allocation.hpp"
86 #include "memory/classLoaderMetaspace.hpp"
87 #include "memory/memoryReserver.hpp"
88 #include "memory/metaspaceUtils.hpp"
89 #include "memory/universe.hpp"
90 #include "nmt/mallocTracker.hpp"
91 #include "nmt/memTracker.hpp"
92 #include "oops/compressedOops.inline.hpp"
93 #include "prims/jvmtiTagMap.hpp"
94 #include "runtime/atomic.hpp"
95 #include "runtime/atomicAccess.hpp"
96 #include "runtime/globals.hpp"
1211 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1212 _retire(ResizeTLAB), _propagator(gc_state) {}
1213
1214 void do_thread(Thread* thread) override {
1215 _propagator.do_thread(thread);
1216 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1217 _retire.do_thread(thread);
1218 }
1219 }
1220 private:
1221 ShenandoahRetireGCLABClosure _retire;
1222 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1223 };
1224
1225 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1226 assert(generation->is_global(), "Only global generation expected here");
1227 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1228 workers()->run_task(&task);
1229 }
1230
1231 class ShenandoahCompleteStackwatermarkHandshakeClosure : public HandshakeClosure {
1232 public:
1233 ShenandoahCompleteStackwatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
1234 void do_thread(Thread* thread) override {
1235 if (thread->is_Java_thread()) {
1236 JavaThread* jt = JavaThread::cast(thread);
1237 StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
1238 }
1239 }
1240 };
1241
1242 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1243 // Make sure the current stack watermark machinery has completed before we drop evac flags.
1244 // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
1245 ShenandoahCompleteStackwatermarkHandshakeClosure cl;
1246 Handshake::execute(&cl);
1247
1248 {
1249 // Java threads take this lock while they are being attached and added to the list of threads.
1250 // If another thread holds this lock before we update the gc state, it will receive a stale
1251 // gc state, but they will have been added to the list of java threads and so will be corrected
1252 // by the following handshake.
1253 MutexLocker lock(Threads_lock);
1254
1255 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1256 set_gc_state_concurrent(EVACUATION, false);
1257 set_gc_state_concurrent(UPDATE_REFS, true);
1258 }
1259
1260 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1261 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1262
1263 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1264 Threads::non_java_threads_do(&prepare_for_update_refs);
1265
1266 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1267 Handshake::execute(&prepare_for_update_refs);
1268
1269 _update_refs_iterator.reset();
1270 }
1271
1272 void ShenandoahHeap::op_conc_roots() {
1273 // Make sure the current stack watermark machinery has completed before we drop flags.
1274 ShenandoahCompleteStackwatermarkHandshakeClosure cl;
1275 Handshake::execute(&cl);
1276
1277 {
1278 // Java threads take this lock while they are being attached and added to the list of threads.
1279 // If another thread holds this lock before we update the gc state, it will receive a stale
1280 // gc state, but they will have been added to the list of java threads and so will be corrected
1281 // by the following handshake.
1282 MutexLocker lock(Threads_lock);
1283
1284 set_gc_state_concurrent(WEAK_ROOTS, false);
1285 }
1286
1287 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1288 Threads::non_java_threads_do(&propagator);
1289 Handshake::execute(&propagator);
1290 }
1291
1292 void ShenandoahHeap::op_final_roots() {
1293 // Arm the nmethods to put barriers down.
1294 ShenandoahCodeRoots::arm_nmethods();
1295 ShenandoahStackWatermark::change_epoch_id();
1296 }
1297
1298 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1299 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1300
1301 ShenandoahHeapRegion* r = heap_region_containing(p);
1302 assert(!r->is_humongous(), "never evacuate humongous objects");
1303
1304 ShenandoahAffiliation target_gen = r->affiliation();
1305 return try_evacuate_object(p, thread, r, target_gen);
1306 }
1307
1308 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1309 ShenandoahAffiliation target_gen) {
1310 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1311 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1312 bool alloc_from_lab = true;
1313 HeapWord* copy = nullptr;
1314 size_t size = ShenandoahForwarding::size(p);
1315
1316 #ifdef ASSERT
1317 if (ShenandoahOOMDuringEvacALot &&
|