54 #include "gc/shenandoah/shenandoahFreeSet.hpp"
55 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
56 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
57 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
58 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
59 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
62 #include "gc/shenandoah/shenandoahInitLogger.hpp"
63 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
64 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
65 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
66 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
67 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
68 #include "gc/shenandoah/shenandoahPadding.hpp"
69 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
70 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
71 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
72 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
73 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
74 #include "gc/shenandoah/shenandoahSTWMark.hpp"
75 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
76 #include "gc/shenandoah/shenandoahUtils.hpp"
77 #include "gc/shenandoah/shenandoahVerifier.hpp"
78 #include "gc/shenandoah/shenandoahVMOperations.hpp"
79 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
80 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
81 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
82 #include "memory/allocation.hpp"
83 #include "memory/classLoaderMetaspace.hpp"
84 #include "memory/memoryReserver.hpp"
85 #include "memory/metaspaceUtils.hpp"
86 #include "memory/universe.hpp"
87 #include "nmt/mallocTracker.hpp"
88 #include "nmt/memTracker.hpp"
89 #include "oops/compressedOops.inline.hpp"
90 #include "prims/jvmtiTagMap.hpp"
91 #include "runtime/atomic.hpp"
92 #include "runtime/atomicAccess.hpp"
93 #include "runtime/globals.hpp"
1211 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1212 _retire(ResizeTLAB), _propagator(gc_state) {}
1213
1214 void do_thread(Thread* thread) override {
1215 _propagator.do_thread(thread);
1216 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1217 _retire.do_thread(thread);
1218 }
1219 }
1220 private:
1221 ShenandoahRetireGCLABClosure _retire;
1222 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1223 };
1224
1225 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1226 assert(generation->is_global(), "Only global generation expected here");
1227 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1228 workers()->run_task(&task);
1229 }
1230
1231 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1232 {
1233 // Java threads take this lock while they are being attached and added to the list of threads.
1234 // If another thread holds this lock before we update the gc state, it will receive a stale
1235 // gc state, but they will have been added to the list of java threads and so will be corrected
1236 // by the following handshake.
1237 MutexLocker lock(Threads_lock);
1238
1239 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1240 set_gc_state_concurrent(EVACUATION, false);
1241 set_gc_state_concurrent(WEAK_ROOTS, false);
1242 set_gc_state_concurrent(UPDATE_REFS, true);
1243 }
1244
1245 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1246 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1247
1248 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1249 Threads::non_java_threads_do(&prepare_for_update_refs);
1250
1251 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1252 Handshake::execute(&prepare_for_update_refs);
1253
1254 _update_refs_iterator.reset();
1255 }
1256
1257 class ShenandoahCompositeHandshakeClosure : public HandshakeClosure {
1258 HandshakeClosure* _handshake_1;
1259 HandshakeClosure* _handshake_2;
1260 public:
1261 ShenandoahCompositeHandshakeClosure(HandshakeClosure* handshake_1, HandshakeClosure* handshake_2) :
1262 HandshakeClosure(handshake_2->name()),
1263 _handshake_1(handshake_1), _handshake_2(handshake_2) {}
1264
1265 void do_thread(Thread* thread) override {
1266 _handshake_1->do_thread(thread);
1267 _handshake_2->do_thread(thread);
1268 }
1269 };
1270
1271 void ShenandoahHeap::concurrent_final_roots(HandshakeClosure* handshake_closure) {
1272 {
1273 assert(!is_evacuation_in_progress(), "Should not evacuate for abbreviated or old cycles");
1274 MutexLocker lock(Threads_lock);
1275 set_gc_state_concurrent(WEAK_ROOTS, false);
1276 }
1277
1278 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1279 Threads::non_java_threads_do(&propagator);
1280 if (handshake_closure == nullptr) {
1281 Handshake::execute(&propagator);
1282 } else {
1283 ShenandoahCompositeHandshakeClosure composite(&propagator, handshake_closure);
1284 Handshake::execute(&composite);
1285 }
1286 }
1287
1288 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1289 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1290
1291 ShenandoahHeapRegion* r = heap_region_containing(p);
1292 assert(!r->is_humongous(), "never evacuate humongous objects");
1293
1294 ShenandoahAffiliation target_gen = r->affiliation();
1295 return try_evacuate_object(p, thread, r, target_gen);
1296 }
1297
1298 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1299 ShenandoahAffiliation target_gen) {
1300 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1301 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1302 bool alloc_from_lab = true;
1303 HeapWord* copy = nullptr;
1304 size_t size = ShenandoahForwarding::size(p);
1305
|
54 #include "gc/shenandoah/shenandoahFreeSet.hpp"
55 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
56 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
57 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
58 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
59 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
62 #include "gc/shenandoah/shenandoahInitLogger.hpp"
63 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
64 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
65 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
66 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
67 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
68 #include "gc/shenandoah/shenandoahPadding.hpp"
69 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
70 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
71 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
72 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
73 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
74 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
75 #include "gc/shenandoah/shenandoahSTWMark.hpp"
76 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
77 #include "gc/shenandoah/shenandoahUtils.hpp"
78 #include "gc/shenandoah/shenandoahVerifier.hpp"
79 #include "gc/shenandoah/shenandoahVMOperations.hpp"
80 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
81 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
82 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
83 #include "memory/allocation.hpp"
84 #include "memory/classLoaderMetaspace.hpp"
85 #include "memory/memoryReserver.hpp"
86 #include "memory/metaspaceUtils.hpp"
87 #include "memory/universe.hpp"
88 #include "nmt/mallocTracker.hpp"
89 #include "nmt/memTracker.hpp"
90 #include "oops/compressedOops.inline.hpp"
91 #include "prims/jvmtiTagMap.hpp"
92 #include "runtime/atomic.hpp"
93 #include "runtime/atomicAccess.hpp"
94 #include "runtime/globals.hpp"
1212 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1213 _retire(ResizeTLAB), _propagator(gc_state) {}
1214
1215 void do_thread(Thread* thread) override {
1216 _propagator.do_thread(thread);
1217 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1218 _retire.do_thread(thread);
1219 }
1220 }
1221 private:
1222 ShenandoahRetireGCLABClosure _retire;
1223 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1224 };
1225
1226 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1227 assert(generation->is_global(), "Only global generation expected here");
1228 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1229 workers()->run_task(&task);
1230 }
1231
1232 class ShenandoahCompleteStackwatermarkHandshakeClosure : public HandshakeClosure {
1233 public:
1234 ShenandoahCompleteStackwatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
1235 void do_thread(Thread* thread) override {
1236 if (thread->is_Java_thread()) {
1237 JavaThread* jt = JavaThread::cast(thread);
1238 StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
1239 }
1240 }
1241 };
1242
1243 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1244 // Make sure the current stack watermark machinery has completed before we drop evac flags.
1245 // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
1246 ShenandoahCompleteStackwatermarkHandshakeClosure cl;
1247 Handshake::execute(&cl);
1248
1249 {
1250 // Java threads take this lock while they are being attached and added to the list of threads.
1251 // If another thread holds this lock before we update the gc state, it will receive a stale
1252 // gc state, but they will have been added to the list of java threads and so will be corrected
1253 // by the following handshake.
1254 MutexLocker lock(Threads_lock);
1255
1256 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1257 set_gc_state_concurrent(EVACUATION, false);
1258 set_gc_state_concurrent(UPDATE_REFS, true);
1259 }
1260
1261 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1262 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1263
1264 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1265 Threads::non_java_threads_do(&prepare_for_update_refs);
1266
1267 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1268 Handshake::execute(&prepare_for_update_refs);
1269
1270 _update_refs_iterator.reset();
1271 }
1272
1273 void ShenandoahHeap::op_final_roots(bool at_gc_end) {
1274 set_gc_state_at_safepoint(WEAK_ROOTS, false);
1275 propagate_gc_state_to_all_threads();
1276
1277 if (ShenandoahVerify && at_gc_end) {
1278 verifier()->verify_after_gc(active_generation());
1279 }
1280
1281 // Arm the nmethods to put barriers down.
1282 ShenandoahCodeRoots::arm_nmethods();
1283 ShenandoahStackWatermark::change_epoch_id();
1284 }
1285
1286 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1287 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1288
1289 ShenandoahHeapRegion* r = heap_region_containing(p);
1290 assert(!r->is_humongous(), "never evacuate humongous objects");
1291
1292 ShenandoahAffiliation target_gen = r->affiliation();
1293 return try_evacuate_object(p, thread, r, target_gen);
1294 }
1295
1296 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1297 ShenandoahAffiliation target_gen) {
1298 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1299 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1300 bool alloc_from_lab = true;
1301 HeapWord* copy = nullptr;
1302 size_t size = ShenandoahForwarding::size(p);
1303
|