54 #include "gc/shenandoah/shenandoahFreeSet.hpp"
55 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
56 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
57 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
58 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
59 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
62 #include "gc/shenandoah/shenandoahInitLogger.hpp"
63 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
64 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
65 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
66 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
67 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
68 #include "gc/shenandoah/shenandoahPadding.hpp"
69 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
70 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
71 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
72 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
73 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
74 #include "gc/shenandoah/shenandoahSTWMark.hpp"
75 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
76 #include "gc/shenandoah/shenandoahUtils.hpp"
77 #include "gc/shenandoah/shenandoahVerifier.hpp"
78 #include "gc/shenandoah/shenandoahVMOperations.hpp"
79 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
80 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
81 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
82 #include "memory/allocation.hpp"
83 #include "memory/classLoaderMetaspace.hpp"
84 #include "memory/memoryReserver.hpp"
85 #include "memory/metaspaceUtils.hpp"
86 #include "memory/universe.hpp"
87 #include "nmt/mallocTracker.hpp"
88 #include "nmt/memTracker.hpp"
89 #include "oops/compressedOops.inline.hpp"
90 #include "prims/jvmtiTagMap.hpp"
91 #include "runtime/atomic.hpp"
92 #include "runtime/atomicAccess.hpp"
93 #include "runtime/globals.hpp"
1237 MutexLocker lock(Threads_lock);
1238
1239 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1240 set_gc_state_concurrent(EVACUATION, false);
1241 set_gc_state_concurrent(WEAK_ROOTS, false);
1242 set_gc_state_concurrent(UPDATE_REFS, true);
1243 }
1244
1245 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1246 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1247
1248 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1249 Threads::non_java_threads_do(&prepare_for_update_refs);
1250
1251 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1252 Handshake::execute(&prepare_for_update_refs);
1253
1254 _update_refs_iterator.reset();
1255 }
1256
1257 class ShenandoahCompositeHandshakeClosure : public HandshakeClosure {
1258 HandshakeClosure* _handshake_1;
1259 HandshakeClosure* _handshake_2;
1260 public:
1261 ShenandoahCompositeHandshakeClosure(HandshakeClosure* handshake_1, HandshakeClosure* handshake_2) :
1262 HandshakeClosure(handshake_2->name()),
1263 _handshake_1(handshake_1), _handshake_2(handshake_2) {}
1264
1265 void do_thread(Thread* thread) override {
1266 _handshake_1->do_thread(thread);
1267 _handshake_2->do_thread(thread);
1268 }
1269 };
1270
1271 void ShenandoahHeap::concurrent_final_roots(HandshakeClosure* handshake_closure) {
1272 {
1273 assert(!is_evacuation_in_progress(), "Should not evacuate for abbreviated or old cycles");
1274 MutexLocker lock(Threads_lock);
1275 set_gc_state_concurrent(WEAK_ROOTS, false);
1276 }
1277
1278 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1279 Threads::non_java_threads_do(&propagator);
1280 if (handshake_closure == nullptr) {
1281 Handshake::execute(&propagator);
1282 } else {
1283 ShenandoahCompositeHandshakeClosure composite(&propagator, handshake_closure);
1284 Handshake::execute(&composite);
1285 }
1286 }
1287
1288 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1289 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1290
1291 ShenandoahHeapRegion* r = heap_region_containing(p);
1292 assert(!r->is_humongous(), "never evacuate humongous objects");
1293
1294 ShenandoahAffiliation target_gen = r->affiliation();
1295 return try_evacuate_object(p, thread, r, target_gen);
1296 }
1297
1298 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1299 ShenandoahAffiliation target_gen) {
1300 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1301 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1302 bool alloc_from_lab = true;
1303 HeapWord* copy = nullptr;
1304 size_t size = ShenandoahForwarding::size(p);
1305
|
54 #include "gc/shenandoah/shenandoahFreeSet.hpp"
55 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
56 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
57 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
58 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
59 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
62 #include "gc/shenandoah/shenandoahInitLogger.hpp"
63 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
64 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
65 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
66 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
67 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
68 #include "gc/shenandoah/shenandoahPadding.hpp"
69 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
70 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
71 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
72 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
73 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
74 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
75 #include "gc/shenandoah/shenandoahSTWMark.hpp"
76 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
77 #include "gc/shenandoah/shenandoahUtils.hpp"
78 #include "gc/shenandoah/shenandoahVerifier.hpp"
79 #include "gc/shenandoah/shenandoahVMOperations.hpp"
80 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
81 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
82 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
83 #include "memory/allocation.hpp"
84 #include "memory/classLoaderMetaspace.hpp"
85 #include "memory/memoryReserver.hpp"
86 #include "memory/metaspaceUtils.hpp"
87 #include "memory/universe.hpp"
88 #include "nmt/mallocTracker.hpp"
89 #include "nmt/memTracker.hpp"
90 #include "oops/compressedOops.inline.hpp"
91 #include "prims/jvmtiTagMap.hpp"
92 #include "runtime/atomic.hpp"
93 #include "runtime/atomicAccess.hpp"
94 #include "runtime/globals.hpp"
1238 MutexLocker lock(Threads_lock);
1239
1240 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1241 set_gc_state_concurrent(EVACUATION, false);
1242 set_gc_state_concurrent(WEAK_ROOTS, false);
1243 set_gc_state_concurrent(UPDATE_REFS, true);
1244 }
1245
1246 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1247 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1248
1249 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1250 Threads::non_java_threads_do(&prepare_for_update_refs);
1251
1252 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1253 Handshake::execute(&prepare_for_update_refs);
1254
1255 _update_refs_iterator.reset();
1256 }
1257
1258 void ShenandoahHeap::op_final_roots() {
1259 assert(!is_evacuation_in_progress(), "Should not evacuate for abbreviated or old cycles");
1260 set_gc_state_at_safepoint(WEAK_ROOTS, false);
1261 propagate_gc_state_to_all_threads();
1262
1263 if (ShenandoahVerify) {
1264 verifier()->verify_after_gc(active_generation());
1265 }
1266
1267 // Final pause, arm the nmethods to put barriers down.
1268 ShenandoahCodeRoots::arm_nmethods();
1269 ShenandoahStackWatermark::change_epoch_id();
1270 }
1271
1272 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1273 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1274
1275 ShenandoahHeapRegion* r = heap_region_containing(p);
1276 assert(!r->is_humongous(), "never evacuate humongous objects");
1277
1278 ShenandoahAffiliation target_gen = r->affiliation();
1279 return try_evacuate_object(p, thread, r, target_gen);
1280 }
1281
1282 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1283 ShenandoahAffiliation target_gen) {
1284 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1285 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1286 bool alloc_from_lab = true;
1287 HeapWord* copy = nullptr;
1288 size_t size = ShenandoahForwarding::size(p);
1289
|