56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahSTWMark.hpp"
77 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
78 #include "gc/shenandoah/shenandoahUtils.hpp"
79 #include "gc/shenandoah/shenandoahVerifier.hpp"
80 #include "gc/shenandoah/shenandoahVMOperations.hpp"
81 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
82 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
83 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
84 #include "memory/allocation.hpp"
85 #include "memory/classLoaderMetaspace.hpp"
86 #include "memory/memoryReserver.hpp"
87 #include "memory/metaspaceUtils.hpp"
88 #include "memory/universe.hpp"
89 #include "nmt/mallocTracker.hpp"
90 #include "nmt/memTracker.hpp"
91 #include "oops/compressedOops.inline.hpp"
92 #include "prims/jvmtiTagMap.hpp"
93 #include "runtime/atomic.hpp"
94 #include "runtime/atomicAccess.hpp"
95 #include "runtime/globals.hpp"
1212 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1213 _retire(ResizeTLAB), _propagator(gc_state) {}
1214
1215 void do_thread(Thread* thread) override {
1216 _propagator.do_thread(thread);
1217 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1218 _retire.do_thread(thread);
1219 }
1220 }
1221 private:
1222 ShenandoahRetireGCLABClosure _retire;
1223 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1224 };
1225
1226 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1227 assert(generation->is_global(), "Only global generation expected here");
1228 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1229 workers()->run_task(&task);
1230 }
1231
1232 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1233 {
1234 // Java threads take this lock while they are being attached and added to the list of threads.
1235 // If another thread holds this lock before we update the gc state, it will receive a stale
1236 // gc state, but they will have been added to the list of java threads and so will be corrected
1237 // by the following handshake.
1238 MutexLocker lock(Threads_lock);
1239
1240 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1241 set_gc_state_concurrent(EVACUATION, false);
1242 set_gc_state_concurrent(UPDATE_REFS, true);
1243 }
1244
1245 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1246 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1247
1248 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1249 Threads::non_java_threads_do(&prepare_for_update_refs);
1250
1251 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1252 Handshake::execute(&prepare_for_update_refs);
1253
1254 _update_refs_iterator.reset();
1255 }
1256
1257 void ShenandoahHeap::concurrent_final_roots() {
1258 {
1259 MutexLocker lock(Threads_lock);
1260
1261 #ifdef ASSERT
1262 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1263 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1264 assert(sw == nullptr || sw->processing_completed(),
1265 "Cannot turn off weak roots before stack watermark processing is complete");
1266 }
1267 #endif
1268
1269 set_gc_state_concurrent(WEAK_ROOTS, false);
1270 }
1271
1272 ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
1273 Threads::non_java_threads_do(&propagator);
1274 Handshake::execute(&propagator);
1275 }
1276
1277 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1278 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1279
1280 ShenandoahHeapRegion* r = heap_region_containing(p);
1281 assert(!r->is_humongous(), "never evacuate humongous objects");
1282
1283 ShenandoahAffiliation target_gen = r->affiliation();
1284 return try_evacuate_object(p, thread, r, target_gen);
1285 }
1286
1287 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1288 ShenandoahAffiliation target_gen) {
1289 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1290 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1291 bool alloc_from_lab = true;
1292 HeapWord* copy = nullptr;
1293 size_t size = ShenandoahForwarding::size(p);
1294
2881 }
2882 #endif
2883 return _liveness_cache[worker_id];
2884 }
2885
2886 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2887 assert(worker_id < _max_workers, "sanity");
2888 assert(_liveness_cache != nullptr, "sanity");
2889 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2890 for (uint i = 0; i < num_regions(); i++) {
2891 ShenandoahLiveData live = ld[i];
2892 if (live > 0) {
2893 ShenandoahHeapRegion* r = get_region(i);
2894 r->increase_live_data_gc_words(live);
2895 ld[i] = 0;
2896 }
2897 }
2898 }
2899
2900 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2901 if (is_idle()) return false;
2902
2903 // Objects allocated after marking start are implicitly alive, don't need any barriers during
2904 // marking phase.
2905 if (is_concurrent_mark_in_progress() &&
2906 !marking_context()->allocated_after_mark_start(obj)) {
2907 return true;
2908 }
2909
2910 // Can not guarantee obj is deeply good.
2911 if (has_forwarded_objects()) {
2912 return true;
2913 }
2914
2915 return false;
2916 }
2917
2918 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
2919 #if INCLUDE_CDS_JAVA_HEAP
2920 // CDS wants a raw continuous memory range to load a bunch of objects itself.
2921 // This is an unusual request, since all requested regions should be regular, not humongous.
2922 //
2923 // CDS would guarantee no objects straddle multiple regions, as long as regions are as large
2924 // as MIN_GC_REGION_ALIGNMENT.
2925 guarantee(ShenandoahHeapRegion::region_size_bytes() >= AOTMappedHeapWriter::MIN_GC_REGION_ALIGNMENT, "Must be");
2926
2927 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_cds(size);
2928 return allocate_memory(req);
2929 #else
2930 assert(false, "Archive heap loader should not be available, should not be here");
2931 return nullptr;
|
56 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
57 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
58 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
59 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
60 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
61 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
62 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
63 #include "gc/shenandoah/shenandoahInitLogger.hpp"
64 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
65 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
66 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
67 #include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
68 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
69 #include "gc/shenandoah/shenandoahPadding.hpp"
70 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
71 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
72 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
73 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
74 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
75 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
76 #include "gc/shenandoah/shenandoahStackChunkGCData.inline.hpp"
77 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
78 #include "gc/shenandoah/shenandoahSTWMark.hpp"
79 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
80 #include "gc/shenandoah/shenandoahUtils.hpp"
81 #include "gc/shenandoah/shenandoahVerifier.hpp"
82 #include "gc/shenandoah/shenandoahVMOperations.hpp"
83 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
84 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
85 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
86 #include "memory/allocation.hpp"
87 #include "memory/classLoaderMetaspace.hpp"
88 #include "memory/memoryReserver.hpp"
89 #include "memory/metaspaceUtils.hpp"
90 #include "memory/universe.hpp"
91 #include "nmt/mallocTracker.hpp"
92 #include "nmt/memTracker.hpp"
93 #include "oops/compressedOops.inline.hpp"
94 #include "prims/jvmtiTagMap.hpp"
95 #include "runtime/atomic.hpp"
96 #include "runtime/atomicAccess.hpp"
97 #include "runtime/globals.hpp"
1214 HandshakeClosure("Shenandoah Prepare for Update Refs"),
1215 _retire(ResizeTLAB), _propagator(gc_state) {}
1216
1217 void do_thread(Thread* thread) override {
1218 _propagator.do_thread(thread);
1219 if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1220 _retire.do_thread(thread);
1221 }
1222 }
1223 private:
1224 ShenandoahRetireGCLABClosure _retire;
1225 ShenandoahGCStatePropagatorHandshakeClosure _propagator;
1226 };
1227
1228 void ShenandoahHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
1229 assert(generation->is_global(), "Only global generation expected here");
1230 ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1231 workers()->run_task(&task);
1232 }
1233
1234 class ShenandoahCompleteStackWatermarkHandshakeClosure : public HandshakeClosure {
1235 public:
1236 ShenandoahCompleteStackWatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
1237 void do_thread(Thread* thread) override {
1238 if (thread->is_Java_thread()) {
1239 JavaThread* jt = JavaThread::cast(thread);
1240 StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
1241 }
1242 }
1243 };
1244
1245 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1246 // The stack watermarks are active since op_final_roots().
1247 // Make sure the current stack watermark machinery has completed before we drop evac flags.
1248 // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
1249 ShenandoahCompleteStackWatermarkHandshakeClosure cl;
1250 Handshake::execute(&cl);
1251
1252 {
1253 // Java threads take this lock while they are being attached and added to the list of threads.
1254 // If another thread holds this lock before we update the gc state, it will receive a stale
1255 // gc state, but they will have been added to the list of java threads and so will be corrected
1256 // by the following handshake.
1257 MutexLocker lock(Threads_lock);
1258
1259 // A cancellation at this point means the degenerated cycle must resume from update-refs.
1260 set_gc_state_concurrent(EVACUATION, false);
1261 set_gc_state_concurrent(UPDATE_REFS, true);
1262 }
1263
1264 // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1265 ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
1266
1267 // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1268 Threads::non_java_threads_do(&prepare_for_update_refs);
1269
1270 // Now retire gclabs and plabs and propagate gc_state for mutator threads
1271 Handshake::execute(&prepare_for_update_refs);
1272
1273 _update_refs_iterator.reset();
1274 }
1275
1276 void ShenandoahHeap::op_final_roots() {
1277 #ifdef ASSERT
1278 // Check if stack watermark machinery is in safe state:
1279 // 1. With evac-in-progress, we are about to supersede evac processing with new epoch.
1280 // op_thread_roots() should have completed the stack watermark processing before
1281 // we reach here.
1282 // 2. Without evac-in-progress, we are in abbreviated cycle, and we are switching
1283 // to a new epoch that *also* does not change any oops, which is safe.
1284 if (is_evacuation_in_progress()) {
1285 for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
1286 StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
1287 assert(sw == nullptr || sw->processing_completed(),
1288 "Cannot turn off weak roots before stack watermark processing is complete");
1289 }
1290 }
1291 #endif
1292
1293 set_gc_state_at_safepoint(WEAK_ROOTS, false);
1294
1295 // Arm the nmethods to change the barriers.
1296 CodeCache::arm_all_nmethods();
1297
1298 {
1299 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
1300 propagate_gc_state_to_all_threads();
1301 }
1302 }
1303
1304 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1305 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1306
1307 ShenandoahHeapRegion* r = heap_region_containing(p);
1308 assert(!r->is_humongous(), "never evacuate humongous objects");
1309
1310 ShenandoahAffiliation target_gen = r->affiliation();
1311 return try_evacuate_object(p, thread, r, target_gen);
1312 }
1313
1314 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1315 ShenandoahAffiliation target_gen) {
1316 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1317 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1318 bool alloc_from_lab = true;
1319 HeapWord* copy = nullptr;
1320 size_t size = ShenandoahForwarding::size(p);
1321
2908 }
2909 #endif
2910 return _liveness_cache[worker_id];
2911 }
2912
2913 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2914 assert(worker_id < _max_workers, "sanity");
2915 assert(_liveness_cache != nullptr, "sanity");
2916 ShenandoahLiveData* ld = _liveness_cache[worker_id];
2917 for (uint i = 0; i < num_regions(); i++) {
2918 ShenandoahLiveData live = ld[i];
2919 if (live > 0) {
2920 ShenandoahHeapRegion* r = get_region(i);
2921 r->increase_live_data_gc_words(live);
2922 ld[i] = 0;
2923 }
2924 }
2925 }
2926
2927 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2928 // Can not guarantee obj is deeply good.
2929 if (has_forwarded_objects()) {
2930 return true;
2931 }
2932
2933 // Objects allocated after marking start are implicitly alive.
2934 if (is_concurrent_mark_in_progress() &&
2935 !marking_context()->allocated_after_mark_start(obj)) {
2936 return true;
2937 }
2938
2939 // Nmethods referenced by stack chunk may need the barrier fixups.
2940 if (ShenandoahStackChunkGCData::is_different_epoch(obj)) {
2941 return true;
2942 }
2943
2944 return false;
2945 }
2946
2947 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
2948 #if INCLUDE_CDS_JAVA_HEAP
2949 // CDS wants a raw continuous memory range to load a bunch of objects itself.
2950 // This is an unusual request, since all requested regions should be regular, not humongous.
2951 //
2952 // CDS would guarantee no objects straddle multiple regions, as long as regions are as large
2953 // as MIN_GC_REGION_ALIGNMENT.
2954 guarantee(ShenandoahHeapRegion::region_size_bytes() >= AOTMappedHeapWriter::MIN_GC_REGION_ALIGNMENT, "Must be");
2955
2956 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_cds(size);
2957 return allocate_memory(req);
2958 #else
2959 assert(false, "Archive heap loader should not be available, should not be here");
2960 return nullptr;
|