15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "memory/allocation.hpp"
27 #include "memory/universe.hpp"
28
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shared/gcTimer.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/locationPrinter.inline.hpp"
33 #include "gc/shared/memAllocator.hpp"
34 #include "gc/shared/plab.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36
37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
41 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
42 #include "gc/shenandoah/shenandoahControlThread.hpp"
43 #include "gc/shenandoah/shenandoahFreeSet.hpp"
44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
45 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
48 #include "gc/shenandoah/shenandoahInitLogger.hpp"
49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
50 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
51 #include "gc/shenandoah/shenandoahMetrics.hpp"
52 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
53 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
54 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
174 _soft_max_size = _num_regions * reg_size_bytes;
175
176 _committed = _initial_size;
177
178 size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
179 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
180 size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
181
182 //
183 // Reserve and commit memory for heap
184 //
185
186 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
187 initialize_reserved_region(heap_rs);
188 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
189 _heap_region_special = heap_rs.special();
190
191 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
192 "Misaligned heap: " PTR_FORMAT, p2i(base()));
193
194 #if SHENANDOAH_OPTIMIZED_MARKTASK
195 // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
196 // Fail if we ever attempt to address more than we can.
197 if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
198 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
199 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
200 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
201 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
202 vm_exit_during_initialization("Fatal Error", buf);
203 }
204 #endif
205
206 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
207 if (!_heap_region_special) {
208 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
209 "Cannot commit heap memory");
210 }
211
212 //
213 // Reserve and commit memory for bitmap(s)
930 // Expand and retry allocation
931 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
932 if (result != NULL) {
933 return result;
934 }
935
936 // Out of memory
937 return NULL;
938 }
939
940 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
941 private:
942 ShenandoahHeap* const _heap;
943 Thread* const _thread;
944 public:
945 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
946 _heap(heap), _thread(Thread::current()) {}
947
948 void do_object(oop p) {
949 shenandoah_assert_marked(NULL, p);
950 if (!p->is_forwarded()) {
951 _heap->evacuate_object(p, _thread);
952 }
953 }
954 };
955
956 class ShenandoahEvacuationTask : public WorkerTask {
957 private:
958 ShenandoahHeap* const _sh;
959 ShenandoahCollectionSet* const _cs;
960 bool _concurrent;
961 public:
962 ShenandoahEvacuationTask(ShenandoahHeap* sh,
963 ShenandoahCollectionSet* cs,
964 bool concurrent) :
965 WorkerTask("Shenandoah Evacuation"),
966 _sh(sh),
967 _cs(cs),
968 _concurrent(concurrent)
969 {}
970
1218 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1219 return _free_set->capacity();
1220 }
1221
1222 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1223 private:
1224 MarkBitMap* _bitmap;
1225 ShenandoahScanObjectStack* _oop_stack;
1226 ShenandoahHeap* const _heap;
1227 ShenandoahMarkingContext* const _marking_context;
1228
1229 template <class T>
1230 void do_oop_work(T* p) {
1231 T o = RawAccess<>::oop_load(p);
1232 if (!CompressedOops::is_null(o)) {
1233 oop obj = CompressedOops::decode_not_null(o);
1234 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1235 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1236 return;
1237 }
1238 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1239
1240 assert(oopDesc::is_oop(obj), "must be a valid oop");
1241 if (!_bitmap->is_marked(obj)) {
1242 _bitmap->mark(obj);
1243 _oop_stack->push(obj);
1244 }
1245 }
1246 }
1247 public:
1248 ObjectIterateScanRootClosure(MarkBitMap* bitmap, ShenandoahScanObjectStack* oop_stack) :
1249 _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1250 _marking_context(_heap->marking_context()) {}
1251 void do_oop(oop* p) { do_oop_work(p); }
1252 void do_oop(narrowOop* p) { do_oop_work(p); }
1253 };
1254
1255 /*
1256 * This is public API, used in preparation of object_iterate().
1257 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1258 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1274 * wiped the bitmap in preparation for next marking).
1275 *
1276 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1277 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1278 * is allowed to report dead objects, but is not required to do so.
1279 */
1280 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1281 // Reset bitmap
1282 if (!prepare_aux_bitmap_for_iteration())
1283 return;
1284
1285 ShenandoahScanObjectStack oop_stack;
1286 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1287 // Seed the stack with root scan
1288 scan_roots_for_iteration(&oop_stack, &oops);
1289
1290 // Work through the oop stack to traverse heap
1291 while (! oop_stack.is_empty()) {
1292 oop obj = oop_stack.pop();
1293 assert(oopDesc::is_oop(obj), "must be a valid oop");
1294 cl->do_object(obj);
1295 obj->oop_iterate(&oops);
1296 }
1297
1298 assert(oop_stack.is_empty(), "should be empty");
1299 // Reclaim bitmap
1300 reclaim_aux_bitmap_for_iteration();
1301 }
1302
1303 bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
1304 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1305
1306 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1307 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1308 return false;
1309 }
1310 // Reset bitmap
1311 _aux_bit_map.clear();
1312 return true;
1313 }
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "memory/allocation.hpp"
27 #include "memory/universe.hpp"
28
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shared/gcTimer.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/locationPrinter.inline.hpp"
33 #include "gc/shared/memAllocator.hpp"
34 #include "gc/shared/plab.hpp"
35 #include "gc/shared/slidingForwarding.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37
38 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
39 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
41 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
42 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
43 #include "gc/shenandoah/shenandoahControlThread.hpp"
44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
49 #include "gc/shenandoah/shenandoahInitLogger.hpp"
50 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
51 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
52 #include "gc/shenandoah/shenandoahMetrics.hpp"
53 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
54 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
55 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
175 _soft_max_size = _num_regions * reg_size_bytes;
176
177 _committed = _initial_size;
178
179 size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
180 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
181 size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
182
183 //
184 // Reserve and commit memory for heap
185 //
186
187 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
188 initialize_reserved_region(heap_rs);
189 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
190 _heap_region_special = heap_rs.special();
191
192 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
193 "Misaligned heap: " PTR_FORMAT, p2i(base()));
194
195 _forwarding = new SlidingForwarding(_heap_region, ShenandoahHeapRegion::region_size_words_shift());
196
197 #if SHENANDOAH_OPTIMIZED_MARKTASK
198 // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
199 // Fail if we ever attempt to address more than we can.
200 if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
201 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
202 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
203 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
204 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
205 vm_exit_during_initialization("Fatal Error", buf);
206 }
207 #endif
208
209 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
210 if (!_heap_region_special) {
211 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
212 "Cannot commit heap memory");
213 }
214
215 //
216 // Reserve and commit memory for bitmap(s)
933 // Expand and retry allocation
934 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
935 if (result != NULL) {
936 return result;
937 }
938
939 // Out of memory
940 return NULL;
941 }
942
943 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
944 private:
945 ShenandoahHeap* const _heap;
946 Thread* const _thread;
947 public:
948 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
949 _heap(heap), _thread(Thread::current()) {}
950
951 void do_object(oop p) {
952 shenandoah_assert_marked(NULL, p);
953 if (!ShenandoahForwarding::is_forwarded(p)) {
954 _heap->evacuate_object(p, _thread);
955 }
956 }
957 };
958
959 class ShenandoahEvacuationTask : public WorkerTask {
960 private:
961 ShenandoahHeap* const _sh;
962 ShenandoahCollectionSet* const _cs;
963 bool _concurrent;
964 public:
965 ShenandoahEvacuationTask(ShenandoahHeap* sh,
966 ShenandoahCollectionSet* cs,
967 bool concurrent) :
968 WorkerTask("Shenandoah Evacuation"),
969 _sh(sh),
970 _cs(cs),
971 _concurrent(concurrent)
972 {}
973
1221 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1222 return _free_set->capacity();
1223 }
1224
1225 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1226 private:
1227 MarkBitMap* _bitmap;
1228 ShenandoahScanObjectStack* _oop_stack;
1229 ShenandoahHeap* const _heap;
1230 ShenandoahMarkingContext* const _marking_context;
1231
1232 template <class T>
1233 void do_oop_work(T* p) {
1234 T o = RawAccess<>::oop_load(p);
1235 if (!CompressedOops::is_null(o)) {
1236 oop obj = CompressedOops::decode_not_null(o);
1237 if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1238 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1239 return;
1240 }
1241
1242 // We must not expose from-space oops to the rest of runtime, or else it
1243 // will call klass() on it, which might fail because of unexpected header.
1244 obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1245
1246 assert(oopDesc::is_oop(obj), "must be a valid oop");
1247 if (!_bitmap->is_marked(obj)) {
1248 _bitmap->mark(obj);
1249 _oop_stack->push(obj);
1250 }
1251 }
1252 }
1253 public:
1254 ObjectIterateScanRootClosure(MarkBitMap* bitmap, ShenandoahScanObjectStack* oop_stack) :
1255 _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1256 _marking_context(_heap->marking_context()) {}
1257 void do_oop(oop* p) { do_oop_work(p); }
1258 void do_oop(narrowOop* p) { do_oop_work(p); }
1259 };
1260
1261 /*
1262 * This is public API, used in preparation of object_iterate().
1263 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1264 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1280 * wiped the bitmap in preparation for next marking).
1281 *
1282 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1283 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1284 * is allowed to report dead objects, but is not required to do so.
1285 */
1286 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1287 // Reset bitmap
1288 if (!prepare_aux_bitmap_for_iteration())
1289 return;
1290
1291 ShenandoahScanObjectStack oop_stack;
1292 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1293 // Seed the stack with root scan
1294 scan_roots_for_iteration(&oop_stack, &oops);
1295
1296 // Work through the oop stack to traverse heap
1297 while (! oop_stack.is_empty()) {
1298 oop obj = oop_stack.pop();
1299 assert(oopDesc::is_oop(obj), "must be a valid oop");
1300 shenandoah_assert_not_in_cset_except(NULL, obj, cancelled_gc());
1301 cl->do_object(obj);
1302 obj->oop_iterate(&oops);
1303 }
1304
1305 assert(oop_stack.is_empty(), "should be empty");
1306 // Reclaim bitmap
1307 reclaim_aux_bitmap_for_iteration();
1308 }
1309
1310 bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
1311 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1312
1313 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1314 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1315 return false;
1316 }
1317 // Reset bitmap
1318 _aux_bit_map.clear();
1319 return true;
1320 }
|