1 /* 2 * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP 28 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP 29 30 #include "gc/shared/markBitMap.hpp" 31 #include "gc/shared/softRefPolicy.hpp" 32 #include "gc/shared/collectedHeap.hpp" 33 #include "gc/shenandoah/shenandoahAsserts.hpp" 34 #include "gc/shenandoah/shenandoahAllocRequest.hpp" 35 #include "gc/shenandoah/shenandoahLock.hpp" 36 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" 37 #include "gc/shenandoah/shenandoahEvacTracker.hpp" 38 #include "gc/shenandoah/shenandoahGenerationType.hpp" 39 #include "gc/shenandoah/shenandoahMmuTracker.hpp" 40 #include "gc/shenandoah/shenandoahPadding.hpp" 41 #include "gc/shenandoah/shenandoahSharedVariables.hpp" 42 #include "gc/shenandoah/shenandoahUnload.hpp" 43 #include "gc/shenandoah/shenandoahScanRemembered.hpp" 44 #include "memory/metaspace.hpp" 45 #include "services/memoryManager.hpp" 46 #include "utilities/globalDefinitions.hpp" 47 #include "utilities/stack.hpp" 48 49 class ConcurrentGCTimer; 50 class ObjectIterateScanRootClosure; 51 class PLAB; 52 class ShenandoahCollectorPolicy; 53 class ShenandoahControlThread; 54 class ShenandoahRegulatorThread; 55 class ShenandoahGCSession; 56 class ShenandoahGCStateResetter; 57 class ShenandoahGeneration; 58 class ShenandoahYoungGeneration; 59 class ShenandoahOldGeneration; 60 class ShenandoahHeuristics; 61 class ShenandoahOldHeuristics; 62 class ShenandoahMarkingContext; 63 class ShenandoahPhaseTimings; 64 class ShenandoahHeap; 65 class ShenandoahHeapRegion; 66 class ShenandoahHeapRegionClosure; 67 class ShenandoahCollectionSet; 68 class ShenandoahFreeSet; 69 class ShenandoahConcurrentMark; 70 class ShenandoahFullGC; 71 class ShenandoahMonitoringSupport; 72 class ShenandoahMode; 73 class ShenandoahPacer; 74 class ShenandoahReferenceProcessor; 75 class ShenandoahVerifier; 76 class ShenandoahWorkerThreads; 77 class VMStructs; 78 79 // Used for buffering per-region liveness data. 80 // Needed since ShenandoahHeapRegion uses atomics to update liveness. 81 // The ShenandoahHeap array has max-workers elements, each of which is an array of 82 // uint16_t * max_regions. The choice of uint16_t is not accidental: 83 // there is a tradeoff between static/dynamic footprint that translates 84 // into cache pressure (which is already high during marking), and 85 // too many atomic updates. uint32_t is too large, uint8_t is too small. 86 typedef uint16_t ShenandoahLiveData; 87 #define SHENANDOAH_LIVEDATA_MAX ((ShenandoahLiveData)-1) 88 89 class ShenandoahRegionIterator : public StackObj { 90 private: 91 ShenandoahHeap* _heap; 92 93 shenandoah_padding(0); 94 volatile size_t _index; 95 shenandoah_padding(1); 96 97 // No implicit copying: iterators should be passed by reference to capture the state 98 NONCOPYABLE(ShenandoahRegionIterator); 99 100 public: 101 ShenandoahRegionIterator(); 102 ShenandoahRegionIterator(ShenandoahHeap* heap); 103 104 // Reset iterator to default state 105 void reset(); 106 107 // Returns next region, or null if there are no more regions. 108 // This is multi-thread-safe. 109 inline ShenandoahHeapRegion* next(); 110 111 // This is *not* MT safe. However, in the absence of multithreaded access, it 112 // can be used to determine if there is more work to do. 113 bool has_next() const; 114 }; 115 116 class ShenandoahHeapRegionClosure : public StackObj { 117 public: 118 virtual void heap_region_do(ShenandoahHeapRegion* r) = 0; 119 virtual bool is_thread_safe() { return false; } 120 }; 121 122 template<ShenandoahGenerationType GENERATION> 123 class ShenandoahGenerationRegionClosure : public ShenandoahHeapRegionClosure { 124 public: 125 explicit ShenandoahGenerationRegionClosure(ShenandoahHeapRegionClosure* cl) : _cl(cl) {} 126 void heap_region_do(ShenandoahHeapRegion* r); 127 virtual bool is_thread_safe() { return _cl->is_thread_safe(); } 128 private: 129 ShenandoahHeapRegionClosure* _cl; 130 }; 131 132 typedef ShenandoahLock ShenandoahHeapLock; 133 typedef ShenandoahLocker ShenandoahHeapLocker; 134 typedef Stack<oop, mtGC> ShenandoahScanObjectStack; 135 136 // Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers 137 // to encode forwarding data. See BrooksPointer for details on forwarding data encoding. 138 // See ShenandoahControlThread for GC cycle structure. 139 // 140 class ShenandoahHeap : public CollectedHeap { 141 friend class ShenandoahAsserts; 142 friend class VMStructs; 143 friend class ShenandoahGCSession; 144 friend class ShenandoahGCStateResetter; 145 friend class ShenandoahParallelObjectIterator; 146 friend class ShenandoahSafepoint; 147 // Supported GC 148 friend class ShenandoahConcurrentGC; 149 friend class ShenandoahOldGC; 150 friend class ShenandoahDegenGC; 151 friend class ShenandoahFullGC; 152 friend class ShenandoahUnload; 153 154 // ---------- Locks that guard important data structures in Heap 155 // 156 private: 157 ShenandoahHeapLock _lock; 158 ShenandoahGeneration* _gc_generation; 159 160 // true iff we are concurrently coalescing and filling old-gen HeapRegions 161 bool _prepare_for_old_mark; 162 163 public: 164 ShenandoahHeapLock* lock() { 165 return &_lock; 166 } 167 168 ShenandoahGeneration* active_generation() const { 169 // last or latest generation might be a better name here. 170 return _gc_generation; 171 } 172 173 void set_gc_generation(ShenandoahGeneration* generation) { 174 _gc_generation = generation; 175 } 176 177 ShenandoahOldHeuristics* old_heuristics(); 178 179 bool doing_mixed_evacuations(); 180 bool is_old_bitmap_stable() const; 181 bool is_gc_generation_young() const; 182 183 // ---------- Initialization, termination, identification, printing routines 184 // 185 public: 186 static ShenandoahHeap* heap(); 187 188 const char* name() const override { return "Shenandoah"; } 189 ShenandoahHeap::Name kind() const override { return CollectedHeap::Shenandoah; } 190 191 ShenandoahHeap(ShenandoahCollectorPolicy* policy); 192 jint initialize() override; 193 void post_initialize() override; 194 void initialize_heuristics_generations(); 195 196 void initialize_serviceability() override; 197 198 void print_on(outputStream* st) const override; 199 void print_extended_on(outputStream *st) const override; 200 void print_tracing_info() const override; 201 void print_heap_regions_on(outputStream* st) const; 202 203 void stop() override; 204 205 void prepare_for_verify() override; 206 void verify(VerifyOption vo) override; 207 208 bool verify_generation_usage(bool verify_old, size_t old_regions, size_t old_bytes, size_t old_waste, 209 bool verify_young, size_t young_regions, size_t young_bytes, size_t young_waste); 210 211 // WhiteBox testing support. 212 bool supports_concurrent_gc_breakpoints() const override { 213 return true; 214 } 215 216 // ---------- Heap counters and metrics 217 // 218 private: 219 size_t _initial_size; 220 size_t _minimum_size; 221 size_t _promotion_potential; 222 size_t _promotion_in_place_potential; 223 size_t _pad_for_promote_in_place; // bytes of filler 224 size_t _promotable_humongous_regions; 225 size_t _promotable_humongous_usage; 226 size_t _regular_regions_promoted_in_place; 227 size_t _regular_usage_promoted_in_place; 228 229 volatile size_t _soft_max_size; 230 shenandoah_padding(0); 231 volatile size_t _committed; 232 shenandoah_padding(1); 233 234 void increase_used(const ShenandoahAllocRequest& req); 235 236 public: 237 void increase_used(ShenandoahGeneration* generation, size_t bytes); 238 void decrease_used(ShenandoahGeneration* generation, size_t bytes); 239 void increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes); 240 void decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes); 241 242 void increase_committed(size_t bytes); 243 void decrease_committed(size_t bytes); 244 245 void reset_bytes_allocated_since_gc_start(); 246 247 size_t min_capacity() const; 248 size_t max_capacity() const override; 249 size_t soft_max_capacity() const; 250 size_t initial_capacity() const; 251 size_t capacity() const override; 252 size_t used() const override; 253 size_t committed() const; 254 255 void set_soft_max_capacity(size_t v); 256 257 // ---------- Workers handling 258 // 259 private: 260 uint _max_workers; 261 ShenandoahWorkerThreads* _workers; 262 ShenandoahWorkerThreads* _safepoint_workers; 263 264 public: 265 uint max_workers(); 266 void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN; 267 268 WorkerThreads* workers() const; 269 WorkerThreads* safepoint_workers() override; 270 271 void gc_threads_do(ThreadClosure* tcl) const override; 272 273 // ---------- Heap regions handling machinery 274 // 275 private: 276 MemRegion _heap_region; 277 bool _heap_region_special; 278 size_t _num_regions; 279 ShenandoahHeapRegion** _regions; 280 uint8_t* _affiliations; // Holds array of enum ShenandoahAffiliation, including FREE status in non-generational mode 281 ShenandoahRegionIterator _update_refs_iterator; 282 283 public: 284 285 inline HeapWord* base() const { return _heap_region.start(); } 286 287 inline size_t num_regions() const { return _num_regions; } 288 inline bool is_heap_region_special() { return _heap_region_special; } 289 290 inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const; 291 inline size_t heap_region_index_containing(const void* addr) const; 292 293 inline ShenandoahHeapRegion* const get_region(size_t region_idx) const; 294 295 void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; 296 void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; 297 298 inline ShenandoahMmuTracker* const mmu_tracker() { return &_mmu_tracker; }; 299 300 // ---------- GC state machinery 301 // 302 // GC state describes the important parts of collector state, that may be 303 // used to make barrier selection decisions in the native and generated code. 304 // Multiple bits can be set at once. 305 // 306 // Important invariant: when GC state is zero, the heap is stable, and no barriers 307 // are required. 308 // 309 public: 310 enum GCStateBitPos { 311 // Heap has forwarded objects: needs LRB barriers. 312 HAS_FORWARDED_BITPOS = 0, 313 314 // Young regions are under marking: needs SATB barriers. 315 YOUNG_MARKING_BITPOS = 1, 316 317 // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED) 318 EVACUATION_BITPOS = 2, 319 320 // Heap is under updating: needs no additional barriers. 321 UPDATEREFS_BITPOS = 3, 322 323 // Heap is under weak-reference/roots processing: needs weak-LRB barriers. 324 WEAK_ROOTS_BITPOS = 4, 325 326 // Old regions are under marking, still need SATB barriers. 327 OLD_MARKING_BITPOS = 5 328 }; 329 330 enum GCState { 331 STABLE = 0, 332 HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS, 333 YOUNG_MARKING = 1 << YOUNG_MARKING_BITPOS, 334 EVACUATION = 1 << EVACUATION_BITPOS, 335 UPDATEREFS = 1 << UPDATEREFS_BITPOS, 336 WEAK_ROOTS = 1 << WEAK_ROOTS_BITPOS, 337 OLD_MARKING = 1 << OLD_MARKING_BITPOS 338 }; 339 340 private: 341 ShenandoahSharedBitmap _gc_state; 342 ShenandoahSharedFlag _degenerated_gc_in_progress; 343 ShenandoahSharedFlag _full_gc_in_progress; 344 ShenandoahSharedFlag _full_gc_move_in_progress; 345 ShenandoahSharedFlag _progress_last_gc; 346 ShenandoahSharedFlag _concurrent_strong_root_in_progress; 347 348 // TODO: Revisit the following comment. It may not accurately represent the true behavior when evacuations fail due to 349 // difficulty finding memory to hold evacuated objects. 350 // 351 // Note that the typical total expenditure on evacuation is less than the associated evacuation reserve because we generally 352 // reserve ShenandoahEvacWaste (> 1.0) times the anticipated evacuation need. In the case that there is an excessive amount 353 // of waste, it may be that one thread fails to grab a new GCLAB, this does not necessarily doom the associated evacuation 354 // effort. If this happens, the requesting thread blocks until some other thread manages to evacuate the offending object. 355 // Only after "all" threads fail to evacuate an object do we consider the evacuation effort to have failed. 356 357 // How many full-gc cycles have been completed? 358 volatile size_t _completed_fullgc_cycles; 359 360 size_t _promoted_reserve; // Bytes reserved within old-gen to hold the results of promotion 361 volatile size_t _promoted_expended; // Bytes of old-gen memory expended on promotions 362 363 // Allocation of old GCLABs (aka PLABs) assures that _old_evac_expended + request-size < _old_evac_reserved. If the allocation 364 // is authorized, increment _old_evac_expended by request size. This allocation ignores old_gen->available(). 365 366 size_t _old_evac_reserve; // Bytes reserved within old-gen to hold evacuated objects from old-gen collection set 367 volatile size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations 368 369 size_t _young_evac_reserve; // Bytes reserved within young-gen to hold evacuated objects from young-gen collection set 370 371 size_t _captured_old_usage; // What was old usage (bytes) when last captured? 372 373 size_t _previous_promotion; // Bytes promoted during previous evacuation 374 375 bool _upgraded_to_full; 376 377 // At the end of final mark, but before we begin evacuating, heuristics calculate how much memory is required to 378 // hold the results of evacuating to young-gen and to old-gen. These quantitites, stored in _promoted_reserve, 379 // _old_evac_reserve, and _young_evac_reserve, are consulted prior to rebuilding the free set (ShenandoahFreeSet) 380 // in preparation for evacuation. When the free set is rebuilt, we make sure to reserve sufficient memory in the 381 // collector and old_collector sets to hold if _has_evacuation_reserve_quantities is true. The other time we 382 // rebuild the freeset is at the end of GC, as we prepare to idle GC until the next trigger. In this case, 383 // _has_evacuation_reserve_quantities is false because we don't yet know how much memory will need to be evacuated 384 // in the next GC cycle. When _has_evacuation_reserve_quantities is false, the free set rebuild operation reserves 385 // for the collector and old_collector sets based on alternative mechanisms, such as ShenandoahEvacReserve, 386 // ShenandoahOldEvacReserve, and ShenandoahOldCompactionReserve. In a future planned enhancement, the reserve 387 // for old_collector set when not _has_evacuation_reserve_quantities is based in part on anticipated promotion as 388 // determined by analysis of live data found during the previous GC pass which is one less than the current tenure age. 389 bool _has_evacuation_reserve_quantities; 390 391 void set_gc_state_all_threads(char state); 392 void set_gc_state_mask(uint mask, bool value); 393 394 public: 395 char gc_state() const; 396 static address gc_state_addr(); 397 398 void set_evacuation_reserve_quantities(bool is_valid); 399 void set_concurrent_young_mark_in_progress(bool in_progress); 400 void set_concurrent_old_mark_in_progress(bool in_progress); 401 void set_evacuation_in_progress(bool in_progress); 402 void set_update_refs_in_progress(bool in_progress); 403 void set_degenerated_gc_in_progress(bool in_progress); 404 void set_full_gc_in_progress(bool in_progress); 405 void set_full_gc_move_in_progress(bool in_progress); 406 void set_has_forwarded_objects(bool cond); 407 void set_concurrent_strong_root_in_progress(bool cond); 408 void set_concurrent_weak_root_in_progress(bool cond); 409 void set_prepare_for_old_mark_in_progress(bool cond); 410 void set_aging_cycle(bool cond); 411 412 413 inline bool is_stable() const; 414 inline bool is_idle() const; 415 inline bool has_evacuation_reserve_quantities() const; 416 inline bool is_concurrent_mark_in_progress() const; 417 inline bool is_concurrent_young_mark_in_progress() const; 418 inline bool is_concurrent_old_mark_in_progress() const; 419 inline bool is_update_refs_in_progress() const; 420 inline bool is_evacuation_in_progress() const; 421 inline bool is_degenerated_gc_in_progress() const; 422 inline bool is_full_gc_in_progress() const; 423 inline bool is_full_gc_move_in_progress() const; 424 inline bool has_forwarded_objects() const; 425 inline bool is_gc_in_progress_mask(uint mask) const; 426 inline bool is_stw_gc_in_progress() const; 427 inline bool is_concurrent_strong_root_in_progress() const; 428 inline bool is_concurrent_weak_root_in_progress() const; 429 inline bool is_prepare_for_old_mark_in_progress() const; 430 inline bool is_aging_cycle() const; 431 inline bool upgraded_to_full() { return _upgraded_to_full; } 432 inline void start_conc_gc() { _upgraded_to_full = false; } 433 inline void record_upgrade_to_full() { _upgraded_to_full = true; } 434 435 inline size_t capture_old_usage(size_t usage); 436 inline void set_previous_promotion(size_t promoted_bytes); 437 inline size_t get_previous_promotion() const; 438 439 inline void clear_promotion_potential() { _promotion_potential = 0; }; 440 inline void set_promotion_potential(size_t val) { _promotion_potential = val; }; 441 inline size_t get_promotion_potential() { return _promotion_potential; }; 442 443 inline void clear_promotion_in_place_potential() { _promotion_in_place_potential = 0; }; 444 inline void set_promotion_in_place_potential(size_t val) { _promotion_in_place_potential = val; }; 445 inline size_t get_promotion_in_place_potential() { return _promotion_in_place_potential; }; 446 447 inline void set_pad_for_promote_in_place(size_t pad) { _pad_for_promote_in_place = pad; } 448 inline size_t get_pad_for_promote_in_place() { return _pad_for_promote_in_place; } 449 450 inline void reserve_promotable_humongous_regions(size_t region_count) { _promotable_humongous_regions = region_count; } 451 inline void reserve_promotable_humongous_usage(size_t bytes) { _promotable_humongous_usage = bytes; } 452 inline void reserve_promotable_regular_regions(size_t region_count) { _regular_regions_promoted_in_place = region_count; } 453 inline void reserve_promotable_regular_usage(size_t used_bytes) { _regular_usage_promoted_in_place = used_bytes; } 454 455 inline size_t get_promotable_humongous_regions() { return _promotable_humongous_regions; } 456 inline size_t get_promotable_humongous_usage() { return _promotable_humongous_usage; } 457 inline size_t get_regular_regions_promoted_in_place() { return _regular_regions_promoted_in_place; } 458 inline size_t get_regular_usage_promoted_in_place() { return _regular_usage_promoted_in_place; } 459 460 // Returns previous value 461 inline size_t set_promoted_reserve(size_t new_val); 462 inline size_t get_promoted_reserve() const; 463 inline void augment_promo_reserve(size_t increment); 464 465 inline void reset_promoted_expended(); 466 inline size_t expend_promoted(size_t increment); 467 inline size_t unexpend_promoted(size_t decrement); 468 inline size_t get_promoted_expended(); 469 470 // Returns previous value 471 inline size_t set_old_evac_reserve(size_t new_val); 472 inline size_t get_old_evac_reserve() const; 473 inline void augment_old_evac_reserve(size_t increment); 474 475 inline void reset_old_evac_expended(); 476 inline size_t expend_old_evac(size_t increment); 477 inline size_t get_old_evac_expended(); 478 479 // Returns previous value 480 inline size_t set_young_evac_reserve(size_t new_val); 481 inline size_t get_young_evac_reserve() const; 482 483 private: 484 void manage_satb_barrier(bool active); 485 486 enum CancelState { 487 // Normal state. GC has not been cancelled and is open for cancellation. 488 // Worker threads can suspend for safepoint. 489 CANCELLABLE, 490 491 // GC has been cancelled. Worker threads can not suspend for 492 // safepoint but must finish their work as soon as possible. 493 CANCELLED 494 }; 495 496 double _cancel_requested_time; 497 ShenandoahSharedEnumFlag<CancelState> _cancelled_gc; 498 499 // Returns true if cancel request was successfully communicated. 500 // Returns false if some other thread already communicated cancel 501 // request. A true return value does not mean GC has been 502 // cancelled, only that the process of cancelling GC has begun. 503 bool try_cancel_gc(); 504 505 public: 506 inline bool cancelled_gc() const; 507 inline bool check_cancelled_gc_and_yield(bool sts_active = true); 508 509 inline void clear_cancelled_gc(bool clear_oom_handler = true); 510 511 void cancel_concurrent_mark(); 512 void cancel_gc(GCCause::Cause cause); 513 514 public: 515 // Elastic heap support 516 void entry_uncommit(double shrink_before, size_t shrink_until); 517 void op_uncommit(double shrink_before, size_t shrink_until); 518 519 private: 520 // GC support 521 // Evacuation 522 void evacuate_collection_set(bool concurrent); 523 // Concurrent root processing 524 void prepare_concurrent_roots(); 525 void finish_concurrent_roots(); 526 // Concurrent class unloading support 527 void do_class_unloading(); 528 // Reference updating 529 void prepare_update_heap_references(bool concurrent); 530 void update_heap_references(bool concurrent); 531 // Final update region states 532 void update_heap_region_states(bool concurrent); 533 534 void rendezvous_threads(); 535 void recycle_trash(); 536 public: 537 void rebuild_free_set(bool concurrent); 538 void notify_gc_progress() { _progress_last_gc.set(); } 539 void notify_gc_no_progress() { _progress_last_gc.unset(); } 540 541 // 542 // Mark support 543 private: 544 ShenandoahYoungGeneration* _young_generation; 545 ShenandoahGeneration* _global_generation; 546 ShenandoahOldGeneration* _old_generation; 547 548 ShenandoahControlThread* _control_thread; 549 ShenandoahRegulatorThread* _regulator_thread; 550 ShenandoahCollectorPolicy* _shenandoah_policy; 551 ShenandoahMode* _gc_mode; 552 ShenandoahFreeSet* _free_set; 553 ShenandoahPacer* _pacer; 554 ShenandoahVerifier* _verifier; 555 556 ShenandoahPhaseTimings* _phase_timings; 557 ShenandoahEvacuationTracker* _evac_tracker; 558 ShenandoahMmuTracker _mmu_tracker; 559 ShenandoahGenerationSizer _generation_sizer; 560 561 ShenandoahRegulatorThread* regulator_thread() { return _regulator_thread; } 562 563 public: 564 ShenandoahControlThread* control_thread() { return _control_thread; } 565 ShenandoahYoungGeneration* young_generation() const { return _young_generation; } 566 ShenandoahGeneration* global_generation() const { return _global_generation; } 567 ShenandoahOldGeneration* old_generation() const { return _old_generation; } 568 ShenandoahGeneration* generation_for(ShenandoahAffiliation affiliation) const; 569 const ShenandoahGenerationSizer* generation_sizer() const { return &_generation_sizer; } 570 571 size_t max_size_for(ShenandoahGeneration* generation) const; 572 size_t min_size_for(ShenandoahGeneration* generation) const; 573 574 ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } 575 ShenandoahMode* mode() const { return _gc_mode; } 576 ShenandoahFreeSet* free_set() const { return _free_set; } 577 ShenandoahPacer* pacer() const { return _pacer; } 578 579 ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } 580 ShenandoahEvacuationTracker* evac_tracker() const { return _evac_tracker; } 581 582 void on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation); 583 void on_cycle_end(ShenandoahGeneration* generation); 584 585 ShenandoahVerifier* verifier(); 586 587 // ---------- VM subsystem bindings 588 // 589 private: 590 ShenandoahMonitoringSupport* _monitoring_support; 591 MemoryPool* _memory_pool; 592 MemoryPool* _young_gen_memory_pool; 593 MemoryPool* _old_gen_memory_pool; 594 595 GCMemoryManager _stw_memory_manager; 596 GCMemoryManager _cycle_memory_manager; 597 ConcurrentGCTimer* _gc_timer; 598 SoftRefPolicy _soft_ref_policy; 599 600 // For exporting to SA 601 int _log_min_obj_alignment_in_bytes; 602 public: 603 ShenandoahMonitoringSupport* monitoring_support() const { return _monitoring_support; } 604 GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; } 605 GCMemoryManager* stw_memory_manager() { return &_stw_memory_manager; } 606 SoftRefPolicy* soft_ref_policy() override { return &_soft_ref_policy; } 607 608 GrowableArray<GCMemoryManager*> memory_managers() override; 609 GrowableArray<MemoryPool*> memory_pools() override; 610 MemoryUsage memory_usage() override; 611 GCTracer* tracer(); 612 ConcurrentGCTimer* gc_timer() const; 613 614 // ---------- Class Unloading 615 // 616 private: 617 ShenandoahSharedFlag _is_aging_cycle; 618 ShenandoahSharedFlag _unload_classes; 619 ShenandoahUnload _unloader; 620 621 public: 622 void set_unload_classes(bool uc); 623 bool unload_classes() const; 624 625 // Perform STW class unloading and weak root cleaning 626 void parallel_cleaning(bool full_gc); 627 628 private: 629 void stw_unload_classes(bool full_gc); 630 void stw_process_weak_roots(bool full_gc); 631 void stw_weak_refs(bool full_gc); 632 633 inline void assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation, 634 ShenandoahAffiliation new_affiliation); 635 636 // Heap iteration support 637 void scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops); 638 bool prepare_aux_bitmap_for_iteration(); 639 void reclaim_aux_bitmap_for_iteration(); 640 641 // ---------- Generic interface hooks 642 // Minor things that super-interface expects us to implement to play nice with 643 // the rest of runtime. Some of the things here are not required to be implemented, 644 // and can be stubbed out. 645 // 646 public: 647 bool is_maximal_no_gc() const override shenandoah_not_implemented_return(false); 648 649 inline bool is_in(const void* p) const override; 650 651 inline bool is_in_active_generation(oop obj) const; 652 inline bool is_in_young(const void* p) const; 653 inline bool is_in_old(const void* p) const; 654 inline bool is_old(oop pobj) const; 655 656 inline ShenandoahAffiliation region_affiliation(const ShenandoahHeapRegion* r); 657 inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation); 658 659 inline ShenandoahAffiliation region_affiliation(size_t index); 660 inline void set_affiliation(size_t index, ShenandoahAffiliation new_affiliation); 661 662 bool requires_barriers(stackChunkOop obj) const override; 663 664 MemRegion reserved_region() const { return _reserved; } 665 bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); } 666 667 void collect(GCCause::Cause cause) override; 668 void do_full_collection(bool clear_all_soft_refs) override; 669 670 // Used for parsing heap during error printing 671 HeapWord* block_start(const void* addr) const; 672 bool block_is_obj(const HeapWord* addr) const; 673 bool print_location(outputStream* st, void* addr) const override; 674 675 // Used for native heap walkers: heap dumpers, mostly 676 void object_iterate(ObjectClosure* cl) override; 677 // Parallel heap iteration support 678 ParallelObjectIteratorImpl* parallel_object_iterator(uint workers) override; 679 680 // Keep alive an object that was loaded with AS_NO_KEEPALIVE. 681 void keep_alive(oop obj) override; 682 683 // ---------- Safepoint interface hooks 684 // 685 public: 686 void safepoint_synchronize_begin() override; 687 void safepoint_synchronize_end() override; 688 689 // ---------- Code roots handling hooks 690 // 691 public: 692 void register_nmethod(nmethod* nm) override; 693 void unregister_nmethod(nmethod* nm) override; 694 void verify_nmethod(nmethod* nm) override {} 695 696 // ---------- Pinning hooks 697 // 698 public: 699 // Shenandoah supports per-object (per-region) pinning 700 void pin_object(JavaThread* thread, oop obj) override; 701 void unpin_object(JavaThread* thread, oop obj) override; 702 703 void sync_pinned_region_status(); 704 void assert_pinned_region_status() NOT_DEBUG_RETURN; 705 706 // ---------- Concurrent Stack Processing support 707 // 708 public: 709 bool uses_stack_watermark_barrier() const override { return true; } 710 711 // ---------- Allocation support 712 // 713 private: 714 // How many bytes to transfer between old and young after we have finished recycling collection set regions? 715 size_t _old_regions_surplus; 716 size_t _old_regions_deficit; 717 718 HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region, bool is_promotion); 719 720 inline HeapWord* allocate_from_gclab(Thread* thread, size_t size); 721 HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); 722 HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size); 723 724 inline HeapWord* allocate_from_plab(Thread* thread, size_t size, bool is_promotion); 725 HeapWord* allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion); 726 HeapWord* allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size); 727 728 public: 729 HeapWord* allocate_memory(ShenandoahAllocRequest& request, bool is_promotion); 730 HeapWord* mem_allocate(size_t size, bool* what) override; 731 MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, 732 size_t size, 733 Metaspace::MetadataType mdtype) override; 734 735 void notify_mutator_alloc_words(size_t words, size_t waste); 736 737 HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override; 738 size_t tlab_capacity(Thread *thr) const override; 739 size_t unsafe_max_tlab_alloc(Thread *thread) const override; 740 size_t max_tlab_size() const override; 741 size_t tlab_used(Thread* ignored) const override; 742 743 void ensure_parsability(bool retire_labs) override; 744 745 void labs_make_parsable(); 746 void tlabs_retire(bool resize); 747 void gclabs_retire(bool resize); 748 749 void set_young_lab_region_flags(); 750 751 inline void set_old_region_surplus(size_t surplus) { _old_regions_surplus = surplus; }; 752 inline void set_old_region_deficit(size_t deficit) { _old_regions_deficit = deficit; }; 753 754 inline size_t get_old_region_surplus() { return _old_regions_surplus; }; 755 inline size_t get_old_region_deficit() { return _old_regions_deficit; }; 756 757 // ---------- Marking support 758 // 759 private: 760 ShenandoahMarkingContext* _marking_context; 761 MemRegion _bitmap_region; 762 MemRegion _aux_bitmap_region; 763 MarkBitMap _verification_bit_map; 764 MarkBitMap _aux_bit_map; 765 766 size_t _bitmap_size; 767 size_t _bitmap_regions_per_slice; 768 size_t _bitmap_bytes_per_slice; 769 770 size_t _pretouch_heap_page_size; 771 size_t _pretouch_bitmap_page_size; 772 773 bool _bitmap_region_special; 774 bool _aux_bitmap_region_special; 775 776 ShenandoahLiveData** _liveness_cache; 777 778 public: 779 inline ShenandoahMarkingContext* complete_marking_context() const; 780 inline ShenandoahMarkingContext* marking_context() const; 781 782 template<class T> 783 inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl); 784 785 template<class T> 786 inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit); 787 788 template<class T> 789 inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit); 790 791 // SATB barriers hooks 792 inline bool requires_marking(const void* entry) const; 793 794 // Support for bitmap uncommits 795 bool commit_bitmap_slice(ShenandoahHeapRegion *r); 796 bool uncommit_bitmap_slice(ShenandoahHeapRegion *r); 797 bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false); 798 799 // Liveness caching support 800 ShenandoahLiveData* get_liveness_cache(uint worker_id); 801 void flush_liveness_cache(uint worker_id); 802 803 size_t pretouch_heap_page_size() { return _pretouch_heap_page_size; } 804 805 // ---------- Evacuation support 806 // 807 private: 808 ShenandoahCollectionSet* _collection_set; 809 ShenandoahEvacOOMHandler _oom_evac_handler; 810 ShenandoahSharedFlag _old_gen_oom_evac; 811 812 inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen); 813 void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion); 814 void handle_old_evacuation_failure(); 815 816 public: 817 void handle_promotion_failure(); 818 void report_promotion_failure(Thread* thread, size_t size); 819 820 static address in_cset_fast_test_addr(); 821 822 ShenandoahCollectionSet* collection_set() const { return _collection_set; } 823 824 // Checks if object is in the collection set. 825 inline bool in_collection_set(oop obj) const; 826 827 // Checks if location is in the collection set. Can be interior pointer, not the oop itself. 828 inline bool in_collection_set_loc(void* loc) const; 829 830 // Evacuates or promotes object src. Returns the evacuated object, either evacuated 831 // by this thread, or by some other thread. 832 inline oop evacuate_object(oop src, Thread* thread); 833 834 // Call before/after evacuation. 835 inline void enter_evacuation(Thread* t); 836 inline void leave_evacuation(Thread* t); 837 838 inline bool clear_old_evacuation_failure(); 839 840 // ---------- Generational support 841 // 842 private: 843 RememberedScanner* _card_scan; 844 845 public: 846 inline RememberedScanner* card_scan() { return _card_scan; } 847 void clear_cards_for(ShenandoahHeapRegion* region); 848 void dirty_cards(HeapWord* start, HeapWord* end); 849 void clear_cards(HeapWord* start, HeapWord* end); 850 void mark_card_as_dirty(void* location); 851 void retire_plab(PLAB* plab); 852 void retire_plab(PLAB* plab, Thread* thread); 853 void cancel_old_gc(); 854 bool is_old_gc_active(); 855 void coalesce_and_fill_old_regions(); 856 void adjust_generation_sizes_for_next_cycle(size_t old_xfer_limit, size_t young_cset_regions, size_t old_cset_regions); 857 858 // ---------- Helper functions 859 // 860 public: 861 template <class T> 862 inline void conc_update_with_forwarded(T* p); 863 864 template <class T> 865 inline void update_with_forwarded(T* p); 866 867 static inline void atomic_update_oop(oop update, oop* addr, oop compare); 868 static inline void atomic_update_oop(oop update, narrowOop* addr, oop compare); 869 static inline void atomic_update_oop(oop update, narrowOop* addr, narrowOop compare); 870 871 static inline bool atomic_update_oop_check(oop update, oop* addr, oop compare); 872 static inline bool atomic_update_oop_check(oop update, narrowOop* addr, oop compare); 873 static inline bool atomic_update_oop_check(oop update, narrowOop* addr, narrowOop compare); 874 875 static inline void atomic_clear_oop( oop* addr, oop compare); 876 static inline void atomic_clear_oop(narrowOop* addr, oop compare); 877 static inline void atomic_clear_oop(narrowOop* addr, narrowOop compare); 878 879 size_t trash_humongous_region_at(ShenandoahHeapRegion *r); 880 881 static inline void increase_object_age(oop obj, uint additional_age); 882 static inline uint get_object_age(oop obj); 883 884 void transfer_old_pointers_from_satb(); 885 886 void log_heap_status(const char *msg) const; 887 888 private: 889 void trash_cset_regions(); 890 891 // ---------- Testing helpers functions 892 // 893 private: 894 ShenandoahSharedFlag _inject_alloc_failure; 895 896 void try_inject_alloc_failure(); 897 bool should_inject_alloc_failure(); 898 }; 899 900 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP