1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/softRefPolicy.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shenandoah/markBitMap.hpp"
  30 #include "gc/shenandoah/shenandoahAsserts.hpp"
  31 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  32 #include "gc/shenandoah/shenandoahLock.hpp"
  33 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  34 #include "gc/shenandoah/shenandoahPadding.hpp"
  35 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  36 #include "services/memoryManager.hpp"
  37 
  38 class ConcurrentGCTimer;
  39 class ReferenceProcessor;
  40 class ShenandoahCollectorPolicy;
  41 class ShenandoahControlThread;
  42 class ShenandoahGCSession;
  43 class ShenandoahGCStateResetter;
  44 class ShenandoahHeuristics;
  45 class ShenandoahMarkingContext;
  46 class ShenandoahMarkCompact;
  47 class ShenandoahMode;
  48 class ShenandoahPhaseTimings;
  49 class ShenandoahHeap;
  50 class ShenandoahHeapRegion;
  51 class ShenandoahHeapRegionClosure;
  52 class ShenandoahCollectionSet;
  53 class ShenandoahFreeSet;
  54 class ShenandoahConcurrentMark;
  55 class ShenandoahMarkCompact;
  56 class ShenandoahMonitoringSupport;
  57 class ShenandoahPacer;
  58 class ShenandoahVerifier;
  59 class ShenandoahWorkGang;
  60 class VMStructs;
  61 
  62 // Used for buffering per-region liveness data.
  63 // Needed since ShenandoahHeapRegion uses atomics to update liveness.
  64 // The ShenandoahHeap array has max-workers elements, each of which is an array of
  65 // uint16_t * max_regions. The choice of uint16_t is not accidental:
  66 // there is a tradeoff between static/dynamic footprint that translates
  67 // into cache pressure (which is already high during marking), and
  68 // too many atomic updates. uint32_t is too large, uint8_t is too small.
  69 typedef uint16_t ShenandoahLiveData;
  70 #define SHENANDOAH_LIVEDATA_MAX ((ShenandoahLiveData)-1)
  71 
  72 class ShenandoahRegionIterator : public StackObj {
  73 private:
  74   ShenandoahHeap* _heap;
  75 
  76   shenandoah_padding(0);
  77   volatile size_t _index;
  78   shenandoah_padding(1);
  79 
  80   // No implicit copying: iterators should be passed by reference to capture the state
  81   ShenandoahRegionIterator(const ShenandoahRegionIterator& that);
  82   ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);
  83 
  84 public:
  85   ShenandoahRegionIterator();
  86   ShenandoahRegionIterator(ShenandoahHeap* heap);
  87 
  88   // Reset iterator to default state
  89   void reset();
  90 
  91   // Returns next region, or NULL if there are no more regions.
  92   // This is multi-thread-safe.
  93   inline ShenandoahHeapRegion* next();
  94 
  95   // This is *not* MT safe. However, in the absence of multithreaded access, it
  96   // can be used to determine if there is more work to do.
  97   bool has_next() const;
  98 };
  99 
 100 class ShenandoahHeapRegionClosure : public StackObj {
 101 public:
 102   virtual void heap_region_do(ShenandoahHeapRegion* r) = 0;
 103   virtual bool is_thread_safe() { return false; }
 104 };
 105 
 106 #ifdef ASSERT
 107 class ShenandoahAssertToSpaceClosure : public OopClosure {
 108 private:
 109   template <class T>
 110   void do_oop_work(T* p);
 111 public:
 112   void do_oop(narrowOop* p);
 113   void do_oop(oop* p);
 114 };
 115 #endif
 116 
 117 typedef ShenandoahLock    ShenandoahHeapLock;
 118 typedef ShenandoahLocker  ShenandoahHeapLocker;
 119 
 120 // Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers
 121 // to encode forwarding data. See BrooksPointer for details on forwarding data encoding.
 122 // See ShenandoahControlThread for GC cycle structure.
 123 //
 124 class ShenandoahHeap : public CollectedHeap {
 125   friend class ShenandoahAsserts;
 126   friend class VMStructs;
 127   friend class ShenandoahGCSession;
 128   friend class ShenandoahGCStateResetter;
 129 
 130 // ---------- Locks that guard important data structures in Heap
 131 //
 132 private:
 133   ShenandoahHeapLock _lock;
 134 
 135 public:
 136   ShenandoahHeapLock* lock() {
 137     return &_lock;
 138   }
 139 
 140 // ---------- Initialization, termination, identification, printing routines
 141 //
 142 private:
 143   static ShenandoahHeap* _heap;
 144 
 145 public:
 146   static ShenandoahHeap* heap();
 147 
 148   const char* name()          const { return "Shenandoah"; }
 149   ShenandoahHeap::Name kind() const { return CollectedHeap::Shenandoah; }
 150 
 151   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 152   jint initialize();
 153   void post_initialize();
 154   void initialize_heuristics();
 155 
 156   void initialize_serviceability();
 157 
 158   void print_on(outputStream* st)              const;
 159   void print_extended_on(outputStream *st)     const;
 160   void print_tracing_info()                    const;
 161   void print_gc_threads_on(outputStream* st)   const;
 162   void print_heap_regions_on(outputStream* st) const;
 163 
 164   void stop();
 165 
 166   void prepare_for_verify();
 167   void verify(VerifyOption vo);
 168 
 169 // ---------- Heap counters and metrics
 170 //
 171 private:
 172            size_t _initial_size;
 173            size_t _minimum_size;
 174   shenandoah_padding(0);
 175   volatile size_t _used;
 176   volatile size_t _committed;
 177   volatile size_t _bytes_allocated_since_gc_start;
 178   shenandoah_padding(1);
 179 
 180 public:
 181   void increase_used(size_t bytes);
 182   void decrease_used(size_t bytes);
 183   void set_used(size_t bytes);
 184 
 185   void increase_committed(size_t bytes);
 186   void decrease_committed(size_t bytes);
 187   void increase_allocated(size_t bytes);
 188 
 189   size_t bytes_allocated_since_gc_start();
 190   void reset_bytes_allocated_since_gc_start();
 191 
 192   size_t min_capacity()     const;
 193   size_t max_capacity()     const;
 194   size_t initial_capacity() const;
 195   size_t capacity()         const;
 196   size_t used()             const;
 197   size_t committed()        const;
 198 
 199 // ---------- Workers handling
 200 //
 201 private:
 202   uint _max_workers;
 203   ShenandoahWorkGang* _workers;
 204   ShenandoahWorkGang* _safepoint_workers;
 205 
 206 public:
 207   uint max_workers();
 208   void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;
 209 
 210   WorkGang* workers() const;
 211   WorkGang* get_safepoint_workers();
 212 
 213   void gc_threads_do(ThreadClosure* tcl) const;
 214 
 215 // ---------- Heap regions handling machinery
 216 //
 217 private:
 218   MemRegion _heap_region;
 219   bool      _heap_region_special;
 220   size_t    _num_regions;
 221   ShenandoahHeapRegion** _regions;
 222   ShenandoahRegionIterator _update_refs_iterator;
 223 
 224 public:
 225   inline size_t num_regions() const { return _num_regions; }
 226   inline bool is_heap_region_special() { return _heap_region_special; }
 227 
 228   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 229   inline size_t heap_region_index_containing(const void* addr) const;
 230 
 231   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 232 
 233   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 234   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 235 
 236 // ---------- GC state machinery
 237 //
 238 // GC state describes the important parts of collector state, that may be
 239 // used to make barrier selection decisions in the native and generated code.
 240 // Multiple bits can be set at once.
 241 //
 242 // Important invariant: when GC state is zero, the heap is stable, and no barriers
 243 // are required.
 244 //
 245 public:
 246   enum GCStateBitPos {
 247     // Heap has forwarded objects: needs LRB barriers.
 248     HAS_FORWARDED_BITPOS   = 0,
 249 
 250     // Heap is under marking: needs SATB barriers.
 251     MARKING_BITPOS    = 1,
 252 
 253     // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
 254     EVACUATION_BITPOS = 2,
 255 
 256     // Heap is under updating: needs no additional barriers.
 257     UPDATEREFS_BITPOS = 3,
 258   };
 259 
 260   enum GCState {
 261     STABLE        = 0,
 262     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 263     MARKING       = 1 << MARKING_BITPOS,
 264     EVACUATION    = 1 << EVACUATION_BITPOS,
 265     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 266   };
 267 
 268 private:
 269   ShenandoahSharedBitmap _gc_state;
 270   ShenandoahSharedFlag   _degenerated_gc_in_progress;
 271   ShenandoahSharedFlag   _full_gc_in_progress;
 272   ShenandoahSharedFlag   _full_gc_move_in_progress;
 273   ShenandoahSharedFlag   _progress_last_gc;
 274 
 275   void set_gc_state_all_threads(char state);
 276   void set_gc_state_mask(uint mask, bool value);
 277 
 278 public:
 279   char gc_state() const;
 280   static address gc_state_addr();
 281 
 282   void set_concurrent_mark_in_progress(bool in_progress);
 283   void set_evacuation_in_progress(bool in_progress);
 284   void set_update_refs_in_progress(bool in_progress);
 285   void set_degenerated_gc_in_progress(bool in_progress);
 286   void set_full_gc_in_progress(bool in_progress);
 287   void set_full_gc_move_in_progress(bool in_progress);
 288   void set_has_forwarded_objects(bool cond);
 289 
 290   inline bool is_stable() const;
 291   inline bool is_idle() const;
 292   inline bool is_concurrent_mark_in_progress() const;
 293   inline bool is_update_refs_in_progress() const;
 294   inline bool is_evacuation_in_progress() const;
 295   inline bool is_degenerated_gc_in_progress() const;
 296   inline bool is_full_gc_in_progress() const;
 297   inline bool is_full_gc_move_in_progress() const;
 298   inline bool has_forwarded_objects() const;
 299   inline bool is_gc_in_progress_mask(uint mask) const;
 300 
 301 // ---------- GC cancellation and degeneration machinery
 302 //
 303 // Cancelled GC flag is used to notify concurrent phases that they should terminate.
 304 //
 305 public:
 306   enum ShenandoahDegenPoint {
 307     _degenerated_unset,
 308     _degenerated_outside_cycle,
 309     _degenerated_mark,
 310     _degenerated_evac,
 311     _degenerated_updaterefs,
 312     _DEGENERATED_LIMIT
 313   };
 314 
 315   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 316     switch (point) {
 317       case _degenerated_unset:
 318         return "<UNSET>";
 319       case _degenerated_outside_cycle:
 320         return "Outside of Cycle";
 321       case _degenerated_mark:
 322         return "Mark";
 323       case _degenerated_evac:
 324         return "Evacuation";
 325       case _degenerated_updaterefs:
 326         return "Update Refs";
 327       default:
 328         ShouldNotReachHere();
 329         return "ERROR";
 330     }
 331   };
 332 
 333 private:
 334   enum CancelState {
 335     // Normal state. GC has not been cancelled and is open for cancellation.
 336     // Worker threads can suspend for safepoint.
 337     CANCELLABLE,
 338 
 339     // GC has been cancelled. Worker threads can not suspend for
 340     // safepoint but must finish their work as soon as possible.
 341     CANCELLED,
 342 
 343     // GC has not been cancelled and must not be cancelled. At least
 344     // one worker thread checks for pending safepoint and may suspend
 345     // if a safepoint is pending.
 346     NOT_CANCELLED
 347   };
 348 
 349   ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;
 350   inline bool try_cancel_gc();
 351 
 352 public:
 353   static address cancelled_gc_addr();
 354 
 355   inline bool cancelled_gc() const;
 356   inline bool check_cancelled_gc_and_yield(bool sts_active = true);
 357 
 358   inline void clear_cancelled_gc();
 359 
 360   void cancel_gc(GCCause::Cause cause);
 361 
 362 // ---------- GC operations entry points
 363 //
 364 public:
 365   // Entry points to STW GC operations, these cause a related safepoint, that then
 366   // call the entry method below
 367   void vmop_entry_init_mark();
 368   void vmop_entry_final_mark();
 369   void vmop_entry_init_updaterefs();
 370   void vmop_entry_final_updaterefs();
 371   void vmop_entry_full(GCCause::Cause cause);
 372   void vmop_degenerated(ShenandoahDegenPoint point);
 373 
 374   // Entry methods to normally STW GC operations. These set up logging, monitoring
 375   // and workers for net VM operation
 376   void entry_init_mark();
 377   void entry_final_mark();
 378   void entry_init_updaterefs();
 379   void entry_final_updaterefs();
 380   void entry_full(GCCause::Cause cause);
 381   void entry_degenerated(int point);
 382 
 383   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 384   // for concurrent operation.
 385   void entry_reset();
 386   void entry_mark();
 387   void entry_preclean();
 388   void entry_cleanup_early();
 389   void entry_evac();
 390   void entry_updaterefs();
 391   void entry_cleanup_complete();
 392   void entry_uncommit(double shrink_before);
 393 
 394 private:
 395   // Actual work for the phases
 396   void op_init_mark();
 397   void op_final_mark();
 398   void op_init_updaterefs();
 399   void op_final_updaterefs();
 400   void op_full(GCCause::Cause cause);
 401   void op_degenerated(ShenandoahDegenPoint point);
 402   void op_degenerated_fail();
 403   void op_degenerated_futile();
 404 
 405   void op_reset();
 406   void op_mark();
 407   void op_preclean();
 408   void op_cleanup_early();
 409   void op_conc_evac();
 410   void op_stw_evac();
 411   void op_updaterefs();
 412   void op_cleanup_complete();
 413   void op_uncommit(double shrink_before);
 414 
 415   // Messages for GC trace events, they have to be immortal for
 416   // passing around the logging/tracing systems
 417   const char* init_mark_event_message() const;
 418   const char* final_mark_event_message() const;
 419   const char* conc_mark_event_message() const;
 420   const char* degen_event_message(ShenandoahDegenPoint point) const;
 421 
 422 // ---------- GC subsystems
 423 //
 424 private:
 425   ShenandoahControlThread*   _control_thread;
 426   ShenandoahCollectorPolicy* _shenandoah_policy;
 427   ShenandoahMode*            _gc_mode;
 428   ShenandoahHeuristics*      _heuristics;
 429   ShenandoahFreeSet*         _free_set;
 430   ShenandoahConcurrentMark*  _scm;
 431   ShenandoahMarkCompact*     _full_gc;
 432   ShenandoahPacer*           _pacer;
 433   ShenandoahVerifier*        _verifier;
 434 
 435   ShenandoahPhaseTimings*    _phase_timings;
 436 
 437   ShenandoahControlThread*   control_thread()          { return _control_thread;    }
 438   ShenandoahMarkCompact*     full_gc()                 { return _full_gc;           }
 439 
 440 public:
 441   ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
 442   ShenandoahHeuristics*      heuristics()        const { return _heuristics;        }
 443   ShenandoahFreeSet*         free_set()          const { return _free_set;          }
 444   ShenandoahConcurrentMark*  concurrent_mark()         { return _scm;               }
 445   ShenandoahPacer*           pacer()             const { return _pacer;             }
 446 
 447   ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }
 448 
 449   ShenandoahVerifier*        verifier();
 450 
 451 // ---------- VM subsystem bindings
 452 //
 453 private:
 454   ShenandoahMonitoringSupport* _monitoring_support;
 455   MemoryPool*                  _memory_pool;
 456   GCMemoryManager              _stw_memory_manager;
 457   GCMemoryManager              _cycle_memory_manager;
 458   ConcurrentGCTimer*           _gc_timer;
 459   SoftRefPolicy                _soft_ref_policy;
 460 
 461 public:
 462   ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support;    }
 463   GCMemoryManager* cycle_memory_manager()           { return &_cycle_memory_manager; }
 464   GCMemoryManager* stw_memory_manager()             { return &_stw_memory_manager;   }
 465   SoftRefPolicy* soft_ref_policy()                  { return &_soft_ref_policy;      }
 466 
 467   GrowableArray<GCMemoryManager*> memory_managers();
 468   GrowableArray<MemoryPool*> memory_pools();
 469   MemoryUsage memory_usage();
 470   GCTracer* tracer();
 471   ConcurrentGCTimer* gc_timer() const;
 472   CollectorPolicy* collector_policy() const;
 473 
 474 // ---------- Reference processing
 475 //
 476 private:
 477   AlwaysTrueClosure    _subject_to_discovery;
 478   ReferenceProcessor*  _ref_processor;
 479   ShenandoahSharedFlag _process_references;
 480 
 481   void ref_processing_init();
 482 
 483 public:
 484   ReferenceProcessor* ref_processor() { return _ref_processor; }
 485   void set_process_references(bool pr);
 486   bool process_references() const;
 487 
 488 // ---------- Class Unloading
 489 //
 490 private:
 491   ShenandoahSharedFlag _unload_classes;
 492 
 493 public:
 494   void set_unload_classes(bool uc);
 495   bool unload_classes() const;
 496 
 497   // Perform STW class unloading and weak root cleaning
 498   void parallel_cleaning(bool full_gc);
 499 
 500 private:
 501   void stw_unload_classes(bool full_gc);
 502   void stw_process_weak_roots(bool full_gc);
 503 
 504 // ---------- Generic interface hooks
 505 // Minor things that super-interface expects us to implement to play nice with
 506 // the rest of runtime. Some of the things here are not required to be implemented,
 507 // and can be stubbed out.
 508 //
 509 public:
 510   AdaptiveSizePolicy* size_policy() shenandoah_not_implemented_return(NULL);
 511   bool is_maximal_no_gc() const shenandoah_not_implemented_return(false);
 512 
 513   bool is_in(const void* p) const;
 514 
 515   // All objects can potentially move
 516   bool is_scavengable(oop obj) { return true; };
 517 
 518   void collect(GCCause::Cause cause);
 519   void do_full_collection(bool clear_all_soft_refs);
 520 
 521   // Used for parsing heap during error printing
 522   HeapWord* block_start(const void* addr) const;
 523   size_t block_size(const HeapWord* addr) const;
 524   bool block_is_obj(const HeapWord* addr) const;
 525 
 526   // Used for native heap walkers: heap dumpers, mostly
 527   void object_iterate(ObjectClosure* cl);
 528   void safe_object_iterate(ObjectClosure* cl);
 529 
 530   // Used by RMI
 531   jlong millis_since_last_gc();
 532 
 533 // ---------- Safepoint interface hooks
 534 //
 535 public:
 536   void safepoint_synchronize_begin();
 537   void safepoint_synchronize_end();
 538 
 539 // ---------- Code roots handling hooks
 540 //
 541 public:
 542   void register_nmethod(nmethod* nm);
 543   void unregister_nmethod(nmethod* nm);
 544 
 545 // ---------- Pinning hooks
 546 //
 547 public:
 548   // Shenandoah supports per-object (per-region) pinning
 549   bool supports_object_pinning() const { return true; }
 550 
 551   oop pin_object(JavaThread* thread, oop obj);
 552   void unpin_object(JavaThread* thread, oop obj);
 553 
 554   void sync_pinned_region_status();
 555   void assert_pinned_region_status() NOT_DEBUG_RETURN;
 556 
 557 // ---------- Allocation support
 558 //
 559 private:
 560   HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region);
 561   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 562   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 563   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
 564   void retire_and_reset_gclabs();
 565 
 566 public:
 567   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
 568   HeapWord* mem_allocate(size_t size, bool* what);
 569   MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 570                                                size_t size,
 571                                                Metaspace::MetadataType mdtype);
 572 
 573   void notify_mutator_alloc_words(size_t words, bool waste);
 574 
 575   // Shenandoah supports TLAB allocation
 576   bool supports_tlab_allocation() const { return true; }
 577 
 578   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);
 579   size_t tlab_capacity(Thread *thr) const;
 580   size_t unsafe_max_tlab_alloc(Thread *thread) const;
 581   size_t max_tlab_size() const;
 582   size_t tlab_used(Thread* ignored) const;
 583 
 584   void accumulate_statistics_tlabs();
 585   void resize_tlabs();
 586 
 587   void ensure_parsability(bool retire_tlabs);
 588   void make_parsable(bool retire_tlabs);
 589 
 590 // ---------- Marking support
 591 //
 592 private:
 593   ShenandoahMarkingContext* _marking_context;
 594   MemRegion  _bitmap_region;
 595   MemRegion  _aux_bitmap_region;
 596   MarkBitMap _verification_bit_map;
 597   MarkBitMap _aux_bit_map;
 598 
 599   size_t _bitmap_size;
 600   size_t _bitmap_regions_per_slice;
 601   size_t _bitmap_bytes_per_slice;
 602 
 603   size_t _pretouch_heap_page_size;
 604   size_t _pretouch_bitmap_page_size;
 605 
 606   bool _bitmap_region_special;
 607   bool _aux_bitmap_region_special;
 608 
 609   ShenandoahLiveData** _liveness_cache;
 610 
 611 public:
 612   inline ShenandoahMarkingContext* complete_marking_context() const;
 613   inline ShenandoahMarkingContext* marking_context() const;
 614   inline void mark_complete_marking_context();
 615   inline void mark_incomplete_marking_context();
 616 
 617   template<class T>
 618   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 619 
 620   template<class T>
 621   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 622 
 623   template<class T>
 624   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 625 
 626   void reset_mark_bitmap();
 627 
 628   // SATB barriers hooks
 629   inline bool requires_marking(const void* entry) const;
 630   void force_satb_flush_all_threads();
 631 
 632   // Support for bitmap uncommits
 633   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 634   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 635   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 636 
 637   // Liveness caching support
 638   ShenandoahLiveData* get_liveness_cache(uint worker_id);
 639   void flush_liveness_cache(uint worker_id);
 640 
 641   size_t pretouch_heap_page_size() { return _pretouch_heap_page_size; }
 642 
 643 // ---------- Evacuation support
 644 //
 645 private:
 646   ShenandoahCollectionSet* _collection_set;
 647   ShenandoahEvacOOMHandler _oom_evac_handler;
 648 
 649   void evacuate_and_update_roots();
 650 
 651 public:
 652   static address in_cset_fast_test_addr();
 653 
 654   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 655 
 656   // Checks if object is in the collection set.
 657   inline bool in_collection_set(oop obj) const;
 658 
 659   // Checks if location is in the collection set. Can be interior pointer, not the oop itself.
 660   inline bool in_collection_set_loc(void* loc) const;
 661 
 662   // Evacuates object src. Returns the evacuated object, either evacuated
 663   // by this thread, or by some other thread.
 664   inline oop evacuate_object(oop src, Thread* thread);
 665 
 666   // Call before/after evacuation.
 667   void enter_evacuation();
 668   void leave_evacuation();
 669 
 670 // ---------- Helper functions
 671 //
 672 public:
 673   template <class T>
 674   inline oop evac_update_with_forwarded(T* p);
 675 
 676   template <class T>
 677   inline oop maybe_update_with_forwarded(T* p);
 678 
 679   template <class T>
 680   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 681 
 682   template <class T>
 683   inline oop update_with_forwarded_not_null(T* p, oop obj);
 684 
 685   static inline oop cas_oop(oop n, narrowOop* addr, oop c);
 686   static inline oop cas_oop(oop n, oop* addr, oop c);
 687   static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c);
 688 
 689   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 690 
 691   void deduplicate_string(oop str);
 692 
 693 private:
 694   void trash_cset_regions();
 695   void update_heap_references(bool concurrent);
 696 
 697 // ---------- Testing helpers functions
 698 //
 699 private:
 700   ShenandoahSharedFlag _inject_alloc_failure;
 701 
 702   void try_inject_alloc_failure();
 703   bool should_inject_alloc_failure();
 704 };
 705 
 706 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP