1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/markBitMap.hpp"
  28 #include "gc/shared/softRefPolicy.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "gc/shenandoah/shenandoahAsserts.hpp"
  31 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  32 #include "gc/shenandoah/shenandoahLock.hpp"
  33 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  34 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  35 #include "gc/shenandoah/shenandoahUnload.hpp"
  36 #include "services/memoryManager.hpp"
  37 
  38 class ConcurrentGCTimer;
  39 class ReferenceProcessor;
  40 class ShenandoahAllocTracker;
  41 class ShenandoahCollectorPolicy;
  42 class ShenandoahControlThread;
  43 class ShenandoahGCSession;
  44 class ShenandoahGCStateResetter;
  45 class ShenandoahHeuristics;
  46 class ShenandoahMarkingContext;
  47 class ShenandoahMarkCompact;
  48 class ShenandoahMode;
  49 class ShenandoahPhaseTimings;
  50 class ShenandoahHeap;
  51 class ShenandoahHeapRegion;
  52 class ShenandoahHeapRegionClosure;
  53 class ShenandoahCollectionSet;
  54 class ShenandoahFreeSet;
  55 class ShenandoahConcurrentMark;
  56 class ShenandoahMarkCompact;
  57 class ShenandoahMonitoringSupport;
  58 class ShenandoahPacer;
  59 class ShenandoahTraversalGC;
  60 class ShenandoahVerifier;
  61 class ShenandoahWorkGang;
  62 class VMStructs;
  63 
  64 class ShenandoahRegionIterator : public StackObj {
  65 private:
  66   ShenandoahHeap* _heap;
  67 
  68   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
  69   volatile size_t _index;
  70   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
  71 
  72   // No implicit copying: iterators should be passed by reference to capture the state
  73   ShenandoahRegionIterator(const ShenandoahRegionIterator& that);
  74   ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);
  75 
  76 public:
  77   ShenandoahRegionIterator();
  78   ShenandoahRegionIterator(ShenandoahHeap* heap);
  79 
  80   // Reset iterator to default state
  81   void reset();
  82 
  83   // Returns next region, or NULL if there are no more regions.
  84   // This is multi-thread-safe.
  85   inline ShenandoahHeapRegion* next();
  86 
  87   // This is *not* MT safe. However, in the absence of multithreaded access, it
  88   // can be used to determine if there is more work to do.
  89   bool has_next() const;
  90 };
  91 
  92 class ShenandoahHeapRegionClosure : public StackObj {
  93 public:
  94   virtual void heap_region_do(ShenandoahHeapRegion* r) = 0;
  95   virtual bool is_thread_safe() { return false; }
  96 };
  97 
  98 #ifdef ASSERT
  99 class ShenandoahAssertToSpaceClosure : public OopClosure {
 100 private:
 101   template <class T>
 102   void do_oop_work(T* p);
 103 public:
 104   void do_oop(narrowOop* p);
 105   void do_oop(oop* p);
 106 };
 107 #endif
 108 
 109 typedef ShenandoahLock    ShenandoahHeapLock;
 110 typedef ShenandoahLocker  ShenandoahHeapLocker;
 111 
 112 // Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers
 113 // to encode forwarding data. See BrooksPointer for details on forwarding data encoding.
 114 // See ShenandoahControlThread for GC cycle structure.
 115 //
 116 class ShenandoahHeap : public CollectedHeap {
 117   friend class ShenandoahAsserts;
 118   friend class VMStructs;
 119   friend class ShenandoahGCSession;
 120   friend class ShenandoahGCStateResetter;
 121 
 122 // ---------- Locks that guard important data structures in Heap
 123 //
 124 private:
 125   ShenandoahHeapLock _lock;
 126 
 127 public:
 128   ShenandoahHeapLock* lock() {
 129     return &_lock;
 130   }
 131 
 132   void assert_heaplock_owned_by_current_thread()     NOT_DEBUG_RETURN;
 133   void assert_heaplock_not_owned_by_current_thread() NOT_DEBUG_RETURN;
 134   void assert_heaplock_or_safepoint()                NOT_DEBUG_RETURN;
 135 
 136 // ---------- Initialization, termination, identification, printing routines
 137 //
 138 public:
 139   static ShenandoahHeap* heap();
 140   static ShenandoahHeap* heap_no_check();
 141 
 142   const char* name()          const { return "Shenandoah"; }
 143   ShenandoahHeap::Name kind() const { return CollectedHeap::Shenandoah; }
 144 
 145   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 146   jint initialize();
 147   void post_initialize();
 148   void initialize_heuristics();
 149 
 150   void initialize_serviceability();
 151 
 152   void print_on(outputStream* st)              const;
 153   void print_extended_on(outputStream *st)     const;
 154   void print_tracing_info()                    const;
 155   void print_gc_threads_on(outputStream* st)   const;
 156   void print_heap_regions_on(outputStream* st) const;
 157 
 158   void stop();
 159 
 160   void prepare_for_verify();
 161   void verify(VerifyOption vo);
 162 
 163 // ---------- Heap counters and metrics
 164 //
 165 private:
 166            size_t _initial_size;
 167            size_t _minimum_size;
 168   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
 169   volatile size_t _used;
 170   volatile size_t _committed;
 171   volatile size_t _bytes_allocated_since_gc_start;
 172   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
 173 
 174 public:
 175   void increase_used(size_t bytes);
 176   void decrease_used(size_t bytes);
 177   void set_used(size_t bytes);
 178 
 179   void increase_committed(size_t bytes);
 180   void decrease_committed(size_t bytes);
 181   void increase_allocated(size_t bytes);
 182 
 183   size_t bytes_allocated_since_gc_start();
 184   void reset_bytes_allocated_since_gc_start();
 185 
 186   size_t min_capacity()     const;
 187   size_t max_capacity()     const;
 188   size_t initial_capacity() const;
 189   size_t capacity()         const;
 190   size_t used()             const;
 191   size_t committed()        const;
 192 
 193 // ---------- Workers handling
 194 //
 195 private:
 196   uint _max_workers;
 197   ShenandoahWorkGang* _workers;
 198   ShenandoahWorkGang* _safepoint_workers;
 199 
 200 public:
 201   uint max_workers();
 202   void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;
 203 
 204   WorkGang* workers() const;
 205   WorkGang* get_safepoint_workers();
 206 
 207   void gc_threads_do(ThreadClosure* tcl) const;
 208 
 209 // ---------- Heap regions handling machinery
 210 //
 211 private:
 212   MemRegion _heap_region;
 213   bool      _heap_region_special;
 214   size_t    _num_regions;
 215   ShenandoahHeapRegion** _regions;
 216   ShenandoahRegionIterator _update_refs_iterator;
 217 
 218 public:
 219 
 220   inline HeapWord* base() const { return _heap_region.start(); }
 221 
 222   inline size_t num_regions() const { return _num_regions; }
 223   inline bool is_heap_region_special() { return _heap_region_special; }
 224 
 225   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 226   inline size_t heap_region_index_containing(const void* addr) const;
 227 
 228   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 229 
 230   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 231   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 232 
 233 // ---------- GC state machinery
 234 //
 235 // GC state describes the important parts of collector state, that may be
 236 // used to make barrier selection decisions in the native and generated code.
 237 // Multiple bits can be set at once.
 238 //
 239 // Important invariant: when GC state is zero, the heap is stable, and no barriers
 240 // are required.
 241 //
 242 public:
 243   enum GCStateBitPos {
 244     // Heap has forwarded objects: needs LRB barriers.
 245     HAS_FORWARDED_BITPOS   = 0,
 246 
 247     // Heap is under marking: needs SATB barriers.
 248     MARKING_BITPOS    = 1,
 249 
 250     // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
 251     EVACUATION_BITPOS = 2,
 252 
 253     // Heap is under updating: needs no additional barriers.
 254     UPDATEREFS_BITPOS = 3,
 255 
 256     // Heap is under traversal collection
 257     TRAVERSAL_BITPOS  = 4
 258   };
 259 
 260   enum GCState {
 261     STABLE        = 0,
 262     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 263     MARKING       = 1 << MARKING_BITPOS,
 264     EVACUATION    = 1 << EVACUATION_BITPOS,
 265     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 266     TRAVERSAL     = 1 << TRAVERSAL_BITPOS
 267   };
 268 
 269 private:
 270   ShenandoahSharedBitmap _gc_state;
 271   ShenandoahSharedFlag   _degenerated_gc_in_progress;
 272   ShenandoahSharedFlag   _full_gc_in_progress;
 273   ShenandoahSharedFlag   _full_gc_move_in_progress;
 274   ShenandoahSharedFlag   _progress_last_gc;
 275 
 276   void set_gc_state_all_threads(char state);
 277   void set_gc_state_mask(uint mask, bool value);
 278 
 279 public:
 280   char gc_state() const;
 281   static address gc_state_addr();
 282 
 283   void set_concurrent_mark_in_progress(bool in_progress);
 284   void set_evacuation_in_progress(bool in_progress);
 285   void set_update_refs_in_progress(bool in_progress);
 286   void set_degenerated_gc_in_progress(bool in_progress);
 287   void set_full_gc_in_progress(bool in_progress);
 288   void set_full_gc_move_in_progress(bool in_progress);
 289   void set_concurrent_traversal_in_progress(bool in_progress);
 290   void set_has_forwarded_objects(bool cond);
 291 
 292   inline bool is_stable() const;
 293   inline bool is_idle() const;
 294   inline bool is_concurrent_mark_in_progress() const;
 295   inline bool is_update_refs_in_progress() const;
 296   inline bool is_evacuation_in_progress() const;
 297   inline bool is_degenerated_gc_in_progress() const;
 298   inline bool is_full_gc_in_progress() const;
 299   inline bool is_full_gc_move_in_progress() const;
 300   inline bool is_concurrent_traversal_in_progress() const;
 301   inline bool has_forwarded_objects() const;
 302   inline bool is_gc_in_progress_mask(uint mask) const;
 303   inline bool is_stw_gc_in_progress() const;
 304 
 305 // ---------- GC cancellation and degeneration machinery
 306 //
 307 // Cancelled GC flag is used to notify concurrent phases that they should terminate.
 308 //
 309 public:
 310   enum ShenandoahDegenPoint {
 311     _degenerated_unset,
 312     _degenerated_traversal,
 313     _degenerated_outside_cycle,
 314     _degenerated_mark,
 315     _degenerated_evac,
 316     _degenerated_updaterefs,
 317     _DEGENERATED_LIMIT
 318   };
 319 
 320   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 321     switch (point) {
 322       case _degenerated_unset:
 323         return "<UNSET>";
 324       case _degenerated_traversal:
 325         return "Traversal";
 326       case _degenerated_outside_cycle:
 327         return "Outside of Cycle";
 328       case _degenerated_mark:
 329         return "Mark";
 330       case _degenerated_evac:
 331         return "Evacuation";
 332       case _degenerated_updaterefs:
 333         return "Update Refs";
 334       default:
 335         ShouldNotReachHere();
 336         return "ERROR";
 337     }
 338   };
 339 
 340 private:
 341   enum CancelState {
 342     // Normal state. GC has not been cancelled and is open for cancellation.
 343     // Worker threads can suspend for safepoint.
 344     CANCELLABLE,
 345 
 346     // GC has been cancelled. Worker threads can not suspend for
 347     // safepoint but must finish their work as soon as possible.
 348     CANCELLED,
 349 
 350     // GC has not been cancelled and must not be cancelled. At least
 351     // one worker thread checks for pending safepoint and may suspend
 352     // if a safepoint is pending.
 353     NOT_CANCELLED
 354   };
 355 
 356   ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;
 357   bool try_cancel_gc();
 358 
 359 public:
 360   static address cancelled_gc_addr();
 361 
 362   inline bool cancelled_gc() const;
 363   inline bool check_cancelled_gc_and_yield(bool sts_active = true);
 364 
 365   inline void clear_cancelled_gc();
 366 
 367   void cancel_gc(GCCause::Cause cause);
 368 
 369 // ---------- GC operations entry points
 370 //
 371 public:
 372   // Entry points to STW GC operations, these cause a related safepoint, that then
 373   // call the entry method below
 374   void vmop_entry_init_mark();
 375   void vmop_entry_final_mark();
 376   void vmop_entry_final_evac();
 377   void vmop_entry_init_updaterefs();
 378   void vmop_entry_final_updaterefs();
 379   void vmop_entry_init_traversal();
 380   void vmop_entry_final_traversal();
 381   void vmop_entry_full(GCCause::Cause cause);
 382   void vmop_degenerated(ShenandoahDegenPoint point);
 383 
 384   // Entry methods to normally STW GC operations. These set up logging, monitoring
 385   // and workers for net VM operation
 386   void entry_init_mark();
 387   void entry_final_mark();
 388   void entry_final_evac();
 389   void entry_init_updaterefs();
 390   void entry_final_updaterefs();
 391   void entry_init_traversal();
 392   void entry_final_traversal();
 393   void entry_full(GCCause::Cause cause);
 394   void entry_degenerated(int point);
 395 
 396   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 397   // for concurrent operation.
 398   void entry_reset();
 399   void entry_mark();
 400   void entry_preclean();
 401   void entry_roots();
 402   void entry_cleanup();
 403   void entry_evac();
 404   void entry_updaterefs();
 405   void entry_traversal();
 406   void entry_uncommit(double shrink_before);
 407 
 408 private:
 409   // Actual work for the phases
 410   void op_init_mark();
 411   void op_final_mark();
 412   void op_final_evac();
 413   void op_init_updaterefs();
 414   void op_final_updaterefs();
 415   void op_init_traversal();
 416   void op_final_traversal();
 417   void op_full(GCCause::Cause cause);
 418   void op_degenerated(ShenandoahDegenPoint point);
 419   void op_degenerated_fail();
 420   void op_degenerated_futile();
 421 
 422   void op_reset();
 423   void op_mark();
 424   void op_preclean();
 425   void op_roots();
 426   void op_cleanup();
 427   void op_conc_evac();
 428   void op_stw_evac();
 429   void op_updaterefs();
 430   void op_traversal();
 431   void op_uncommit(double shrink_before);
 432 
 433   // Messages for GC trace events, they have to be immortal for
 434   // passing around the logging/tracing systems
 435   const char* init_mark_event_message() const;
 436   const char* final_mark_event_message() const;
 437   const char* conc_mark_event_message() const;
 438   const char* degen_event_message(ShenandoahDegenPoint point) const;
 439 
 440 // ---------- GC subsystems
 441 //
 442 private:
 443   ShenandoahControlThread*   _control_thread;
 444   ShenandoahCollectorPolicy* _shenandoah_policy;
 445   ShenandoahMode*            _gc_mode;
 446   ShenandoahHeuristics*      _heuristics;
 447   ShenandoahFreeSet*         _free_set;
 448   ShenandoahConcurrentMark*  _scm;
 449   ShenandoahTraversalGC*     _traversal_gc;
 450   ShenandoahMarkCompact*     _full_gc;
 451   ShenandoahPacer*           _pacer;
 452   ShenandoahVerifier*        _verifier;
 453 
 454   ShenandoahAllocTracker*    _alloc_tracker;
 455   ShenandoahPhaseTimings*    _phase_timings;
 456 
 457   ShenandoahControlThread*   control_thread()          { return _control_thread;    }
 458   ShenandoahMarkCompact*     full_gc()                 { return _full_gc;           }
 459 
 460 public:
 461   ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
 462   ShenandoahHeuristics*      heuristics()        const { return _heuristics;        }
 463   ShenandoahFreeSet*         free_set()          const { return _free_set;          }
 464   ShenandoahConcurrentMark*  concurrent_mark()         { return _scm;               }
 465   ShenandoahTraversalGC*     traversal_gc()      const { return _traversal_gc;      }
 466   bool                       is_traversal_mode() const { return _traversal_gc != NULL; }
 467   ShenandoahPacer*           pacer()             const { return _pacer;             }
 468 
 469   ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }
 470   ShenandoahAllocTracker*    alloc_tracker()     const { return _alloc_tracker;     }
 471 
 472   ShenandoahVerifier*        verifier();
 473 
 474 // ---------- VM subsystem bindings
 475 //
 476 private:
 477   ShenandoahMonitoringSupport* _monitoring_support;
 478   MemoryPool*                  _memory_pool;
 479   GCMemoryManager              _stw_memory_manager;
 480   GCMemoryManager              _cycle_memory_manager;
 481   ConcurrentGCTimer*           _gc_timer;
 482   SoftRefPolicy                _soft_ref_policy;
 483 
 484   // For exporting to SA
 485   int                          _log_min_obj_alignment_in_bytes;
 486 public:
 487   ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support;    }
 488   GCMemoryManager* cycle_memory_manager()           { return &_cycle_memory_manager; }
 489   GCMemoryManager* stw_memory_manager()             { return &_stw_memory_manager;   }
 490   SoftRefPolicy* soft_ref_policy()                  { return &_soft_ref_policy;      }
 491 
 492   GrowableArray<GCMemoryManager*> memory_managers();
 493   GrowableArray<MemoryPool*> memory_pools();
 494   MemoryUsage memory_usage();
 495   GCTracer* tracer();
 496   GCTimer* gc_timer() const;
 497 
 498 // ---------- Reference processing
 499 //
 500 private:
 501   AlwaysTrueClosure    _subject_to_discovery;
 502   ReferenceProcessor*  _ref_processor;
 503   ShenandoahSharedFlag _process_references;
 504 
 505   void ref_processing_init();
 506 
 507 public:
 508   ReferenceProcessor* ref_processor() { return _ref_processor; }
 509   void set_process_references(bool pr);
 510   bool process_references() const;
 511 
 512 // ---------- Class Unloading
 513 //
 514 private:
 515   ShenandoahSharedFlag _unload_classes;
 516   ShenandoahUnload     _unloader;
 517 
 518 public:
 519   void set_unload_classes(bool uc);
 520   bool unload_classes() const;
 521 
 522   // Perform STW class unloading and weak root cleaning
 523   void parallel_cleaning(bool full_gc);
 524 
 525 private:
 526   void stw_unload_classes(bool full_gc);
 527   void stw_process_weak_roots(bool full_gc);
 528 
 529   // Prepare and finish concurrent unloading
 530   void prepare_concurrent_unloading();
 531   void finish_concurrent_unloading();
 532 
 533 // ---------- Generic interface hooks
 534 // Minor things that super-interface expects us to implement to play nice with
 535 // the rest of runtime. Some of the things here are not required to be implemented,
 536 // and can be stubbed out.
 537 //
 538 public:
 539   AdaptiveSizePolicy* size_policy() shenandoah_not_implemented_return(NULL);
 540   bool is_maximal_no_gc() const shenandoah_not_implemented_return(false);
 541 
 542   bool is_in(const void* p) const;
 543 
 544   MemRegion reserved_region() const { return _reserved; }
 545   bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); }
 546 
 547   void collect(GCCause::Cause cause);
 548   void do_full_collection(bool clear_all_soft_refs);
 549 
 550   // Used for parsing heap during error printing
 551   HeapWord* block_start(const void* addr) const;
 552   bool block_is_obj(const HeapWord* addr) const;
 553   bool print_location(outputStream* st, void* addr) const;
 554 
 555   // Used for native heap walkers: heap dumpers, mostly
 556   void object_iterate(ObjectClosure* cl);
 557   void safe_object_iterate(ObjectClosure* cl);
 558 
 559   // Used by RMI
 560   jlong millis_since_last_gc();
 561 
 562 // ---------- Safepoint interface hooks
 563 //
 564 public:
 565   void safepoint_synchronize_begin();
 566   void safepoint_synchronize_end();
 567 
 568 // ---------- Code roots handling hooks
 569 //
 570 public:
 571   void register_nmethod(nmethod* nm);
 572   void unregister_nmethod(nmethod* nm);
 573   void flush_nmethod(nmethod* nm);
 574   void verify_nmethod(nmethod* nm) {}
 575 
 576 // ---------- Pinning hooks
 577 //
 578 public:
 579   // Shenandoah supports per-object (per-region) pinning
 580   bool supports_object_pinning() const { return true; }
 581 
 582   oop pin_object(JavaThread* thread, oop obj);
 583   void unpin_object(JavaThread* thread, oop obj);
 584 
 585   void sync_pinned_region_status();
 586   void assert_pinned_region_status() NOT_DEBUG_RETURN;
 587 
 588 // ---------- Allocation support
 589 //
 590 private:
 591   HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region);
 592   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 593   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 594   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
 595   void retire_and_reset_gclabs();
 596 
 597 public:
 598   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
 599   HeapWord* mem_allocate(size_t size, bool* what);
 600   MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 601                                                size_t size,
 602                                                Metaspace::MetadataType mdtype);
 603 
 604   void notify_mutator_alloc_words(size_t words, bool waste);
 605 
 606   // Shenandoah supports TLAB allocation
 607   bool supports_tlab_allocation() const { return true; }
 608 
 609   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);
 610   size_t tlab_capacity(Thread *thr) const;
 611   size_t unsafe_max_tlab_alloc(Thread *thread) const;
 612   size_t max_tlab_size() const;
 613   size_t tlab_used(Thread* ignored) const;
 614 
 615   void resize_tlabs();
 616 
 617   void ensure_parsability(bool retire_tlabs);
 618   void make_parsable(bool retire_tlabs);
 619 
 620 // ---------- Marking support
 621 //
 622 private:
 623   ShenandoahMarkingContext* _marking_context;
 624   MemRegion  _bitmap_region;
 625   MemRegion  _aux_bitmap_region;
 626   MarkBitMap _verification_bit_map;
 627   MarkBitMap _aux_bit_map;
 628 
 629   size_t _bitmap_size;
 630   size_t _bitmap_regions_per_slice;
 631   size_t _bitmap_bytes_per_slice;
 632 
 633   bool _bitmap_region_special;
 634   bool _aux_bitmap_region_special;
 635 
 636   // Used for buffering per-region liveness data.
 637   // Needed since ShenandoahHeapRegion uses atomics to update liveness.
 638   //
 639   // The array has max-workers elements, each of which is an array of
 640   // jushort * max_regions. The choice of jushort is not accidental:
 641   // there is a tradeoff between static/dynamic footprint that translates
 642   // into cache pressure (which is already high during marking), and
 643   // too many atomic updates. size_t/jint is too large, jbyte is too small.
 644   jushort** _liveness_cache;
 645 
 646 public:
 647   inline ShenandoahMarkingContext* complete_marking_context() const;
 648   inline ShenandoahMarkingContext* marking_context() const;
 649   inline void mark_complete_marking_context();
 650   inline void mark_incomplete_marking_context();
 651 
 652   template<class T>
 653   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 654 
 655   template<class T>
 656   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 657 
 658   template<class T>
 659   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 660 
 661   void reset_mark_bitmap();
 662 
 663   // SATB barriers hooks
 664   template<bool RESOLVE>
 665   inline bool requires_marking(const void* entry) const;
 666   void force_satb_flush_all_threads();
 667 
 668   // Support for bitmap uncommits
 669   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 670   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 671   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 672 
 673   // Liveness caching support
 674   jushort* get_liveness_cache(uint worker_id);
 675   void flush_liveness_cache(uint worker_id);
 676 
 677 // ---------- Evacuation support
 678 //
 679 private:
 680   ShenandoahCollectionSet* _collection_set;
 681   ShenandoahEvacOOMHandler _oom_evac_handler;
 682 
 683   void evacuate_and_update_roots();
 684 
 685 public:
 686   static address in_cset_fast_test_addr();
 687 
 688   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 689 
 690   template <class T>
 691   inline bool in_collection_set(T obj) const;
 692 
 693   // Avoid accidentally calling the method above with ShenandoahHeapRegion*, which would be *wrong*.
 694   inline bool in_collection_set(ShenandoahHeapRegion* r) shenandoah_not_implemented_return(false);
 695 
 696   // Evacuates object src. Returns the evacuated object, either evacuated
 697   // by this thread, or by some other thread.
 698   inline oop evacuate_object(oop src, Thread* thread);
 699 
 700   // Call before/after evacuation.
 701   void enter_evacuation();
 702   void leave_evacuation();
 703 
 704 // ---------- Helper functions
 705 //
 706 public:
 707   template <class T>
 708   inline oop evac_update_with_forwarded(T* p);
 709 
 710   template <class T>
 711   inline oop maybe_update_with_forwarded(T* p);
 712 
 713   template <class T>
 714   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 715 
 716   template <class T>
 717   inline oop update_with_forwarded_not_null(T* p, oop obj);
 718 
 719   static inline oop cas_oop(oop n, narrowOop* addr, oop c);
 720   static inline oop cas_oop(oop n, oop* addr, oop c);
 721   static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c);
 722 
 723   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 724 
 725   void deduplicate_string(oop str);
 726 
 727 private:
 728   void trash_cset_regions();
 729   void update_heap_references(bool concurrent);
 730 
 731 // ---------- Testing helpers functions
 732 //
 733 private:
 734   ShenandoahSharedFlag _inject_alloc_failure;
 735 
 736   void try_inject_alloc_failure();
 737   bool should_inject_alloc_failure();
 738 };
 739 
 740 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
--- EOF ---