1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/markBitMap.hpp"
  28 #include "gc/shared/softRefPolicy.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "gc/shenandoah/shenandoahAsserts.hpp"
  31 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  32 #include "gc/shenandoah/shenandoahHeapLock.hpp"
  33 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  34 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  35 #include "services/memoryManager.hpp"
  36 
  37 class ConcurrentGCTimer;
  38 class ReferenceProcessor;
  39 class ShenandoahAllocTracker;
  40 class ShenandoahCollectorPolicy;
  41 class ShenandoahControlThread;
  42 class ShenandoahGCSession;
  43 class ShenandoahHeuristics;
  44 class ShenandoahMarkingContext;
  45 class ShenandoahPhaseTimings;
  46 class ShenandoahHeap;
  47 class ShenandoahHeapRegion;
  48 class ShenandoahHeapRegionClosure;
  49 class ShenandoahCollectionSet;
  50 class ShenandoahFreeSet;
  51 class ShenandoahConcurrentMark;
  52 class ShenandoahMarkCompact;
  53 class ShenandoahMonitoringSupport;
  54 class ShenandoahPacer;
  55 class ShenandoahTraversalGC;
  56 class ShenandoahVerifier;
  57 class ShenandoahWorkGang;
  58 class VMStructs;
  59 
  60 class ShenandoahRegionIterator : public StackObj {
  61 private:
  62   ShenandoahHeap* _heap;
  63 
  64   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
  65   volatile size_t _index;
  66   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
  67 
  68   // No implicit copying: iterators should be passed by reference to capture the state
  69   ShenandoahRegionIterator(const ShenandoahRegionIterator& that);
  70   ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);
  71 
  72 public:
  73   ShenandoahRegionIterator();
  74   ShenandoahRegionIterator(ShenandoahHeap* heap);
  75 
  76   // Reset iterator to default state
  77   void reset();
  78 
  79   // Returns next region, or NULL if there are no more regions.
  80   // This is multi-thread-safe.
  81   inline ShenandoahHeapRegion* next();
  82 
  83   // This is *not* MT safe. However, in the absence of multithreaded access, it
  84   // can be used to determine if there is more work to do.
  85   bool has_next() const;
  86 };
  87 
  88 class ShenandoahHeapRegionClosure : public StackObj {
  89 public:
  90   virtual void heap_region_do(ShenandoahHeapRegion* r) = 0;
  91   virtual bool is_thread_safe() { return false; }
  92 };
  93 
  94 class ShenandoahUpdateRefsClosure: public OopClosure {
  95 private:
  96   ShenandoahHeap* _heap;
  97 
  98   template <class T>
  99   inline void do_oop_work(T* p);
 100 
 101 public:
 102   ShenandoahUpdateRefsClosure();
 103   inline void do_oop(oop* p);
 104   inline void do_oop(narrowOop* p);
 105 };
 106 
 107 #ifdef ASSERT
 108 class ShenandoahAssertToSpaceClosure : public OopClosure {
 109 private:
 110   template <class T>
 111   void do_oop_work(T* p);
 112 public:
 113   void do_oop(narrowOop* p);
 114   void do_oop(oop* p);
 115 };
 116 #endif
 117 
 118 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
 119 public:
 120   bool do_object_b(oop p) { return true; }
 121 };
 122 
 123 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
 124 private:
 125   ShenandoahMarkingContext* const _mark_context;
 126 public:
 127   ShenandoahForwardedIsAliveClosure();
 128   bool do_object_b(oop obj);
 129 };
 130 
 131 class ShenandoahIsAliveClosure: public BoolObjectClosure {
 132 private:
 133   ShenandoahMarkingContext* const _mark_context;
 134 public:
 135   ShenandoahIsAliveClosure();
 136   bool do_object_b(oop obj);
 137 };
 138 
 139 class ShenandoahIsAliveSelector : public StackObj {
 140 private:
 141   ShenandoahIsAliveClosure _alive_cl;
 142   ShenandoahForwardedIsAliveClosure _fwd_alive_cl;
 143 public:
 144   BoolObjectClosure* is_alive_closure();
 145 };
 146 
 147 // Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers
 148 // to encode forwarding data. See BrooksPointer for details on forwarding data encoding.
 149 // See ShenandoahControlThread for GC cycle structure.
 150 //
 151 class ShenandoahHeap : public CollectedHeap {
 152   friend class ShenandoahAsserts;
 153   friend class VMStructs;
 154   friend class ShenandoahGCSession;
 155 
 156 // ---------- Locks that guard important data structures in Heap
 157 //
 158 private:
 159   ShenandoahHeapLock _lock;
 160 
 161 public:
 162   ShenandoahHeapLock* lock() {
 163     return &_lock;
 164   }
 165 
 166   void assert_heaplock_owned_by_current_thread()     NOT_DEBUG_RETURN;
 167   void assert_heaplock_not_owned_by_current_thread() NOT_DEBUG_RETURN;
 168   void assert_heaplock_or_safepoint()                NOT_DEBUG_RETURN;
 169 
 170 // ---------- Initialization, termination, identification, printing routines
 171 //
 172 public:
 173   static ShenandoahHeap* heap();
 174   static ShenandoahHeap* heap_no_check();
 175 
 176   const char* name()          const { return "Shenandoah"; }
 177   ShenandoahHeap::Name kind() const { return CollectedHeap::Shenandoah; }
 178 
 179   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 180   jint initialize();
 181   void post_initialize();
 182   void initialize_heuristics();
 183 
 184   void initialize_serviceability();
 185 
 186   void print_on(outputStream* st)              const;
 187   void print_extended_on(outputStream *st)     const;
 188   void print_tracing_info()                    const;
 189   void print_gc_threads_on(outputStream* st)   const;
 190   void print_heap_regions_on(outputStream* st) const;
 191 
 192   void stop();
 193 
 194   void prepare_for_verify();
 195   void verify(VerifyOption vo);
 196 
 197 // ---------- Heap counters and metrics
 198 //
 199 private:
 200            size_t _initial_size;
 201   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
 202   volatile size_t _used;
 203   volatile size_t _committed;
 204   volatile size_t _bytes_allocated_since_gc_start;
 205   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
 206 
 207 public:
 208   void increase_used(size_t bytes);
 209   void decrease_used(size_t bytes);
 210   void set_used(size_t bytes);
 211 
 212   void increase_committed(size_t bytes);
 213   void decrease_committed(size_t bytes);
 214   void increase_allocated(size_t bytes);
 215 
 216   size_t bytes_allocated_since_gc_start();
 217   void reset_bytes_allocated_since_gc_start();
 218 
 219   size_t max_capacity()     const;
 220   size_t initial_capacity() const;
 221   size_t capacity()         const;
 222   size_t used()             const;
 223   size_t committed()        const;
 224 
 225 // ---------- Workers handling
 226 //
 227 private:
 228   uint _max_workers;
 229   ShenandoahWorkGang* _workers;
 230   ShenandoahWorkGang* _safepoint_workers;
 231 
 232 public:
 233   uint max_workers();
 234   void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;
 235 
 236   WorkGang* workers() const;
 237   WorkGang* get_safepoint_workers();
 238 
 239   void gc_threads_do(ThreadClosure* tcl) const;
 240 
 241 // ---------- Heap regions handling machinery
 242 //
 243 private:
 244   MemRegion _heap_region;
 245   bool      _heap_region_special;
 246   size_t    _num_regions;
 247   ShenandoahHeapRegion** _regions;
 248   ShenandoahRegionIterator _update_refs_iterator;
 249 
 250 public:
 251   inline size_t num_regions() const { return _num_regions; }
 252   inline bool is_heap_region_special() { return _heap_region_special; }
 253 
 254   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 255   inline size_t heap_region_index_containing(const void* addr) const;
 256 
 257   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 258 
 259   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 260   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 261 
 262 // ---------- GC state machinery
 263 //
 264 // GC state describes the important parts of collector state, that may be
 265 // used to make barrier selection decisions in the native and generated code.
 266 // Multiple bits can be set at once.
 267 //
 268 // Important invariant: when GC state is zero, the heap is stable, and no barriers
 269 // are required.
 270 //
 271 public:
 272   enum GCStateBitPos {
 273     // Heap has forwarded objects: needs LRB barriers.
 274     HAS_FORWARDED_BITPOS   = 0,
 275 
 276     // Heap is under marking: needs SATB barriers.
 277     MARKING_BITPOS    = 1,
 278 
 279     // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
 280     EVACUATION_BITPOS = 2,
 281 
 282     // Heap is under updating: needs no additional barriers.
 283     UPDATEREFS_BITPOS = 3,
 284 
 285     // Heap is under traversal collection
 286     TRAVERSAL_BITPOS  = 4,
 287   };
 288 
 289   enum GCState {
 290     STABLE        = 0,
 291     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 292     MARKING       = 1 << MARKING_BITPOS,
 293     EVACUATION    = 1 << EVACUATION_BITPOS,
 294     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 295     TRAVERSAL     = 1 << TRAVERSAL_BITPOS,
 296   };
 297 
 298 private:
 299   ShenandoahSharedBitmap _gc_state;
 300   ShenandoahSharedFlag   _degenerated_gc_in_progress;
 301   ShenandoahSharedFlag   _full_gc_in_progress;
 302   ShenandoahSharedFlag   _full_gc_move_in_progress;
 303   ShenandoahSharedFlag   _progress_last_gc;
 304 
 305   void set_gc_state_all_threads(char state);
 306   void set_gc_state_mask(uint mask, bool value);
 307 
 308 public:
 309   char gc_state() const;
 310   static address gc_state_addr();
 311 
 312   void set_concurrent_mark_in_progress(bool in_progress);
 313   void set_evacuation_in_progress(bool in_progress);
 314   void set_update_refs_in_progress(bool in_progress);
 315   void set_degenerated_gc_in_progress(bool in_progress);
 316   void set_full_gc_in_progress(bool in_progress);
 317   void set_full_gc_move_in_progress(bool in_progress);
 318   void set_concurrent_traversal_in_progress(bool in_progress);
 319   void set_has_forwarded_objects(bool cond);
 320 
 321   inline bool is_stable() const;
 322   inline bool is_idle() const;
 323   inline bool is_concurrent_mark_in_progress() const;
 324   inline bool is_update_refs_in_progress() const;
 325   inline bool is_evacuation_in_progress() const;
 326   inline bool is_degenerated_gc_in_progress() const;
 327   inline bool is_full_gc_in_progress() const;
 328   inline bool is_full_gc_move_in_progress() const;
 329   inline bool is_concurrent_traversal_in_progress() const;
 330   inline bool has_forwarded_objects() const;
 331   inline bool is_gc_in_progress_mask(uint mask) const;
 332 
 333 // ---------- GC cancellation and degeneration machinery
 334 //
 335 // Cancelled GC flag is used to notify concurrent phases that they should terminate.
 336 //
 337 public:
 338   enum ShenandoahDegenPoint {
 339     _degenerated_unset,
 340     _degenerated_traversal,
 341     _degenerated_outside_cycle,
 342     _degenerated_mark,
 343     _degenerated_evac,
 344     _degenerated_updaterefs,
 345     _DEGENERATED_LIMIT,
 346   };
 347 
 348   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 349     switch (point) {
 350       case _degenerated_unset:
 351         return "<UNSET>";
 352       case _degenerated_traversal:
 353         return "Traversal";
 354       case _degenerated_outside_cycle:
 355         return "Outside of Cycle";
 356       case _degenerated_mark:
 357         return "Mark";
 358       case _degenerated_evac:
 359         return "Evacuation";
 360       case _degenerated_updaterefs:
 361         return "Update Refs";
 362       default:
 363         ShouldNotReachHere();
 364         return "ERROR";
 365     }
 366   };
 367 
 368 private:
 369   enum CancelState {
 370     // Normal state. GC has not been cancelled and is open for cancellation.
 371     // Worker threads can suspend for safepoint.
 372     CANCELLABLE,
 373 
 374     // GC has been cancelled. Worker threads can not suspend for
 375     // safepoint but must finish their work as soon as possible.
 376     CANCELLED,
 377 
 378     // GC has not been cancelled and must not be cancelled. At least
 379     // one worker thread checks for pending safepoint and may suspend
 380     // if a safepoint is pending.
 381     NOT_CANCELLED
 382   };
 383 
 384   ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;
 385   bool try_cancel_gc();
 386 
 387 public:
 388   static address cancelled_gc_addr();
 389 
 390   inline bool cancelled_gc() const;
 391   inline bool check_cancelled_gc_and_yield(bool sts_active = true);
 392 
 393   inline void clear_cancelled_gc();
 394 
 395   void cancel_gc(GCCause::Cause cause);
 396 
 397 // ---------- GC operations entry points
 398 //
 399 public:
 400   // Entry points to STW GC operations, these cause a related safepoint, that then
 401   // call the entry method below
 402   void vmop_entry_init_mark();
 403   void vmop_entry_final_mark();
 404   void vmop_entry_final_evac();
 405   void vmop_entry_init_updaterefs();
 406   void vmop_entry_final_updaterefs();
 407   void vmop_entry_init_traversal();
 408   void vmop_entry_final_traversal();
 409   void vmop_entry_full(GCCause::Cause cause);
 410   void vmop_degenerated(ShenandoahDegenPoint point);
 411 
 412   // Entry methods to normally STW GC operations. These set up logging, monitoring
 413   // and workers for net VM operation
 414   void entry_init_mark();
 415   void entry_final_mark();
 416   void entry_final_evac();
 417   void entry_init_updaterefs();
 418   void entry_final_updaterefs();
 419   void entry_init_traversal();
 420   void entry_final_traversal();
 421   void entry_full(GCCause::Cause cause);
 422   void entry_degenerated(int point);
 423 
 424   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 425   // for concurrent operation.
 426   void entry_reset();
 427   void entry_mark();
 428   void entry_preclean();
 429   void entry_cleanup();
 430   void entry_evac();
 431   void entry_updaterefs();
 432   void entry_traversal();
 433   void entry_uncommit(double shrink_before);
 434 
 435 private:
 436   // Actual work for the phases
 437   void op_init_mark();
 438   void op_final_mark();
 439   void op_final_evac();
 440   void op_init_updaterefs();
 441   void op_final_updaterefs();
 442   void op_init_traversal();
 443   void op_final_traversal();
 444   void op_full(GCCause::Cause cause);
 445   void op_degenerated(ShenandoahDegenPoint point);
 446   void op_degenerated_fail();
 447   void op_degenerated_futile();
 448 
 449   void op_reset();
 450   void op_mark();
 451   void op_preclean();
 452   void op_cleanup();
 453   void op_conc_evac();
 454   void op_stw_evac();
 455   void op_updaterefs();
 456   void op_traversal();
 457   void op_uncommit(double shrink_before);
 458 
 459   // Messages for GC trace events, they have to be immortal for
 460   // passing around the logging/tracing systems
 461   const char* init_mark_event_message() const;
 462   const char* final_mark_event_message() const;
 463   const char* conc_mark_event_message() const;
 464   const char* degen_event_message(ShenandoahDegenPoint point) const;
 465 
 466 // ---------- GC subsystems
 467 //
 468 private:
 469   ShenandoahControlThread*   _control_thread;
 470   ShenandoahCollectorPolicy* _shenandoah_policy;
 471   ShenandoahHeuristics*      _heuristics;
 472   ShenandoahFreeSet*         _free_set;
 473   ShenandoahConcurrentMark*  _scm;
 474   ShenandoahTraversalGC*     _traversal_gc;
 475   ShenandoahMarkCompact*     _full_gc;
 476   ShenandoahPacer*           _pacer;
 477   ShenandoahVerifier*        _verifier;
 478 
 479   ShenandoahAllocTracker*    _alloc_tracker;
 480   ShenandoahPhaseTimings*    _phase_timings;
 481 
 482   ShenandoahControlThread*   control_thread()          { return _control_thread;    }
 483   ShenandoahMarkCompact*     full_gc()                 { return _full_gc;           }
 484 
 485 public:
 486   ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
 487   ShenandoahHeuristics*      heuristics()        const { return _heuristics;        }
 488   ShenandoahFreeSet*         free_set()          const { return _free_set;          }
 489   ShenandoahConcurrentMark*  concurrent_mark()         { return _scm;               }
 490   ShenandoahTraversalGC*     traversal_gc()            { return _traversal_gc;      }
 491   ShenandoahPacer*           pacer()             const { return _pacer;             }
 492 
 493   ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }
 494   ShenandoahAllocTracker*    alloc_tracker()     const { return _alloc_tracker;     }
 495 
 496   ShenandoahVerifier*        verifier();
 497 
 498 // ---------- VM subsystem bindings
 499 //
 500 private:
 501   ShenandoahMonitoringSupport* _monitoring_support;
 502   MemoryPool*                  _memory_pool;
 503   GCMemoryManager              _stw_memory_manager;
 504   GCMemoryManager              _cycle_memory_manager;
 505   ConcurrentGCTimer*           _gc_timer;
 506   SoftRefPolicy                _soft_ref_policy;
 507 
 508   // For exporting to SA
 509   int                          _log_min_obj_alignment_in_bytes;
 510 public:
 511   ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support;    }
 512   GCMemoryManager* cycle_memory_manager()           { return &_cycle_memory_manager; }
 513   GCMemoryManager* stw_memory_manager()             { return &_stw_memory_manager;   }
 514   SoftRefPolicy* soft_ref_policy()                  { return &_soft_ref_policy;      }
 515 
 516   GrowableArray<GCMemoryManager*> memory_managers();
 517   GrowableArray<MemoryPool*> memory_pools();
 518   MemoryUsage memory_usage();
 519   GCTracer* tracer();
 520   GCTimer* gc_timer() const;
 521   CollectorPolicy* collector_policy() const;
 522 
 523 // ---------- Reference processing
 524 //
 525 private:
 526   AlwaysTrueClosure    _subject_to_discovery;
 527   ReferenceProcessor*  _ref_processor;
 528   ShenandoahSharedFlag _process_references;
 529 
 530   void ref_processing_init();
 531 
 532 public:
 533   ReferenceProcessor* ref_processor() { return _ref_processor; }
 534   void set_process_references(bool pr);
 535   bool process_references() const;
 536 
 537 // ---------- Class Unloading
 538 //
 539 private:
 540   ShenandoahSharedFlag _unload_classes;
 541 
 542 public:
 543   void set_unload_classes(bool uc);
 544   bool unload_classes() const;
 545 
 546   // Delete entries for dead interned string and clean up unreferenced symbols
 547   // in symbol table, possibly in parallel.
 548   void unload_classes_and_cleanup_tables(bool full_gc);
 549 
 550 // ---------- Generic interface hooks
 551 // Minor things that super-interface expects us to implement to play nice with
 552 // the rest of runtime. Some of the things here are not required to be implemented,
 553 // and can be stubbed out.
 554 //
 555 public:
 556   AdaptiveSizePolicy* size_policy() shenandoah_not_implemented_return(NULL);
 557   bool is_maximal_no_gc() const shenandoah_not_implemented_return(false);
 558 
 559   bool is_in(const void* p) const;
 560 
 561   size_t obj_size(oop obj) const;
 562   virtual ptrdiff_t cell_header_size() const;
 563 
 564   void collect(GCCause::Cause cause);
 565   void do_full_collection(bool clear_all_soft_refs);
 566 
 567   // Used for parsing heap during error printing
 568   HeapWord* block_start(const void* addr) const;
 569   bool block_is_obj(const HeapWord* addr) const;
 570 
 571   // Used for native heap walkers: heap dumpers, mostly
 572   void object_iterate(ObjectClosure* cl);
 573   void safe_object_iterate(ObjectClosure* cl);
 574 
 575   // Used by RMI
 576   jlong millis_since_last_gc();
 577 
 578 // ---------- Safepoint interface hooks
 579 //
 580 public:
 581   void safepoint_synchronize_begin();
 582   void safepoint_synchronize_end();
 583 
 584 // ---------- Code roots handling hooks
 585 //
 586 public:
 587   void register_nmethod(nmethod* nm);
 588   void unregister_nmethod(nmethod* nm);
 589   void flush_nmethod(nmethod* nm) {}
 590   void verify_nmethod(nmethod* nm) {}
 591 
 592 // ---------- Pinning hooks
 593 //
 594 public:
 595   // Shenandoah supports per-object (per-region) pinning
 596   bool supports_object_pinning() const { return true; }
 597 
 598   oop pin_object(JavaThread* thread, oop obj);
 599   void unpin_object(JavaThread* thread, oop obj);
 600 
 601 // ---------- Allocation support
 602 //
 603 private:
 604   HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region);
 605   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 606   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 607   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
 608   void retire_and_reset_gclabs();
 609 
 610 public:
 611   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
 612   HeapWord* mem_allocate(size_t size, bool* what);
 613   MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 614                                                size_t size,
 615                                                Metaspace::MetadataType mdtype);
 616 
 617   oop obj_allocate(Klass* klass, int size, TRAPS);
 618   oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS);
 619   oop class_allocate(Klass* klass, int size, TRAPS);
 620 
 621   void notify_mutator_alloc_words(size_t words, bool waste);
 622 
 623   // Shenandoah supports TLAB allocation
 624   bool supports_tlab_allocation() const { return true; }
 625 
 626   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);
 627   size_t tlab_capacity(Thread *thr) const;
 628   size_t unsafe_max_tlab_alloc(Thread *thread) const;
 629   size_t max_tlab_size() const;
 630   size_t tlab_used(Thread* ignored) const;
 631 
 632   HeapWord* tlab_post_allocation_setup(HeapWord* obj);
 633   void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
 634   size_t min_dummy_object_size() const;
 635 
 636   void resize_tlabs();
 637 
 638   void ensure_parsability(bool retire_tlabs);
 639   void make_parsable(bool retire_tlabs);
 640 
 641 // ---------- Marking support
 642 //
 643 private:
 644   ShenandoahMarkingContext* _marking_context;
 645   MemRegion  _bitmap_region;
 646   MemRegion  _aux_bitmap_region;
 647   MarkBitMap _verification_bit_map;
 648   MarkBitMap _aux_bit_map;
 649 
 650   size_t _bitmap_size;
 651   size_t _bitmap_regions_per_slice;
 652   size_t _bitmap_bytes_per_slice;
 653 
 654   bool _bitmap_region_special;
 655   bool _aux_bitmap_region_special;
 656 
 657   // Used for buffering per-region liveness data.
 658   // Needed since ShenandoahHeapRegion uses atomics to update liveness.
 659   //
 660   // The array has max-workers elements, each of which is an array of
 661   // jushort * max_regions. The choice of jushort is not accidental:
 662   // there is a tradeoff between static/dynamic footprint that translates
 663   // into cache pressure (which is already high during marking), and
 664   // too many atomic updates. size_t/jint is too large, jbyte is too small.
 665   jushort** _liveness_cache;
 666 
 667 public:
 668   inline ShenandoahMarkingContext* complete_marking_context() const;
 669   inline ShenandoahMarkingContext* marking_context() const;
 670   inline void mark_complete_marking_context();
 671   inline void mark_incomplete_marking_context();
 672 
 673   template<class T>
 674   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 675 
 676   template<class T>
 677   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 678 
 679   template<class T>
 680   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 681 
 682   void reset_mark_bitmap();
 683 
 684   // SATB barriers hooks
 685   template<bool RESOLVE>
 686   inline bool requires_marking(const void* entry) const;
 687   void force_satb_flush_all_threads();
 688 
 689   // Support for bitmap uncommits
 690   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 691   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 692   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 693 
 694   // Liveness caching support
 695   jushort* get_liveness_cache(uint worker_id);
 696   void flush_liveness_cache(uint worker_id);
 697 
 698 // ---------- Evacuation support
 699 //
 700 private:
 701   ShenandoahCollectionSet* _collection_set;
 702   ShenandoahEvacOOMHandler _oom_evac_handler;
 703 
 704   void evacuate_and_update_roots();
 705 
 706 public:
 707   static address in_cset_fast_test_addr();
 708 
 709   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 710 
 711   template <class T>
 712   inline bool in_collection_set(T obj) const;
 713 
 714   // Avoid accidentally calling the method above with ShenandoahHeapRegion*, which would be *wrong*.
 715   inline bool in_collection_set(ShenandoahHeapRegion* r) shenandoah_not_implemented_return(false);
 716 
 717   // Evacuates object src. Returns the evacuated object, either evacuated
 718   // by this thread, or by some other thread.
 719   inline oop evacuate_object(oop src, Thread* thread);
 720 
 721   // Call before/after evacuation.
 722   void enter_evacuation();
 723   void leave_evacuation();
 724 
 725 // ---------- Helper functions
 726 //
 727 public:
 728   template <class T>
 729   inline oop evac_update_with_forwarded(T* p);
 730 
 731   template <class T>
 732   inline oop maybe_update_with_forwarded(T* p);
 733 
 734   template <class T>
 735   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 736 
 737   template <class T>
 738   inline oop update_with_forwarded_not_null(T* p, oop obj);
 739 
 740   inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
 741   inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
 742 
 743   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 744 
 745   void deduplicate_string(oop str);
 746 
 747   void stop_concurrent_marking();
 748 
 749 private:
 750   void trash_cset_regions();
 751   void update_heap_references(bool concurrent);
 752 
 753 // ---------- Testing helpers functions
 754 //
 755 private:
 756   ShenandoahSharedFlag _inject_alloc_failure;
 757 
 758   void try_inject_alloc_failure();
 759   bool should_inject_alloc_failure();
 760 };
 761 
 762 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP