26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
27 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
28
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "utilities/debug.hpp"
42 #include "utilities/sizes.hpp"
43
44 class ShenandoahThreadLocalData {
45 private:
46 char _gc_state;
47 // Evacuation OOM state
48 uint8_t _oom_scope_nesting_level;
49 bool _oom_during_evac;
50
51 SATBMarkQueue _satb_mark_queue;
52
53 // Current active CardTable's byte_map_base for this thread.
54 CardTable::CardValue* _card_table;
55
56 // Thread-local allocation buffer for object evacuations.
57 // In generational mode, it is exclusive to the young generation.
58 PLAB* _gclab;
59 size_t _gclab_size;
60
61 // Thread-local allocation buffer only used in generational mode.
62 // Used both by mutator threads and by GC worker threads
63 // for evacuations within the old generation and
64 // for promotions from the young generation into the old generation.
65 PLAB* _plab;
66
67 // Heuristics will grow the desired size of plabs.
68 size_t _plab_desired_size;
69
70 // Once the plab has been allocated, and we know the actual size, we record it here.
71 size_t _plab_actual_size;
72
73 // As the plab is used for promotions, this value is incremented. When the plab is
74 // retired, the difference between 'actual_size' and 'promoted' will be returned to
75 // the old generation's promotion reserve (i.e., it will be 'unexpended').
76 size_t _plab_promoted;
77
78 // If false, no more promotion by this thread during this evacuation phase.
79 bool _plab_allows_promotion;
80
81 // If true, evacuations may attempt to allocate a smaller plab if the original size fails.
82 bool _plab_retries_enabled;
83
84 ShenandoahEvacuationStats* _evacuation_stats;
85
86 ShenandoahThreadLocalData();
87 ~ShenandoahThreadLocalData();
88
89 static ShenandoahThreadLocalData* data(Thread* thread) {
90 assert(UseShenandoahGC, "Sanity");
91 return thread->gc_data<ShenandoahThreadLocalData>();
92 }
93
94 static ByteSize satb_mark_queue_offset() {
95 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
96 }
97
98 public:
99 static void create(Thread* thread) {
100 new (data(thread)) ShenandoahThreadLocalData();
101 }
102
103 static void destroy(Thread* thread) {
104 data(thread)->~ShenandoahThreadLocalData();
105 }
106
107 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
108 return data(thread)->_satb_mark_queue;
109 }
110
111 static void set_gc_state(Thread* thread, char gc_state) {
112 data(thread)->_gc_state = gc_state;
113 }
114
115 static char gc_state(Thread* thread) {
116 return data(thread)->_gc_state;
117 }
118
119 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
120 return (gc_state(thread) & state) != 0;
121 }
122
123 static bool is_gc_state(ShenandoahHeap::GCState state) {
124 return is_gc_state(Thread::current(), state);
125 }
126
127 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
128 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
129 data(thread)->_card_table = ct;
130 }
131
132 static CardTable::CardValue* card_table(Thread* thread) {
260 return level;
261 }
262
263 static bool is_evac_allowed(Thread* thread) {
264 return evac_oom_scope_level(thread) > 0;
265 }
266
267 // Offsets
268 static ByteSize satb_mark_queue_index_offset() {
269 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
270 }
271
272 static ByteSize satb_mark_queue_buffer_offset() {
273 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
274 }
275
276 static ByteSize gc_state_offset() {
277 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
278 }
279
280 static ByteSize card_table_offset() {
281 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
282 }
283 };
284
285 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
286
287 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
|
26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
27 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
28
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "utilities/debug.hpp"
42 #include "utilities/sizes.hpp"
43
44 class ShenandoahThreadLocalData {
45 private:
46 // Thread-local mirror for global GC state
47 char _gc_state;
48
49 // Quickened version of GC state, use single bit to check the group of states
50 char _gc_state_fast;
51
52 // Evacuation OOM state
53 uint8_t _oom_scope_nesting_level;
54 bool _oom_during_evac;
55
56 SATBMarkQueue _satb_mark_queue;
57
58 // Current active CardTable's byte_map_base for this thread.
59 CardTable::CardValue* _card_table;
60
61 // Thread-local allocation buffer for object evacuations.
62 // In generational mode, it is exclusive to the young generation.
63 PLAB* _gclab;
64 size_t _gclab_size;
65
66 // Thread-local allocation buffer only used in generational mode.
67 // Used both by mutator threads and by GC worker threads
68 // for evacuations within the old generation and
69 // for promotions from the young generation into the old generation.
70 PLAB* _plab;
71
72 // Heuristics will grow the desired size of plabs.
73 size_t _plab_desired_size;
74
75 // Once the plab has been allocated, and we know the actual size, we record it here.
76 size_t _plab_actual_size;
77
78 // As the plab is used for promotions, this value is incremented. When the plab is
79 // retired, the difference between 'actual_size' and 'promoted' will be returned to
80 // the old generation's promotion reserve (i.e., it will be 'unexpended').
81 size_t _plab_promoted;
82
83 // If false, no more promotion by this thread during this evacuation phase.
84 bool _plab_allows_promotion;
85
86 // If true, evacuations may attempt to allocate a smaller plab if the original size fails.
87 bool _plab_retries_enabled;
88
89 enum FastGCState {
90 FORWARDED = ShenandoahHeap::HAS_FORWARDED,
91 MARKING = ShenandoahHeap::MARKING,
92 WEAK = ShenandoahHeap::WEAK_ROOTS,
93 FORWARDED_OR_MARKING = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING,
94 FORWARDED_OR_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::WEAK_ROOTS,
95 MARKING_OR_WEAK = ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
96 FORWARDED_OR_MARKING_OR_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
97 };
98
99 enum FastGCStateBitPos {
100 FORWARDED_BITPOS = 0,
101 MARKING_BITPOS = 1,
102 WEAK_BITPOS = 2,
103 FORWARDED_OR_MARKING_BITPOS = 3,
104 FORWARDED_OR_WEAK_BITPOS = 4,
105 MARKING_OR_WEAK_BITPOS = 5,
106 FORWARDED_OR_MARKING_OR_WEAK_BITPOS = 6,
107 };
108
109 ShenandoahEvacuationStats* _evacuation_stats;
110
111 ShenandoahThreadLocalData();
112 ~ShenandoahThreadLocalData();
113
114 static ShenandoahThreadLocalData* data(Thread* thread) {
115 assert(UseShenandoahGC, "Sanity");
116 return thread->gc_data<ShenandoahThreadLocalData>();
117 }
118
119 static ByteSize satb_mark_queue_offset() {
120 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
121 }
122
123 public:
124 static void create(Thread* thread) {
125 new (data(thread)) ShenandoahThreadLocalData();
126 }
127
128 static void destroy(Thread* thread) {
129 data(thread)->~ShenandoahThreadLocalData();
130 }
131
132 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
133 return data(thread)->_satb_mark_queue;
134 }
135
136 static char gc_state_to_fast_bit(char gc_state) {
137 if (gc_state == FORWARDED) return FORWARDED_BITPOS;
138 if (gc_state == MARKING) return MARKING_BITPOS;
139 if (gc_state == WEAK) return WEAK_BITPOS;
140 if (gc_state == FORWARDED_OR_MARKING) return FORWARDED_OR_MARKING_BITPOS;
141 if (gc_state == FORWARDED_OR_WEAK) return FORWARDED_OR_WEAK_BITPOS;
142 if (gc_state == MARKING_OR_WEAK) return MARKING_OR_WEAK_BITPOS;
143 if (gc_state == FORWARDED_OR_MARKING_OR_WEAK) return FORWARDED_OR_MARKING_OR_WEAK_BITPOS;
144 ShouldNotReachHere();
145 return 0;
146 }
147
148 static char gc_state_to_fast(char gc_state) {
149 return 1 << gc_state_to_fast_bit(gc_state);
150 }
151
152 static char compute_gc_state_fast(char gc_state) {
153 char fast = 0;
154 if ((gc_state & FORWARDED) > 0) fast |= (1 << FORWARDED_BITPOS);
155 if ((gc_state & MARKING) > 0) fast |= (1 << MARKING_BITPOS);
156 if ((gc_state & WEAK) > 0) fast |= (1 << WEAK_BITPOS);
157 if ((gc_state & FORWARDED_OR_MARKING) > 0) fast |= (1 << FORWARDED_OR_MARKING_BITPOS);
158 if ((gc_state & FORWARDED_OR_WEAK) > 0) fast |= (1 << FORWARDED_OR_WEAK_BITPOS);
159 if ((gc_state & MARKING_OR_WEAK) > 0) fast |= (1 << MARKING_OR_WEAK_BITPOS);
160 if ((gc_state & FORWARDED_OR_MARKING_OR_WEAK) > 0) fast |= (1 << FORWARDED_OR_MARKING_OR_WEAK_BITPOS);
161 return fast;
162 }
163
164 static void set_gc_state(Thread* thread, char gc_state, char gc_state_fast) {
165 data(thread)->_gc_state = gc_state;
166 data(thread)->_gc_state_fast = gc_state_fast;
167 }
168
169 static void set_gc_state(Thread* thread, char gc_state) {
170 set_gc_state(thread, gc_state, compute_gc_state_fast(gc_state));
171 }
172
173 static char gc_state(Thread* thread) {
174 return data(thread)->_gc_state;
175 }
176
177 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
178 return (gc_state(thread) & state) != 0;
179 }
180
181 static bool is_gc_state(ShenandoahHeap::GCState state) {
182 return is_gc_state(Thread::current(), state);
183 }
184
185 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
186 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
187 data(thread)->_card_table = ct;
188 }
189
190 static CardTable::CardValue* card_table(Thread* thread) {
318 return level;
319 }
320
321 static bool is_evac_allowed(Thread* thread) {
322 return evac_oom_scope_level(thread) > 0;
323 }
324
325 // Offsets
326 static ByteSize satb_mark_queue_index_offset() {
327 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
328 }
329
330 static ByteSize satb_mark_queue_buffer_offset() {
331 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
332 }
333
334 static ByteSize gc_state_offset() {
335 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
336 }
337
338 static ByteSize gc_state_fast_offset() {
339 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state_fast);
340 }
341
342 static ByteSize card_table_offset() {
343 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
344 }
345 };
346
347 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
348
349 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
|