30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahPLAB.hpp"
40 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/sizes.hpp"
44
45 class ShenandoahThreadLocalData {
46 private:
47 // Thread-local mirror for global GC state
48 char _gc_state;
49
50 // Quickened version of GC state.
51 // This allows all architectures to quickly check the group of states by using a single byte load.
52 enum FastGCState {
53 FORWARDED = ShenandoahHeap::HAS_FORWARDED,
54 MARKING = ShenandoahHeap::MARKING,
55 WEAK = ShenandoahHeap::WEAK_ROOTS,
56 FORWARDED_MARKING = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING,
57 FORWARDED_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::WEAK_ROOTS,
58 MARKING_WEAK = ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
59 FORWARDED_MARKING_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS
60 };
61
62 enum FastGCStatePos {
63 POS_FORWARDED = 0,
64 POS_MARKING = 1,
65 POS_WEAK = 2,
66 POS_FORWARDED_MARKING = 3,
67 POS_FORWARDED_WEAK = 4,
68 POS_MARKING_WEAK = 5,
69 POS_FORWARDED_MARKING_WEAK = 6,
70 POS_MAX
71 };
72
73 char _gc_state_fast_array[POS_MAX];
74
75 SATBMarkQueue _satb_mark_queue;
76
77 // Current active CardTable's byte_map_base for this thread.
78 CardTable::CardValue* _card_table;
79
80 // Thread-local allocation buffer for object evacuations.
81 // In generational mode, it is exclusive to the young generation.
82 PLAB* _gclab;
83 size_t _gclab_size;
84
85 // Thread-local allocation buffer only used in generational mode.
86 // Used both by mutator threads and by GC worker threads
87 // for evacuations within the old generation and
88 // for promotions from the young generation into the old generation.
89 ShenandoahPLAB* _shenandoah_plab;
90
91 ShenandoahEvacuationStats* _evacuation_stats;
92
93 Atomic<HeapWord*> _invisible_root;
94 Atomic<size_t> _invisible_root_word_size;
107 return thread->gc_data<ShenandoahThreadLocalData>();
108 }
109
110 static ByteSize satb_mark_queue_offset() {
111 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
112 }
113
114 public:
115 static void create(Thread* thread) {
116 new (data(thread)) ShenandoahThreadLocalData();
117 }
118
119 static void destroy(Thread* thread) {
120 data(thread)->~ShenandoahThreadLocalData();
121 }
122
123 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
124 return data(thread)->_satb_mark_queue;
125 }
126
127 static char gc_state_to_fast_array_index(char gc_state) {
128 if (gc_state == FORWARDED) return POS_FORWARDED;
129 if (gc_state == MARKING) return POS_MARKING;
130 if (gc_state == WEAK) return POS_WEAK;
131 if (gc_state == FORWARDED_MARKING) return POS_FORWARDED_MARKING;
132 if (gc_state == FORWARDED_WEAK) return POS_FORWARDED_WEAK;
133 if (gc_state == MARKING_WEAK) return POS_MARKING_WEAK;
134 if (gc_state == FORWARDED_MARKING_WEAK) return POS_FORWARDED_MARKING_WEAK;
135 ShouldNotReachHere();
136 return 0;
137 }
138
139 static void set_gc_state(Thread* thread, char gc_state) {
140 ShenandoahThreadLocalData* d = data(thread);
141 d->_gc_state = gc_state;
142 d->_gc_state_fast_array[POS_FORWARDED] = (gc_state & FORWARDED) != 0;
143 d->_gc_state_fast_array[POS_MARKING] = (gc_state & MARKING) != 0;
144 d->_gc_state_fast_array[POS_WEAK] = (gc_state & WEAK) != 0;
145 d->_gc_state_fast_array[POS_FORWARDED_MARKING] = (gc_state & FORWARDED_MARKING) != 0;
146 d->_gc_state_fast_array[POS_FORWARDED_WEAK] = (gc_state & FORWARDED_WEAK) != 0;
147 d->_gc_state_fast_array[POS_MARKING_WEAK] = (gc_state & MARKING_WEAK) != 0;
148 d->_gc_state_fast_array[POS_FORWARDED_MARKING_WEAK] = (gc_state & FORWARDED_MARKING_WEAK) != 0;
149 }
150
151 static char gc_state(Thread* thread) {
152 return data(thread)->_gc_state;
153 }
154
155 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
156 return (gc_state(thread) & state) != 0;
157 }
158
159 static bool is_gc_state(ShenandoahHeap::GCState state) {
160 return is_gc_state(Thread::current(), state);
161 }
162
163 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
164 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
165 data(thread)->_card_table = ct;
166 }
167
168 static CardTable::CardValue* card_table(Thread* thread) {
205 return data(thread)->_evacuation_stats;
206 }
207
208 static ShenandoahPLAB* shenandoah_plab(Thread* thread) {
209 return data(thread)->_shenandoah_plab;
210 }
211
212 // Offsets
213 static ByteSize satb_mark_queue_index_offset() {
214 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
215 }
216
217 static ByteSize satb_mark_queue_buffer_offset() {
218 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
219 }
220
221 static ByteSize gc_state_offset() {
222 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
223 }
224
225 static ByteSize gc_state_fast_array_offset(char gc_state) {
226 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state_fast_array) + in_ByteSize(gc_state_to_fast_array_index(gc_state));
227 }
228
229 static ByteSize card_table_offset() {
230 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
231 }
232
233 // invisible root are the partially initialized obj array set by ShenandoahObjArrayAllocator
234 static void set_invisible_root(Thread* thread, HeapWord* invisible_root, size_t word_size) {
235 data(thread)->_invisible_root.store_relaxed(invisible_root);
236 data(thread)->_invisible_root_word_size.store_relaxed(word_size);
237 }
238
239 static void clear_invisible_root(Thread* thread) {
240 data(thread)->_invisible_root.store_relaxed(nullptr);
241 data(thread)->_invisible_root_word_size.store_relaxed(0);
242 }
243
244 static HeapWord* get_invisible_root(Thread* thread) {
245 return data(thread)->_invisible_root.load_relaxed();
246 }
247
248 static size_t get_invisible_root_word_size(Thread* thread) {
|
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahPLAB.hpp"
40 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/sizes.hpp"
44
45 class ShenandoahThreadLocalData {
46 private:
47 // Thread-local mirror for global GC state
48 char _gc_state;
49
50 SATBMarkQueue _satb_mark_queue;
51
52 // Current active CardTable's byte_map_base for this thread.
53 CardTable::CardValue* _card_table;
54
55 // Thread-local allocation buffer for object evacuations.
56 // In generational mode, it is exclusive to the young generation.
57 PLAB* _gclab;
58 size_t _gclab_size;
59
60 // Thread-local allocation buffer only used in generational mode.
61 // Used both by mutator threads and by GC worker threads
62 // for evacuations within the old generation and
63 // for promotions from the young generation into the old generation.
64 ShenandoahPLAB* _shenandoah_plab;
65
66 ShenandoahEvacuationStats* _evacuation_stats;
67
68 Atomic<HeapWord*> _invisible_root;
69 Atomic<size_t> _invisible_root_word_size;
82 return thread->gc_data<ShenandoahThreadLocalData>();
83 }
84
85 static ByteSize satb_mark_queue_offset() {
86 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
87 }
88
89 public:
90 static void create(Thread* thread) {
91 new (data(thread)) ShenandoahThreadLocalData();
92 }
93
94 static void destroy(Thread* thread) {
95 data(thread)->~ShenandoahThreadLocalData();
96 }
97
98 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
99 return data(thread)->_satb_mark_queue;
100 }
101
102 static void set_gc_state(Thread* thread, char gc_state) {
103 ShenandoahThreadLocalData* d = data(thread);
104 d->_gc_state = gc_state;
105 }
106
107 static char gc_state(Thread* thread) {
108 return data(thread)->_gc_state;
109 }
110
111 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
112 return (gc_state(thread) & state) != 0;
113 }
114
115 static bool is_gc_state(ShenandoahHeap::GCState state) {
116 return is_gc_state(Thread::current(), state);
117 }
118
119 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
120 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
121 data(thread)->_card_table = ct;
122 }
123
124 static CardTable::CardValue* card_table(Thread* thread) {
161 return data(thread)->_evacuation_stats;
162 }
163
164 static ShenandoahPLAB* shenandoah_plab(Thread* thread) {
165 return data(thread)->_shenandoah_plab;
166 }
167
168 // Offsets
169 static ByteSize satb_mark_queue_index_offset() {
170 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
171 }
172
173 static ByteSize satb_mark_queue_buffer_offset() {
174 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
175 }
176
177 static ByteSize gc_state_offset() {
178 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
179 }
180
181 static ByteSize card_table_offset() {
182 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
183 }
184
185 // invisible root are the partially initialized obj array set by ShenandoahObjArrayAllocator
186 static void set_invisible_root(Thread* thread, HeapWord* invisible_root, size_t word_size) {
187 data(thread)->_invisible_root.store_relaxed(invisible_root);
188 data(thread)->_invisible_root_word_size.store_relaxed(word_size);
189 }
190
191 static void clear_invisible_root(Thread* thread) {
192 data(thread)->_invisible_root.store_relaxed(nullptr);
193 data(thread)->_invisible_root_word_size.store_relaxed(0);
194 }
195
196 static HeapWord* get_invisible_root(Thread* thread) {
197 return data(thread)->_invisible_root.load_relaxed();
198 }
199
200 static size_t get_invisible_root_word_size(Thread* thread) {
|