1 /*
2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
27 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
28
29 #include "gc/shared/plab.hpp"
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/gc_globals.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahCardTable.hpp"
34 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
35 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
36 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
37 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
38 #include "gc/shenandoah/mode/shenandoahMode.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "utilities/debug.hpp"
41 #include "utilities/sizes.hpp"
42
43 class ShenandoahThreadLocalData {
44 private:
45 char _gc_state;
46 // Evacuation OOM state
47 uint8_t _oom_scope_nesting_level;
48 bool _oom_during_evac;
49
50 SATBMarkQueue _satb_mark_queue;
51
52 // Current active CardTable's byte_map_base for this thread.
53 CardTable::CardValue* _card_table;
54
55 // Thread-local allocation buffer for object evacuations.
56 // In generational mode, it is exclusive to the young generation.
57 PLAB* _gclab;
58 size_t _gclab_size;
59
60 double _paced_time;
61
62 // Thread-local allocation buffer only used in generational mode.
63 // Used both by mutator threads and by GC worker threads
64 // for evacuations within the old generation and
65 // for promotions from the young generation into the old generation.
66 PLAB* _plab;
67
68 // Heuristics will grow the desired size of plabs.
69 size_t _plab_desired_size;
70
71 // Once the plab has been allocated, and we know the actual size, we record it here.
72 size_t _plab_actual_size;
73
74 // As the plab is used for promotions, this value is incremented. When the plab is
75 // retired, the difference between 'actual_size' and 'promoted' will be returned to
76 // the old generation's promotion reserve (i.e., it will be 'unexpended').
77 size_t _plab_promoted;
78
79 // If false, no more promotion by this thread during this evacuation phase.
80 bool _plab_allows_promotion;
81
82 // If true, evacuations may attempt to allocate a smaller plab if the original size fails.
83 bool _plab_retries_enabled;
84
85 ShenandoahEvacuationStats* _evacuation_stats;
86
87 ShenandoahThreadLocalData();
88 ~ShenandoahThreadLocalData();
89
90 static ShenandoahThreadLocalData* data(Thread* thread) {
91 assert(UseShenandoahGC, "Sanity");
92 return thread->gc_data<ShenandoahThreadLocalData>();
93 }
94
95 static ByteSize satb_mark_queue_offset() {
96 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
97 }
98
99 public:
100 static void create(Thread* thread) {
101 new (data(thread)) ShenandoahThreadLocalData();
102 }
103
104 static void destroy(Thread* thread) {
105 data(thread)->~ShenandoahThreadLocalData();
106 }
107
108 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
109 return data(thread)->_satb_mark_queue;
110 }
111
112 static void set_gc_state(Thread* thread, char gc_state) {
113 data(thread)->_gc_state = gc_state;
114 }
115
116 static char gc_state(Thread* thread) {
117 return data(thread)->_gc_state;
118 }
119
120 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
121 return (gc_state(thread) & state) != 0;
122 }
123
124 static bool is_gc_state(ShenandoahHeap::GCState state) {
125 return is_gc_state(Thread::current(), state);
126 }
127
128 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
129 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
130 data(thread)->_card_table = ct;
131 }
132
133 static CardTable::CardValue* card_table(Thread* thread) {
134 CardTable::CardValue* ct = data(thread)->_card_table;
135 assert(ct != nullptr, "returning a null thread local card_table pointer.");
136 return ct;
137 }
138
139 static void initialize_gclab(Thread* thread) {
140 assert(data(thread)->_gclab == nullptr, "Only initialize once");
141 data(thread)->_gclab = new PLAB(PLAB::min_size());
142 data(thread)->_gclab_size = 0;
143
144 if (ShenandoahHeap::heap()->mode()->is_generational()) {
145 data(thread)->_plab = new PLAB(align_up(PLAB::min_size(), CardTable::card_size_in_words()));
146 data(thread)->_plab_desired_size = 0;
147 }
148 }
149
150 static PLAB* gclab(Thread* thread) {
151 return data(thread)->_gclab;
152 }
153
154 static size_t gclab_size(Thread* thread) {
155 return data(thread)->_gclab_size;
156 }
157
158 static void set_gclab_size(Thread* thread, size_t v) {
159 data(thread)->_gclab_size = v;
160 }
161
162 static void begin_evacuation(Thread* thread, size_t bytes) {
163 data(thread)->_evacuation_stats->begin_evacuation(bytes);
164 }
165
166 static void end_evacuation(Thread* thread, size_t bytes) {
167 data(thread)->_evacuation_stats->end_evacuation(bytes);
168 }
169
170 static void record_age(Thread* thread, size_t bytes, uint age) {
171 data(thread)->_evacuation_stats->record_age(bytes, age);
172 }
173
174 static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) {
175 shenandoah_assert_generational();
176 return data(thread)->_evacuation_stats;
177 }
178
179 static PLAB* plab(Thread* thread) {
180 return data(thread)->_plab;
181 }
182
183 static size_t plab_size(Thread* thread) {
184 return data(thread)->_plab_desired_size;
185 }
186
187 static void set_plab_size(Thread* thread, size_t v) {
188 data(thread)->_plab_desired_size = v;
189 }
190
191 static void enable_plab_retries(Thread* thread) {
192 data(thread)->_plab_retries_enabled = true;
193 }
194
195 static void disable_plab_retries(Thread* thread) {
196 data(thread)->_plab_retries_enabled = false;
197 }
198
199 static bool plab_retries_enabled(Thread* thread) {
200 return data(thread)->_plab_retries_enabled;
201 }
202
203 static void enable_plab_promotions(Thread* thread) {
204 data(thread)->_plab_allows_promotion = true;
205 }
206
207 static void disable_plab_promotions(Thread* thread) {
208 data(thread)->_plab_allows_promotion = false;
209 }
210
211 static bool allow_plab_promotions(Thread* thread) {
212 return data(thread)->_plab_allows_promotion;
213 }
214
215 static void reset_plab_promoted(Thread* thread) {
216 data(thread)->_plab_promoted = 0;
217 }
218
219 static void add_to_plab_promoted(Thread* thread, size_t increment) {
220 data(thread)->_plab_promoted += increment;
221 }
222
223 static void subtract_from_plab_promoted(Thread* thread, size_t increment) {
224 assert(data(thread)->_plab_promoted >= increment, "Cannot subtract more than remaining promoted");
225 data(thread)->_plab_promoted -= increment;
226 }
227
228 static size_t get_plab_promoted(Thread* thread) {
229 return data(thread)->_plab_promoted;
230 }
231
232 static void set_plab_actual_size(Thread* thread, size_t value) {
233 data(thread)->_plab_actual_size = value;
234 }
235
236 static size_t get_plab_actual_size(Thread* thread) {
237 return data(thread)->_plab_actual_size;
238 }
239
240 static void add_paced_time(Thread* thread, double v) {
241 data(thread)->_paced_time += v;
242 }
243
244 static double paced_time(Thread* thread) {
245 return data(thread)->_paced_time;
246 }
247
248 static void reset_paced_time(Thread* thread) {
249 data(thread)->_paced_time = 0;
250 }
251
252 // Evacuation OOM handling
253 static bool is_oom_during_evac(Thread* thread) {
254 return data(thread)->_oom_during_evac;
255 }
256
257 static void set_oom_during_evac(Thread* thread, bool oom) {
258 data(thread)->_oom_during_evac = oom;
259 }
260
261 static uint8_t evac_oom_scope_level(Thread* thread) {
262 return data(thread)->_oom_scope_nesting_level;
263 }
264
265 // Push the scope one level deeper, return previous level
266 static uint8_t push_evac_oom_scope(Thread* thread) {
267 uint8_t level = evac_oom_scope_level(thread);
268 assert(level < 254, "Overflow nesting level"); // UINT8_MAX = 255
269 data(thread)->_oom_scope_nesting_level = level + 1;
270 return level;
271 }
272
273 // Pop the scope by one level, return previous level
274 static uint8_t pop_evac_oom_scope(Thread* thread) {
275 uint8_t level = evac_oom_scope_level(thread);
276 assert(level > 0, "Underflow nesting level");
277 data(thread)->_oom_scope_nesting_level = level - 1;
278 return level;
279 }
280
281 static bool is_evac_allowed(Thread* thread) {
282 return evac_oom_scope_level(thread) > 0;
283 }
284
285 // Offsets
286 static ByteSize satb_mark_queue_active_offset() {
287 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active();
288 }
289
290 static ByteSize satb_mark_queue_index_offset() {
291 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
292 }
293
294 static ByteSize satb_mark_queue_buffer_offset() {
295 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
296 }
297
298 static ByteSize gc_state_offset() {
299 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
300 }
301
302 static ByteSize card_table_offset() {
303 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
304 }
305 };
306
307 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
308
309 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP