1 /*
2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
27 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
28
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahPLAB.hpp"
40 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/sizes.hpp"
44
45 class ShenandoahThreadLocalData {
46 private:
47 // Thread-local mirror for global GC state
48 char _gc_state;
49
50 // Quickened version of GC state, use single bit to check the group of states
51 char _gc_state_fast;
52
53 // Evacuation OOM state
54 uint8_t _oom_scope_nesting_level;
55 bool _oom_during_evac;
56
57 SATBMarkQueue _satb_mark_queue;
58
59 // Current active CardTable's byte_map_base for this thread.
60 CardTable::CardValue* _card_table;
61
62 // Thread-local allocation buffer for object evacuations.
63 // In generational mode, it is exclusive to the young generation.
64 PLAB* _gclab;
65 size_t _gclab_size;
66
67 // Thread-local allocation buffer only used in generational mode.
68 // Used both by mutator threads and by GC worker threads
69 // for evacuations within the old generation and
70 // for promotions from the young generation into the old generation.
71 ShenandoahPLAB* _shenandoah_plab;
72
73 enum FastGCState {
74 FORWARDED = ShenandoahHeap::HAS_FORWARDED,
75 MARKING = ShenandoahHeap::MARKING,
76 WEAK = ShenandoahHeap::WEAK_ROOTS,
77 FORWARDED_OR_MARKING = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING,
78 FORWARDED_OR_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::WEAK_ROOTS,
79 MARKING_OR_WEAK = ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
80 FORWARDED_OR_MARKING_OR_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
81 };
82
83 enum FastGCStateBitPos {
84 FORWARDED_BITPOS = 0,
85 MARKING_BITPOS = 1,
86 WEAK_BITPOS = 2,
87 FORWARDED_OR_MARKING_BITPOS = 3,
88 FORWARDED_OR_WEAK_BITPOS = 4,
89 MARKING_OR_WEAK_BITPOS = 5,
90 FORWARDED_OR_MARKING_OR_WEAK_BITPOS = 6,
91 };
92
93 ShenandoahEvacuationStats* _evacuation_stats;
94
95 ShenandoahThreadLocalData();
96 ~ShenandoahThreadLocalData();
97
98 static ShenandoahThreadLocalData* data(Thread* thread) {
99 assert(UseShenandoahGC, "Sanity");
100 return thread->gc_data<ShenandoahThreadLocalData>();
101 }
102
103 static ByteSize satb_mark_queue_offset() {
104 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
105 }
106
107 public:
108 static void create(Thread* thread) {
109 new (data(thread)) ShenandoahThreadLocalData();
110 }
111
112 static void destroy(Thread* thread) {
113 data(thread)->~ShenandoahThreadLocalData();
114 }
115
116 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
117 return data(thread)->_satb_mark_queue;
118 }
119
120 static char gc_state_to_fast_bit(char gc_state) {
121 if (gc_state == FORWARDED) return FORWARDED_BITPOS;
122 if (gc_state == MARKING) return MARKING_BITPOS;
123 if (gc_state == WEAK) return WEAK_BITPOS;
124 if (gc_state == FORWARDED_OR_MARKING) return FORWARDED_OR_MARKING_BITPOS;
125 if (gc_state == FORWARDED_OR_WEAK) return FORWARDED_OR_WEAK_BITPOS;
126 if (gc_state == MARKING_OR_WEAK) return MARKING_OR_WEAK_BITPOS;
127 if (gc_state == FORWARDED_OR_MARKING_OR_WEAK) return FORWARDED_OR_MARKING_OR_WEAK_BITPOS;
128 ShouldNotReachHere();
129 return 0;
130 }
131
132 static char gc_state_to_fast(char gc_state) {
133 return 1 << gc_state_to_fast_bit(gc_state);
134 }
135
136 static char compute_gc_state_fast(char gc_state) {
137 char fast = 0;
138 if ((gc_state & FORWARDED) > 0) fast |= (1 << FORWARDED_BITPOS);
139 if ((gc_state & MARKING) > 0) fast |= (1 << MARKING_BITPOS);
140 if ((gc_state & WEAK) > 0) fast |= (1 << WEAK_BITPOS);
141 if ((gc_state & FORWARDED_OR_MARKING) > 0) fast |= (1 << FORWARDED_OR_MARKING_BITPOS);
142 if ((gc_state & FORWARDED_OR_WEAK) > 0) fast |= (1 << FORWARDED_OR_WEAK_BITPOS);
143 if ((gc_state & MARKING_OR_WEAK) > 0) fast |= (1 << MARKING_OR_WEAK_BITPOS);
144 if ((gc_state & FORWARDED_OR_MARKING_OR_WEAK) > 0) fast |= (1 << FORWARDED_OR_MARKING_OR_WEAK_BITPOS);
145 return fast;
146 }
147
148 static void set_gc_state(Thread* thread, char gc_state, char gc_state_fast) {
149 data(thread)->_gc_state = gc_state;
150 data(thread)->_gc_state_fast = gc_state_fast;
151 }
152
153 static void set_gc_state(Thread* thread, char gc_state) {
154 set_gc_state(thread, gc_state, compute_gc_state_fast(gc_state));
155 }
156
157 static char gc_state(Thread* thread) {
158 return data(thread)->_gc_state;
159 }
160
161 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
162 return (gc_state(thread) & state) != 0;
163 }
164
165 static bool is_gc_state(ShenandoahHeap::GCState state) {
166 return is_gc_state(Thread::current(), state);
167 }
168
169 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
170 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
171 data(thread)->_card_table = ct;
172 }
173
174 static CardTable::CardValue* card_table(Thread* thread) {
175 CardTable::CardValue* ct = data(thread)->_card_table;
176 assert(ct != nullptr, "returning a null thread local card_table pointer.");
177 return ct;
178 }
179
180 static void initialize_gclab(Thread* thread) {
181 assert(data(thread)->_gclab == nullptr, "Only initialize once");
182 data(thread)->_gclab = new PLAB(PLAB::min_size());
183 data(thread)->_gclab_size = 0;
184
185 if (ShenandoahHeap::heap()->mode()->is_generational()) {
186 data(thread)->_shenandoah_plab = new ShenandoahPLAB();
187 }
188 }
189
190 static PLAB* gclab(Thread* thread) {
191 return data(thread)->_gclab;
192 }
193
194 static size_t gclab_size(Thread* thread) {
195 return data(thread)->_gclab_size;
196 }
197
198 static void set_gclab_size(Thread* thread, size_t v) {
199 data(thread)->_gclab_size = v;
200 }
201
202 static void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
203 data(thread)->_evacuation_stats->begin_evacuation(bytes, from, to);
204 }
205
206 static void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
207 data(thread)->_evacuation_stats->end_evacuation(bytes, from, to);
208 }
209
210 static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) {
211 return data(thread)->_evacuation_stats;
212 }
213
214 static ShenandoahPLAB* shenandoah_plab(Thread* thread) {
215 return data(thread)->_shenandoah_plab;
216 }
217
218 // Evacuation OOM handling
219 static bool is_oom_during_evac(Thread* thread) {
220 return data(thread)->_oom_during_evac;
221 }
222
223 static void set_oom_during_evac(Thread* thread, bool oom) {
224 data(thread)->_oom_during_evac = oom;
225 }
226
227 static uint8_t evac_oom_scope_level(Thread* thread) {
228 return data(thread)->_oom_scope_nesting_level;
229 }
230
231 // Push the scope one level deeper, return previous level
232 static uint8_t push_evac_oom_scope(Thread* thread) {
233 uint8_t level = evac_oom_scope_level(thread);
234 assert(level < 254, "Overflow nesting level"); // UINT8_MAX = 255
235 data(thread)->_oom_scope_nesting_level = level + 1;
236 return level;
237 }
238
239 // Pop the scope by one level, return previous level
240 static uint8_t pop_evac_oom_scope(Thread* thread) {
241 uint8_t level = evac_oom_scope_level(thread);
242 assert(level > 0, "Underflow nesting level");
243 data(thread)->_oom_scope_nesting_level = level - 1;
244 return level;
245 }
246
247 static bool is_evac_allowed(Thread* thread) {
248 return evac_oom_scope_level(thread) > 0;
249 }
250
251 // Offsets
252 static ByteSize satb_mark_queue_index_offset() {
253 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
254 }
255
256 static ByteSize satb_mark_queue_buffer_offset() {
257 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
258 }
259
260 static ByteSize gc_state_offset() {
261 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
262 }
263
264 static ByteSize gc_state_fast_offset() {
265 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state_fast);
266 }
267
268 static ByteSize card_table_offset() {
269 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
270 }
271 };
272
273 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
274
275 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP