1 /*
2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
27 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
28
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/gcThreadLocalData.hpp"
31 #include "gc/shared/plab.hpp"
32 #include "gc/shenandoah/mode/shenandoahMode.hpp"
33 #include "gc/shenandoah/shenandoahAffiliation.hpp"
34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
35 #include "gc/shenandoah/shenandoahCardTable.hpp"
36 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
37 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahPLAB.hpp"
40 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/sizes.hpp"
44
45 class ShenandoahThreadLocalData {
46 private:
47 // Thread-local mirror for global GC state
48 char _gc_state;
49
50 // Quickened version of GC state.
51 // This allows all architectures to quickly check the group of states by using a single byte load.
52 enum FastGCState {
53 FORWARDED = ShenandoahHeap::HAS_FORWARDED,
54 MARKING = ShenandoahHeap::MARKING,
55 WEAK = ShenandoahHeap::WEAK_ROOTS,
56 FORWARDED_MARKING = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING,
57 FORWARDED_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::WEAK_ROOTS,
58 MARKING_WEAK = ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS,
59 FORWARDED_MARKING_WEAK = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING | ShenandoahHeap::WEAK_ROOTS
60 };
61
62 enum FastGCStatePos {
63 POS_FORWARDED = 0,
64 POS_MARKING = 1,
65 POS_WEAK = 2,
66 POS_FORWARDED_MARKING = 3,
67 POS_FORWARDED_WEAK = 4,
68 POS_MARKING_WEAK = 5,
69 POS_FORWARDED_MARKING_WEAK = 6,
70 POS_MAX
71 };
72
73 char _gc_state_fast_array[POS_MAX];
74
75 SATBMarkQueue _satb_mark_queue;
76
77 // Current active CardTable's byte_map_base for this thread.
78 CardTable::CardValue* _card_table;
79
80 // Thread-local allocation buffer for object evacuations.
81 // In generational mode, it is exclusive to the young generation.
82 PLAB* _gclab;
83 size_t _gclab_size;
84
85 // Thread-local allocation buffer only used in generational mode.
86 // Used both by mutator threads and by GC worker threads
87 // for evacuations within the old generation and
88 // for promotions from the young generation into the old generation.
89 ShenandoahPLAB* _shenandoah_plab;
90
91 ShenandoahEvacuationStats* _evacuation_stats;
92
93 Atomic<HeapWord*> _invisible_root;
94 Atomic<size_t> _invisible_root_word_size;
95
96 ShenandoahThreadLocalData();
97 ~ShenandoahThreadLocalData();
98
99 static ShenandoahThreadLocalData* data(Thread* thread) {
100 assert(UseShenandoahGC, "Sanity");
101 return thread->gc_data<ShenandoahThreadLocalData>();
102 }
103
104 static ByteSize satb_mark_queue_offset() {
105 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _satb_mark_queue);
106 }
107
108 public:
109 static void create(Thread* thread) {
110 new (data(thread)) ShenandoahThreadLocalData();
111 }
112
113 static void destroy(Thread* thread) {
114 data(thread)->~ShenandoahThreadLocalData();
115 }
116
117 static SATBMarkQueue& satb_mark_queue(Thread* thread) {
118 return data(thread)->_satb_mark_queue;
119 }
120
121 static char gc_state_to_fast_array_index(char gc_state) {
122 if (gc_state == FORWARDED) return POS_FORWARDED;
123 if (gc_state == MARKING) return POS_MARKING;
124 if (gc_state == WEAK) return POS_WEAK;
125 if (gc_state == FORWARDED_MARKING) return POS_FORWARDED_MARKING;
126 if (gc_state == FORWARDED_WEAK) return POS_FORWARDED_WEAK;
127 if (gc_state == MARKING_WEAK) return POS_MARKING_WEAK;
128 if (gc_state == FORWARDED_MARKING_WEAK) return POS_FORWARDED_MARKING_WEAK;
129 ShouldNotReachHere();
130 return 0;
131 }
132
133 static char fast_array_index_to_gc_state(int index) {
134 if (index == POS_FORWARDED) return FORWARDED;
135 if (index == POS_MARKING) return MARKING;
136 if (index == POS_WEAK) return WEAK;
137 if (index == POS_FORWARDED_MARKING) return FORWARDED_MARKING;
138 if (index == POS_FORWARDED_WEAK) return FORWARDED_WEAK;
139 if (index == POS_MARKING_WEAK) return MARKING_WEAK;
140 if (index == POS_FORWARDED_MARKING_WEAK) return FORWARDED_MARKING_WEAK;
141 ShouldNotReachHere();
142 return 0;
143 }
144
145 static void set_gc_state(Thread* thread, char gc_state) {
146 ShenandoahThreadLocalData* d = data(thread);
147 d->_gc_state = gc_state;
148 d->_gc_state_fast_array[POS_FORWARDED] = (gc_state & FORWARDED) != 0;
149 d->_gc_state_fast_array[POS_MARKING] = (gc_state & MARKING) != 0;
150 d->_gc_state_fast_array[POS_WEAK] = (gc_state & WEAK) != 0;
151 d->_gc_state_fast_array[POS_FORWARDED_MARKING] = (gc_state & FORWARDED_MARKING) != 0;
152 d->_gc_state_fast_array[POS_FORWARDED_WEAK] = (gc_state & FORWARDED_WEAK) != 0;
153 d->_gc_state_fast_array[POS_MARKING_WEAK] = (gc_state & MARKING_WEAK) != 0;
154 d->_gc_state_fast_array[POS_FORWARDED_MARKING_WEAK] = (gc_state & FORWARDED_MARKING_WEAK) != 0;
155 }
156
157 static char gc_state(Thread* thread) {
158 return data(thread)->_gc_state;
159 }
160
161 static bool is_gc_state(Thread* thread, ShenandoahHeap::GCState state) {
162 return (gc_state(thread) & state) != 0;
163 }
164
165 static bool is_gc_state(ShenandoahHeap::GCState state) {
166 return is_gc_state(Thread::current(), state);
167 }
168
169 static void set_card_table(Thread* thread, CardTable::CardValue* ct) {
170 assert(ct != nullptr, "trying to set thread local card_table pointer to nullptr.");
171 data(thread)->_card_table = ct;
172 }
173
174 static CardTable::CardValue* card_table(Thread* thread) {
175 CardTable::CardValue* ct = data(thread)->_card_table;
176 assert(ct != nullptr, "returning a null thread local card_table pointer.");
177 return ct;
178 }
179
180 static void initialize_gclab(Thread* thread) {
181 assert(data(thread)->_gclab == nullptr, "Only initialize once");
182 data(thread)->_gclab = new PLAB(PLAB::min_size());
183 data(thread)->_gclab_size = 0;
184
185 if (ShenandoahHeap::heap()->mode()->is_generational()) {
186 data(thread)->_shenandoah_plab = new ShenandoahPLAB();
187 }
188 }
189
190 static PLAB* gclab(Thread* thread) {
191 return data(thread)->_gclab;
192 }
193
194 static size_t gclab_size(Thread* thread) {
195 return data(thread)->_gclab_size;
196 }
197
198 static void set_gclab_size(Thread* thread, size_t v) {
199 data(thread)->_gclab_size = v;
200 }
201
202 static void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
203 data(thread)->_evacuation_stats->begin_evacuation(bytes, from, to);
204 }
205
206 static void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
207 data(thread)->_evacuation_stats->end_evacuation(bytes, from, to);
208 }
209
210 static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) {
211 return data(thread)->_evacuation_stats;
212 }
213
214 static ShenandoahPLAB* shenandoah_plab(Thread* thread) {
215 return data(thread)->_shenandoah_plab;
216 }
217
218 // Offsets
219 static ByteSize satb_mark_queue_index_offset() {
220 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
221 }
222
223 static ByteSize satb_mark_queue_buffer_offset() {
224 return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf();
225 }
226
227 static ByteSize gc_state_offset() {
228 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state);
229 }
230
231 static ByteSize gc_state_fast_array_offset(char gc_state) {
232 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state_fast_array) + in_ByteSize(gc_state_to_fast_array_index(gc_state));
233 }
234
235 static ByteSize card_table_offset() {
236 return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
237 }
238
239 // invisible root are the partially initialized obj array set by ShenandoahObjArrayAllocator
240 static void set_invisible_root(Thread* thread, HeapWord* invisible_root, size_t word_size) {
241 data(thread)->_invisible_root.store_relaxed(invisible_root);
242 data(thread)->_invisible_root_word_size.store_relaxed(word_size);
243 }
244
245 static void clear_invisible_root(Thread* thread) {
246 data(thread)->_invisible_root.store_relaxed(nullptr);
247 data(thread)->_invisible_root_word_size.store_relaxed(0);
248 }
249
250 static HeapWord* get_invisible_root(Thread* thread) {
251 return data(thread)->_invisible_root.load_relaxed();
252 }
253
254 static size_t get_invisible_root_word_size(Thread* thread) {
255 return data(thread)->_invisible_root_word_size.load_relaxed();
256 }
257 };
258
259 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
260
261 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP