1 /*
2 * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahClosures.hpp"
29
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "gc/shenandoah/shenandoahAsserts.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
36 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
38 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
39 #include "memory/iterator.inline.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "runtime/atomicAccess.hpp"
42 #include "runtime/javaThread.hpp"
43
44 //
45 // ========= Super
46 //
47
48 ShenandoahSuperClosure::ShenandoahSuperClosure() :
49 MetadataVisitingOopIterateClosure(), _heap(ShenandoahHeap::heap()) {}
50
51 ShenandoahSuperClosure::ShenandoahSuperClosure(ShenandoahReferenceProcessor* rp) :
52 MetadataVisitingOopIterateClosure(rp), _heap(ShenandoahHeap::heap()) {}
53
54 void ShenandoahSuperClosure::do_nmethod(nmethod* nm) {
55 nm->run_nmethod_entry_barrier();
56 }
57
58 //
59 // ========= Marking
60 //
61 ShenandoahFlushSATBHandshakeClosure::ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
62 HandshakeClosure("Shenandoah Flush SATB"),
63 _qset(qset) {}
64
65 void ShenandoahFlushSATBHandshakeClosure::do_thread(Thread* thread) {
66 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
67 }
68
69 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,
70 ShenandoahReferenceProcessor* rp,
71 ShenandoahObjToScanQueue* old_q) :
72 ShenandoahSuperClosure(rp),
73 _queue(q),
74 _old_queue(old_q),
75 _mark_context(ShenandoahHeap::heap()->marking_context()),
76 _weak(false) {}
77
78 template<class T, ShenandoahGenerationType GENERATION>
79 inline void ShenandoahMarkRefsSuperClosure::work(T* p) {
80 ShenandoahMark::mark_through_ref<T, GENERATION>(p, _queue, _old_queue, _mark_context, _weak);
81 }
82
83 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
84 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
85
86 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
87 if (CompressedOops::is_null(obj)) {
88 return false;
89 }
90 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
91 shenandoah_assert_not_forwarded_if(nullptr, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
92 return _mark_context->is_marked_or_old(obj);
93 }
94
95 ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
96 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
97
98 bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
99 if (CompressedOops::is_null(obj)) {
100 return false;
101 }
102 shenandoah_assert_not_forwarded(nullptr, obj);
103 return _mark_context->is_marked_or_old(obj);
104 }
105
106 BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
107 return ShenandoahHeap::heap()->has_forwarded_objects() ?
108 reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl) :
109 reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
110 }
111
112 ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
113 _bs(ShenandoahBarrierSet::barrier_set()) {}
114
115 template <typename T>
116 void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
117 assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase");
118 assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || !ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
119
120 T o = RawAccess<>::oop_load(p);
121 if (!CompressedOops::is_null(o)) {
122 oop obj = CompressedOops::decode_not_null(o);
123 _bs->enqueue(obj);
124 }
125 }
126
127
128 //
129 // ========= Evacuating + Roots
130 //
131
132 template <bool CONCURRENT, bool STABLE_THREAD>
133 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop(oop* p) {
134 do_oop_work(p);
135 }
136
137 template <bool CONCURRENT, bool STABLE_THREAD>
138 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop(narrowOop* p) {
139 do_oop_work(p);
140 }
141
142 template <bool CONCURRENT, bool STABLE_THREAD>
143 template <class T>
144 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop_work(T* p) {
145 assert(_heap->is_evacuation_in_progress(), "Performance: no reason to call this otherwise");
146
147 T o = RawAccess<>::oop_load(p);
148 if (!CompressedOops::is_null(o)) {
149 oop obj = CompressedOops::decode_not_null(o);
150 if (_heap->in_collection_set(obj)) {
151 shenandoah_assert_marked(p, obj);
152 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
153 if (resolved == obj) {
154 Thread* thr = STABLE_THREAD ? _thread : Thread::current();
155 assert(thr == Thread::current(), "Wrong thread");
156
157 resolved = _heap->evacuate_object(obj, thr);
158 }
159 if (CONCURRENT) {
160 ShenandoahHeap::atomic_update_oop(resolved, p, o);
161 } else {
162 RawAccess<IS_NOT_NULL | MO_UNORDERED>::oop_store(p, resolved);
163 }
164 }
165 }
166 }
167
168 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
169 ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::ShenandoahCleanUpdateWeakOopsClosure(IsAlive* is_alive, KeepAlive* keep_alive) :
170 _is_alive(is_alive), _keep_alive(keep_alive) {
171 if (!CONCURRENT) {
172 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
173 }
174 }
175
176 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
177 void ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::do_oop(oop* p) {
178 oop obj = RawAccess<>::oop_load(p);
179 if (!CompressedOops::is_null(obj)) {
180 if (_is_alive->do_object_b(obj)) {
181 _keep_alive->do_oop(p);
182 } else {
183 if (CONCURRENT) {
184 ShenandoahHeap::atomic_clear_oop(p, obj);
185 } else {
186 RawAccess<IS_NOT_NULL>::oop_store(p, oop());
187 }
188 }
189 }
190 }
191
192 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
193 void ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::do_oop(narrowOop* p) {
194 ShouldNotReachHere();
195 }
196
197 ShenandoahNMethodAndDisarmClosure::ShenandoahNMethodAndDisarmClosure(OopClosure* cl) :
198 NMethodToOopClosure(cl, true /* fix_relocations */) {}
199
200 void ShenandoahNMethodAndDisarmClosure::do_nmethod(nmethod* nm) {
201 assert(nm != nullptr, "Sanity");
202 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
203 ShenandoahNMethodLocker locker(data->lock());
204 assert(!data->is_unregistered(), "Should not be here");
205 NMethodToOopClosure::do_nmethod(nm);
206 ShenandoahNMethod::complete_and_disarm_nmethod(nm);
207 }
208
209 //
210 // ========= Update References
211 //
212
213 template <ShenandoahGenerationType GENERATION>
214 ShenandoahMarkUpdateRefsClosure<GENERATION>::ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q,
215 ShenandoahReferenceProcessor* rp,
216 ShenandoahObjToScanQueue* old_q) :
217 ShenandoahMarkRefsSuperClosure(q, rp, old_q) {
218 assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC");
219 }
220
221 template<ShenandoahGenerationType GENERATION>
222 template<class T>
223 inline void ShenandoahMarkUpdateRefsClosure<GENERATION>::work(T* p) {
224 // Update the location
225 _heap->non_conc_update_with_forwarded(p);
226
227 // ...then do the usual thing
228 ShenandoahMarkRefsSuperClosure::work<T, GENERATION>(p);
229 }
230
231 template<class T>
232 inline void ShenandoahNonConcUpdateRefsClosure::work(T* p) {
233 _heap->non_conc_update_with_forwarded(p);
234 }
235
236 template<class T>
237 inline void ShenandoahConcUpdateRefsClosure::work(T* p) {
238 _heap->conc_update_with_forwarded(p);
239 }
240
241 inline void ShenandoahFlushSATB::do_thread(Thread* thread) {
242 // Transfer any partial buffer to the qset for completed buffer processing.
243 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
244 }
245
246 //
247 // ========= Utilities
248 //
249
250 #ifdef ASSERT
251 template <class T>
252 void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {
253 T o = RawAccess<>::oop_load(p);
254 if (!CompressedOops::is_null(o)) {
255 oop obj = CompressedOops::decode_not_null(o);
256 shenandoah_assert_not_forwarded(p, obj);
257 }
258 }
259
260 void ShenandoahAssertNotForwardedClosure::do_oop(narrowOop* p) { do_oop_work(p); }
261 void ShenandoahAssertNotForwardedClosure::do_oop(oop* p) { do_oop_work(p); }
262 #endif
263
264 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP