1 /*
2 * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahClosures.hpp"
29
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "gc/shenandoah/shenandoahAsserts.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahMark.inline.hpp"
36 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
39 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
40 #include "memory/iterator.inline.hpp"
41 #include "oops/compressedOops.inline.hpp"
42 #include "runtime/atomicAccess.hpp"
43 #include "runtime/javaThread.hpp"
44
45 //
46 // ========= Super
47 //
48
49 ShenandoahSuperClosure::ShenandoahSuperClosure() :
50 MetadataVisitingOopIterateClosure(), _heap(ShenandoahHeap::heap()) {}
51
52 ShenandoahSuperClosure::ShenandoahSuperClosure(ShenandoahReferenceProcessor* rp) :
53 MetadataVisitingOopIterateClosure(rp), _heap(ShenandoahHeap::heap()) {}
54
55 void ShenandoahSuperClosure::do_nmethod(nmethod* nm) {
56 nm->run_nmethod_entry_barrier();
57 }
58
59 //
60 // ========= Marking
61 //
62 ShenandoahFlushSATBHandshakeClosure::ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
63 HandshakeClosure("Shenandoah Flush SATB"),
64 _qset(qset) {}
65
66 void ShenandoahFlushSATBHandshakeClosure::do_thread(Thread* thread) {
67 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
68 }
69
70 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,
71 ShenandoahReferenceProcessor* rp,
72 ShenandoahObjToScanQueue* old_q) :
73 ShenandoahSuperClosure(rp),
74 _queue(q),
75 _old_queue(old_q),
76 _mark_context(ShenandoahHeap::heap()->marking_context()),
77 _weak(false) {}
78
79 template<class T, ShenandoahGenerationType GENERATION>
80 ALWAYSINLINE
81 void ShenandoahMarkRefsSuperClosure::work(T* p) {
82 ShenandoahMark::mark_through_ref<T, GENERATION>(p, _queue, _old_queue, _mark_context, _weak);
83 }
84
85 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
86 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
87
88 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
89 if (CompressedOops::is_null(obj)) {
90 return false;
91 }
92 obj = ShenandoahForwarding::get_forwardee(obj);
93 shenandoah_assert_not_forwarded_if(nullptr, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
94 return _mark_context->is_marked_or_old(obj);
95 }
96
97 ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
98 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
99
100 bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
101 if (CompressedOops::is_null(obj)) {
102 return false;
103 }
104 shenandoah_assert_not_forwarded(nullptr, obj);
105 return _mark_context->is_marked_or_old(obj);
106 }
107
108 BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
109 return ShenandoahHeap::heap()->has_forwarded_objects() ?
110 reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl) :
111 reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
112 }
113
114 ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
115 _bs(ShenandoahBarrierSet::barrier_set()) {}
116
117 template <typename T>
118 void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
119 assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase");
120 assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || !ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
121
122 T o = RawAccess<>::oop_load(p);
123 if (!CompressedOops::is_null(o)) {
124 oop obj = CompressedOops::decode_not_null(o);
125 _bs->enqueue(obj);
126 }
127 }
128
129
130 //
131 // ========= Evacuating + Roots
132 //
133
134 template <bool CONCURRENT, bool STABLE_THREAD>
135 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop(oop* p) {
136 do_oop_work(p);
137 }
138
139 template <bool CONCURRENT, bool STABLE_THREAD>
140 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop(narrowOop* p) {
141 do_oop_work(p);
142 }
143
144 template <bool CONCURRENT, bool STABLE_THREAD>
145 template <class T>
146 void ShenandoahEvacuateUpdateRootClosureBase<CONCURRENT, STABLE_THREAD>::do_oop_work(T* p) {
147 assert((_heap->is_concurrent_weak_root_in_progress() && _heap->is_evacuation_in_progress()) ||
148 _heap->is_concurrent_strong_root_in_progress(),
149 "Only do this in root processing phase");
150
151 T o = RawAccess<>::oop_load(p);
152 if (!CompressedOops::is_null(o)) {
153 oop obj = CompressedOops::decode_not_null(o);
154 if (_heap->in_collection_set(obj)) {
155 assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
156 shenandoah_assert_marked(p, obj);
157 oop resolved = ShenandoahForwarding::get_forwardee(obj);
158 if (resolved == obj) {
159 Thread* thr = STABLE_THREAD ? _thread : Thread::current();
160 assert(thr == Thread::current(), "Wrong thread");
161
162 resolved = _heap->evacuate_object(obj, thr);
163 }
164 if (CONCURRENT) {
165 ShenandoahHeap::atomic_update_oop(resolved, p, o);
166 } else {
167 RawAccess<IS_NOT_NULL | MO_UNORDERED>::oop_store(p, resolved);
168 }
169 }
170 }
171 }
172
173 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
174 ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::ShenandoahCleanUpdateWeakOopsClosure(IsAlive* is_alive, KeepAlive* keep_alive) :
175 _is_alive(is_alive), _keep_alive(keep_alive) {
176 if (!CONCURRENT) {
177 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
178 }
179 }
180
181 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
182 void ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::do_oop(oop* p) {
183 oop obj = RawAccess<>::oop_load(p);
184 if (!CompressedOops::is_null(obj)) {
185 if (_is_alive->do_object_b(obj)) {
186 _keep_alive->do_oop(p);
187 } else {
188 if (CONCURRENT) {
189 ShenandoahHeap::atomic_clear_oop(p, obj);
190 } else {
191 RawAccess<IS_NOT_NULL>::oop_store(p, oop());
192 }
193 }
194 }
195 }
196
197 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>
198 void ShenandoahCleanUpdateWeakOopsClosure<CONCURRENT, IsAlive, KeepAlive>::do_oop(narrowOop* p) {
199 ShouldNotReachHere();
200 }
201
202 //
203 // ========= Update References
204 //
205
206 template <ShenandoahGenerationType GENERATION>
207 ShenandoahMarkUpdateRefsClosure<GENERATION>::ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q,
208 ShenandoahReferenceProcessor* rp,
209 ShenandoahObjToScanQueue* old_q) :
210 ShenandoahMarkRefsSuperClosure(q, rp, old_q) {
211 assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC");
212 }
213
214 template<ShenandoahGenerationType GENERATION>
215 template<class T>
216 inline void ShenandoahMarkUpdateRefsClosure<GENERATION>::work(T* p) {
217 // Update the location
218 _heap->non_conc_update_with_forwarded(p);
219
220 // ...then do the usual thing
221 ShenandoahMarkRefsSuperClosure::work<T, GENERATION>(p);
222 }
223
224 template<class T>
225 inline void ShenandoahNonConcUpdateRefsClosure::work(T* p) {
226 _heap->non_conc_update_with_forwarded(p);
227 }
228
229 template<class T>
230 inline void ShenandoahConcUpdateRefsClosure::work(T* p) {
231 _heap->conc_update_with_forwarded(p);
232 }
233
234 inline void ShenandoahFlushSATB::do_thread(Thread* thread) {
235 // Transfer any partial buffer to the qset for completed buffer processing.
236 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
237 }
238
239 //
240 // ========= Utilities
241 //
242
243 #ifdef ASSERT
244 template <class T>
245 void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {
246 T o = RawAccess<>::oop_load(p);
247 if (!CompressedOops::is_null(o)) {
248 oop obj = CompressedOops::decode_not_null(o);
249 shenandoah_assert_not_forwarded(p, obj);
250 }
251 }
252
253 void ShenandoahAssertNotForwardedClosure::do_oop(narrowOop* p) { do_oop_work(p); }
254 void ShenandoahAssertNotForwardedClosure::do_oop(oop* p) { do_oop_work(p); }
255 #endif
256
257 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP