1 /*
2 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "code/codeCache.hpp"
26 #include "code/nmethod.hpp"
27 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
28 #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "runtime/vmThread.hpp"
31
32 ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
33 _claimed_idx(0), _finished(false) {
34 }
35
36 void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
37 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
38
39 /*
40 * Parallel code heap walk.
41 *
42 * This code makes all threads scan all code heaps, but only one thread would execute the
43 * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
44 * had claimed the block, it can process all blobs in it. Others have to fast-forward to
45 * next attempt without processing.
46 *
47 * Late threads would return immediately if iterator is finished.
48 */
49
50 if (_finished) {
51 return;
52 }
53
54 int stride = 256; // educated guess
55 int stride_mask = stride - 1;
56 assert (is_power_of_2(stride), "sanity");
57
58 int count = 0;
59 bool process_block = true;
60
61 for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
62 int current = count++;
63 if ((current & stride_mask) == 0) {
64 process_block = (current >= _claimed_idx) &&
65 (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
66 }
67 if (process_block) {
68 if (cb->is_alive()) {
69 f->do_code_blob(cb);
70 #ifdef ASSERT
71 if (cb->is_nmethod())
72 ((nmethod*)cb)->verify_scavenge_root_oops();
73 #endif
74 }
75 }
76 }
77
78 _finished = true;
79 }
80
81 class ShenandoahNMethodOopDetector : public OopClosure {
82 private:
83 ResourceMark rm; // For growable array allocation below.
84 GrowableArray<oop*> _oops;
85
86 public:
87 ShenandoahNMethodOopDetector() : _oops(10) {};
88
89 void do_oop(oop* o) {
90 _oops.append(o);
91 }
92
93 void do_oop(narrowOop* o) {
94 fatal("NMethods should not have compressed oops embedded.");
95 }
96
97 GrowableArray<oop*>* oops() {
98 return &_oops;
99 }
100
101 bool has_oops() {
102 return !_oops.is_empty();
103 }
104 };
105
106 ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock;
107 GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
108
109 void ShenandoahCodeRoots::initialize() {
110 _recorded_nms_lock._lock = 0;
111 _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
112 }
113
114 void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
115 switch (ShenandoahCodeRootsStyle) {
116 case 0:
117 case 1:
118 break;
119 case 2: {
120 ShenandoahNMethodOopDetector detector;
121 nm->oops_do(&detector);
122
123 if (detector.has_oops()) {
124 ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
125 nmr->assert_alive_and_correct();
126
127 ShenandoahCodeRootsLock lock(true);
128
129 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
130 if (idx != -1) {
131 ShenandoahNMethod* old = _recorded_nms->at(idx);
132 _recorded_nms->at_put(idx, nmr);
133 delete old;
134 } else {
135 _recorded_nms->append(nmr);
136 }
137 }
138 break;
139 }
140 default:
141 ShouldNotReachHere();
142 }
143 };
144
145 void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
146 switch (ShenandoahCodeRootsStyle) {
147 case 0:
148 case 1: {
149 break;
150 }
151 case 2: {
152 ShenandoahNMethodOopDetector detector;
153 nm->oops_do(&detector, /* allow_zombie = */ true);
154
155 if (detector.has_oops()) {
156 ShenandoahCodeRootsLock lock(true);
157
158 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
159 assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
160 ShenandoahNMethod* old = _recorded_nms->at(idx);
161 old->assert_same_oops(detector.oops());
162 _recorded_nms->delete_at(idx);
163 delete old;
164 }
165 break;
166 }
167 default:
168 ShouldNotReachHere();
169 }
170 }
171
172 ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
173 _heap(ShenandoahHeap::heap()),
174 _claimed(0) {
175 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
176 assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
177 switch (ShenandoahCodeRootsStyle) {
178 case 0:
179 case 1:
180 break;
181 case 2:
182 ShenandoahCodeRoots::acquire_lock(false);
183 break;
184 default:
185 ShouldNotReachHere();
186 }
187 }
188
189 ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
190 switch (ShenandoahCodeRootsStyle) {
191 case 0:
192 case 1: {
193 // No need to do anything here
194 break;
195 }
196 case 2: {
197 ShenandoahCodeRoots::release_lock(false);
198 break;
199 }
200 default:
201 ShouldNotReachHere();
202 }
203 }
204
205 template<bool CSET_FILTER>
206 void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
207 switch (ShenandoahCodeRootsStyle) {
208 case 0: {
209 if (_seq_claimed.try_set()) {
210 CodeCache::blobs_do(f);
211 }
212 break;
213 }
214 case 1: {
215 _par_iterator.parallel_blobs_do(f);
216 break;
217 }
218 case 2: {
219 ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
220 break;
221 }
222 default:
223 ShouldNotReachHere();
224 }
225 }
226
227 void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
228 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
229 }
230
231 void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
232 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
233 }
234
235 template <bool CSET_FILTER>
236 void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
237 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
238
239 size_t stride = 256; // educated guess
240
241 GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
242
243 jlong max = list->length();
244 while (_claimed < max) {
245 size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
246 size_t start = cur;
247 size_t end = MIN2(cur + stride, (size_t) max);
248 if (start >= (size_t)max) break;
249
250 for (size_t idx = start; idx < end; idx++) {
251 ShenandoahNMethod* nmr = list->at((int) idx);
252 nmr->assert_alive_and_correct();
253
254 if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
255 continue;
256 }
257
258 f->do_code_blob(nmr->nm());
259 }
260 }
261 }
262
263 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
264 _nm = nm;
265 _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
266 _oops_count = oops->length();
267 for (int c = 0; c < _oops_count; c++) {
268 _oops[c] = oops->at(c);
269 }
270 }
271
272 ShenandoahNMethod::~ShenandoahNMethod() {
273 if (_oops != NULL) {
274 FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
275 }
276 }
277
278 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
279 for (int c = 0; c < _oops_count; c++) {
280 oop o = oopDesc::load_heap_oop(_oops[c]);
281 if (heap->in_collection_set(o)) {
282 return true;
283 }
284 }
285 return false;
286 }
287
288 #ifdef ASSERT
289 void ShenandoahNMethod::assert_alive_and_correct() {
290 assert(_nm->is_alive(), "only alive nmethods here");
291 assert(_oops_count > 0, "should have filtered nmethods without oops before");
292 ShenandoahHeap* heap = ShenandoahHeap::heap();
293 for (int c = 0; c < _oops_count; c++) {
294 oop *loc = _oops[c];
295 assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
296 oop o = oopDesc::load_heap_oop(loc);
297 shenandoah_assert_correct_except(loc, o,
298 o == NULL ||
299 heap->is_full_gc_move_in_progress() ||
300 (VMThread::vm_operation() != NULL) && (VMThread::vm_operation()->type() == VM_Operation::VMOp_HeapWalkOperation)
301 );
302 }
303 }
304
305 void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
306 assert(_oops_count == oops->length(), "should have the same number of oop*");
307 for (int c = 0; c < _oops_count; c++) {
308 assert(_oops[c] == oops->at(c), "should be the same oop*");
309 }
310 }
311 #endif