12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/tlab_globals.hpp"
27 #include "gc/shenandoah/shenandoahAsserts.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
32 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
33 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
34 #include "gc/shenandoah/shenandoahUtils.hpp"
35 #include "gc/shenandoah/shenandoahVerifier.hpp"
36 #include "memory/allocation.hpp"
37 #include "memory/iterator.inline.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/compressedOops.inline.hpp"
40 #include "runtime/atomic.hpp"
41 #include "runtime/orderAccess.hpp"
42 #include "runtime/threads.hpp"
43 #include "utilities/align.hpp"
44
45 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
46 #ifdef verify_oop
47 #undef verify_oop
48 #endif
49
50 static bool is_instance_ref_klass(Klass* k) {
51 return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;
81 _interior_loc(nullptr),
82 _loc(nullptr) {
83 if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
84 options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
85 set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
86 }
87 }
88
89 private:
90 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
91 if (!test) {
92 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
93 }
94 }
95
96 template <class T>
97 void do_oop_work(T* p) {
98 T o = RawAccess<>::oop_load(p);
99 if (!CompressedOops::is_null(o)) {
100 oop obj = CompressedOops::decode_not_null(o);
101 if (is_instance_ref_klass(obj->klass())) {
102 obj = ShenandoahForwarding::get_forwardee(obj);
103 }
104 // Single threaded verification can use faster non-atomic stack and bitmap
105 // methods.
106 //
107 // For performance reasons, only fully verify non-marked field values.
108 // We are here when the host object for *p is already marked.
109
110 if (_map->par_mark(obj)) {
111 verify_oop_at(p, obj);
112 _stack->push(ShenandoahVerifierTask(obj));
113 }
114 }
115 }
116
117 void verify_oop(oop obj) {
118 // Perform consistency checks with gradually decreasing safety level. This guarantees
119 // that failure report would not try to touch something that was not yet verified to be
120 // safe to process.
121
122 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
123 "oop must be in heap");
124 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
125 "oop must be aligned");
126
127 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
128 Klass* obj_klass = obj->klass_or_null();
129
130 // Verify that obj is not in dead space:
131 {
132 // Do this before touching obj->size()
133 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
134 "Object klass pointer should not be null");
135 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
136 "Object klass pointer must go to metaspace");
137
138 HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
139 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
140 "Object start should be within the region");
141
142 if (!obj_reg->is_humongous()) {
143 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
144 "Object end should be within the region");
145 } else {
146 size_t humongous_start = obj_reg->index();
147 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
148 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
149 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
150 "Humongous object is in continuation that fits it");
151 }
152 }
153
154 // ------------ obj is safe at this point --------------
155
156 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
157 "Object should be in active region");
158
159 switch (_options._verify_liveness) {
160 case ShenandoahVerifier::_verify_liveness_disable:
161 // skip
162 break;
163 case ShenandoahVerifier::_verify_liveness_complete:
164 Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
165 // fallthrough for fast failure for un-live regions:
166 case ShenandoahVerifier::_verify_liveness_conservative:
167 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
168 "Object must belong to region with live data");
169 break;
170 default:
171 assert(false, "Unhandled liveness verification");
172 }
173 }
174
175 oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
176
177 ShenandoahHeapRegion* fwd_reg = nullptr;
178
179 if (obj != fwd) {
180 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
181 "Forwardee must be in heap");
182 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
183 "Forwardee is set");
184 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
185 "Forwardee must be aligned");
186
187 // Do this before touching fwd->size()
188 Klass* fwd_klass = fwd->klass_or_null();
189 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
190 "Forwardee klass pointer should not be null");
191 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
192 "Forwardee klass pointer must go to metaspace");
193 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
194 "Forwardee klass pointer must go to metaspace");
195
196 fwd_reg = _heap->heap_region_containing(fwd);
197
198 // Verify that forwardee is not in the dead space:
199 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
200 "Should have no humongous forwardees");
201
202 HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
203 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
204 "Forwardee start should be within the region");
205 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
206 "Forwardee end should be within the region");
207
208 oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
209 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
210 "Double forwarding");
211 } else {
212 fwd_reg = obj_reg;
213 }
214
215 // ------------ obj and fwd are safe at this point --------------
216
217 switch (_options._verify_marked) {
218 case ShenandoahVerifier::_verify_marked_disable:
219 // skip
220 break;
221 case ShenandoahVerifier::_verify_marked_incomplete:
222 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
223 "Must be marked in incomplete bitmap");
224 break;
225 case ShenandoahVerifier::_verify_marked_complete:
288 }
289
290 /**
291 * Verify object without known interior reference.
292 * Useful when picking up the object at known offset in heap,
293 * but without knowing what objects reference it.
294 * @param obj verified object
295 */
296 void verify_oop_standalone(oop obj) {
297 _interior_loc = nullptr;
298 verify_oop(obj);
299 _interior_loc = nullptr;
300 }
301
302 /**
303 * Verify oop fields from this object.
304 * @param obj host object for verified fields
305 */
306 void verify_oops_from(oop obj) {
307 _loc = obj;
308 obj->oop_iterate(this);
309 _loc = nullptr;
310 }
311
312 virtual void do_oop(oop* p) { do_oop_work(p); }
313 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
314 };
315
316 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
317 private:
318 size_t _used, _committed, _garbage;
319 public:
320 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
321
322 void heap_region_do(ShenandoahHeapRegion* r) {
323 _used += r->used();
324 _garbage += r->garbage();
325 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
326 }
327
328 size_t used() { return _used; }
568 HeapWord* addr = tams;
569
570 while (addr < limit) {
571 verify_and_follow(addr, stack, cl, &processed);
572 addr += cast_to_oop(addr)->size();
573 }
574 }
575
576 Atomic::add(&_processed, processed, memory_order_relaxed);
577 }
578
579 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
580 if (!_bitmap->par_mark(addr)) return;
581
582 // Verify the object itself:
583 oop obj = cast_to_oop(addr);
584 cl.verify_oop_standalone(obj);
585
586 // Verify everything reachable from that object too, hopefully realizing
587 // everything was already marked, and never touching further:
588 if (!is_instance_ref_klass(obj->klass())) {
589 cl.verify_oops_from(obj);
590 (*processed)++;
591 }
592 while (!stack.is_empty()) {
593 ShenandoahVerifierTask task = stack.pop();
594 cl.verify_oops_from(task.obj());
595 (*processed)++;
596 }
597 }
598 };
599
600 class VerifyThreadGCState : public ThreadClosure {
601 private:
602 const char* const _label;
603 char const _expected;
604
605 public:
606 VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
607 void do_thread(Thread* t) {
608 char actual = ShenandoahThreadLocalData::gc_state(t);
|
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/tlab_globals.hpp"
27 #include "gc/shenandoah/shenandoahAsserts.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
32 #include "gc/shenandoah/shenandoahObjectUtils.inline.hpp"
33 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
34 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
35 #include "gc/shenandoah/shenandoahUtils.hpp"
36 #include "gc/shenandoah/shenandoahVerifier.hpp"
37 #include "memory/allocation.hpp"
38 #include "memory/iterator.inline.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "runtime/atomic.hpp"
42 #include "runtime/orderAccess.hpp"
43 #include "runtime/threads.hpp"
44 #include "utilities/align.hpp"
45
46 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
47 #ifdef verify_oop
48 #undef verify_oop
49 #endif
50
51 static bool is_instance_ref_klass(Klass* k) {
52 return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;
82 _interior_loc(nullptr),
83 _loc(nullptr) {
84 if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
85 options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
86 set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
87 }
88 }
89
90 private:
91 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
92 if (!test) {
93 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
94 }
95 }
96
97 template <class T>
98 void do_oop_work(T* p) {
99 T o = RawAccess<>::oop_load(p);
100 if (!CompressedOops::is_null(o)) {
101 oop obj = CompressedOops::decode_not_null(o);
102 if (is_instance_ref_klass(ShenandoahObjectUtils::klass(obj))) {
103 obj = ShenandoahForwarding::get_forwardee(obj);
104 }
105 // Single threaded verification can use faster non-atomic stack and bitmap
106 // methods.
107 //
108 // For performance reasons, only fully verify non-marked field values.
109 // We are here when the host object for *p is already marked.
110
111 if (_map->par_mark(obj)) {
112 verify_oop_at(p, obj);
113 _stack->push(ShenandoahVerifierTask(obj));
114 }
115 }
116 }
117
118 void verify_oop(oop obj) {
119 // Perform consistency checks with gradually decreasing safety level. This guarantees
120 // that failure report would not try to touch something that was not yet verified to be
121 // safe to process.
122
123 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
124 "oop must be in heap");
125 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
126 "oop must be aligned");
127
128 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
129 Klass* obj_klass = ShenandoahObjectUtils::klass(obj);
130
131 // Verify that obj is not in dead space:
132 {
133 // Do this before touching obj->size()
134 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
135 "Object klass pointer should not be null");
136 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
137 "Object klass pointer must go to metaspace");
138
139 HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
140 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
141 "Object start should be within the region");
142
143 if (!obj_reg->is_humongous()) {
144 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + ShenandoahObjectUtils::size(obj)) <= obj_reg->top(),
145 "Object end should be within the region");
146 } else {
147 size_t humongous_start = obj_reg->index();
148 size_t humongous_end = humongous_start + (ShenandoahObjectUtils::size(obj) >> ShenandoahHeapRegion::region_size_words_shift());
149 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
150 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
151 "Humongous object is in continuation that fits it");
152 }
153 }
154
155 // ------------ obj is safe at this point --------------
156
157 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
158 "Object should be in active region");
159
160 switch (_options._verify_liveness) {
161 case ShenandoahVerifier::_verify_liveness_disable:
162 // skip
163 break;
164 case ShenandoahVerifier::_verify_liveness_complete:
165 Atomic::add(&_ld[obj_reg->index()], (uint) ShenandoahObjectUtils::size(obj), memory_order_relaxed);
166 // fallthrough for fast failure for un-live regions:
167 case ShenandoahVerifier::_verify_liveness_conservative:
168 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
169 "Object must belong to region with live data");
170 break;
171 default:
172 assert(false, "Unhandled liveness verification");
173 }
174 }
175
176 oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
177
178 ShenandoahHeapRegion* fwd_reg = nullptr;
179
180 if (obj != fwd) {
181 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
182 "Forwardee must be in heap");
183 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
184 "Forwardee is set");
185 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
186 "Forwardee must be aligned");
187
188 // Do this before touching fwd->size()
189 Klass* fwd_klass = fwd->klass_or_null();
190 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
191 "Forwardee klass pointer should not be null");
192 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
193 "Forwardee klass pointer must go to metaspace");
194 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
195 "Forwardee klass pointer must go to metaspace");
196
197 fwd_reg = _heap->heap_region_containing(fwd);
198
199 // Verify that forwardee is not in the dead space:
200 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
201 "Should have no humongous forwardees");
202
203 HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
204 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
205 "Forwardee start should be within the region");
206 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + ShenandoahObjectUtils::size(fwd)) <= fwd_reg->top(),
207 "Forwardee end should be within the region");
208
209 oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
210 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
211 "Double forwarding");
212 } else {
213 fwd_reg = obj_reg;
214 }
215
216 // ------------ obj and fwd are safe at this point --------------
217
218 switch (_options._verify_marked) {
219 case ShenandoahVerifier::_verify_marked_disable:
220 // skip
221 break;
222 case ShenandoahVerifier::_verify_marked_incomplete:
223 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
224 "Must be marked in incomplete bitmap");
225 break;
226 case ShenandoahVerifier::_verify_marked_complete:
289 }
290
291 /**
292 * Verify object without known interior reference.
293 * Useful when picking up the object at known offset in heap,
294 * but without knowing what objects reference it.
295 * @param obj verified object
296 */
297 void verify_oop_standalone(oop obj) {
298 _interior_loc = nullptr;
299 verify_oop(obj);
300 _interior_loc = nullptr;
301 }
302
303 /**
304 * Verify oop fields from this object.
305 * @param obj host object for verified fields
306 */
307 void verify_oops_from(oop obj) {
308 _loc = obj;
309 Klass* klass = ShenandoahObjectUtils::klass(obj);
310 obj->oop_iterate_backwards(this, klass);
311 _loc = nullptr;
312 }
313
314 virtual void do_oop(oop* p) { do_oop_work(p); }
315 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
316 };
317
318 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
319 private:
320 size_t _used, _committed, _garbage;
321 public:
322 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
323
324 void heap_region_do(ShenandoahHeapRegion* r) {
325 _used += r->used();
326 _garbage += r->garbage();
327 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
328 }
329
330 size_t used() { return _used; }
570 HeapWord* addr = tams;
571
572 while (addr < limit) {
573 verify_and_follow(addr, stack, cl, &processed);
574 addr += cast_to_oop(addr)->size();
575 }
576 }
577
578 Atomic::add(&_processed, processed, memory_order_relaxed);
579 }
580
581 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
582 if (!_bitmap->par_mark(addr)) return;
583
584 // Verify the object itself:
585 oop obj = cast_to_oop(addr);
586 cl.verify_oop_standalone(obj);
587
588 // Verify everything reachable from that object too, hopefully realizing
589 // everything was already marked, and never touching further:
590 if (!is_instance_ref_klass(ShenandoahObjectUtils::klass(obj))) {
591 cl.verify_oops_from(obj);
592 (*processed)++;
593 }
594 while (!stack.is_empty()) {
595 ShenandoahVerifierTask task = stack.pop();
596 cl.verify_oops_from(task.obj());
597 (*processed)++;
598 }
599 }
600 };
601
602 class VerifyThreadGCState : public ThreadClosure {
603 private:
604 const char* const _label;
605 char const _expected;
606
607 public:
608 VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
609 void do_thread(Thread* t) {
610 char actual = ShenandoahThreadLocalData::gc_state(t);
|