81 _interior_loc(nullptr),
82 _loc(nullptr) {
83 if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
84 options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
85 set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
86 }
87 }
88
89 private:
90 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
91 if (!test) {
92 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
93 }
94 }
95
96 template <class T>
97 void do_oop_work(T* p) {
98 T o = RawAccess<>::oop_load(p);
99 if (!CompressedOops::is_null(o)) {
100 oop obj = CompressedOops::decode_not_null(o);
101 if (is_instance_ref_klass(obj->klass())) {
102 obj = ShenandoahForwarding::get_forwardee(obj);
103 }
104 // Single threaded verification can use faster non-atomic stack and bitmap
105 // methods.
106 //
107 // For performance reasons, only fully verify non-marked field values.
108 // We are here when the host object for *p is already marked.
109
110 if (_map->par_mark(obj)) {
111 verify_oop_at(p, obj);
112 _stack->push(ShenandoahVerifierTask(obj));
113 }
114 }
115 }
116
117 void verify_oop(oop obj) {
118 // Perform consistency checks with gradually decreasing safety level. This guarantees
119 // that failure report would not try to touch something that was not yet verified to be
120 // safe to process.
121
122 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
123 "oop must be in heap");
124 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
125 "oop must be aligned");
126
127 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
128 Klass* obj_klass = obj->klass_or_null();
129
130 // Verify that obj is not in dead space:
131 {
132 // Do this before touching obj->size()
133 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
134 "Object klass pointer should not be null");
135 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
136 "Object klass pointer must go to metaspace");
137
138 HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
139 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
140 "Object start should be within the region");
141
142 if (!obj_reg->is_humongous()) {
143 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
144 "Object end should be within the region");
145 } else {
146 size_t humongous_start = obj_reg->index();
147 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
148 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
149 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
150 "Humongous object is in continuation that fits it");
151 }
152 }
153
154 // ------------ obj is safe at this point --------------
155
156 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
157 "Object should be in active region");
158
159 switch (_options._verify_liveness) {
160 case ShenandoahVerifier::_verify_liveness_disable:
161 // skip
162 break;
163 case ShenandoahVerifier::_verify_liveness_complete:
164 Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
165 // fallthrough for fast failure for un-live regions:
166 case ShenandoahVerifier::_verify_liveness_conservative:
167 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
168 "Object must belong to region with live data");
169 break;
170 default:
171 assert(false, "Unhandled liveness verification");
172 }
173 }
174
175 oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
176
177 ShenandoahHeapRegion* fwd_reg = nullptr;
178
179 if (obj != fwd) {
180 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
181 "Forwardee must be in heap");
182 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
183 "Forwardee is set");
184 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
185 "Forwardee must be aligned");
186
187 // Do this before touching fwd->size()
188 Klass* fwd_klass = fwd->klass_or_null();
189 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
190 "Forwardee klass pointer should not be null");
191 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
192 "Forwardee klass pointer must go to metaspace");
193 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
194 "Forwardee klass pointer must go to metaspace");
195
196 fwd_reg = _heap->heap_region_containing(fwd);
197
198 // Verify that forwardee is not in the dead space:
199 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
200 "Should have no humongous forwardees");
201
202 HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
203 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
204 "Forwardee start should be within the region");
205 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
206 "Forwardee end should be within the region");
207
208 oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
209 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
210 "Double forwarding");
211 } else {
212 fwd_reg = obj_reg;
213 }
214
215 // ------------ obj and fwd are safe at this point --------------
216
217 switch (_options._verify_marked) {
218 case ShenandoahVerifier::_verify_marked_disable:
219 // skip
220 break;
221 case ShenandoahVerifier::_verify_marked_incomplete:
222 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
223 "Must be marked in incomplete bitmap");
224 break;
225 case ShenandoahVerifier::_verify_marked_complete:
288 }
289
290 /**
291 * Verify object without known interior reference.
292 * Useful when picking up the object at known offset in heap,
293 * but without knowing what objects reference it.
294 * @param obj verified object
295 */
296 void verify_oop_standalone(oop obj) {
297 _interior_loc = nullptr;
298 verify_oop(obj);
299 _interior_loc = nullptr;
300 }
301
302 /**
303 * Verify oop fields from this object.
304 * @param obj host object for verified fields
305 */
306 void verify_oops_from(oop obj) {
307 _loc = obj;
308 obj->oop_iterate(this);
309 _loc = nullptr;
310 }
311
312 virtual void do_oop(oop* p) { do_oop_work(p); }
313 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
314 };
315
316 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
317 private:
318 size_t _used, _committed, _garbage;
319 public:
320 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
321
322 void heap_region_do(ShenandoahHeapRegion* r) {
323 _used += r->used();
324 _garbage += r->garbage();
325 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
326 }
327
328 size_t used() { return _used; }
568 HeapWord* addr = tams;
569
570 while (addr < limit) {
571 verify_and_follow(addr, stack, cl, &processed);
572 addr += cast_to_oop(addr)->size();
573 }
574 }
575
576 Atomic::add(&_processed, processed, memory_order_relaxed);
577 }
578
579 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
580 if (!_bitmap->par_mark(addr)) return;
581
582 // Verify the object itself:
583 oop obj = cast_to_oop(addr);
584 cl.verify_oop_standalone(obj);
585
586 // Verify everything reachable from that object too, hopefully realizing
587 // everything was already marked, and never touching further:
588 if (!is_instance_ref_klass(obj->klass())) {
589 cl.verify_oops_from(obj);
590 (*processed)++;
591 }
592 while (!stack.is_empty()) {
593 ShenandoahVerifierTask task = stack.pop();
594 cl.verify_oops_from(task.obj());
595 (*processed)++;
596 }
597 }
598 };
599
600 class VerifyThreadGCState : public ThreadClosure {
601 private:
602 const char* const _label;
603 char const _expected;
604
605 public:
606 VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
607 void do_thread(Thread* t) {
608 char actual = ShenandoahThreadLocalData::gc_state(t);
|
81 _interior_loc(nullptr),
82 _loc(nullptr) {
83 if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
84 options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
85 set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
86 }
87 }
88
89 private:
90 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
91 if (!test) {
92 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
93 }
94 }
95
96 template <class T>
97 void do_oop_work(T* p) {
98 T o = RawAccess<>::oop_load(p);
99 if (!CompressedOops::is_null(o)) {
100 oop obj = CompressedOops::decode_not_null(o);
101 if (is_instance_ref_klass(obj->forward_safe_klass())) {
102 obj = ShenandoahForwarding::get_forwardee(obj);
103 }
104 // Single threaded verification can use faster non-atomic stack and bitmap
105 // methods.
106 //
107 // For performance reasons, only fully verify non-marked field values.
108 // We are here when the host object for *p is already marked.
109
110 if (_map->par_mark(obj)) {
111 verify_oop_at(p, obj);
112 _stack->push(ShenandoahVerifierTask(obj));
113 }
114 }
115 }
116
117 void verify_oop(oop obj) {
118 // Perform consistency checks with gradually decreasing safety level. This guarantees
119 // that failure report would not try to touch something that was not yet verified to be
120 // safe to process.
121
122 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
123 "oop must be in heap");
124 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
125 "oop must be aligned");
126
127 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
128 Klass* obj_klass = obj->forward_safe_klass();
129
130 // Verify that obj is not in dead space:
131 {
132 // Do this before touching obj->size()
133 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
134 "Object klass pointer should not be null");
135 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
136 "Object klass pointer must go to metaspace");
137
138 HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
139 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
140 "Object start should be within the region");
141
142 if (!obj_reg->is_humongous()) {
143 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->forward_safe_size()) <= obj_reg->top(),
144 "Object end should be within the region");
145 } else {
146 size_t humongous_start = obj_reg->index();
147 size_t humongous_end = humongous_start + (obj->forward_safe_size() >> ShenandoahHeapRegion::region_size_words_shift());
148 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
149 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
150 "Humongous object is in continuation that fits it");
151 }
152 }
153
154 // ------------ obj is safe at this point --------------
155
156 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
157 "Object should be in active region");
158
159 switch (_options._verify_liveness) {
160 case ShenandoahVerifier::_verify_liveness_disable:
161 // skip
162 break;
163 case ShenandoahVerifier::_verify_liveness_complete:
164 Atomic::add(&_ld[obj_reg->index()], (uint) obj->forward_safe_size(), memory_order_relaxed);
165 // fallthrough for fast failure for un-live regions:
166 case ShenandoahVerifier::_verify_liveness_conservative:
167 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
168 "Object must belong to region with live data");
169 break;
170 default:
171 assert(false, "Unhandled liveness verification");
172 }
173 }
174
175 oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
176
177 ShenandoahHeapRegion* fwd_reg = nullptr;
178
179 if (obj != fwd) {
180 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
181 "Forwardee must be in heap");
182 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
183 "Forwardee is set");
184 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
185 "Forwardee must be aligned");
186
187 // Do this before touching fwd->size()
188 Klass* fwd_klass = fwd->klass_or_null();
189 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
190 "Forwardee klass pointer should not be null");
191 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
192 "Forwardee klass pointer must go to metaspace");
193 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
194 "Forwardee klass pointer must go to metaspace");
195
196 fwd_reg = _heap->heap_region_containing(fwd);
197
198 // Verify that forwardee is not in the dead space:
199 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
200 "Should have no humongous forwardees");
201
202 HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
203 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
204 "Forwardee start should be within the region");
205 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->forward_safe_size()) <= fwd_reg->top(),
206 "Forwardee end should be within the region");
207
208 oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
209 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
210 "Double forwarding");
211 } else {
212 fwd_reg = obj_reg;
213 }
214
215 // ------------ obj and fwd are safe at this point --------------
216
217 switch (_options._verify_marked) {
218 case ShenandoahVerifier::_verify_marked_disable:
219 // skip
220 break;
221 case ShenandoahVerifier::_verify_marked_incomplete:
222 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
223 "Must be marked in incomplete bitmap");
224 break;
225 case ShenandoahVerifier::_verify_marked_complete:
288 }
289
290 /**
291 * Verify object without known interior reference.
292 * Useful when picking up the object at known offset in heap,
293 * but without knowing what objects reference it.
294 * @param obj verified object
295 */
296 void verify_oop_standalone(oop obj) {
297 _interior_loc = nullptr;
298 verify_oop(obj);
299 _interior_loc = nullptr;
300 }
301
302 /**
303 * Verify oop fields from this object.
304 * @param obj host object for verified fields
305 */
306 void verify_oops_from(oop obj) {
307 _loc = obj;
308 Klass* klass = obj->forward_safe_klass();
309 obj->oop_iterate_backwards(this, klass);
310 _loc = nullptr;
311 }
312
313 virtual void do_oop(oop* p) { do_oop_work(p); }
314 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
315 };
316
317 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
318 private:
319 size_t _used, _committed, _garbage;
320 public:
321 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
322
323 void heap_region_do(ShenandoahHeapRegion* r) {
324 _used += r->used();
325 _garbage += r->garbage();
326 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
327 }
328
329 size_t used() { return _used; }
569 HeapWord* addr = tams;
570
571 while (addr < limit) {
572 verify_and_follow(addr, stack, cl, &processed);
573 addr += cast_to_oop(addr)->size();
574 }
575 }
576
577 Atomic::add(&_processed, processed, memory_order_relaxed);
578 }
579
580 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
581 if (!_bitmap->par_mark(addr)) return;
582
583 // Verify the object itself:
584 oop obj = cast_to_oop(addr);
585 cl.verify_oop_standalone(obj);
586
587 // Verify everything reachable from that object too, hopefully realizing
588 // everything was already marked, and never touching further:
589 if (!is_instance_ref_klass(obj->forward_safe_klass())) {
590 cl.verify_oops_from(obj);
591 (*processed)++;
592 }
593 while (!stack.is_empty()) {
594 ShenandoahVerifierTask task = stack.pop();
595 cl.verify_oops_from(task.obj());
596 (*processed)++;
597 }
598 }
599 };
600
601 class VerifyThreadGCState : public ThreadClosure {
602 private:
603 const char* const _label;
604 char const _expected;
605
606 public:
607 VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
608 void do_thread(Thread* t) {
609 char actual = ShenandoahThreadLocalData::gc_state(t);
|