< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

Print this page

 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/tlab_globals.hpp"
 27 #include "gc/shenandoah/shenandoahAsserts.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"

 32 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
 33 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 34 #include "gc/shenandoah/shenandoahUtils.hpp"
 35 #include "gc/shenandoah/shenandoahVerifier.hpp"
 36 #include "memory/allocation.hpp"
 37 #include "memory/iterator.inline.hpp"
 38 #include "memory/resourceArea.hpp"
 39 #include "oops/compressedOops.inline.hpp"
 40 #include "runtime/atomic.hpp"
 41 #include "runtime/orderAccess.hpp"
 42 #include "runtime/threads.hpp"
 43 #include "utilities/align.hpp"
 44 
 45 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
 46 #ifdef verify_oop
 47 #undef verify_oop
 48 #endif
 49 
 50 static bool is_instance_ref_klass(Klass* k) {
 51   return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;

 81     _interior_loc(nullptr),
 82     _loc(nullptr) {
 83     if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
 84         options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
 85       set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
 86     }
 87   }
 88 
 89 private:
 90   void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
 91     if (!test) {
 92       ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
 93     }
 94   }
 95 
 96   template <class T>
 97   void do_oop_work(T* p) {
 98     T o = RawAccess<>::oop_load(p);
 99     if (!CompressedOops::is_null(o)) {
100       oop obj = CompressedOops::decode_not_null(o);
101       if (is_instance_ref_klass(obj->klass())) {
102         obj = ShenandoahForwarding::get_forwardee(obj);
103       }
104       // Single threaded verification can use faster non-atomic stack and bitmap
105       // methods.
106       //
107       // For performance reasons, only fully verify non-marked field values.
108       // We are here when the host object for *p is already marked.
109 
110       if (_map->par_mark(obj)) {
111         verify_oop_at(p, obj);
112         _stack->push(ShenandoahVerifierTask(obj));
113       }
114     }
115   }
116 
117   void verify_oop(oop obj) {
118     // Perform consistency checks with gradually decreasing safety level. This guarantees
119     // that failure report would not try to touch something that was not yet verified to be
120     // safe to process.
121 
122     check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
123               "oop must be in heap");
124     check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
125               "oop must be aligned");
126 
127     ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
128     Klass* obj_klass = obj->klass_or_null();
129 
130     // Verify that obj is not in dead space:
131     {
132       // Do this before touching obj->size()
133       check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
134              "Object klass pointer should not be null");
135       check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
136              "Object klass pointer must go to metaspace");
137 
138       HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
139       check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
140              "Object start should be within the region");
141 
142       if (!obj_reg->is_humongous()) {
143         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
144                "Object end should be within the region");
145       } else {
146         size_t humongous_start = obj_reg->index();
147         size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
148         for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
149           check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
150                  "Humongous object is in continuation that fits it");
151         }
152       }
153 
154       // ------------ obj is safe at this point --------------
155 
156       check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
157             "Object should be in active region");
158 
159       switch (_options._verify_liveness) {
160         case ShenandoahVerifier::_verify_liveness_disable:
161           // skip
162           break;
163         case ShenandoahVerifier::_verify_liveness_complete:
164           Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
165           // fallthrough for fast failure for un-live regions:
166         case ShenandoahVerifier::_verify_liveness_conservative:
167           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
168                    "Object must belong to region with live data");
169           break;
170         default:
171           assert(false, "Unhandled liveness verification");
172       }
173     }
174 
175     oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
176 
177     ShenandoahHeapRegion* fwd_reg = nullptr;
178 
179     if (obj != fwd) {
180       check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
181              "Forwardee must be in heap");
182       check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
183              "Forwardee is set");
184       check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
185              "Forwardee must be aligned");
186 
187       // Do this before touching fwd->size()
188       Klass* fwd_klass = fwd->klass_or_null();
189       check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
190              "Forwardee klass pointer should not be null");
191       check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
192              "Forwardee klass pointer must go to metaspace");
193       check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
194              "Forwardee klass pointer must go to metaspace");
195 
196       fwd_reg = _heap->heap_region_containing(fwd);
197 
198       // Verify that forwardee is not in the dead space:
199       check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
200              "Should have no humongous forwardees");
201 
202       HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
203       check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
204              "Forwardee start should be within the region");
205       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
206              "Forwardee end should be within the region");
207 
208       oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
209       check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
210              "Double forwarding");
211     } else {
212       fwd_reg = obj_reg;
213     }
214 
215     // ------------ obj and fwd are safe at this point --------------
216 
217     switch (_options._verify_marked) {
218       case ShenandoahVerifier::_verify_marked_disable:
219         // skip
220         break;
221       case ShenandoahVerifier::_verify_marked_incomplete:
222         check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
223                "Must be marked in incomplete bitmap");
224         break;
225       case ShenandoahVerifier::_verify_marked_complete:

288   }
289 
290   /**
291    * Verify object without known interior reference.
292    * Useful when picking up the object at known offset in heap,
293    * but without knowing what objects reference it.
294    * @param obj verified object
295    */
296   void verify_oop_standalone(oop obj) {
297     _interior_loc = nullptr;
298     verify_oop(obj);
299     _interior_loc = nullptr;
300   }
301 
302   /**
303    * Verify oop fields from this object.
304    * @param obj host object for verified fields
305    */
306   void verify_oops_from(oop obj) {
307     _loc = obj;
308     obj->oop_iterate(this);

309     _loc = nullptr;
310   }
311 
312   virtual void do_oop(oop* p) { do_oop_work(p); }
313   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
314 };
315 
316 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
317 private:
318   size_t _used, _committed, _garbage;
319 public:
320   ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
321 
322   void heap_region_do(ShenandoahHeapRegion* r) {
323     _used += r->used();
324     _garbage += r->garbage();
325     _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
326   }
327 
328   size_t used() { return _used; }

568       HeapWord* addr = tams;
569 
570       while (addr < limit) {
571         verify_and_follow(addr, stack, cl, &processed);
572         addr += cast_to_oop(addr)->size();
573       }
574     }
575 
576     Atomic::add(&_processed, processed, memory_order_relaxed);
577   }
578 
579   void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
580     if (!_bitmap->par_mark(addr)) return;
581 
582     // Verify the object itself:
583     oop obj = cast_to_oop(addr);
584     cl.verify_oop_standalone(obj);
585 
586     // Verify everything reachable from that object too, hopefully realizing
587     // everything was already marked, and never touching further:
588     if (!is_instance_ref_klass(obj->klass())) {
589       cl.verify_oops_from(obj);
590       (*processed)++;
591     }
592     while (!stack.is_empty()) {
593       ShenandoahVerifierTask task = stack.pop();
594       cl.verify_oops_from(task.obj());
595       (*processed)++;
596     }
597   }
598 };
599 
600 class VerifyThreadGCState : public ThreadClosure {
601 private:
602   const char* const _label;
603          char const _expected;
604 
605 public:
606   VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
607   void do_thread(Thread* t) {
608     char actual = ShenandoahThreadLocalData::gc_state(t);

 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/tlab_globals.hpp"
 27 #include "gc/shenandoah/shenandoahAsserts.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
 32 #include "gc/shenandoah/shenandoahObjectUtils.inline.hpp"
 33 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
 34 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 35 #include "gc/shenandoah/shenandoahUtils.hpp"
 36 #include "gc/shenandoah/shenandoahVerifier.hpp"
 37 #include "memory/allocation.hpp"
 38 #include "memory/iterator.inline.hpp"
 39 #include "memory/resourceArea.hpp"
 40 #include "oops/compressedOops.inline.hpp"
 41 #include "runtime/atomic.hpp"
 42 #include "runtime/orderAccess.hpp"
 43 #include "runtime/threads.hpp"
 44 #include "utilities/align.hpp"
 45 
 46 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
 47 #ifdef verify_oop
 48 #undef verify_oop
 49 #endif
 50 
 51 static bool is_instance_ref_klass(Klass* k) {
 52   return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;

 82     _interior_loc(nullptr),
 83     _loc(nullptr) {
 84     if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
 85         options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
 86       set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
 87     }
 88   }
 89 
 90 private:
 91   void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
 92     if (!test) {
 93       ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
 94     }
 95   }
 96 
 97   template <class T>
 98   void do_oop_work(T* p) {
 99     T o = RawAccess<>::oop_load(p);
100     if (!CompressedOops::is_null(o)) {
101       oop obj = CompressedOops::decode_not_null(o);
102       if (is_instance_ref_klass(ShenandoahObjectUtils::klass(obj))) {
103         obj = ShenandoahForwarding::get_forwardee(obj);
104       }
105       // Single threaded verification can use faster non-atomic stack and bitmap
106       // methods.
107       //
108       // For performance reasons, only fully verify non-marked field values.
109       // We are here when the host object for *p is already marked.
110 
111       if (_map->par_mark(obj)) {
112         verify_oop_at(p, obj);
113         _stack->push(ShenandoahVerifierTask(obj));
114       }
115     }
116   }
117 
118   void verify_oop(oop obj) {
119     // Perform consistency checks with gradually decreasing safety level. This guarantees
120     // that failure report would not try to touch something that was not yet verified to be
121     // safe to process.
122 
123     check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
124               "oop must be in heap");
125     check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
126               "oop must be aligned");
127 
128     ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
129     Klass* obj_klass = ShenandoahObjectUtils::klass(obj);
130 
131     // Verify that obj is not in dead space:
132     {
133       // Do this before touching obj->size()
134       check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
135              "Object klass pointer should not be null");
136       check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
137              "Object klass pointer must go to metaspace");
138 
139       HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
140       check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
141              "Object start should be within the region");
142 
143       if (!obj_reg->is_humongous()) {
144         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + ShenandoahObjectUtils::size(obj)) <= obj_reg->top(),
145                "Object end should be within the region");
146       } else {
147         size_t humongous_start = obj_reg->index();
148         size_t humongous_end = humongous_start + (ShenandoahObjectUtils::size(obj) >> ShenandoahHeapRegion::region_size_words_shift());
149         for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
150           check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
151                  "Humongous object is in continuation that fits it");
152         }
153       }
154 
155       // ------------ obj is safe at this point --------------
156 
157       check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
158             "Object should be in active region");
159 
160       switch (_options._verify_liveness) {
161         case ShenandoahVerifier::_verify_liveness_disable:
162           // skip
163           break;
164         case ShenandoahVerifier::_verify_liveness_complete:
165           Atomic::add(&_ld[obj_reg->index()], (uint) ShenandoahObjectUtils::size(obj), memory_order_relaxed);
166           // fallthrough for fast failure for un-live regions:
167         case ShenandoahVerifier::_verify_liveness_conservative:
168           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
169                    "Object must belong to region with live data");
170           break;
171         default:
172           assert(false, "Unhandled liveness verification");
173       }
174     }
175 
176     oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
177 
178     ShenandoahHeapRegion* fwd_reg = nullptr;
179 
180     if (obj != fwd) {
181       check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
182              "Forwardee must be in heap");
183       check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
184              "Forwardee is set");
185       check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
186              "Forwardee must be aligned");
187 
188       // Do this before touching fwd->size()
189       Klass* fwd_klass = fwd->klass_or_null();
190       check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != nullptr,
191              "Forwardee klass pointer should not be null");
192       check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
193              "Forwardee klass pointer must go to metaspace");
194       check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
195              "Forwardee klass pointer must go to metaspace");
196 
197       fwd_reg = _heap->heap_region_containing(fwd);
198 
199       // Verify that forwardee is not in the dead space:
200       check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
201              "Should have no humongous forwardees");
202 
203       HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
204       check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
205              "Forwardee start should be within the region");
206       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + ShenandoahObjectUtils::size(fwd)) <= fwd_reg->top(),
207              "Forwardee end should be within the region");
208 
209       oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
210       check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
211              "Double forwarding");
212     } else {
213       fwd_reg = obj_reg;
214     }
215 
216     // ------------ obj and fwd are safe at this point --------------
217 
218     switch (_options._verify_marked) {
219       case ShenandoahVerifier::_verify_marked_disable:
220         // skip
221         break;
222       case ShenandoahVerifier::_verify_marked_incomplete:
223         check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
224                "Must be marked in incomplete bitmap");
225         break;
226       case ShenandoahVerifier::_verify_marked_complete:

289   }
290 
291   /**
292    * Verify object without known interior reference.
293    * Useful when picking up the object at known offset in heap,
294    * but without knowing what objects reference it.
295    * @param obj verified object
296    */
297   void verify_oop_standalone(oop obj) {
298     _interior_loc = nullptr;
299     verify_oop(obj);
300     _interior_loc = nullptr;
301   }
302 
303   /**
304    * Verify oop fields from this object.
305    * @param obj host object for verified fields
306    */
307   void verify_oops_from(oop obj) {
308     _loc = obj;
309     Klass* klass = ShenandoahObjectUtils::klass(obj);
310     obj->oop_iterate_backwards(this, klass);
311     _loc = nullptr;
312   }
313 
314   virtual void do_oop(oop* p) { do_oop_work(p); }
315   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
316 };
317 
318 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
319 private:
320   size_t _used, _committed, _garbage;
321 public:
322   ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
323 
324   void heap_region_do(ShenandoahHeapRegion* r) {
325     _used += r->used();
326     _garbage += r->garbage();
327     _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
328   }
329 
330   size_t used() { return _used; }

570       HeapWord* addr = tams;
571 
572       while (addr < limit) {
573         verify_and_follow(addr, stack, cl, &processed);
574         addr += cast_to_oop(addr)->size();
575       }
576     }
577 
578     Atomic::add(&_processed, processed, memory_order_relaxed);
579   }
580 
581   void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
582     if (!_bitmap->par_mark(addr)) return;
583 
584     // Verify the object itself:
585     oop obj = cast_to_oop(addr);
586     cl.verify_oop_standalone(obj);
587 
588     // Verify everything reachable from that object too, hopefully realizing
589     // everything was already marked, and never touching further:
590     if (!is_instance_ref_klass(ShenandoahObjectUtils::klass(obj))) {
591       cl.verify_oops_from(obj);
592       (*processed)++;
593     }
594     while (!stack.is_empty()) {
595       ShenandoahVerifierTask task = stack.pop();
596       cl.verify_oops_from(task.obj());
597       (*processed)++;
598     }
599   }
600 };
601 
602 class VerifyThreadGCState : public ThreadClosure {
603 private:
604   const char* const _label;
605          char const _expected;
606 
607 public:
608   VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
609   void do_thread(Thread* t) {
610     char actual = ShenandoahThreadLocalData::gc_state(t);
< prev index next >