1 /*
  2  * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/tlab_globals.hpp"
 27 #include "gc/shenandoah/shenandoahAsserts.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
 32 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
 33 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 34 #include "gc/shenandoah/shenandoahUtils.hpp"
 35 #include "gc/shenandoah/shenandoahVerifier.hpp"
 36 #include "memory/allocation.hpp"
 37 #include "memory/iterator.inline.hpp"
 38 #include "memory/resourceArea.hpp"
 39 #include "oops/compressedOops.inline.hpp"
 40 #include "runtime/atomic.hpp"
 41 #include "runtime/orderAccess.hpp"
 42 #include "utilities/align.hpp"
 43 
 44 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
 45 #ifdef verify_oop
 46 #undef verify_oop
 47 #endif
 48 
 49 static bool is_instance_ref_klass(Klass* k) {
 50   return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;
 51 }
 52 
 53 class ShenandoahIgnoreReferenceDiscoverer : public ReferenceDiscoverer {
 54 public:
 55   virtual bool discover_reference(oop obj, ReferenceType type) {
 56     return true;
 57   }
 58 };
 59 
 60 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
 61 private:
 62   const char* _phase;
 63   ShenandoahVerifier::VerifyOptions _options;
 64   ShenandoahVerifierStack* _stack;
 65   ShenandoahHeap* _heap;
 66   MarkBitMap* _map;
 67   ShenandoahLivenessData* _ld;
 68   void* _interior_loc;
 69   oop _loc;
 70 
 71 public:
 72   ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
 73                              const char* phase, ShenandoahVerifier::VerifyOptions options) :
 74     _phase(phase),
 75     _options(options),
 76     _stack(stack),
 77     _heap(ShenandoahHeap::heap()),
 78     _map(map),
 79     _ld(ld),
 80     _interior_loc(NULL),
 81     _loc(NULL) {
 82     if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
 83         options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
 84       set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
 85     }
 86   }
 87 
 88 private:
 89   void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
 90     if (!test) {
 91       ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
 92     }
 93   }
 94 
 95   template <class T>
 96   void do_oop_work(T* p) {
 97     T o = RawAccess<>::oop_load(p);
 98     if (!CompressedOops::is_null(o)) {
 99       oop obj = CompressedOops::decode_not_null(o);
100       if (is_instance_ref_klass(obj->klass())) {
101         obj = ShenandoahForwarding::get_forwardee(obj);
102       }
103       // Single threaded verification can use faster non-atomic stack and bitmap
104       // methods.
105       //
106       // For performance reasons, only fully verify non-marked field values.
107       // We are here when the host object for *p is already marked.
108 
109       if (_map->par_mark(obj)) {
110         verify_oop_at(p, obj);
111         _stack->push(ShenandoahVerifierTask(obj));
112       }
113     }
114   }
115 
116   void verify_oop(oop obj) {
117     // Perform consistency checks with gradually decreasing safety level. This guarantees
118     // that failure report would not try to touch something that was not yet verified to be
119     // safe to process.
120 
121     check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
122               "oop must be in heap");
123     check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
124               "oop must be aligned");
125 
126     ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
127     Klass* obj_klass = obj->klass_or_null();
128 
129     // Verify that obj is not in dead space:
130     {
131       // Do this before touching obj->size()
132       check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
133              "Object klass pointer should not be NULL");
134       check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
135              "Object klass pointer must go to metaspace");
136 
137       HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
138       check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
139              "Object start should be within the region");
140 
141       if (!obj_reg->is_humongous()) {
142         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
143                "Object end should be within the region");
144       } else {
145         size_t humongous_start = obj_reg->index();
146         size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
147         for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
148           check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
149                  "Humongous object is in continuation that fits it");
150         }
151       }
152 
153       // ------------ obj is safe at this point --------------
154 
155       check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
156             "Object should be in active region");
157 
158       switch (_options._verify_liveness) {
159         case ShenandoahVerifier::_verify_liveness_disable:
160           // skip
161           break;
162         case ShenandoahVerifier::_verify_liveness_complete:
163           Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
164           // fallthrough for fast failure for un-live regions:
165         case ShenandoahVerifier::_verify_liveness_conservative:
166           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
167                    "Object must belong to region with live data");
168           break;
169         default:
170           assert(false, "Unhandled liveness verification");
171       }
172     }
173 
174     oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
175 
176     ShenandoahHeapRegion* fwd_reg = NULL;
177 
178     if (obj != fwd) {
179       check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
180              "Forwardee must be in heap");
181       check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
182              "Forwardee is set");
183       check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
184              "Forwardee must be aligned");
185 
186       // Do this before touching fwd->size()
187       Klass* fwd_klass = fwd->klass_or_null();
188       check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
189              "Forwardee klass pointer should not be NULL");
190       check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
191              "Forwardee klass pointer must go to metaspace");
192       check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
193              "Forwardee klass pointer must go to metaspace");
194 
195       fwd_reg = _heap->heap_region_containing(fwd);
196 
197       // Verify that forwardee is not in the dead space:
198       check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
199              "Should have no humongous forwardees");
200 
201       HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
202       check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
203              "Forwardee start should be within the region");
204       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
205              "Forwardee end should be within the region");
206 
207       oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
208       check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
209              "Double forwarding");
210     } else {
211       fwd_reg = obj_reg;
212     }
213 
214     // ------------ obj and fwd are safe at this point --------------
215 
216     switch (_options._verify_marked) {
217       case ShenandoahVerifier::_verify_marked_disable:
218         // skip
219         break;
220       case ShenandoahVerifier::_verify_marked_incomplete:
221         check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
222                "Must be marked in incomplete bitmap");
223         break;
224       case ShenandoahVerifier::_verify_marked_complete:
225         check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
226                "Must be marked in complete bitmap");
227         break;
228       case ShenandoahVerifier::_verify_marked_complete_except_references:
229         check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
230               "Must be marked in complete bitmap, except j.l.r.Reference referents");
231         break;
232       default:
233         assert(false, "Unhandled mark verification");
234     }
235 
236     switch (_options._verify_forwarded) {
237       case ShenandoahVerifier::_verify_forwarded_disable:
238         // skip
239         break;
240       case ShenandoahVerifier::_verify_forwarded_none: {
241         check(ShenandoahAsserts::_safe_all, obj, (obj == fwd),
242                "Should not be forwarded");
243         break;
244       }
245       case ShenandoahVerifier::_verify_forwarded_allow: {
246         if (obj != fwd) {
247           check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
248                  "Forwardee should be in another region");
249         }
250         break;
251       }
252       default:
253         assert(false, "Unhandled forwarding verification");
254     }
255 
256     switch (_options._verify_cset) {
257       case ShenandoahVerifier::_verify_cset_disable:
258         // skip
259         break;
260       case ShenandoahVerifier::_verify_cset_none:
261         check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
262                "Should not have references to collection set");
263         break;
264       case ShenandoahVerifier::_verify_cset_forwarded:
265         if (_heap->in_collection_set(obj)) {
266           check(ShenandoahAsserts::_safe_all, obj, (obj != fwd),
267                  "Object in collection set, should have forwardee");
268         }
269         break;
270       default:
271         assert(false, "Unhandled cset verification");
272     }
273 
274   }
275 
276 public:
277   /**
278    * Verify object with known interior reference.
279    * @param p interior reference where the object is referenced from; can be off-heap
280    * @param obj verified object
281    */
282   template <class T>
283   void verify_oop_at(T* p, oop obj) {
284     _interior_loc = p;
285     verify_oop(obj);
286     _interior_loc = NULL;
287   }
288 
289   /**
290    * Verify object without known interior reference.
291    * Useful when picking up the object at known offset in heap,
292    * but without knowing what objects reference it.
293    * @param obj verified object
294    */
295   void verify_oop_standalone(oop obj) {
296     _interior_loc = NULL;
297     verify_oop(obj);
298     _interior_loc = NULL;
299   }
300 
301   /**
302    * Verify oop fields from this object.
303    * @param obj host object for verified fields
304    */
305   void verify_oops_from(oop obj) {
306     _loc = obj;
307     obj->oop_iterate(this);
308     _loc = NULL;
309   }
310 
311   virtual void do_oop(oop* p) { do_oop_work(p); }
312   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
313 };
314 
315 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
316 private:
317   size_t _used, _committed, _garbage;
318 public:
319   ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
320 
321   void heap_region_do(ShenandoahHeapRegion* r) {
322     _used += r->used();
323     _garbage += r->garbage();
324     _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
325   }
326 
327   size_t used() { return _used; }
328   size_t committed() { return _committed; }
329   size_t garbage() { return _garbage; }
330 };
331 
332 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
333 private:
334   ShenandoahHeap* _heap;
335   const char* _phase;
336   ShenandoahVerifier::VerifyRegions _regions;
337 public:
338   ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
339     _heap(ShenandoahHeap::heap()),
340     _phase(phase),
341     _regions(regions) {};
342 
343   void print_failure(ShenandoahHeapRegion* r, const char* label) {
344     ResourceMark rm;
345 
346     ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
347 
348     stringStream ss;
349     r->print_on(&ss);
350     msg.append("%s", ss.as_string());
351 
352     report_vm_error(__FILE__, __LINE__, msg.buffer());
353   }
354 
355   void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
356     if (!test) {
357       print_failure(r, msg);
358     }
359   }
360 
361   void heap_region_do(ShenandoahHeapRegion* r) {
362     switch (_regions) {
363       case ShenandoahVerifier::_verify_regions_disable:
364         break;
365       case ShenandoahVerifier::_verify_regions_notrash:
366         verify(r, !r->is_trash(),
367                "Should not have trash regions");
368         break;
369       case ShenandoahVerifier::_verify_regions_nocset:
370         verify(r, !r->is_cset(),
371                "Should not have cset regions");
372         break;
373       case ShenandoahVerifier::_verify_regions_notrash_nocset:
374         verify(r, !r->is_trash(),
375                "Should not have trash regions");
376         verify(r, !r->is_cset(),
377                "Should not have cset regions");
378         break;
379       default:
380         ShouldNotReachHere();
381     }
382 
383     verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
384            "Capacity should match region size");
385 
386     verify(r, r->bottom() <= r->top(),
387            "Region top should not be less than bottom");
388 
389     verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r),
390            "Region TAMS should not be less than bottom");
391 
392     verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(),
393            "Complete TAMS should not be larger than top");
394 
395     verify(r, r->get_live_data_bytes() <= r->capacity(),
396            "Live data cannot be larger than capacity");
397 
398     verify(r, r->garbage() <= r->capacity(),
399            "Garbage cannot be larger than capacity");
400 
401     verify(r, r->used() <= r->capacity(),
402            "Used cannot be larger than capacity");
403 
404     verify(r, r->get_shared_allocs() <= r->capacity(),
405            "Shared alloc count should not be larger than capacity");
406 
407     verify(r, r->get_tlab_allocs() <= r->capacity(),
408            "TLAB alloc count should not be larger than capacity");
409 
410     verify(r, r->get_gclab_allocs() <= r->capacity(),
411            "GCLAB alloc count should not be larger than capacity");
412 
413     verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
414            "Accurate accounting: shared + TLAB + GCLAB = used");
415 
416     verify(r, !r->is_empty() || !r->has_live(),
417            "Empty regions should not have live data");
418 
419     verify(r, r->is_cset() == _heap->collection_set()->is_in(r),
420            "Transitional: region flags and collection set agree");
421   }
422 };
423 
424 class ShenandoahVerifierReachableTask : public WorkerTask {
425 private:
426   const char* _label;
427   ShenandoahVerifier::VerifyOptions _options;
428   ShenandoahHeap* _heap;
429   ShenandoahLivenessData* _ld;
430   MarkBitMap* _bitmap;
431   volatile size_t _processed;
432 
433 public:
434   ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
435                                   ShenandoahLivenessData* ld,
436                                   const char* label,
437                                   ShenandoahVerifier::VerifyOptions options) :
438     WorkerTask("Shenandoah Verifier Reachable Objects"),
439     _label(label),
440     _options(options),
441     _heap(ShenandoahHeap::heap()),
442     _ld(ld),
443     _bitmap(bitmap),
444     _processed(0) {};
445 
446   size_t processed() {
447     return _processed;
448   }
449 
450   virtual void work(uint worker_id) {
451     ResourceMark rm;
452     ShenandoahVerifierStack stack;
453 
454     // On level 2, we need to only check the roots once.
455     // On level 3, we want to check the roots, and seed the local stack.
456     // It is a lesser evil to accept multiple root scans at level 3, because
457     // extended parallelism would buy us out.
458     if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
459         || (ShenandoahVerifyLevel >= 3)) {
460         ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
461                                       ShenandoahMessageBuffer("%s, Roots", _label),
462                                       _options);
463         if (_heap->unload_classes()) {
464           ShenandoahRootVerifier::strong_roots_do(&cl);
465         } else {
466           ShenandoahRootVerifier::roots_do(&cl);
467         }
468     }
469 
470     size_t processed = 0;
471 
472     if (ShenandoahVerifyLevel >= 3) {
473       ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
474                                     ShenandoahMessageBuffer("%s, Reachable", _label),
475                                     _options);
476       while (!stack.is_empty()) {
477         processed++;
478         ShenandoahVerifierTask task = stack.pop();
479         cl.verify_oops_from(task.obj());
480       }
481     }
482 
483     Atomic::add(&_processed, processed, memory_order_relaxed);
484   }
485 };
486 
487 class ShenandoahVerifierMarkedRegionTask : public WorkerTask {
488 private:
489   const char* _label;
490   ShenandoahVerifier::VerifyOptions _options;
491   ShenandoahHeap *_heap;
492   MarkBitMap* _bitmap;
493   ShenandoahLivenessData* _ld;
494   volatile size_t _claimed;
495   volatile size_t _processed;
496 
497 public:
498   ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
499                                      ShenandoahLivenessData* ld,
500                                      const char* label,
501                                      ShenandoahVerifier::VerifyOptions options) :
502           WorkerTask("Shenandoah Verifier Marked Objects"),
503           _label(label),
504           _options(options),
505           _heap(ShenandoahHeap::heap()),
506           _bitmap(bitmap),
507           _ld(ld),
508           _claimed(0),
509           _processed(0) {};
510 
511   size_t processed() {
512     return Atomic::load(&_processed);
513   }
514 
515   virtual void work(uint worker_id) {
516     ShenandoahVerifierStack stack;
517     ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
518                                   ShenandoahMessageBuffer("%s, Marked", _label),
519                                   _options);
520 
521     while (true) {
522       size_t v = Atomic::fetch_and_add(&_claimed, 1u, memory_order_relaxed);
523       if (v < _heap->num_regions()) {
524         ShenandoahHeapRegion* r = _heap->get_region(v);
525         if (!r->is_humongous() && !r->is_trash()) {
526           work_regular(r, stack, cl);
527         } else if (r->is_humongous_start()) {
528           work_humongous(r, stack, cl);
529         }
530       } else {
531         break;
532       }
533     }
534   }
535 
536   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
537     size_t processed = 0;
538     HeapWord* obj = r->bottom();
539     if (_heap->complete_marking_context()->is_marked(cast_to_oop(obj))) {
540       verify_and_follow(obj, stack, cl, &processed);
541     }
542     Atomic::add(&_processed, processed, memory_order_relaxed);
543   }
544 
545   virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
546     size_t processed = 0;
547     ShenandoahMarkingContext* ctx = _heap->complete_marking_context();
548     HeapWord* tams = ctx->top_at_mark_start(r);
549 
550     // Bitmaps, before TAMS
551     if (tams > r->bottom()) {
552       HeapWord* start = r->bottom();
553       HeapWord* addr = ctx->get_next_marked_addr(start, tams);
554 
555       while (addr < tams) {
556         verify_and_follow(addr, stack, cl, &processed);
557         addr += 1;
558         if (addr < tams) {
559           addr = ctx->get_next_marked_addr(addr, tams);
560         }
561       }
562     }
563 
564     // Size-based, after TAMS
565     {
566       HeapWord* limit = r->top();
567       HeapWord* addr = tams;
568 
569       while (addr < limit) {
570         verify_and_follow(addr, stack, cl, &processed);
571         addr += cast_to_oop(addr)->size();
572       }
573     }
574 
575     Atomic::add(&_processed, processed, memory_order_relaxed);
576   }
577 
578   void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
579     if (!_bitmap->par_mark(addr)) return;
580 
581     // Verify the object itself:
582     oop obj = cast_to_oop(addr);
583     cl.verify_oop_standalone(obj);
584 
585     // Verify everything reachable from that object too, hopefully realizing
586     // everything was already marked, and never touching further:
587     if (!is_instance_ref_klass(obj->klass())) {
588       cl.verify_oops_from(obj);
589       (*processed)++;
590     }
591     while (!stack.is_empty()) {
592       ShenandoahVerifierTask task = stack.pop();
593       cl.verify_oops_from(task.obj());
594       (*processed)++;
595     }
596   }
597 };
598 
599 class VerifyThreadGCState : public ThreadClosure {
600 private:
601   const char* const _label;
602          char const _expected;
603 
604 public:
605   VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {}
606   void do_thread(Thread* t) {
607     char actual = ShenandoahThreadLocalData::gc_state(t);
608     if (actual != _expected) {
609       fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual);
610     }
611   }
612 };
613 
614 void ShenandoahVerifier::verify_at_safepoint(const char *label,
615                                              VerifyForwarded forwarded, VerifyMarked marked,
616                                              VerifyCollectionSet cset,
617                                              VerifyLiveness liveness, VerifyRegions regions,
618                                              VerifyGCState gcstate) {
619   guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
620   guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
621 
622   // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
623   ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
624 
625   log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
626 
627   // GC state checks
628   {
629     char expected = -1;
630     bool enabled;
631     switch (gcstate) {
632       case _verify_gcstate_disable:
633         enabled = false;
634         break;
635       case _verify_gcstate_forwarded:
636         enabled = true;
637         expected = ShenandoahHeap::HAS_FORWARDED;
638         break;
639       case _verify_gcstate_evacuation:
640         enabled = true;
641         expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION;
642         if (!_heap->is_stw_gc_in_progress()) {
643           // Only concurrent GC sets this.
644           expected |= ShenandoahHeap::WEAK_ROOTS;
645         }
646         break;
647       case _verify_gcstate_stable:
648         enabled = true;
649         expected = ShenandoahHeap::STABLE;
650         break;
651       case _verify_gcstate_stable_weakroots:
652         enabled = true;
653         expected = ShenandoahHeap::STABLE;
654         if (!_heap->is_stw_gc_in_progress()) {
655           // Only concurrent GC sets this.
656           expected |= ShenandoahHeap::WEAK_ROOTS;
657         }
658         break;
659       default:
660         enabled = false;
661         assert(false, "Unhandled gc-state verification");
662     }
663 
664     if (enabled) {
665       char actual = _heap->gc_state();
666       if (actual != expected) {
667         fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual);
668       }
669 
670       VerifyThreadGCState vtgcs(label, expected);
671       Threads::java_threads_do(&vtgcs);
672     }
673   }
674 
675   // Deactivate barriers temporarily: Verifier wants plain heap accesses
676   ShenandoahGCStateResetter resetter;
677 
678   // Heap size checks
679   {
680     ShenandoahHeapLocker lock(_heap->lock());
681 
682     ShenandoahCalculateRegionStatsClosure cl;
683     _heap->heap_region_iterate(&cl);
684     size_t heap_used = _heap->used();
685     guarantee(cl.used() == heap_used,
686               "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
687               label,
688               byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used),
689               byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used()));
690 
691     size_t heap_committed = _heap->committed();
692     guarantee(cl.committed() == heap_committed,
693               "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s",
694               label,
695               byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed),
696               byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed()));
697   }
698 
699   // Internal heap region checks
700   if (ShenandoahVerifyLevel >= 1) {
701     ShenandoahVerifyHeapRegionClosure cl(label, regions);
702     _heap->heap_region_iterate(&cl);
703   }
704 
705   OrderAccess::fence();
706 
707   if (UseTLAB) {
708     _heap->labs_make_parsable();
709   }
710 
711   // Allocate temporary bitmap for storing marking wavefront:
712   _verification_bit_map->clear();
713 
714   // Allocate temporary array for storing liveness data
715   ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
716   Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
717 
718   const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate);
719 
720   // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
721   // This verifies what application can see, since it only cares about reachable objects.
722   size_t count_reachable = 0;
723   if (ShenandoahVerifyLevel >= 2) {
724     ShenandoahVerifierReachableTask task(_verification_bit_map, ld, label, options);
725     _heap->workers()->run_task(&task);
726     count_reachable = task.processed();
727   }
728 
729   // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
730   // not the application, can see during the region scans. There is no reason to process the objects
731   // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
732   // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
733   // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
734   // version
735 
736   size_t count_marked = 0;
737   if (ShenandoahVerifyLevel >= 4 && (marked == _verify_marked_complete || marked == _verify_marked_complete_except_references)) {
738     guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
739     ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
740     _heap->workers()->run_task(&task);
741     count_marked = task.processed();
742   } else {
743     guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be");
744   }
745 
746   // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
747   // marked objects.
748 
749   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
750     for (size_t i = 0; i < _heap->num_regions(); i++) {
751       ShenandoahHeapRegion* r = _heap->get_region(i);
752 
753       juint verf_live = 0;
754       if (r->is_humongous()) {
755         // For humongous objects, test if start region is marked live, and if so,
756         // all humongous regions in that chain have live data equal to their "used".
757         juint start_live = Atomic::load(&ld[r->humongous_start_region()->index()]);
758         if (start_live > 0) {
759           verf_live = (juint)(r->used() / HeapWordSize);
760         }
761       } else {
762         verf_live = Atomic::load(&ld[r->index()]);
763       }
764 
765       size_t reg_live = r->get_live_data_words();
766       if (reg_live != verf_live) {
767         ResourceMark rm;
768         stringStream ss;
769         r->print_on(&ss);
770         fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
771               label, reg_live, verf_live, ss.as_string());
772       }
773     }
774   }
775 
776   log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
777                label, ShenandoahVerifyLevel, count_reachable, count_marked);
778 
779   FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
780 }
781 
782 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
783   verify_at_safepoint(
784           "Generic Verification",
785           _verify_forwarded_allow,     // conservatively allow forwarded
786           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
787           _verify_cset_disable,        // cset may be inconsistent
788           _verify_liveness_disable,    // no reliable liveness data
789           _verify_regions_disable,     // no reliable region data
790           _verify_gcstate_disable      // no data about gcstate
791   );
792 }
793 
794 void ShenandoahVerifier::verify_before_concmark() {
795     verify_at_safepoint(
796           "Before Mark",
797           _verify_forwarded_none,      // UR should have fixed up
798           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
799           _verify_cset_none,           // UR should have fixed this
800           _verify_liveness_disable,    // no reliable liveness data
801           _verify_regions_notrash,     // no trash regions
802           _verify_gcstate_stable       // there are no forwarded objects
803   );
804 }
805 
806 void ShenandoahVerifier::verify_after_concmark() {
807   verify_at_safepoint(
808           "After Mark",
809           _verify_forwarded_none,      // no forwarded references
810           _verify_marked_complete_except_references, // bitmaps as precise as we can get, except dangling j.l.r.Refs
811           _verify_cset_none,           // no references to cset anymore
812           _verify_liveness_complete,   // liveness data must be complete here
813           _verify_regions_disable,     // trash regions not yet recycled
814           _verify_gcstate_stable_weakroots  // heap is still stable, weakroots are in progress
815   );
816 }
817 
818 void ShenandoahVerifier::verify_before_evacuation() {
819   verify_at_safepoint(
820           "Before Evacuation",
821           _verify_forwarded_none,                    // no forwarded references
822           _verify_marked_complete_except_references, // walk over marked objects too
823           _verify_cset_disable,                      // non-forwarded references to cset expected
824           _verify_liveness_complete,                 // liveness data must be complete here
825           _verify_regions_disable,                   // trash regions not yet recycled
826           _verify_gcstate_stable_weakroots           // heap is still stable, weakroots are in progress
827   );
828 }
829 
830 void ShenandoahVerifier::verify_during_evacuation() {
831   verify_at_safepoint(
832           "During Evacuation",
833           _verify_forwarded_allow,    // some forwarded references are allowed
834           _verify_marked_disable,     // walk only roots
835           _verify_cset_disable,       // some cset references are not forwarded yet
836           _verify_liveness_disable,   // liveness data might be already stale after pre-evacs
837           _verify_regions_disable,    // trash regions not yet recycled
838           _verify_gcstate_evacuation  // evacuation is in progress
839   );
840 }
841 
842 void ShenandoahVerifier::verify_after_evacuation() {
843   verify_at_safepoint(
844           "After Evacuation",
845           _verify_forwarded_allow,     // objects are still forwarded
846           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
847           _verify_cset_forwarded,      // all cset refs are fully forwarded
848           _verify_liveness_disable,    // no reliable liveness data anymore
849           _verify_regions_notrash,     // trash regions have been recycled already
850           _verify_gcstate_forwarded    // evacuation produced some forwarded objects
851   );
852 }
853 
854 void ShenandoahVerifier::verify_before_updaterefs() {
855   verify_at_safepoint(
856           "Before Updating References",
857           _verify_forwarded_allow,     // forwarded references allowed
858           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
859           _verify_cset_forwarded,      // all cset refs are fully forwarded
860           _verify_liveness_disable,    // no reliable liveness data anymore
861           _verify_regions_notrash,     // trash regions have been recycled already
862           _verify_gcstate_forwarded    // evacuation should have produced some forwarded objects
863   );
864 }
865 
866 void ShenandoahVerifier::verify_after_updaterefs() {
867   verify_at_safepoint(
868           "After Updating References",
869           _verify_forwarded_none,      // no forwarded references
870           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
871           _verify_cset_none,           // no cset references, all updated
872           _verify_liveness_disable,    // no reliable liveness data anymore
873           _verify_regions_nocset,      // no cset regions, trash regions have appeared
874           _verify_gcstate_stable       // update refs had cleaned up forwarded objects
875   );
876 }
877 
878 void ShenandoahVerifier::verify_after_degenerated() {
879   verify_at_safepoint(
880           "After Degenerated GC",
881           _verify_forwarded_none,      // all objects are non-forwarded
882           _verify_marked_complete,     // all objects are marked in complete bitmap
883           _verify_cset_none,           // no cset references
884           _verify_liveness_disable,    // no reliable liveness data anymore
885           _verify_regions_notrash_nocset, // no trash, no cset
886           _verify_gcstate_stable       // degenerated refs had cleaned up forwarded objects
887   );
888 }
889 
890 void ShenandoahVerifier::verify_before_fullgc() {
891   verify_at_safepoint(
892           "Before Full GC",
893           _verify_forwarded_allow,     // can have forwarded objects
894           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
895           _verify_cset_disable,        // cset might be foobared
896           _verify_liveness_disable,    // no reliable liveness data anymore
897           _verify_regions_disable,     // no reliable region data here
898           _verify_gcstate_disable      // no reliable gcstate data
899   );
900 }
901 
902 void ShenandoahVerifier::verify_after_fullgc() {
903   verify_at_safepoint(
904           "After Full GC",
905           _verify_forwarded_none,      // all objects are non-forwarded
906           _verify_marked_complete,     // all objects are marked in complete bitmap
907           _verify_cset_none,           // no cset references
908           _verify_liveness_disable,    // no reliable liveness data anymore
909           _verify_regions_notrash_nocset, // no trash, no cset
910           _verify_gcstate_stable        // full gc cleaned up everything
911   );
912 }
913 
914 class ShenandoahVerifyNoForwared : public OopClosure {
915 private:
916   template <class T>
917   void do_oop_work(T* p) {
918     T o = RawAccess<>::oop_load(p);
919     if (!CompressedOops::is_null(o)) {
920       oop obj = CompressedOops::decode_not_null(o);
921       oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
922       if (obj != fwd) {
923         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
924                                          "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
925       }
926     }
927   }
928 
929 public:
930   void do_oop(narrowOop* p) { do_oop_work(p); }
931   void do_oop(oop* p)       { do_oop_work(p); }
932 };
933 
934 class ShenandoahVerifyInToSpaceClosure : public OopClosure {
935 private:
936   template <class T>
937   void do_oop_work(T* p) {
938     T o = RawAccess<>::oop_load(p);
939     if (!CompressedOops::is_null(o)) {
940       oop obj = CompressedOops::decode_not_null(o);
941       ShenandoahHeap* heap = ShenandoahHeap::heap();
942 
943       if (!heap->marking_context()->is_marked(obj)) {
944         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
945                 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
946       }
947 
948       if (heap->in_collection_set(obj)) {
949         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
950                 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
951       }
952 
953       oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
954       if (obj != fwd) {
955         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
956                 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
957       }
958     }
959   }
960 
961 public:
962   void do_oop(narrowOop* p) { do_oop_work(p); }
963   void do_oop(oop* p)       { do_oop_work(p); }
964 };
965 
966 void ShenandoahVerifier::verify_roots_in_to_space() {
967   ShenandoahVerifyInToSpaceClosure cl;
968   ShenandoahRootVerifier::roots_do(&cl);
969 }
970 
971 void ShenandoahVerifier::verify_roots_no_forwarded() {
972   ShenandoahVerifyNoForwared cl;
973   ShenandoahRootVerifier::roots_do(&cl);
974 }