1 /*
   2  * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "compiler/oopMap.inline.hpp"
  27 #include "oops/instanceStackChunkKlass.hpp"
  28 #include "gc/shared/gc_globals.hpp"
  29 #include "memory/resourceArea.hpp"
  30 #include "oops/oopsHierarchy.hpp"
  31 #include "oops/stackChunkOop.hpp"
  32 #include "code/scopeDesc.hpp"
  33 #include "classfile/javaClasses.inline.hpp"
  34 #include "classfile/systemDictionary.hpp"
  35 #include "gc/shared/collectedHeap.inline.hpp"
  36 #include "jfr/jfrEvents.hpp"
  37 #include "memory/iterator.inline.hpp"
  38 #include "memory/oopFactory.hpp"
  39 #include "memory/universe.hpp"
  40 #include "oops/instanceKlass.hpp"
  41 #include "oops/instanceStackChunkKlass.inline.hpp"
  42 #include "oops/instanceOop.hpp"
  43 #include "oops/oop.inline.hpp"
  44 #include "oops/symbol.hpp"
  45 #include "runtime/continuation.hpp"
  46 #include "runtime/globals.hpp"
  47 #include "utilities/bitMap.hpp"
  48 #include "utilities/globalDefinitions.hpp"
  49 #include "utilities/macros.hpp"
  50 
  51 int InstanceStackChunkKlass::_offset_of_stack = 0;
  52 
  53 #if INCLUDE_CDS
  54 void InstanceStackChunkKlass::serialize_offsets(SerializeClosure* f) {
  55   f->do_u4((u4*)&_offset_of_stack);
  56 }
  57 #endif
  58 
  59 typedef void (*MemcpyFnT)(void* src, void* dst, size_t count);
  60 
  61 void InstanceStackChunkKlass::default_memcpy(void* from, void* to, size_t size) {
  62   memcpy(to, from, size << LogBytesPerWord);
  63 }
  64 
  65 MemcpyFnT InstanceStackChunkKlass::memcpy_fn_from_stack_to_chunk = nullptr;
  66 MemcpyFnT InstanceStackChunkKlass::memcpy_fn_from_chunk_to_stack = nullptr;
  67 
  68 void InstanceStackChunkKlass::resolve_memcpy_functions() {
  69   if (!StubRoutines::has_word_memcpy() || UseNewCode) {
  70     memcpy_fn_from_stack_to_chunk = (MemcpyFnT)InstanceStackChunkKlass::default_memcpy;
  71     memcpy_fn_from_chunk_to_stack = (MemcpyFnT)InstanceStackChunkKlass::default_memcpy;
  72   } else {
  73     memcpy_fn_from_stack_to_chunk = UseContinuationStreamingCopy ? (MemcpyFnT)StubRoutines::word_memcpy_up_nt()
  74                                                                  : (MemcpyFnT)StubRoutines::word_memcpy_up();
  75     memcpy_fn_from_chunk_to_stack = UseContinuationStreamingCopy ? (MemcpyFnT)StubRoutines::word_memcpy_down_nt()
  76                                                                  : (MemcpyFnT)StubRoutines::word_memcpy_down();
  77   }
  78   assert (memcpy_fn_from_stack_to_chunk != nullptr, "");
  79   assert (memcpy_fn_from_chunk_to_stack != nullptr, "");
  80 }
  81 
  82 InstanceStackChunkKlass::InstanceStackChunkKlass(const ClassFileParser& parser)
  83  : InstanceKlass(parser, InstanceKlass::_misc_kind_stack_chunk, ID) {
  84    // see oopDesc::size_given_klass TODO perf
  85    const jint lh = Klass::instance_layout_helper(size_helper(), true);
  86    set_layout_helper(lh);
  87    assert (layout_helper_is_instance(layout_helper()), "");
  88    assert (layout_helper_needs_slow_path(layout_helper()), "");
  89 
  90   //  resolve_memcpy_functions(); -- too early here
  91 }
  92 
  93 size_t InstanceStackChunkKlass::oop_size(oop obj) const {
  94   // see oopDesc::size_given_klass
  95   return instance_size(jdk_internal_vm_StackChunk::size(obj));
  96 }
  97 
  98 template <int x> NOINLINE static bool verify_chunk(stackChunkOop c) { return c->verify(); }
  99 
 100 template <bool disjoint>
 101 size_t InstanceStackChunkKlass::copy(oop obj, HeapWord* to_addr, size_t word_size) {
 102   assert (obj->is_stackChunk(), "");
 103   stackChunkOop chunk = (stackChunkOop)obj;
 104 
 105   HeapWord* from_addr = cast_from_oop<HeapWord*>(obj);
 106   assert (from_addr != to_addr, "");
 107   disjoint ? Copy::aligned_disjoint_words(from_addr, to_addr, word_size)
 108            : Copy::aligned_conjoint_words(from_addr, to_addr, word_size);
 109 
 110   stackChunkOop to_chunk = (stackChunkOop) cast_to_oop(to_addr);
 111   assert (!to_chunk->has_bitmap()|| to_chunk->is_gc_mode(), "");
 112   if (!to_chunk->has_bitmap()) {
 113     build_bitmap(to_chunk);
 114   }
 115 
 116   return word_size;
 117 }
 118 
 119 template size_t InstanceStackChunkKlass::copy<false>(oop obj, HeapWord* to_addr, size_t word_size);
 120 template size_t InstanceStackChunkKlass::copy<true>(oop obj, HeapWord* to_addr, size_t word_size);
 121 
 122 size_t InstanceStackChunkKlass::compact_oop_size(oop obj) const {
 123   assert (obj->is_stackChunk(), "");
 124   stackChunkOop chunk = (stackChunkOop)obj;
 125   // We don't trim chunks with ZGC. See copy_compact
 126   // if (UseZGC) return instance_size(chunk->stack_size());
 127   int used_stack_in_words = chunk->stack_size() - chunk->sp() + metadata_words();
 128   assert (used_stack_in_words <= chunk->stack_size(), "");
 129   return align_object_size(size_helper() + used_stack_in_words + bitmap_size(used_stack_in_words));
 130 }
 131 
 132 template size_t InstanceStackChunkKlass::copy_compact<false>(oop obj, HeapWord* to_addr);
 133 template size_t InstanceStackChunkKlass::copy_compact<true>(oop obj, HeapWord* to_addr);
 134 
 135 template <bool disjoint>
 136 size_t InstanceStackChunkKlass::copy_compact(oop obj, HeapWord* to_addr) {
 137   assert (obj->is_stackChunk(), "");
 138   stackChunkOop chunk = (stackChunkOop)obj;
 139 
 140   assert (chunk->verify(), "");
 141 
 142 #ifdef ASSERT
 143   size_t old_compact_size = obj->compact_size();
 144   size_t old_size = obj->size();
 145   assert (old_compact_size <= old_size, "");
 146 #endif
 147 
 148   const int from_sp = chunk->sp();
 149   assert (from_sp >= metadata_words(), "");
 150 
 151   // ZGC usually relocates objects into allocating regions that don't require barriers, which keeps/makes the chunk mutable.
 152   if (from_sp <= metadata_words() /*|| UseZGC*/) {
 153     assert (oop_size(obj) == compact_oop_size(obj), "");
 154     return copy<disjoint>(obj, to_addr, oop_size(obj));
 155   }
 156 
 157   const int header = size_helper();
 158   const int from_stack_size = chunk->stack_size();
 159   const int to_stack_size = from_stack_size - from_sp + metadata_words();
 160   const size_t from_bitmap_size = bitmap_size(from_stack_size);
 161   const size_t to_bitmap_size = bitmap_size(to_stack_size);
 162   const bool has_bitmap = chunk->has_bitmap();
 163   assert (to_stack_size >= 0, "");
 164   assert (to_stack_size <= from_stack_size, "");
 165   assert (to_stack_size > 0 || chunk->argsize() == 0, "");
 166 #ifdef ASSERT
 167   HeapWord* const start_of_stack0 = start_of_stack(obj);
 168   HeapWord* const start_of_bitmap0 = start_of_bitmap(obj);
 169 #endif
 170 
 171   // tty->print_cr(">>> CPY %s %p-%p (%d) -> %p-%p (%d) [%d] -- %d", disjoint ? "DIS" : "CON", cast_from_oop<HeapWord*>(obj), cast_from_oop<HeapWord*>(obj) + header + from_size, from_size, to_addr, to_addr + header + to_stack_size, to_stack_size, old_compact_size, chunk->is_gc_mode());
 172 
 173   stackChunkOop to_chunk = (stackChunkOop) cast_to_oop(to_addr);
 174   HeapWord* from_addr = cast_from_oop<HeapWord*>(obj);
 175 
 176   // copy header and stack in the appropriate order if conjoint; must not touch old chunk after we start b/c this can be a conjoint copy
 177   const int first  = (disjoint || to_addr <= from_addr) ? 0 : 2;
 178   const int stride = 1 - first;
 179   for (int i = first; 0 <= i && i <= 2; i += stride) {
 180     switch(i) { // copy header and update it
 181     case 0:
 182       // tty->print_cr(">>> CPY header %p-%p -> %p-%p (%d)", from_addr, from_addr + header , to_addr, to_addr + header, header);
 183       if (to_addr != from_addr) {
 184         disjoint ? Copy::aligned_disjoint_words(from_addr, to_addr, header)
 185                  : Copy::aligned_conjoint_words(from_addr, to_addr, header);
 186       }
 187 
 188       jdk_internal_vm_StackChunk::set_size(to_addr, to_stack_size);
 189       to_chunk->set_sp(metadata_words());
 190       break;
 191     case 1: // copy stack
 192       if (to_stack_size > 0) {
 193         assert ((from_addr + header) == start_of_stack0, "");
 194         HeapWord* from_start = from_addr + header + from_sp - metadata_words();
 195         HeapWord* to_start = to_addr + header;
 196         // tty->print_cr(">>> CPY stack  %p-%p -> %p-%p (%d)", from_start, from_start + to_stack_size , to_start, to_start + to_stack_size, to_stack_size);
 197         disjoint ? Copy::aligned_disjoint_words(from_start, to_start, to_stack_size)
 198                  : Copy::aligned_conjoint_words(from_start, to_start, to_stack_size);
 199       }
 200       break;
 201     case 2: // copy bitmap
 202       if (to_stack_size > 0 && has_bitmap) {
 203         assert ((from_addr + header + from_stack_size) == start_of_bitmap0, "");
 204         assert (from_bitmap_size >= to_bitmap_size, "");
 205         HeapWord* from_start = from_addr + header + from_stack_size + (from_bitmap_size - to_bitmap_size);
 206         HeapWord* to_start = to_addr + header + to_stack_size;
 207         // tty->print_cr(">>> CPY bitmap  %p-%p -> %p-%p (%d)", from_start, from_start + to_bitmap_size , to_start, to_start + to_bitmap_size, to_bitmap_size);
 208         disjoint ? Copy::aligned_disjoint_words(from_start, to_start, to_bitmap_size)
 209                  : Copy::aligned_conjoint_words(from_start, to_start, to_bitmap_size);
 210       }
 211       break;
 212     }
 213   }
 214 
 215   assert (!to_chunk->has_bitmap()|| to_chunk->is_gc_mode(), "");
 216   assert (to_chunk->has_bitmap() == has_bitmap, "");
 217   if (!to_chunk->has_bitmap()) {
 218     build_bitmap(to_chunk);
 219   }
 220 
 221   assert (to_chunk->size() == old_compact_size, "");
 222   assert (to_chunk->size() == instance_size(to_stack_size), "");
 223   assert (from_sp <= metadata_words() ?  (to_chunk->size() == old_size) : (to_chunk->size() < old_size), "");
 224   assert (to_chunk->verify(), "");
 225 
 226   // assert (to_chunk->requires_barriers(), ""); // G1 sometimes compacts a young region and *then* turns it old ((G1CollectedHeap*)Universe::heap())->heap_region_containing(oop(to_addr))->print();
 227 
 228   return align_object_size(header + to_stack_size + to_bitmap_size);
 229 }
 230 
 231 template <bool mixed>
 232 int InstanceStackChunkKlass::count_frames(stackChunkOop chunk) {
 233   int frames = 0;
 234   for (StackChunkFrameStream<mixed> f(chunk); !f.is_done(); f.next(SmallRegisterMap::instance)) frames++;
 235   return frames;
 236 }
 237 
 238 #ifndef PRODUCT
 239 void InstanceStackChunkKlass::oop_print_on(oop obj, outputStream* st) {
 240   // InstanceKlass::oop_print_on(obj, st);
 241   print_chunk((stackChunkOop)obj, false, st);
 242 }
 243 #endif
 244 
 245 
 246 // We replace derived pointers with offsets; the converse is done in DerelativizeDerivedPointers
 247 template <bool concurrent_gc>
 248 class RelativizeDerivedPointers : public DerivedOopClosure {
 249 public:
 250   RelativizeDerivedPointers() {}
 251 
 252   virtual void do_derived_oop(oop* base_loc, derived_pointer* derived_loc) override {
 253     // The ordering in the following is crucial
 254     OrderAccess::loadload();
 255     oop base = Atomic::load((oop*)base_loc);
 256     // assert (Universe::heap()->is_in_or_null(base), "not an oop"); -- base might be invalid at this point
 257     if (base != (oop)nullptr) {
 258       assert (!CompressedOops::is_base(base), "");
 259 
 260 #if INCLUDE_ZGC
 261       if (concurrent_gc && UseZGC) {
 262         if (ZAddress::is_good(cast_from_oop<uintptr_t>(base)))
 263           return;
 264       }
 265 #endif
 266 #if INCLUDE_SHENANDOAHGC
 267       if (concurrent_gc && UseShenandoahGC) {
 268         if (!ShenandoahHeap::heap()->in_collection_set(base)) {
 269           return;
 270         }
 271       }
 272 #endif
 273 
 274       OrderAccess::loadload();
 275       intptr_t derived_int_val = Atomic::load((intptr_t*)derived_loc); // *derived_loc;
 276       if (derived_int_val <= 0) { // an offset of 0 was observed on AArch64
 277         return;
 278       }
 279 
 280       // at this point, we've seen a non-offset value *after* we've read the base, but we write the offset *before* fixing the base,
 281       // so we are guaranteed that the value in derived_loc is consistent with base (i.e. points into the object).
 282       intptr_t offset = derived_int_val - cast_from_oop<intptr_t>(base);
 283       assert (offset >= 0, "Derived pointer offset is %ld", offset); // an offset of 0 was observed on AArch64
 284       // assert (offset >= 0 && offset <= (base->size() << LogHeapWordSize), "offset: %ld size: %d", offset, (base->size() << LogHeapWordSize)); -- base might be invalid at this point
 285       Atomic::store((intptr_t*)derived_loc, -offset); // there could be a benign race here; we write a negative offset to let the sign bit signify it's an offset rather than an address
 286     } else {
 287       assert (*derived_loc == derived_pointer(0), "");
 288     }
 289   }
 290 };
 291 
 292 class DerelativizeDerivedPointers : public DerivedOopClosure {
 293 public:
 294   virtual void do_derived_oop(oop* base_loc, derived_pointer* derived_loc) override {
 295     // The ordering in the following is crucial
 296     OrderAccess::loadload();
 297     oop base = Atomic::load(base_loc);
 298     if (base != (oop)nullptr) {
 299       assert (!CompressedOops::is_base(base), "");
 300       ZGC_ONLY(assert (ZAddress::is_good(cast_from_oop<uintptr_t>(base)), "");)
 301 
 302       OrderAccess::loadload();
 303       intptr_t offset = Atomic::load((intptr_t*)derived_loc); // *derived_loc;
 304 
 305       // at this point, we've seen a non-offset value *after* we've read the base, but we write the offset *before* fixing the base,
 306       // so we are guaranteed that the value in derived_loc is consistent with base (i.e. points into the object).
 307       if (offset <= 0) { // an offset of 0 was observed on AArch64
 308         offset = -offset;
 309         assert (offset >= 0 && (size_t)offset <= (base->size() << LogHeapWordSize), "");
 310         Atomic::store((intptr_t*)derived_loc, cast_from_oop<intptr_t>(base) + offset);
 311       }
 312   #ifdef ASSERT
 313       else {
 314         offset = offset - cast_from_oop<intptr_t>(base);
 315         assert (offset >= 0 && (size_t)offset <= (base->size() << LogHeapWordSize), "offset: " PTR_FORMAT " size: %zu", offset, (base->size() << LogHeapWordSize));
 316       }
 317   #endif
 318     }
 319   }
 320 };
 321 
 322 template <bool store, bool compressedOopsWithBitmap>
 323 class BarrierClosure: public OopClosure {
 324   NOT_PRODUCT(intptr_t* _sp;)
 325 public:
 326   BarrierClosure(intptr_t* sp) NOT_PRODUCT(: _sp(sp)) {}
 327 
 328   virtual void do_oop(oop* p)       override { compressedOopsWithBitmap ? do_oop_work((narrowOop*)p) : do_oop_work(p); }
 329   virtual void do_oop(narrowOop* p) override { do_oop_work(p); }
 330 
 331   template <class T> inline void do_oop_work(T* p) {
 332     oop value = (oop)HeapAccess<>::oop_load(p);
 333     if (store) HeapAccess<>::oop_store(p, value);
 334     log_develop_trace(jvmcont)("barriers_for_oops_in_frame narrow: %d p: " INTPTR_FORMAT " sp offset: " INTPTR_FORMAT, sizeof(T) < sizeof(intptr_t), p2i(p), (intptr_t*)p - _sp);
 335   }
 336 };
 337 
 338 
 339 template<typename OopClosureType>
 340 class StackChunkOopIterateFilterClosure: public OopClosure {
 341 private:
 342   OopClosureType* const _closure;
 343   stackChunkOop _chunk;
 344   MemRegion _bound;
 345 
 346 public:
 347   StackChunkOopIterateFilterClosure(OopClosureType* closure, stackChunkOop chunk, MemRegion bound)
 348     : _closure(closure),
 349       _chunk(chunk),
 350       _bound(bound),
 351       _mutated(false),
 352       _num_oops(0) {}
 353 
 354   virtual void do_oop(oop* p)       override { do_oop_work(p); }
 355   virtual void do_oop(narrowOop* p) override { do_oop_work(p); }
 356 
 357   bool _mutated;
 358   int _num_oops;
 359 
 360   template <typename T>
 361   void do_oop_work(T* p) {
 362     if (_bound.contains(p)) {
 363       T before = *p;
 364       Devirtualizer::do_oop(_closure, p);
 365       _mutated |= before != *p;
 366       _num_oops++;
 367     }
 368   }
 369 };
 370 
 371 template <bool concurrent_gc, typename OopClosureType>
 372 class OopOopIterateStackClosure {
 373   stackChunkOop _chunk;
 374   const bool _do_destructive_processing;
 375   OopClosureType * const _closure;
 376   MemRegion _bound;
 377 
 378 public:
 379   int _num_frames, _num_oops;
 380   OopOopIterateStackClosure(stackChunkOop chunk, bool do_destructive_processing, OopClosureType* closure, MemRegion mr)
 381     : _chunk(chunk),
 382       _do_destructive_processing(do_destructive_processing),
 383       _closure(closure),
 384       _bound(mr),
 385       _num_frames(0),
 386       _num_oops(0) {}
 387 
 388   template <bool mixed, typename RegisterMapT>
 389   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 390     log_develop_trace(jvmcont)("stack_chunk_iterate_stack sp: " INTPTR_FORMAT " pc: " INTPTR_FORMAT, f.sp() - _chunk->start_address(), p2i(f.pc()));
 391     // if (Continuation::is_return_barrier_entry(f.pc())) {
 392     //   assert ((int)(f.sp() - chunk->start_address(chunk)) < chunk->sp(), ""); // only happens when starting from gcSP
 393     //   return;
 394     // }
 395 
 396     _num_frames++;
 397     assert (_closure != nullptr, "");
 398 
 399     assert (mixed || !f.is_deoptimized(), "");
 400     if (mixed && f.is_compiled()) f.handle_deopted();
 401 
 402     if (Devirtualizer::do_metadata(_closure)) {
 403       if (f.is_interpreted()) {
 404         Method* im = f.to_frame().interpreter_frame_method();
 405         _closure->do_method(im);
 406       } else if (f.is_compiled()) {
 407         nmethod* nm = f.cb()->as_nmethod();
 408         // The do_nmethod function takes care of having the right synchronization
 409         // when keeping the nmethod alive during concurrent execution.
 410         _closure->do_nmethod(nm);
 411       }
 412     }
 413 
 414     if (_do_destructive_processing) { // evacuation always takes place at a safepoint; for concurrent iterations, we skip derived pointers, which is ok b/c coarse card marking is used for chunks
 415       assert (!f.is_compiled() || f.oopmap()->has_derived_oops() == f.oopmap()->has_any(OopMapValue::derived_oop_value), "");
 416       if (f.is_compiled() && f.oopmap()->has_derived_oops()) {
 417         if (concurrent_gc) {
 418           _chunk->set_gc_mode(true);
 419           OrderAccess::storestore();
 420         }
 421         InstanceStackChunkKlass::relativize_derived_pointers<concurrent_gc>(f, map);
 422         // OrderAccess::storestore();
 423       }
 424     }
 425 
 426     StackChunkOopIterateFilterClosure<OopClosureType> cl(_closure, _chunk, _bound);
 427     f.iterate_oops(&cl, map);
 428     bool mutated_oops = cl._mutated;
 429     _num_oops += cl._num_oops;// f.oopmap()->num_oops();
 430 
 431     // if (FIX_DERIVED_POINTERS && concurrent_gc && mutated_oops && _chunk->is_gc_mode()) { // TODO: this is a ZGC-specific optimization that depends on the one in iterate_derived_pointers
 432     //   InstanceStackChunkKlass::derelativize_derived_pointers(f, map);
 433     // }
 434     return true;
 435   }
 436 };
 437 
 438 template <bool concurrent_gc>
 439 void InstanceStackChunkKlass::oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr) {
 440   assert (Continuation::debug_is_stack_chunk(chunk), "");
 441   log_develop_trace(jvmcont)("stack_chunk_iterate_stack requires_barriers: %d", !chunk->requires_barriers());
 442 
 443   bool do_destructive_processing; // should really be `= closure.is_destructive()`, if we had such a thing
 444   if (concurrent_gc) {
 445     do_destructive_processing = true;
 446   } else {
 447     if (SafepointSynchronize::is_at_safepoint() /*&& !chunk->is_gc_mode()*/) {
 448       do_destructive_processing = true;
 449       chunk->set_gc_mode(true);
 450     } else {
 451       do_destructive_processing = false;
 452     }
 453     assert (!SafepointSynchronize::is_at_safepoint() || chunk->is_gc_mode(), "gc_mode: %d is_at_safepoint: %d", chunk->is_gc_mode(), SafepointSynchronize::is_at_safepoint());
 454   }
 455 
 456   OopOopIterateStackClosure<concurrent_gc, OopIterateClosure> frame_closure(chunk, do_destructive_processing, closure, mr);
 457   chunk->iterate_stack(&frame_closure);
 458 
 459   // if (FIX_DERIVED_POINTERS && concurrent_gc) {
 460   //   OrderAccess::storestore(); // to preserve that we set the offset *before* fixing the base oop
 461   //   chunk->set_gc_mode(false);
 462   // }
 463 
 464   assert (frame_closure._num_frames >= 0, "");
 465   assert (frame_closure._num_oops >= 0, "");
 466 
 467   if (closure != nullptr) {
 468     Continuation::emit_chunk_iterate_event(chunk, frame_closure._num_frames, frame_closure._num_oops);
 469   }
 470 
 471   log_develop_trace(jvmcont)("stack_chunk_iterate_stack ------- end -------");
 472 }
 473 
 474 template void InstanceStackChunkKlass::oop_oop_iterate_stack_slow<false>(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr);
 475 template void InstanceStackChunkKlass::oop_oop_iterate_stack_slow<true> (stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr);
 476 
 477 class MarkMethodsStackClosure {
 478   OopIterateClosure* _closure;
 479 
 480 public:
 481   MarkMethodsStackClosure(OopIterateClosure* cl) : _closure(cl) {}
 482 
 483   template <bool mixed, typename RegisterMapT>
 484   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 485     if (f.is_interpreted()) {
 486       Method* im = f.to_frame().interpreter_frame_method();
 487       _closure->do_method(im);
 488     } else if (f.is_compiled()) {
 489       nmethod* nm = f.cb()->as_nmethod();
 490       // The do_nmethod function takes care of having the right synchronization
 491       // when keeping the nmethod alive during concurrent execution.
 492       _closure->do_nmethod(nm);
 493     }
 494     return true;
 495   }
 496 };
 497 
 498 void InstanceStackChunkKlass::mark_methods(stackChunkOop chunk, OopIterateClosure* cl) {
 499   MarkMethodsStackClosure closure(cl);
 500   chunk->iterate_stack(&closure);
 501 }
 502 
 503 template <bool concurrent_gc, bool mixed, typename RegisterMapT>
 504 void InstanceStackChunkKlass::relativize_derived_pointers(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 505   RelativizeDerivedPointers<concurrent_gc> derived_closure;
 506   f.iterate_derived_pointers(&derived_closure, map);
 507 }
 508 
 509 template <bool mixed, typename RegisterMapT>
 510 void InstanceStackChunkKlass::derelativize_derived_pointers(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 511   DerelativizeDerivedPointers derived_closure;
 512   f.iterate_derived_pointers(&derived_closure, map);
 513 }
 514 
 515 template void InstanceStackChunkKlass::relativize_derived_pointers<false>(const StackChunkFrameStream<true >& f, const RegisterMap* map);
 516 template void InstanceStackChunkKlass::relativize_derived_pointers<true> (const StackChunkFrameStream<true >& f, const RegisterMap* map);
 517 template void InstanceStackChunkKlass::relativize_derived_pointers<false>(const StackChunkFrameStream<false>& f, const RegisterMap* map);
 518 template void InstanceStackChunkKlass::relativize_derived_pointers<true> (const StackChunkFrameStream<false>& f, const RegisterMap* map);
 519 template void InstanceStackChunkKlass::relativize_derived_pointers<false>(const StackChunkFrameStream<true >& f, const SmallRegisterMap* map);
 520 template void InstanceStackChunkKlass::relativize_derived_pointers<true> (const StackChunkFrameStream<true >& f, const SmallRegisterMap* map);
 521 template void InstanceStackChunkKlass::relativize_derived_pointers<false>(const StackChunkFrameStream<false>& f, const SmallRegisterMap* map);
 522 template void InstanceStackChunkKlass::relativize_derived_pointers<true> (const StackChunkFrameStream<false>& f, const SmallRegisterMap* map);
 523 
 524 
 525 template <bool store, bool mixed, typename RegisterMapT>
 526 void InstanceStackChunkKlass::do_barriers(stackChunkOop chunk, const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 527   // we need to invoke the write barriers so as not to miss oops in old chunks that haven't yet been concurrently scanned
 528   if (f.is_done()) return;
 529   log_develop_trace(jvmcont)("InstanceStackChunkKlass::invoke_barriers sp: " INTPTR_FORMAT " pc: " INTPTR_FORMAT, p2i(f.sp()), p2i(f.pc()));
 530 
 531   if (log_develop_is_enabled(Trace, jvmcont) && !mixed && f.is_interpreted()) f.cb()->print_value_on(tty);
 532 
 533   if (mixed) f.handle_deopted(); // we could freeze deopted frames in slow mode.
 534 
 535   if (f.is_interpreted()) {
 536     Method* m = f.to_frame().interpreter_frame_method();
 537     m->record_marking_cycle();
 538   } else if (f.is_compiled()) {
 539     nmethod* nm = f.cb()->as_nmethod();
 540     // The entry barrier takes care of having the right synchronization
 541     // when keeping the nmethod alive during concurrent execution.
 542     nm->run_nmethod_entry_barrier();
 543   }
 544 
 545   assert (!f.is_compiled() || f.oopmap()->has_derived_oops() == f.oopmap()->has_any(OopMapValue::derived_oop_value), "");
 546   bool has_derived = f.is_compiled() && f.oopmap()->has_derived_oops();
 547   if (has_derived) {
 548     if (UseZGC || UseShenandoahGC) {
 549       relativize_derived_pointers<true>(f, map);
 550     }
 551   }
 552 
 553   if (chunk->has_bitmap() && UseCompressedOops) {
 554     BarrierClosure<store, true> oops_closure(f.sp());
 555     f.iterate_oops(&oops_closure, map);
 556   } else {
 557     BarrierClosure<store, false> oops_closure(f.sp());
 558     f.iterate_oops(&oops_closure, map);
 559   }
 560   OrderAccess::loadload(); // observing the barriers will prevent derived pointers from being derelativized concurrently
 561 
 562   // if (has_derived) { // we do this in fix_thawed_frame
 563   //   derelativize_derived_pointers(f, map);
 564   // }
 565 }
 566 
 567 template void InstanceStackChunkKlass::do_barriers<false>(stackChunkOop chunk, const StackChunkFrameStream<true >& f, const RegisterMap* map);
 568 template void InstanceStackChunkKlass::do_barriers<true> (stackChunkOop chunk, const StackChunkFrameStream<true >& f, const RegisterMap* map);
 569 template void InstanceStackChunkKlass::do_barriers<false>(stackChunkOop chunk, const StackChunkFrameStream<false>& f, const RegisterMap* map);
 570 template void InstanceStackChunkKlass::do_barriers<true> (stackChunkOop chunk, const StackChunkFrameStream<false>& f, const RegisterMap* map);
 571 template void InstanceStackChunkKlass::do_barriers<false>(stackChunkOop chunk, const StackChunkFrameStream<true >& f, const SmallRegisterMap* map);
 572 template void InstanceStackChunkKlass::do_barriers<true> (stackChunkOop chunk, const StackChunkFrameStream<true >& f, const SmallRegisterMap* map);
 573 template void InstanceStackChunkKlass::do_barriers<false>(stackChunkOop chunk, const StackChunkFrameStream<false>& f, const SmallRegisterMap* map);
 574 template void InstanceStackChunkKlass::do_barriers<true> (stackChunkOop chunk, const StackChunkFrameStream<false>& f, const SmallRegisterMap* map);
 575 
 576 template void InstanceStackChunkKlass::fix_thawed_frame(stackChunkOop chunk, const frame& f, const RegisterMap* map);
 577 template void InstanceStackChunkKlass::fix_thawed_frame(stackChunkOop chunk, const frame& f, const SmallRegisterMap* map);
 578 
 579 template <bool store>
 580 class DoBarriersStackClosure {
 581   const stackChunkOop _chunk;
 582 public:
 583   DoBarriersStackClosure(stackChunkOop chunk) : _chunk(chunk) {}
 584 
 585   template <bool mixed, typename RegisterMapT>
 586   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 587     InstanceStackChunkKlass::do_barriers<store>(_chunk, f, map);
 588     return true;
 589   }
 590 };
 591 
 592 template void InstanceStackChunkKlass::do_barriers<false>(stackChunkOop chunk);
 593 template void InstanceStackChunkKlass::do_barriers<true>(stackChunkOop chunk);
 594 
 595 template <bool store>
 596 void InstanceStackChunkKlass::do_barriers(stackChunkOop chunk) {
 597   DoBarriersStackClosure<store> closure(chunk);
 598   chunk->iterate_stack(&closure);
 599 }
 600 
 601 #ifdef ASSERT
 602 template<class P>
 603 static inline oop safe_load(P *addr) {
 604   oop obj = (oop)RawAccess<>::oop_load(addr);
 605   obj = (oop)NativeAccess<>::oop_load(&obj);
 606   return obj;
 607 }
 608 
 609 // Returns true iff the address p is readable and *(intptr_t*)p != errvalue
 610 extern "C" bool dbg_is_safe(const void* p, intptr_t errvalue);
 611 static bool is_good_oop(oop o) { return dbg_is_safe(o, -1) && dbg_is_safe(o->klass(), -1) && oopDesc::is_oop(o) && o->klass()->is_klass(); }
 612 #endif
 613 
 614 class FixCompressedOopClosure : public OopClosure {
 615   void do_oop(oop* p) override {
 616     assert (UseCompressedOops, "");
 617     oop obj = CompressedOops::decode(*(narrowOop*)p);
 618     assert (obj == nullptr || is_good_oop(obj), "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT, p2i(p), p2i((oopDesc*)obj));
 619     *p = obj;
 620   }
 621 
 622   void do_oop(narrowOop* p) override {}
 623 };
 624 
 625 template <bool compressedOops>
 626 class BuildBitmapOopClosure : public OopClosure {
 627   intptr_t* const _stack_start;
 628   const BitMap::idx_t _bit_offset;
 629   BitMapView _bm;
 630 public:
 631   BuildBitmapOopClosure(intptr_t* stack_start, BitMap::idx_t bit_offset, BitMapView bm) : _stack_start(stack_start), _bit_offset(bit_offset), _bm(bm) {}
 632 
 633   virtual void do_oop(oop* p) override {
 634     assert (p >= (oop*)_stack_start, "");
 635     if (compressedOops) {
 636       // Convert all oops to narrow before marking bit
 637       oop obj = *p;
 638       *p = nullptr;
 639       // assuming little endian
 640       *(narrowOop*)p = CompressedOops::encode(obj);
 641       do_oop((narrowOop*)p);
 642     } else {
 643       BitMap::idx_t index = _bit_offset + (p - (oop*)_stack_start);
 644       log_develop_trace(jvmcont)("Build bitmap wide oop p: " INTPTR_FORMAT " index: " SIZE_FORMAT " bit_offset: " SIZE_FORMAT, p2i(p), index, _bit_offset);
 645       assert (!_bm.at(index), "");
 646       _bm.set_bit(index);
 647     }
 648   }
 649 
 650   virtual void do_oop(narrowOop* p) override {
 651     assert (p >= (narrowOop*)_stack_start, "");
 652     BitMap::idx_t index = _bit_offset + (p - (narrowOop*)_stack_start);
 653     log_develop_trace(jvmcont)("Build bitmap narrow oop p: " INTPTR_FORMAT " index: " SIZE_FORMAT " bit_offset: " SIZE_FORMAT, p2i(p), index, _bit_offset);
 654     assert (!_bm.at(index), "");
 655     _bm.set_bit(index);
 656   }
 657 };
 658 
 659 template <bool compressedOops>
 660 class BuildBitmapStackClosure {
 661   stackChunkOop _chunk;
 662   const BitMap::idx_t _bit_offset;
 663 public:
 664   BuildBitmapStackClosure(stackChunkOop chunk) : _chunk(chunk), _bit_offset(chunk->bit_offset()) {}
 665 
 666   template <bool mixed, typename RegisterMapT>
 667   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 668     if (!_chunk->is_gc_mode() && f.is_compiled() && f.oopmap()->has_derived_oops()) {
 669       RelativizeDerivedPointers<false> derived_oops_closure;
 670       f.iterate_derived_pointers(&derived_oops_closure, map);
 671     }
 672 
 673     if (UseChunkBitmaps) {
 674       BuildBitmapOopClosure<compressedOops> oops_closure(_chunk->start_address(), _chunk->bit_offset(), _chunk->bitmap());
 675       f.iterate_oops(&oops_closure, map);
 676     }
 677 
 678     return true;
 679   }
 680 };
 681 
 682 void InstanceStackChunkKlass::build_bitmap(stackChunkOop chunk) {
 683   assert (!chunk->has_bitmap(), "");
 684   if (UseChunkBitmaps) {
 685     chunk->set_has_bitmap(true);
 686     BitMapView bm = chunk->bitmap();
 687     bm.clear();
 688   }
 689 
 690   if (UseCompressedOops) {
 691     BuildBitmapStackClosure<true> closure(chunk);
 692     chunk->iterate_stack(&closure);
 693   } else {
 694     BuildBitmapStackClosure<false> closure(chunk);
 695     chunk->iterate_stack(&closure);
 696   }
 697 
 698   chunk->set_gc_mode(true); // must be set *after* the above closure
 699 }
 700 
 701 // template <bool store>
 702 // class BarriersIterateStackClosure {
 703 // public:
 704 //   template <bool mixed, typename RegisterMapT>
 705 //   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 706 //     InstanceStackChunkKlass::barriers_for_oops_in_frame<mixed, store>(f, map);
 707 //     return true;
 708 //   }
 709 // };
 710 
 711 // template <bool store>
 712 // void InstanceStackChunkKlass::barriers_for_oops_in_chunk(stackChunkOop chunk) {
 713 //   BarriersIterateStackClosure<store> frame_closure;
 714 //   chunk->iterate_stack(&frame_closure);
 715 // }
 716 
 717 // NOINLINE void InstanceStackChunkKlass::fix_chunk(stackChunkOop chunk) {
 718 //   log_develop_trace(jvmcont)("fix_stack_chunk young: %d", !chunk->requires_barriers());
 719 //   FixChunkIterateStackClosure frame_closure(chunk);
 720 //   chunk->iterate_stack(&frame_closure);
 721 //   log_develop_trace(jvmcont)("fix_stack_chunk ------- end -------");
 722 // }
 723 
 724 template <typename RegisterMapT>
 725 void InstanceStackChunkKlass::fix_thawed_frame(stackChunkOop chunk, const frame& f, const RegisterMapT* map) {
 726   if (chunk->has_bitmap() && UseCompressedOops) {
 727     FixCompressedOopClosure oop_closure;
 728     if (f.is_interpreted_frame()) {
 729       f.oops_interpreted_do(&oop_closure, nullptr);
 730     } else {
 731       OopMapDo<OopClosure, DerelativizeDerivedPointers, SkipNullValue> visitor(&oop_closure, nullptr);
 732       visitor.oops_do(&f, map, f.oop_map());
 733     }
 734   }
 735 
 736   if (f.is_compiled_frame() && f.oop_map()->has_derived_oops()) {
 737     DerelativizeDerivedPointers derived_closure;
 738     OopMapDo<OopClosure, DerelativizeDerivedPointers, SkipNullValue> visitor(nullptr, &derived_closure);
 739     visitor.oops_do(&f, map, f.oop_map());
 740   }
 741 }
 742 
 743 #ifdef ASSERT
 744 
 745 template <typename OopT>
 746 class StackChunkVerifyBitmapClosure : public BitMapClosure {
 747   stackChunkOop _chunk;
 748 public:
 749   int _count;
 750 
 751   StackChunkVerifyBitmapClosure(stackChunkOop chunk) : _chunk(chunk), _count(0) {}
 752 
 753   bool do_bit(BitMap::idx_t index) override {
 754     OopT* p = _chunk->address_for_bit<OopT>(index);
 755     _count++;
 756 
 757     log_develop_trace(jvmcont)("debug_verify_stack_chunk bitmap p: " INTPTR_FORMAT " i: " SIZE_FORMAT, p2i(p), index);
 758     
 759     if (!SafepointSynchronize::is_at_safepoint()) {
 760       oop obj = safe_load(p);
 761       assert (obj == nullptr || is_good_oop(obj),
 762               "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT " index: " SIZE_FORMAT " bit_offset: " SIZE_FORMAT,
 763               p2i(p), p2i((oopDesc*)obj), index, _chunk->bit_offset());
 764     }
 765 
 766     return true; // continue processing
 767   }
 768 };
 769 
 770 class StackChunkVerifyOopsClosure : public OopClosure {
 771   stackChunkOop _chunk;
 772   intptr_t* _sp;
 773   int _count;
 774 public:
 775   StackChunkVerifyOopsClosure(stackChunkOop chunk, intptr_t* sp) : _chunk(chunk), _sp(sp), _count(0) {}
 776   int count() { return _count; }
 777   void do_oop(oop* p) override { (_chunk->has_bitmap() && UseCompressedOops) ? do_oop_work((narrowOop*)p) : do_oop_work(p); }
 778   void do_oop(narrowOop* p) override { do_oop_work(p); }
 779 
 780   template <class T> inline void do_oop_work(T* p) {
 781     log_develop_trace(jvmcont)("debug_verify_stack_chunk oop narrow: %d p: " INTPTR_FORMAT, sizeof(T) < sizeof(intptr_t), p2i(p));
 782      _count++;
 783     if (SafepointSynchronize::is_at_safepoint()) return;
 784 
 785     oop obj = safe_load(p);
 786     assert (obj == nullptr || is_good_oop(obj), "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT, p2i(p), p2i((oopDesc*)obj));
 787     if (_chunk->has_bitmap()) {
 788       BitMap::idx_t index = (p - (T*)_chunk->start_address()) + _chunk->bit_offset();
 789       assert (_chunk->bitmap().at(index), "Bit not set at index " SIZE_FORMAT " corresponding to " INTPTR_FORMAT, index, p2i(p));
 790     }
 791   }
 792 };
 793 
 794 class StackChunkVerifyDerivedPointersClosure : public DerivedOopClosure {
 795   stackChunkOop _chunk;
 796 public:
 797   StackChunkVerifyDerivedPointersClosure(stackChunkOop chunk) : _chunk(chunk) {}
 798 
 799   virtual void do_derived_oop(oop* base_loc, derived_pointer* derived_loc) override {
 800     log_develop_trace(jvmcont)("debug_verify_stack_chunk base: " INTPTR_FORMAT " derived: " INTPTR_FORMAT, p2i(base_loc), p2i(derived_loc));
 801     if (SafepointSynchronize::is_at_safepoint()) return;
 802 
 803     oop base = (_chunk->has_bitmap() && UseCompressedOops) ? CompressedOops::decode(Atomic::load((narrowOop*)base_loc)) : Atomic::load((oop*)base_loc);
 804     // (oop)NativeAccess<>::oop_load((oop*)base_loc); //
 805     if (base != nullptr) {
 806       ZGC_ONLY(if (UseZGC && !ZAddress::is_good(cast_from_oop<uintptr_t>(base))) return;)
 807       assert (!CompressedOops::is_base(base), "");
 808       assert (oopDesc::is_oop(base), "");
 809       ZGC_ONLY(assert (!UseZGC || ZAddress::is_good(cast_from_oop<uintptr_t>(base)), "");)
 810       OrderAccess::loadload();
 811       intptr_t offset = Atomic::load((intptr_t*)derived_loc);
 812       offset = offset <= 0 // an offset of 0 was observed on AArch64
 813                   ? -offset
 814                   : offset - cast_from_oop<intptr_t>(base);
 815 
 816       // Has been seen to fail on AArch64 for some reason
 817       // assert (offset >= 0 && offset <= (intptr_t)(base->size() << LogHeapWordSize), "offset: %ld base->size: %zu relative: %d", offset, base->size() << LogHeapWordSize, *(intptr_t*)derived_loc <= 0);
 818     } else {
 819       assert (*derived_loc == derived_pointer(0), "");
 820     }
 821   }
 822 };
 823 
 824 class VerifyStackClosure {
 825   stackChunkOop _chunk;
 826 public:
 827   intptr_t* _sp;
 828   CodeBlob* _cb;
 829   bool _callee_interpreted;
 830   int _size;
 831   int _argsize;
 832   int _num_oops, _num_frames, _num_interpreted_frames, _num_i2c;
 833   VerifyStackClosure(stackChunkOop chunk, int num_frames, int size)
 834     : _chunk(chunk), _sp(nullptr), _cb(nullptr), _callee_interpreted(false),
 835       _size(size), _argsize(0), _num_oops(0), _num_frames(num_frames), _num_interpreted_frames(0), _num_i2c(0) {}
 836 
 837   template <bool mixed, typename RegisterMapT>
 838   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 839     _sp = f.sp();
 840     _cb = f.cb();
 841 
 842     int fsize = f.frame_size() - ((f.is_interpreted() == _callee_interpreted) ? _argsize : 0);
 843     int num_oops = f.num_oops();
 844     assert (num_oops >= 0, "");
 845     // tty->print_cr(">>> fsize: %d f.frame_size(): %d callee_interpreted: %d callee_argsize: %d", fsize, f.frame_size(), _callee_interpreted, _argsize);
 846 
 847     _argsize   = f.stack_argsize();
 848     _size     += fsize;
 849     _num_oops += num_oops;
 850     if (f.is_interpreted()) {
 851       _num_interpreted_frames++;
 852     }
 853 
 854     log_develop_trace(jvmcont)("debug_verify_stack_chunk frame: %d sp: " INTPTR_FORMAT " pc: " INTPTR_FORMAT " interpreted: %d size: %d argsize: %d oops: %d", _num_frames, f.sp() - _chunk->start_address(), p2i(f.pc()), f.is_interpreted(), fsize, _argsize, num_oops);
 855     if (log_develop_is_enabled(Trace, jvmcont)) f.print_on(tty);
 856     assert (f.pc() != nullptr,
 857       "young: %d num_frames: %d sp: " INTPTR_FORMAT " start: " INTPTR_FORMAT " end: " INTPTR_FORMAT,
 858       !_chunk->requires_barriers(), _num_frames, p2i(f.sp()), p2i(_chunk->start_address()), p2i(_chunk->bottom_address()));
 859 
 860     if (_num_frames == 0) {
 861       assert (f.pc() == _chunk->pc(), "");
 862     }
 863 
 864     if (_num_frames > 0 && !_callee_interpreted && f.is_interpreted()) {
 865       log_develop_trace(jvmcont)("debug_verify_stack_chunk i2c");
 866       _num_i2c++;
 867     }
 868 
 869     // if (_cb != nullptr && _cb->is_nmethod()) {
 870     //   nmethod* nm = cb->as_nmethod();
 871     //   if (check_deopt && nm->is_marked_for_deoptimization() && nm->is_not_entrant()) {
 872     //     tty->print_cr("-- FOUND NON ENTRANT NMETHOD IN CHUNK: ");
 873     //     if (nm->method() != nullptr) nm->method()->print_on(tty);
 874     //     nm->print_on(tty);
 875     //   }
 876     // }
 877 
 878     StackChunkVerifyOopsClosure oops_closure(_chunk, f.sp());
 879     f.iterate_oops(&oops_closure, map);
 880     assert (oops_closure.count() == num_oops, "oops: %d oopmap->num_oops(): %d", oops_closure.count(), num_oops);
 881 
 882     StackChunkVerifyDerivedPointersClosure derived_oops_closure(_chunk);
 883     f.iterate_derived_pointers(&derived_oops_closure, map);
 884 
 885     _callee_interpreted = f.is_interpreted();
 886     _num_frames++;
 887     return true;
 888   }
 889 };
 890 
 891 // verifies the consistency of the chunk's data
 892 bool InstanceStackChunkKlass::verify(oop obj, size_t* out_size, int* out_oops, int* out_frames, int* out_interpreted_frames) {
 893   DEBUG_ONLY(if (!VerifyContinuations) return true;)
 894 
 895   assert (oopDesc::is_oop(obj), "");
 896   assert (obj->is_stackChunk(), "");
 897   stackChunkOop chunk = (stackChunkOop)obj;
 898 
 899   log_develop_trace(jvmcont)("debug_verify_stack_chunk barriers: %d", chunk->requires_barriers());
 900   // chunk->print_on(true, tty);
 901 
 902   assert (chunk->is_stackChunk(), "");
 903   assert (chunk->stack_size() >= 0, "");
 904   assert (chunk->argsize() >= 0, "");
 905   assert (!chunk->has_bitmap() || chunk->is_gc_mode(), "");
 906 
 907   if (chunk->is_empty()) {
 908     assert (chunk->argsize() == 0, "");
 909     assert (chunk->max_size() == 0, "");
 910   }
 911 
 912   if (!SafepointSynchronize::is_at_safepoint()) {
 913     assert (oopDesc::is_oop_or_null(chunk->parent()), "");
 914   }
 915 
 916   bool check_deopt = false;
 917   if (Thread::current()->is_Java_thread() && !SafepointSynchronize::is_at_safepoint()) {
 918     if (JavaThread::cast(Thread::current())->cont_fastpath_thread_state())
 919       check_deopt = true;
 920   }
 921 
 922   const bool concurrent = !SafepointSynchronize::is_at_safepoint() && !Thread::current()->is_Java_thread();
 923   const bool gc_mode = chunk->is_gc_mode();
 924   const bool is_last = chunk->parent() == nullptr;
 925   const bool mixed = chunk->has_mixed_frames();
 926 
 927   // if argsize == 0 and the chunk isn't mixed, the chunk contains the metadata (pc, fp -- frame::sender_sp_offset) for the top frame (below sp), and *not* for the bottom frame
 928   int size = chunk->stack_size() - chunk->argsize() - chunk->sp();
 929   assert (size >= 0, "");
 930   assert ((size == 0) == chunk->is_empty(), "");
 931 
 932   const StackChunkFrameStream<true> first(chunk);
 933   const bool has_safepoint_stub_frame = first.is_stub();
 934 
 935   VerifyStackClosure closure(chunk,
 936     has_safepoint_stub_frame ? 1 : 0, // iterate_stack skips the safepoint stub
 937     has_safepoint_stub_frame ? first.frame_size() : 0);
 938   chunk->iterate_stack(&closure);
 939 
 940   assert (!chunk->is_empty() || closure._cb == nullptr, "");
 941   if (closure._cb != nullptr && closure._cb->is_compiled()) {
 942     assert (chunk->argsize() == (closure._cb->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord,
 943       "chunk argsize: %d bottom frame argsize: %d", chunk->argsize(), (closure._cb->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord);
 944   } 
 945 
 946   assert (closure._num_interpreted_frames == 0 || chunk->has_mixed_frames(), "");
 947 
 948   if (!concurrent) {
 949     assert (closure._size <= size + chunk->argsize() + metadata_words(), "size: %d argsize: %d closure.size: %d end sp: " PTR_FORMAT " start sp: %d chunk size: %d", size, chunk->argsize(), closure._size, closure._sp - chunk->start_address(), chunk->sp(), chunk->stack_size());
 950     assert (chunk->argsize() == closure._argsize, "chunk->argsize(): %d closure.argsize: %d closure.callee_interpreted: %d", chunk->argsize(), closure._argsize, closure._callee_interpreted);
 951 
 952     int max_size = closure._size + closure._num_i2c * align_wiggle();
 953     assert (chunk->max_size() == max_size, "max_size(): %d max_size: %d argsize: %d num_i2c: %d", chunk->max_size(), max_size, closure._argsize, closure._num_i2c);
 954 
 955     if (out_size   != nullptr) *out_size   += size;
 956     if (out_oops   != nullptr) *out_oops   += closure._num_oops;
 957     if (out_frames != nullptr) *out_frames += closure._num_frames;
 958     if (out_interpreted_frames != nullptr) *out_interpreted_frames += closure._num_interpreted_frames;
 959   } else assert (out_size == nullptr && out_oops == nullptr && out_frames == nullptr && out_interpreted_frames == nullptr, "");
 960 
 961   if (chunk->has_bitmap()) {
 962     assert (chunk->bitmap().size() == chunk->bit_offset() + (size_t)(chunk->stack_size() << (UseCompressedOops ? 1 : 0)), "bitmap().size(): %zu bit_offset: %zu stack_size: %d", chunk->bitmap().size(), chunk->bit_offset(), chunk->stack_size());
 963     int oop_count;
 964     if (UseCompressedOops) {
 965       StackChunkVerifyBitmapClosure<narrowOop> bitmap_closure(chunk);
 966       chunk->bitmap().iterate(&bitmap_closure, chunk->bit_index_for((narrowOop*)(chunk->sp_address() - metadata_words())), chunk->bit_index_for((narrowOop*)chunk->end_address()));
 967       oop_count = bitmap_closure._count;
 968     } else {
 969       StackChunkVerifyBitmapClosure<oop> bitmap_closure(chunk);
 970       chunk->bitmap().iterate(&bitmap_closure, chunk->bit_index_for((oop*)(chunk->sp_address() - metadata_words())), chunk->bit_index_for((oop*)chunk->end_address()));
 971       oop_count = bitmap_closure._count;
 972     }
 973     assert (oop_count == closure._num_oops, "bitmap_closure._count: %d closure._num_oops: %d", oop_count, closure._num_oops);
 974   }
 975 
 976   return true;
 977 }
 978 
 979 namespace {
 980 class DescribeStackChunkClosure {
 981   stackChunkOop _chunk;
 982   FrameValues _values;
 983   RegisterMap _map;
 984   int _frame_no;
 985 public:
 986   DescribeStackChunkClosure(stackChunkOop chunk) : _chunk(chunk), _map((JavaThread*)nullptr, true, false, true), _frame_no(0) {
 987     _map.set_include_argument_oops(false);
 988   }
 989 
 990   const RegisterMap* get_map(const RegisterMap* map,      intptr_t* sp) { return map; }
 991   const RegisterMap* get_map(const SmallRegisterMap* map, intptr_t* sp) { return map->copy_to_RegisterMap(&_map, sp); }
 992 
 993   template <bool mixed, typename RegisterMapT>
 994   bool do_frame(const StackChunkFrameStream<mixed>& f, const RegisterMapT* map) {
 995     ResetNoHandleMark rnhm;
 996     HandleMark hm(Thread::current());
 997 
 998     frame fr = f.to_frame();
 999     if (_frame_no == 0) fr.describe_top(_values);
1000     fr.template describe<true>(_values, _frame_no++, get_map(map, f.sp()));
1001     return true;
1002   }
1003 
1004   void describe_chunk() {
1005     // _values.describe(-1, _chunk->start_address(), "CHUNK START");
1006     _values.describe(-1, _chunk->sp_address(),      "CHUNK SP");
1007     _values.describe(-1, _chunk->bottom_address() - 1, "CHUNK ARGS");
1008     _values.describe(-1, _chunk->end_address() - 1, "CHUNK END");
1009   }
1010 
1011   void print_on(outputStream* out) {
1012     if (_frame_no > 0) {
1013       describe_chunk();
1014       _values.print_on(_chunk, out);
1015     } else {
1016       out->print_cr(" EMPTY");
1017     }
1018   }
1019 };
1020 }
1021 #endif
1022 
1023 namespace {
1024 class PrintStackChunkClosure {
1025   stackChunkOop _chunk;
1026   outputStream* _st;
1027 public:
1028   PrintStackChunkClosure(stackChunkOop chunk, outputStream* st) : _chunk(chunk), _st(st) {}
1029 
1030   template <bool mixed, typename RegisterMapT>
1031   bool do_frame(const StackChunkFrameStream<mixed>& fs, const RegisterMapT* map) {
1032     frame f = fs.to_frame();
1033     _st->print_cr("-- frame sp: " INTPTR_FORMAT " interpreted: %d size: %d argsize: %d", p2i(fs.sp()), fs.is_interpreted(), f.frame_size(), fs.is_interpreted() ? 0 : f.compiled_frame_stack_argsize());
1034     f.print_on<true>(_st);
1035     const ImmutableOopMap* oopmap = fs.oopmap();
1036     if (oopmap != nullptr) {
1037       oopmap->print_on(_st);
1038       _st->cr();
1039     }
1040     return true;
1041   }
1042 };
1043 }
1044 
1045 void InstanceStackChunkKlass::print_chunk(const stackChunkOop c, bool verbose, outputStream* st) {
1046   if (c == (oop)nullptr) {
1047     st->print_cr("CHUNK NULL");
1048     return;
1049   }
1050   assert(c->is_stackChunk(), "");
1051 
1052   // HeapRegion* hr = G1CollectedHeap::heap()->heap_region_containing(chunk);
1053   st->print_cr("CHUNK " INTPTR_FORMAT " " INTPTR_FORMAT " - " INTPTR_FORMAT " :: " INTPTR_FORMAT, p2i((oopDesc*)c), p2i(c->start_address()), p2i(c->end_address()), c->identity_hash());
1054   st->print_cr("       barriers: %d gc_mode: %d bitmap: %d parent: " INTPTR_FORMAT, c->requires_barriers(), c->is_gc_mode(), c->has_bitmap(), p2i((oopDesc*)c->parent()));
1055   st->print_cr("       flags mixed: %d", c->has_mixed_frames());
1056   st->print_cr("       size: %d argsize: %d max_size: %d sp: %d pc: " INTPTR_FORMAT, c->stack_size(), c->argsize(), c->max_size(), c->sp(), p2i(c->pc()));
1057 
1058   if (verbose) {
1059     st->cr();
1060     st->print_cr("------ chunk frames end: " INTPTR_FORMAT, p2i(c->bottom_address()));
1061     PrintStackChunkClosure closure(c, st);
1062     c->iterate_stack(&closure);
1063     st->print_cr("------");
1064 
1065   #ifdef ASSERT
1066     ResourceMark rm;
1067     DescribeStackChunkClosure describe(c);
1068     c->iterate_stack(&describe);
1069     describe.print_on(st);
1070     st->print_cr("======");
1071   #endif
1072   }
1073 }
1074 
1075 #ifndef PRODUCT
1076 template void StackChunkFrameStream<true >::print_on(outputStream* st) const;
1077 template void StackChunkFrameStream<false>::print_on(outputStream* st) const;
1078 
1079 template <bool mixed>
1080 void StackChunkFrameStream<mixed>::print_on(outputStream* st) const {
1081   st->print_cr("chunk: " INTPTR_FORMAT " index: %d sp offset: %d stack size: %d", p2i(_chunk), _index, _chunk->to_offset(_sp), _chunk->stack_size());
1082   to_frame().template print_on<true>(st);
1083 }
1084 #endif