1 /*
   2  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP
  26 #define SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP
  27 
  28 #include "memory/iterator.hpp"
  29 #include "oops/oop.hpp"
  30 #include "oops/objArrayOop.hpp"
  31 #include "gc/shared/collectorCounters.hpp"
  32 #include "gc/shenandoah/shenandoahCardStats.hpp"
  33 #include "gc/shenandoah/shenandoahCardTable.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  36 #include "gc/shenandoah/shenandoahScanRemembered.hpp"
  37 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  38 
  39 inline size_t
  40 ShenandoahDirectCardMarkRememberedSet::last_valid_index() const {
  41   return _card_table->last_valid_index();
  42 }
  43 
  44 inline size_t
  45 ShenandoahDirectCardMarkRememberedSet::total_cards() const {
  46   return _total_card_count;
  47 }
  48 
  49 inline size_t
  50 ShenandoahDirectCardMarkRememberedSet::card_index_for_addr(HeapWord *p) const {
  51   return _card_table->index_for(p);
  52 }
  53 
  54 inline HeapWord*
  55 ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(size_t card_index) const {
  56   return _whole_heap_base + CardTable::card_size_in_words() * card_index;
  57 }
  58 
  59 inline const CardValue*
  60 ShenandoahDirectCardMarkRememberedSet::get_card_table_byte_map(bool use_write_table) const {
  61   return use_write_table ?
  62            _card_table->write_byte_map()
  63            : _card_table->read_byte_map();
  64 }
  65 
  66 inline bool
  67 ShenandoahDirectCardMarkRememberedSet::is_write_card_dirty(size_t card_index) const {
  68   CardValue* bp = &(_card_table->write_byte_map())[card_index];
  69   return (bp[0] == CardTable::dirty_card_val());
  70 }
  71 
  72 inline bool
  73 ShenandoahDirectCardMarkRememberedSet::is_card_dirty(size_t card_index) const {
  74   CardValue* bp = &(_card_table->read_byte_map())[card_index];
  75   return (bp[0] == CardTable::dirty_card_val());
  76 }
  77 
  78 inline void
  79 ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(size_t card_index) {
  80   CardValue* bp = &(_card_table->write_byte_map())[card_index];
  81   bp[0] = CardTable::dirty_card_val();
  82 }
  83 
  84 inline void
  85 ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(size_t card_index, size_t num_cards) {
  86   CardValue* bp = &(_card_table->write_byte_map())[card_index];
  87   while (num_cards-- > 0) {
  88     *bp++ = CardTable::dirty_card_val();
  89   }
  90 }
  91 
  92 inline void
  93 ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(size_t card_index) {
  94   CardValue* bp = &(_card_table->write_byte_map())[card_index];
  95   bp[0] = CardTable::clean_card_val();
  96 }
  97 
  98 inline void
  99 ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(size_t card_index, size_t num_cards) {
 100   CardValue* bp = &(_card_table->write_byte_map())[card_index];
 101   while (num_cards-- > 0) {
 102     *bp++ = CardTable::clean_card_val();
 103   }
 104 }
 105 
 106 inline bool
 107 ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) const {
 108   size_t index = card_index_for_addr(p);
 109   CardValue* bp = &(_card_table->read_byte_map())[index];
 110   return (bp[0] == CardTable::dirty_card_val());
 111 }
 112 
 113 inline void
 114 ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(HeapWord *p) {
 115   size_t index = card_index_for_addr(p);
 116   CardValue* bp = &(_card_table->write_byte_map())[index];
 117   bp[0] = CardTable::dirty_card_val();
 118 }
 119 
 120 inline void
 121 ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) {
 122   CardValue* bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift];
 123   CardValue* end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift];
 124   // If (p + num_heap_words) is not aligned on card boundary, we also need to dirty last card.
 125   if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size() - 1)) {
 126     end_bp++;
 127   }
 128   while (bp < end_bp) {
 129     *bp++ = CardTable::dirty_card_val();
 130   }
 131 }
 132 
 133 inline void
 134 ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(HeapWord *p) {
 135   size_t index = card_index_for_addr(p);
 136   CardValue* bp = &(_card_table->write_byte_map())[index];
 137   bp[0] = CardTable::clean_card_val();
 138 }
 139 
 140 inline void
 141 ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t num_heap_words) {
 142   CardValue* bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift];
 143   CardValue* end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift];
 144   // If (p + num_heap_words) is not aligned on card boundary, we also need to clean last card.
 145   if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size() - 1)) {
 146     end_bp++;
 147   }
 148   while (bp < end_bp) {
 149     *bp++ = CardTable::clean_card_val();
 150   }
 151 }
 152 
 153 inline size_t
 154 ShenandoahDirectCardMarkRememberedSet::cluster_count() const {
 155   return _cluster_count;
 156 }
 157 
 158 // No lock required because arguments align with card boundaries.
 159 template<typename RememberedSet>
 160 inline void
 161 ShenandoahCardCluster<RememberedSet>::reset_object_range(HeapWord* from, HeapWord* to) {
 162   assert(((((unsigned long long) from) & (CardTable::card_size() - 1)) == 0) &&
 163          ((((unsigned long long) to) & (CardTable::card_size() - 1)) == 0),
 164          "reset_object_range bounds must align with card boundaries");
 165   size_t card_at_start = _rs->card_index_for_addr(from);
 166   size_t num_cards = (to - from) / CardTable::card_size_in_words();
 167 
 168   for (size_t i = 0; i < num_cards; i++) {
 169     object_starts[card_at_start + i].short_word = 0;
 170   }
 171 }
 172 
 173 // Assume only one thread at a time registers objects pertaining to
 174 // each card-table entry's range of memory.
 175 template<typename RememberedSet>
 176 inline void
 177 ShenandoahCardCluster<RememberedSet>::register_object(HeapWord* address) {
 178   shenandoah_assert_heaplocked();
 179 
 180   register_object_without_lock(address);
 181 }
 182 
 183 template<typename RememberedSet>
 184 inline void
 185 ShenandoahCardCluster<RememberedSet>::register_object_without_lock(HeapWord* address) {
 186   size_t card_at_start = _rs->card_index_for_addr(address);
 187   HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start);
 188   uint8_t offset_in_card = address - card_start_address;
 189 
 190   if (!starts_object(card_at_start)) {
 191     set_starts_object_bit(card_at_start);
 192     set_first_start(card_at_start, offset_in_card);
 193     set_last_start(card_at_start, offset_in_card);
 194   } else {
 195     if (offset_in_card < get_first_start(card_at_start))
 196       set_first_start(card_at_start, offset_in_card);
 197     if (offset_in_card > get_last_start(card_at_start))
 198       set_last_start(card_at_start, offset_in_card);
 199   }
 200 }
 201 
 202 template<typename RememberedSet>
 203 inline void
 204 ShenandoahCardCluster<RememberedSet>::coalesce_objects(HeapWord* address, size_t length_in_words) {
 205 
 206   size_t card_at_start = _rs->card_index_for_addr(address);
 207   HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start);
 208   size_t card_at_end = card_at_start + ((address + length_in_words) - card_start_address) / CardTable::card_size_in_words();
 209 
 210   if (card_at_start == card_at_end) {
 211     // There are no changes to the get_first_start array.  Either get_first_start(card_at_start) returns this coalesced object,
 212     // or it returns an object that precedes the coalesced object.
 213     if (card_start_address + get_last_start(card_at_start) < address + length_in_words) {
 214       uint8_t coalesced_offset = static_cast<uint8_t>(address - card_start_address);
 215       // The object that used to be the last object starting within this card is being subsumed within the coalesced
 216       // object.  Since we always coalesce entire objects, this condition only occurs if the last object ends before or at
 217       // the end of the card's memory range and there is no object following this object.  In this case, adjust last_start
 218       // to represent the start of the coalesced range.
 219       set_last_start(card_at_start, coalesced_offset);
 220     }
 221     // Else, no changes to last_starts information.  Either get_last_start(card_at_start) returns the object that immediately
 222     // follows the coalesced object, or it returns an object that follows the object immediately following the coalesced object.
 223   } else {
 224     uint8_t coalesced_offset = static_cast<uint8_t>(address - card_start_address);
 225     if (get_last_start(card_at_start) > coalesced_offset) {
 226       // Existing last start is being coalesced, create new last start
 227       set_last_start(card_at_start, coalesced_offset);
 228     }
 229     // otherwise, get_last_start(card_at_start) must equal coalesced_offset
 230 
 231     // All the cards between first and last get cleared.
 232     for (size_t i = card_at_start + 1; i < card_at_end; i++) {
 233       clear_starts_object_bit(i);
 234     }
 235 
 236     uint8_t follow_offset = static_cast<uint8_t>((address + length_in_words) - _rs->addr_for_card_index(card_at_end));
 237     if (starts_object(card_at_end) && (get_first_start(card_at_end) < follow_offset)) {
 238       // It may be that after coalescing within this last card's memory range, the last card
 239       // no longer holds an object.
 240       if (get_last_start(card_at_end) >= follow_offset) {
 241         set_first_start(card_at_end, follow_offset);
 242       } else {
 243         // last_start is being coalesced so this card no longer has any objects.
 244         clear_starts_object_bit(card_at_end);
 245       }
 246     }
 247     // else
 248     //  card_at_end did not have an object, so it still does not have an object, or
 249     //  card_at_end had an object that starts after the coalesced object, so no changes required for card_at_end
 250 
 251   }
 252 }
 253 
 254 
 255 template<typename RememberedSet>
 256 inline size_t
 257 ShenandoahCardCluster<RememberedSet>::get_first_start(size_t card_index) const {
 258   assert(starts_object(card_index), "Can't get first start because no object starts here");
 259   return object_starts[card_index].offsets.first & FirstStartBits;
 260 }
 261 
 262 template<typename RememberedSet>
 263 inline size_t
 264 ShenandoahCardCluster<RememberedSet>::get_last_start(size_t card_index) const {
 265   assert(starts_object(card_index), "Can't get last start because no object starts here");
 266   return object_starts[card_index].offsets.last;
 267 }
 268 
 269 // Given a card_index, return the starting address of the first block in the heap
 270 // that straddles into this card. If this card is co-initial with an object, then
 271 // this would return the first address of the range that this card covers, which is
 272 // where the card's first object also begins.
 273 // TODO: collect some stats for the size of walks backward over cards.
 274 // For larger objects, a logarithmic BOT such as used by G1 might make the
 275 // backwards walk potentially faster.
 276 template<typename RememberedSet>
 277 HeapWord*
 278 ShenandoahCardCluster<RememberedSet>::block_start(const size_t card_index) const {
 279 
 280   HeapWord* left = _rs->addr_for_card_index(card_index);
 281 
 282 #ifdef ASSERT
 283   assert(ShenandoahHeap::heap()->mode()->is_generational(), "Do not use in non-generational mode");
 284   ShenandoahHeapRegion* region = ShenandoahHeap::heap()->heap_region_containing(left);
 285   assert(region->is_old(), "Do not use for young regions");
 286   // For HumongousRegion:s it's more efficient to jump directly to the
 287   // start region.
 288   assert(!region->is_humongous(), "Use region->humongous_start_region() instead");
 289 #endif
 290   if (starts_object(card_index) && get_first_start(card_index) == 0) {
 291     // This card contains a co-initial object; a fortiori, it covers
 292     // also the case of a card being the first in a region.
 293     assert(oopDesc::is_oop(cast_to_oop(left)), "Should be an object");
 294     return left;
 295   }
 296 
 297   HeapWord* p = nullptr;
 298   oop obj = cast_to_oop(p);
 299   ssize_t cur_index = (ssize_t)card_index;
 300   assert(cur_index >= 0, "Overflow");
 301   assert(cur_index > 0, "Should have returned above");
 302   // Walk backwards over the cards...
 303   while (--cur_index > 0 && !starts_object(cur_index)) {
 304    // ... to the one that starts the object
 305   }
 306   // cur_index should start an object: we should not have walked
 307   // past the left end of the region.
 308   assert(cur_index >= 0 && (cur_index <= (ssize_t)card_index), "Error");
 309   assert(region->bottom() <= _rs->addr_for_card_index(cur_index),
 310          "Fell off the bottom of containing region");
 311   assert(starts_object(cur_index), "Error");
 312   size_t offset = get_last_start(cur_index);
 313   // can avoid call via card size arithmetic below instead
 314   p = _rs->addr_for_card_index(cur_index) + offset;
 315   // Recall that we already dealt with the co-initial object case above
 316   assert(p < left, "obj should start before left");
 317   // While it is safe to ask an object its size in the loop that
 318   // follows, the (ifdef'd out) loop should never be needed.
 319   // 1. we ask this question only for regions in the old generation
 320   // 2. there is no direct allocation ever by mutators in old generation
 321   //    regions. Only GC will ever allocate in old regions, and then
 322   //    too only during promotion/evacuation phases. Thus there is no danger
 323   //    of races between reading from and writing to the object start array,
 324   //    or of asking partially initialized objects their size (in the loop below).
 325   // 3. only GC asks this question during phases when it is not concurrently
 326   //    evacuating/promoting, viz. during concurrent root scanning (before
 327   //    the evacuation phase) and during concurrent update refs (after the
 328   //    evacuation phase) of young collections. This is never called
 329   //    during old or global collections.
 330   // 4. Every allocation under TAMS updates the object start array.
 331   NOT_PRODUCT(obj = cast_to_oop(p);)
 332   assert(oopDesc::is_oop(obj), "Should be an object");
 333 #define WALK_FORWARD_IN_BLOCK_START false
 334   while (WALK_FORWARD_IN_BLOCK_START && p + obj->size() < left) {
 335     p += obj->size();
 336   }
 337 #undef WALK_FORWARD_IN_BLOCK_START // false
 338   assert(p + obj->size() > left, "obj should end after left");
 339   return p;
 340 }
 341 
 342 template<typename RememberedSet>
 343 inline size_t
 344 ShenandoahScanRemembered<RememberedSet>::last_valid_index() { return _rs->last_valid_index(); }
 345 
 346 template<typename RememberedSet>
 347 inline size_t
 348 ShenandoahScanRemembered<RememberedSet>::total_cards() { return _rs->total_cards(); }
 349 
 350 template<typename RememberedSet>
 351 inline size_t
 352 ShenandoahScanRemembered<RememberedSet>::card_index_for_addr(HeapWord *p) { return _rs->card_index_for_addr(p); }
 353 
 354 template<typename RememberedSet>
 355 inline HeapWord *
 356 ShenandoahScanRemembered<RememberedSet>::addr_for_card_index(size_t card_index) { return _rs->addr_for_card_index(card_index); }
 357 
 358 template<typename RememberedSet>
 359 inline bool
 360 ShenandoahScanRemembered<RememberedSet>::is_card_dirty(size_t card_index) { return _rs->is_card_dirty(card_index); }
 361 
 362 template<typename RememberedSet>
 363 inline void
 364 ShenandoahScanRemembered<RememberedSet>::mark_card_as_dirty(size_t card_index) { _rs->mark_card_as_dirty(card_index); }
 365 
 366 template<typename RememberedSet>
 367 inline void
 368 ShenandoahScanRemembered<RememberedSet>::mark_range_as_dirty(size_t card_index, size_t num_cards) { _rs->mark_range_as_dirty(card_index, num_cards); }
 369 
 370 template<typename RememberedSet>
 371 inline void
 372 ShenandoahScanRemembered<RememberedSet>::mark_card_as_clean(size_t card_index) { _rs->mark_card_as_clean(card_index); }
 373 
 374 template<typename RememberedSet>
 375 inline void
 376 ShenandoahScanRemembered<RememberedSet>::mark_range_as_clean(size_t card_index, size_t num_cards) { _rs->mark_range_as_clean(card_index, num_cards); }
 377 
 378 template<typename RememberedSet>
 379 inline bool
 380 ShenandoahScanRemembered<RememberedSet>::is_card_dirty(HeapWord *p) { return _rs->is_card_dirty(p); }
 381 
 382 template<typename RememberedSet>
 383 inline void
 384 ShenandoahScanRemembered<RememberedSet>::mark_card_as_dirty(HeapWord *p) { _rs->mark_card_as_dirty(p); }
 385 
 386 template<typename RememberedSet>
 387 inline void
 388 ShenandoahScanRemembered<RememberedSet>::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) { _rs->mark_range_as_dirty(p, num_heap_words); }
 389 
 390 template<typename RememberedSet>
 391 inline void
 392 ShenandoahScanRemembered<RememberedSet>::mark_card_as_clean(HeapWord *p) { _rs->mark_card_as_clean(p); }
 393 
 394 template<typename RememberedSet>
 395 inline void
 396 ShenandoahScanRemembered<RememberedSet>:: mark_range_as_clean(HeapWord *p, size_t num_heap_words) { _rs->mark_range_as_clean(p, num_heap_words); }
 397 
 398 template<typename RememberedSet>
 399 inline size_t
 400 ShenandoahScanRemembered<RememberedSet>::cluster_count() { return _rs->cluster_count(); }
 401 
 402 template<typename RememberedSet>
 403 inline void
 404 ShenandoahScanRemembered<RememberedSet>::reset_object_range(HeapWord *from, HeapWord *to) {
 405   _scc->reset_object_range(from, to);
 406 }
 407 
 408 template<typename RememberedSet>
 409 inline void
 410 ShenandoahScanRemembered<RememberedSet>::register_object(HeapWord *addr) {
 411   _scc->register_object(addr);
 412 }
 413 
 414 template<typename RememberedSet>
 415 inline void
 416 ShenandoahScanRemembered<RememberedSet>::register_object_without_lock(HeapWord *addr) {
 417   _scc->register_object_without_lock(addr);
 418 }
 419 
 420 template <typename RememberedSet>
 421 inline bool
 422 ShenandoahScanRemembered<RememberedSet>::verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx) {
 423 
 424   size_t index = card_index_for_addr(address);
 425   if (!_scc->starts_object(index)) {
 426     return false;
 427   }
 428   HeapWord* base_addr = addr_for_card_index(index);
 429   size_t offset = _scc->get_first_start(index);
 430   ShenandoahHeap* heap = ShenandoahHeap::heap();
 431 
 432   // Verify that I can find this object within its enclosing card by scanning forward from first_start.
 433   while (base_addr + offset < address) {
 434     oop obj = cast_to_oop(base_addr + offset);
 435     if (!ctx || ctx->is_marked(obj)) {
 436       offset += obj->size();
 437     } else {
 438       // If this object is not live, don't trust its size(); all objects above tams are live.
 439       ShenandoahHeapRegion* r = heap->heap_region_containing(obj);
 440       HeapWord* tams = ctx->top_at_mark_start(r);
 441       offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr;
 442     }
 443   }
 444   if (base_addr + offset != address){
 445     return false;
 446   }
 447 
 448   // At this point, offset represents object whose registration we are verifying.  We know that at least this object resides
 449   // within this card's memory.
 450 
 451   // Make sure that last_offset is properly set for the enclosing card, but we can't verify this for
 452   // candidate collection-set regions during mixed evacuations, so disable this check in general
 453   // during mixed evacuations.
 454 
 455   ShenandoahHeapRegion* r = heap->heap_region_containing(base_addr + offset);
 456   size_t max_offset = r->top() - base_addr;
 457   if (max_offset > CardTable::card_size_in_words()) {
 458     max_offset = CardTable::card_size_in_words();
 459   }
 460   size_t prev_offset;
 461   if (!ctx) {
 462     do {
 463       oop obj = cast_to_oop(base_addr + offset);
 464       prev_offset = offset;
 465       offset += obj->size();
 466     } while (offset < max_offset);
 467     if (_scc->get_last_start(index) != prev_offset) {
 468       return false;
 469     }
 470 
 471     // base + offset represents address of first object that starts on following card, if there is one.
 472 
 473     // Notes: base_addr is addr_for_card_index(index)
 474     //        base_addr + offset is end of the object we are verifying
 475     //        cannot use card_index_for_addr(base_addr + offset) because it asserts arg < end of whole heap
 476     size_t end_card_index = index + offset / CardTable::card_size_in_words();
 477 
 478     if (end_card_index > index && end_card_index <= _rs->last_valid_index()) {
 479       // If there is a following object registered on the next card, it should begin where this object ends.
 480       if (_scc->starts_object(end_card_index) &&
 481           ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) {
 482         return false;
 483       }
 484     }
 485 
 486     // Assure that no other objects are registered "inside" of this one.
 487     for (index++; index < end_card_index; index++) {
 488       if (_scc->starts_object(index)) {
 489         return false;
 490       }
 491     }
 492   } else {
 493     // This is a mixed evacuation or a global collect: rely on mark bits to identify which objects need to be properly registered
 494     assert(!ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Cannot rely on mark context here.");
 495     // If the object reaching or spanning the end of this card's memory is marked, then last_offset for this card
 496     // should represent this object.  Otherwise, last_offset is a don't care.
 497     ShenandoahHeapRegion* region = heap->heap_region_containing(base_addr + offset);
 498     HeapWord* tams = ctx->top_at_mark_start(region);
 499     oop last_obj = nullptr;
 500     do {
 501       oop obj = cast_to_oop(base_addr + offset);
 502       if (ctx->is_marked(obj)) {
 503         prev_offset = offset;
 504         offset += obj->size();
 505         last_obj = obj;
 506       } else {
 507         offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr;
 508         // If there are no marked objects remaining in this region, offset equals tams - base_addr.  If this offset is
 509         // greater than max_offset, we will immediately exit this loop.  Otherwise, the next iteration of the loop will
 510         // treat the object at offset as marked and live (because address >= tams) and we will continue iterating object
 511         // by consulting the size() fields of each.
 512       }
 513     } while (offset < max_offset);
 514     if (last_obj != nullptr && prev_offset + last_obj->size() >= max_offset) {
 515       // last marked object extends beyond end of card
 516       if (_scc->get_last_start(index) != prev_offset) {
 517         return false;
 518       }
 519       // otherwise, the value of _scc->get_last_start(index) is a don't care because it represents a dead object and we
 520       // cannot verify its context
 521     }
 522   }
 523   return true;
 524 }
 525 
 526 template<typename RememberedSet>
 527 inline void
 528 ShenandoahScanRemembered<RememberedSet>::coalesce_objects(HeapWord *addr, size_t length_in_words) {
 529   _scc->coalesce_objects(addr, length_in_words);
 530 }
 531 
 532 template<typename RememberedSet>
 533 inline void
 534 ShenandoahScanRemembered<RememberedSet>::mark_range_as_empty(HeapWord *addr, size_t length_in_words) {
 535   _rs->mark_range_as_clean(addr, length_in_words);
 536   _scc->clear_objects_in_range(addr, length_in_words);
 537 }
 538 
 539 // Process all objects starting within count clusters beginning with first_cluster and for which the start address is
 540 // less than end_of_range.  For any non-array object whose header lies on a dirty card, scan the entire object,
 541 // even if its end reaches beyond end_of_range. Object arrays, on the other hand, are precisely dirtied and
 542 // only the portions of the array on dirty cards need to be scanned.
 543 //
 544 // Do not CANCEL within process_clusters.  It is assumed that if a worker thread accepts responsibility for processing
 545 // a chunk of work, it will finish the work it starts.  Otherwise, the chunk of work will be lost in the transition to
 546 // degenerated execution, leading to dangling references.
 547 template<typename RememberedSet>
 548 template <typename ClosureType>
 549 void ShenandoahScanRemembered<RememberedSet>::process_clusters(size_t first_cluster, size_t count, HeapWord* end_of_range,
 550                                                                ClosureType* cl, bool use_write_table, uint worker_id) {
 551 
 552   // If old-gen evacuation is active, then MarkingContext for old-gen heap regions is valid.  We use the MarkingContext
 553   // bits to determine which objects within a DIRTY card need to be scanned.  This is necessary because old-gen heap
 554   // regions that are in the candidate collection set have not been coalesced and filled.  Thus, these heap regions
 555   // may contain zombie objects.  Zombie objects are known to be dead, but have not yet been "collected".  Scanning
 556   // zombie objects is unsafe because the Klass pointer is not reliable, objects referenced from a zombie may have been
 557   // collected (if dead), or relocated (if live), or if dead but not yet collected, we don't want to "revive" them
 558   // by marking them (when marking) or evacuating them (when updating references).
 559 
 560   // start and end addresses of range of objects to be scanned, clipped to end_of_range
 561   const size_t start_card_index = first_cluster * ShenandoahCardCluster<RememberedSet>::CardsPerCluster;
 562   const HeapWord* start_addr = _rs->addr_for_card_index(start_card_index);
 563   // clip at end_of_range (exclusive)
 564   HeapWord* end_addr = MIN2(end_of_range, (HeapWord*)start_addr + (count * ShenandoahCardCluster<RememberedSet>::CardsPerCluster
 565                                                                    * CardTable::card_size_in_words()));
 566   assert(start_addr < end_addr, "Empty region?");
 567 
 568   const size_t whole_cards = (end_addr - start_addr + CardTable::card_size_in_words() - 1)/CardTable::card_size_in_words();
 569   const size_t end_card_index = start_card_index + whole_cards - 1;
 570   log_debug(gc, remset)("Worker %u: cluster = " SIZE_FORMAT " count = " SIZE_FORMAT " eor = " INTPTR_FORMAT
 571                         " start_addr = " INTPTR_FORMAT " end_addr = " INTPTR_FORMAT " cards = " SIZE_FORMAT,
 572                         worker_id, first_cluster, count, p2i(end_of_range), p2i(start_addr), p2i(end_addr), whole_cards);
 573 
 574   // use_write_table states whether we are using the card table that is being
 575   // marked by the mutators. If false, we are using a snapshot of the card table
 576   // that is not subject to modifications. Even when this arg is true, and
 577   // the card table is being actively marked, SATB marking ensures that we need not
 578   // worry about cards marked after the processing here has passed them.
 579   const CardValue* const ctbm = _rs->get_card_table_byte_map(use_write_table);
 580 
 581   // If old gen evacuation is active, ctx will hold the completed marking of
 582   // old generation objects. We'll only scan objects that are marked live by
 583   // the old generation marking. These include objects allocated since the
 584   // start of old generation marking (being those above TAMS).
 585   const ShenandoahHeap* heap = ShenandoahHeap::heap();
 586   const ShenandoahMarkingContext* ctx = heap->is_old_bitmap_stable() ?
 587                                         heap->marking_context() : nullptr;
 588 
 589   // The region we will scan is the half-open interval [start_addr, end_addr),
 590   // and lies entirely within a single region.
 591   const ShenandoahHeapRegion* region = ShenandoahHeap::heap()->heap_region_containing(start_addr);
 592   assert(region->contains(end_addr - 1), "Slice shouldn't cross regions");
 593 
 594   // This code may have implicit assumptions of examining only old gen regions.
 595   assert(region->is_old(), "We only expect to be processing old regions");
 596   assert(!region->is_humongous(), "Humongous regions can be processed more efficiently;"
 597                                   "see process_humongous_clusters()");
 598   // tams and ctx below are for old generation marking. As such, young gen roots must
 599   // consider everything above tams, since it doesn't represent a TAMS for young gen's
 600   // SATB marking.
 601   const HeapWord* tams = (ctx == nullptr ? region->bottom() : ctx->top_at_mark_start(region));
 602 
 603   NOT_PRODUCT(ShenandoahCardStats stats(whole_cards, card_stats(worker_id));)
 604 
 605   // In the case of imprecise marking, we remember the lowest address
 606   // scanned in a range of dirty cards, as we work our way left from the
 607   // highest end_addr. This serves as another upper bound on the address we will
 608   // scan as we move left over each contiguous range of dirty cards.
 609   HeapWord* upper_bound = nullptr;
 610 
 611   // Starting at the right end of the address range, walk backwards accumulating
 612   // a maximal dirty range of cards, then process those cards.
 613   ssize_t cur_index = (ssize_t) end_card_index;
 614   assert(cur_index >= 0, "Overflow");
 615   assert(((ssize_t)start_card_index) >= 0, "Overflow");
 616   while (cur_index >= (ssize_t)start_card_index) {
 617 
 618     // We'll continue the search starting with the card for the upper bound
 619     // address identified by the last dirty range that we processed, if any,
 620     // skipping any cards at higher addresses.
 621     if (upper_bound != nullptr) {
 622       ssize_t right_index = _rs->card_index_for_addr(upper_bound);
 623       assert(right_index >= 0, "Overflow");
 624       cur_index = MIN2(cur_index, right_index);
 625       assert(upper_bound < end_addr, "Program logic");
 626       end_addr  = upper_bound;   // lower end_addr
 627       upper_bound = nullptr;     // and clear upper_bound
 628       if (end_addr <= start_addr) {
 629         assert(right_index <= (ssize_t)start_card_index, "Program logic");
 630         // We are done with our cluster
 631         return;
 632       }
 633     }
 634 
 635     if (ctbm[cur_index] == CardTable::dirty_card_val()) {
 636       // ==== BEGIN DIRTY card range processing ====
 637 
 638       const size_t dirty_r = cur_index;  // record right end of dirty range (inclusive)
 639       while (--cur_index >= (ssize_t)start_card_index && ctbm[cur_index] == CardTable::dirty_card_val()) {
 640         // walk back over contiguous dirty cards to find left end of dirty range (inclusive)
 641       }
 642       // [dirty_l, dirty_r] is a "maximal" closed interval range of dirty card indices:
 643       // it may not be maximal if we are using the write_table, because of concurrent
 644       // mutations dirtying the card-table. It may also not be maximal if an upper bound
 645       // was established by the scan of the previous chunk.
 646       const size_t dirty_l = cur_index + 1;   // record left end of dirty range (inclusive)
 647       // Check that we identified a boundary on our left
 648       assert(ctbm[dirty_l] == CardTable::dirty_card_val(), "First card in range should be dirty");
 649       assert(dirty_l == start_card_index || use_write_table
 650              || ctbm[dirty_l - 1] == CardTable::clean_card_val(),
 651              "Interval isn't maximal on the left");
 652       assert(dirty_r >= dirty_l, "Error");
 653       assert(ctbm[dirty_r] == CardTable::dirty_card_val(), "Last card in range should be dirty");
 654       // Record alternations, dirty run length, and dirty card count
 655       NOT_PRODUCT(stats.record_dirty_run(dirty_r - dirty_l + 1);)
 656 
 657       // Find first object that starts this range:
 658       // [left, right) is a maximal right-open interval of dirty cards
 659       HeapWord* left = _rs->addr_for_card_index(dirty_l);        // inclusive
 660       HeapWord* right = _rs->addr_for_card_index(dirty_r + 1);   // exclusive
 661       // Clip right to end_addr established above (still exclusive)
 662       right = MIN2(right, end_addr);
 663       assert(right <= region->top() && end_addr <= region->top(), "Busted bounds");
 664       const MemRegion mr(left, right);
 665 
 666       // NOTE: We'll not call block_start() repeatedly
 667       // on a very large object if its head card is dirty. If not,
 668       // (i.e. the head card is clean) we'll call it each time we
 669       // process a new dirty range on the object. This is always
 670       // the case for large object arrays, which are typically more
 671       // common.
 672       // TODO: It is worthwhile to memoize this, so as to avoid that
 673       // overhead, and it is easy to do, but deferred to a follow-up.
 674       HeapWord* p = _scc->block_start(dirty_l);
 675       oop obj = cast_to_oop(p);
 676 
 677       // PREFIX: The object that straddles into this range of dirty cards
 678       // from the left may be subject to special treatment unless
 679       // it is an object array.
 680       if (p < left && !obj->is_objArray()) {
 681         // The mutator (both compiler and interpreter, but not JNI?)
 682         // typically dirty imprecisely (i.e. only the head of an object),
 683         // but GC closures typically dirty the object precisely. (It would
 684         // be nice to have everything be precise for maximum efficiency.)
 685         //
 686         // To handle this, we check the head card of the object here and,
 687         // if dirty, (arrange to) scan the object in its entirety. If we
 688         // find the head card clean, we'll scan only the portion of the
 689         // object lying in the dirty card range below, assuming this was
 690         // the result of precise marking by GC closures.
 691 
 692         // index of the "head card" for p
 693         const size_t hc_index = _rs->card_index_for_addr(p);
 694         if (ctbm[hc_index] == CardTable::dirty_card_val()) {
 695           // Scan or skip the object, depending on location of its
 696           // head card, and remember that we'll have processed all
 697           // the objects back up to p, which is thus an upper bound
 698           // for the next iteration of a dirty card loop.
 699           upper_bound = p;   // remember upper bound for next chunk
 700           if (p < start_addr) {
 701             // if object starts in a previous slice, it'll be handled
 702             // in its entirety by the thread processing that slice; we can
 703             // skip over it and avoid an unnecessary extra scan.
 704             assert(obj == cast_to_oop(p), "Inconsistency detected");
 705             p += obj->size();
 706           } else {
 707             // the object starts in our slice, we scan it in its entirety
 708             assert(obj == cast_to_oop(p), "Inconsistency detected");
 709             if (ctx == nullptr || ctx->is_marked(obj)) {
 710               // Scan the object in its entirety
 711               p += obj->oop_iterate_size(cl);
 712             } else {
 713               assert(p < tams, "Error 1 in ctx/marking/tams logic");
 714               // Skip over any intermediate dead objects
 715               p = ctx->get_next_marked_addr(p, tams);
 716               assert(p <= tams, "Error 2 in ctx/marking/tams logic");
 717             }
 718           }
 719           assert(p > left, "Should have processed into interior of dirty range");
 720         }
 721       }
 722 
 723       size_t i = 0;
 724       HeapWord* last_p = nullptr;
 725 
 726       // BODY: Deal with (other) objects in this dirty card range
 727       while (p < right) {
 728         obj = cast_to_oop(p);
 729         // walk right scanning eligible objects
 730         if (ctx == nullptr || ctx->is_marked(obj)) {
 731           // we need to remember the last object ptr we scanned, in case we need to
 732           // complete a partial suffix scan after mr, see below
 733           last_p = p;
 734           // apply the closure to the oops in the portion of
 735           // the object within mr.
 736           p += obj->oop_iterate_size(cl, mr);
 737           NOT_PRODUCT(i++);
 738         } else {
 739           // forget the last object pointer we remembered
 740           last_p = nullptr;
 741           assert(p < tams, "Tams and above are implicitly marked in ctx");
 742           // object under tams isn't marked: skip to next live object
 743           p = ctx->get_next_marked_addr(p, tams);
 744           assert(p <= tams, "Error 3 in ctx/marking/tams logic");
 745         }
 746       }
 747 
 748       // TODO: if an objArray then only use mr, else just iterate over entire object;
 749       // that would avoid the special treatment of suffix below.
 750 
 751       // SUFFIX: Fix up a possible incomplete scan at right end of window
 752       // by scanning the portion of a non-objArray that wasn't done.
 753       if (p > right && last_p != nullptr) {
 754         assert(last_p < right, "Error");
 755         // check if last_p suffix needs scanning
 756         const oop last_obj = cast_to_oop(last_p);
 757         if (!last_obj->is_objArray()) {
 758           // scan the remaining suffix of the object
 759           const MemRegion last_mr(right, p);
 760           assert(p == last_p + last_obj->size(), "Would miss portion of last_obj");
 761           last_obj->oop_iterate(cl, last_mr);
 762           log_debug(gc, remset)("Fixed up non-objArray suffix scan in [" INTPTR_FORMAT ", " INTPTR_FORMAT ")",
 763                                 p2i(last_mr.start()), p2i(last_mr.end()));
 764         } else {
 765           log_debug(gc, remset)("Skipped suffix scan of objArray in [" INTPTR_FORMAT ", " INTPTR_FORMAT ")",
 766                                 p2i(right), p2i(p));
 767         }
 768       }
 769       NOT_PRODUCT(stats.record_scan_obj_cnt(i);)
 770 
 771       // ==== END   DIRTY card range processing ====
 772     } else {
 773       // ==== BEGIN CLEAN card range processing ====
 774 
 775       assert(ctbm[cur_index] == CardTable::clean_card_val(), "Error");
 776       // walk back over contiguous clean cards
 777       size_t i = 0;
 778       while (--cur_index >= (ssize_t)start_card_index && ctbm[cur_index] == CardTable::clean_card_val()) {
 779         NOT_PRODUCT(i++);
 780       }
 781       // Record alternations, clean run length, and clean card count
 782       NOT_PRODUCT(stats.record_clean_run(i);)
 783 
 784       // ==== END CLEAN card range processing ====
 785     }
 786   }
 787 }
 788 
 789 // Given that this range of clusters is known to span a humongous object spanned by region r, scan the
 790 // portion of the humongous object that corresponds to the specified range.
 791 template<typename RememberedSet>
 792 template <typename ClosureType>
 793 inline void
 794 ShenandoahScanRemembered<RememberedSet>::process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count,
 795                                                                     HeapWord *end_of_range, ClosureType *cl, bool use_write_table) {
 796   ShenandoahHeapRegion* start_region = r->humongous_start_region();
 797   HeapWord* p = start_region->bottom();
 798   oop obj = cast_to_oop(p);
 799   assert(r->is_humongous(), "Only process humongous regions here");
 800   assert(start_region->is_humongous_start(), "Should be start of humongous region");
 801   assert(p + obj->size() >= end_of_range, "Humongous object ends before range ends");
 802 
 803   size_t first_card_index = first_cluster * ShenandoahCardCluster<RememberedSet>::CardsPerCluster;
 804   HeapWord* first_cluster_addr = _rs->addr_for_card_index(first_card_index);
 805   size_t spanned_words = count * ShenandoahCardCluster<RememberedSet>::CardsPerCluster * CardTable::card_size_in_words();
 806   start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, use_write_table);
 807 }
 808 
 809 
 810 // This method takes a region & determines the end of the region that the worker can scan.
 811 template<typename RememberedSet>
 812 template <typename ClosureType>
 813 inline void
 814 ShenandoahScanRemembered<RememberedSet>::process_region_slice(ShenandoahHeapRegion *region, size_t start_offset, size_t clusters,
 815                                                               HeapWord *end_of_range, ClosureType *cl, bool use_write_table,
 816                                                               uint worker_id) {
 817 
 818   // This is called only for young gen collection, when we scan old gen regions
 819   assert(region->is_old(), "Expecting an old region");
 820   HeapWord *start_of_range = region->bottom() + start_offset;
 821   size_t start_cluster_no = cluster_for_addr(start_of_range);
 822   assert(addr_for_cluster(start_cluster_no) == start_of_range, "process_region_slice range must align on cluster boundary");
 823 
 824   // region->end() represents the end of memory spanned by this region, but not all of this
 825   //   memory is eligible to be scanned because some of this memory has not yet been allocated.
 826   //
 827   // region->top() represents the end of allocated memory within this region.  Any addresses
 828   //   beyond region->top() should not be scanned as that memory does not hold valid objects.
 829 
 830   if (use_write_table) {
 831     // This is update-refs servicing.
 832     if (end_of_range > region->get_update_watermark()) {
 833       end_of_range = region->get_update_watermark();
 834     }
 835   } else {
 836     // This is concurrent mark servicing.  Note that TAMS for this region is TAMS at start of old-gen
 837     // collection.  Here, we need to scan up to TAMS for most recently initiated young-gen collection.
 838     // Since all LABs are retired at init mark, and since replacement LABs are allocated lazily, and since no
 839     // promotions occur until evacuation phase, TAMS for most recent young-gen is same as top().
 840     if (end_of_range > region->top()) {
 841       end_of_range = region->top();
 842     }
 843   }
 844 
 845   log_debug(gc)("Remembered set scan processing Region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT ", using %s table",
 846                 region->index(), p2i(start_of_range), p2i(end_of_range),
 847                 use_write_table? "read/write (updating)": "read (marking)");
 848 
 849   // Note that end_of_range may point to the middle of a cluster because we limit scanning to
 850   // region->top() or region->get_update_watermark(). We avoid processing past end_of_range.
 851   // Objects that start between start_of_range and end_of_range, including humongous objects, will
 852   // be fully processed by process_clusters. In no case should we need to scan past end_of_range.
 853   if (start_of_range < end_of_range) {
 854     if (region->is_humongous()) {
 855       ShenandoahHeapRegion* start_region = region->humongous_start_region();
 856       // TODO: ysr : This will be called multiple times with same start_region, but different start_cluster_no.
 857       // Check that it does the right thing here, and doesn't do redundant work. Also see if the call API/interface
 858       // can be simplified.
 859       process_humongous_clusters(start_region, start_cluster_no, clusters, end_of_range, cl, use_write_table);
 860     } else {
 861       // TODO: ysr The start_of_range calculated above is discarded and may be calculated again in process_clusters().
 862       // See if the redundant and wasted calculations can be avoided, and if the call parameters can be cleaned up.
 863       // It almost sounds like this set of methods needs a working class to stash away some useful info that can be
 864       // efficiently passed around amongst these methods, as well as related state. Note that we can't use
 865       // ShenandoahScanRemembered as there seems to be only one instance of that object for the heap which is shared
 866       // by all workers. Note that there are also task methods which call these which may have per worker storage.
 867       // We need to be careful however that if the number of workers changes dynamically that state isn't sequestered
 868       // and become obsolete.
 869       process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, worker_id);
 870     }
 871   }
 872 }
 873 
 874 template<typename RememberedSet>
 875 inline size_t
 876 ShenandoahScanRemembered<RememberedSet>::cluster_for_addr(HeapWordImpl **addr) {
 877   size_t card_index = _rs->card_index_for_addr(addr);
 878   size_t result = card_index / ShenandoahCardCluster<RememberedSet>::CardsPerCluster;
 879   return result;
 880 }
 881 
 882 template<typename RememberedSet>
 883 inline HeapWord*
 884 ShenandoahScanRemembered<RememberedSet>::addr_for_cluster(size_t cluster_no) {
 885   size_t card_index = cluster_no * ShenandoahCardCluster<RememberedSet>::CardsPerCluster;
 886   return addr_for_card_index(card_index);
 887 }
 888 
 889 // This is used only for debug verification so don't worry about making the scan parallel.
 890 template<typename RememberedSet>
 891 void ShenandoahScanRemembered<RememberedSet>::roots_do(OopIterateClosure* cl) {
 892   ShenandoahHeap* heap = ShenandoahHeap::heap();
 893   log_info(gc, remset)("Scan remembered set using bitmap: %s", BOOL_TO_STR(heap->is_old_bitmap_stable()));
 894   for (size_t i = 0, n = heap->num_regions(); i < n; ++i) {
 895     ShenandoahHeapRegion* region = heap->get_region(i);
 896     if (region->is_old() && region->is_active() && !region->is_cset()) {
 897       HeapWord* start_of_range = region->bottom();
 898       HeapWord* end_of_range = region->top();
 899       size_t start_cluster_no = cluster_for_addr(start_of_range);
 900       size_t num_heapwords = end_of_range - start_of_range;
 901       unsigned int cluster_size = CardTable::card_size_in_words() *
 902                                   ShenandoahCardCluster<ShenandoahDirectCardMarkRememberedSet>::CardsPerCluster;
 903       size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size);
 904 
 905       // Remembered set scanner
 906       if (region->is_humongous()) {
 907         process_humongous_clusters(region->humongous_start_region(), start_cluster_no, num_clusters, end_of_range, cl,
 908                                    false /* use_write_table */);
 909       } else {
 910         process_clusters(start_cluster_no, num_clusters, end_of_range, cl,
 911                          false /* use_write_table */, 0 /* fake worker id */);
 912       }
 913     }
 914   }
 915 }
 916 
 917 #ifndef PRODUCT
 918 // Log given card stats
 919 template<typename RememberedSet>
 920 inline void ShenandoahScanRemembered<RememberedSet>::log_card_stats(HdrSeq* stats) {
 921   for (int i = 0; i < MAX_CARD_STAT_TYPE; i++) {
 922     log_info(gc, remset)("%18s: [ %8.2f %8.2f %8.2f %8.2f %8.2f ]",
 923       _card_stats_name[i],
 924       stats[i].percentile(0), stats[i].percentile(25),
 925       stats[i].percentile(50), stats[i].percentile(75),
 926       stats[i].maximum());
 927   }
 928 }
 929 
 930 // Log card stats for all nworkers for a specific phase t
 931 template<typename RememberedSet>
 932 void ShenandoahScanRemembered<RememberedSet>::log_card_stats(uint nworkers, CardStatLogType t) {
 933   assert(ShenandoahEnableCardStats, "Do not call");
 934   HdrSeq* sum_stats = card_stats_for_phase(t);
 935   log_info(gc, remset)("%s", _card_stat_log_type[t]);
 936   for (uint i = 0; i < nworkers; i++) {
 937     log_worker_card_stats(i, sum_stats);
 938   }
 939 
 940   // Every so often, log the cumulative global stats
 941   if (++_card_stats_log_counter[t] >= ShenandoahCardStatsLogInterval) {
 942     _card_stats_log_counter[t] = 0;
 943     log_info(gc, remset)("Cumulative stats");
 944     log_card_stats(sum_stats);
 945   }
 946 }
 947 
 948 // Log card stats for given worker_id, & clear them after merging into given cumulative stats
 949 template<typename RememberedSet>
 950 void ShenandoahScanRemembered<RememberedSet>::log_worker_card_stats(uint worker_id, HdrSeq* sum_stats) {
 951   assert(ShenandoahEnableCardStats, "Do not call");
 952 
 953   HdrSeq* worker_card_stats = card_stats(worker_id);
 954   log_info(gc, remset)("Worker %u Card Stats: ", worker_id);
 955   log_card_stats(worker_card_stats);
 956   // Merge worker stats into the cumulative stats & clear worker stats
 957   merge_worker_card_stats_cumulative(worker_card_stats, sum_stats);
 958 }
 959 
 960 template<typename RememberedSet>
 961 void ShenandoahScanRemembered<RememberedSet>::merge_worker_card_stats_cumulative(
 962   HdrSeq* worker_stats, HdrSeq* sum_stats) {
 963   for (int i = 0; i < MAX_CARD_STAT_TYPE; i++) {
 964     sum_stats[i].add(worker_stats[i]);
 965     worker_stats[i].clear();
 966   }
 967 }
 968 #endif
 969 
 970 inline bool ShenandoahRegionChunkIterator::has_next() const {
 971   return _index < _total_chunks;
 972 }
 973 
 974 inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *assignment) {
 975   if (_index >= _total_chunks) {
 976     return false;
 977   }
 978   size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed);
 979   if (new_index > _total_chunks) {
 980     // First worker that hits new_index == _total_chunks continues, other
 981     // contending workers return false.
 982     return false;
 983   }
 984   // convert to zero-based indexing
 985   new_index--;
 986   assert(new_index < _total_chunks, "Error");
 987 
 988   // Find the group number for the assigned chunk index
 989   size_t group_no;
 990   for (group_no = 0; new_index >= _group_entries[group_no]; group_no++)
 991     ;
 992   assert(group_no < _num_groups, "Cannot have group no greater or equal to _num_groups");
 993 
 994   // All size computations measured in HeapWord
 995   size_t region_size_words = ShenandoahHeapRegion::region_size_words();
 996   size_t group_region_index = _region_index[group_no];
 997   size_t group_region_offset = _group_offset[group_no];
 998 
 999   size_t index_within_group = (group_no == 0)? new_index: new_index - _group_entries[group_no - 1];
1000   size_t group_chunk_size = _group_chunk_size[group_no];
1001   size_t offset_of_this_chunk = group_region_offset + index_within_group * group_chunk_size;
1002   size_t regions_spanned_by_chunk_offset = offset_of_this_chunk / region_size_words;
1003   size_t offset_within_region = offset_of_this_chunk % region_size_words;
1004 
1005   size_t region_index = group_region_index + regions_spanned_by_chunk_offset;
1006 
1007   assignment->_r = _heap->get_region(region_index);
1008   assignment->_chunk_offset = offset_within_region;
1009   assignment->_chunk_size = group_chunk_size;
1010   return true;
1011 }
1012 
1013 #endif   // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP