1 /*
  2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/vmClasses.hpp"
 27 #include "classfile/vmSymbols.hpp"
 28 #include "gc/shared/collectedHeap.inline.hpp"
 29 #include "gc/shared/genCollectedHeap.hpp"
 30 #include "gc/shared/slidingForwarding.inline.hpp"
 31 #include "gc/shared/space.hpp"
 32 #include "gc/shared/space.inline.hpp"
 33 #include "gc/shared/spaceDecorator.inline.hpp"
 34 #include "memory/iterator.inline.hpp"
 35 #include "memory/universe.hpp"
 36 #include "oops/oop.inline.hpp"
 37 #include "runtime/atomic.hpp"
 38 #include "runtime/java.hpp"
 39 #include "runtime/prefetch.inline.hpp"
 40 #include "runtime/safepoint.hpp"
 41 #include "utilities/align.hpp"
 42 #include "utilities/copy.hpp"
 43 #include "utilities/globalDefinitions.hpp"
 44 #include "utilities/macros.hpp"
 45 #if INCLUDE_SERIALGC
 46 #include "gc/serial/serialBlockOffsetTable.inline.hpp"
 47 #include "gc/serial/defNewGeneration.hpp"
 48 #endif
 49 
 50 HeapWord* DirtyCardToOopClosure::get_actual_top(HeapWord* top,
 51                                                 HeapWord* top_obj) {
 52   if (top_obj != nullptr && top_obj < (_sp->toContiguousSpace())->top()) {
 53     if (cast_to_oop(top_obj)->is_objArray() || cast_to_oop(top_obj)->is_typeArray()) {
 54       // An arrayOop is starting on the dirty card - since we do exact
 55       // store checks for objArrays we are done.
 56     } else {
 57       // Otherwise, it is possible that the object starting on the dirty
 58       // card spans the entire card, and that the store happened on a
 59       // later card.  Figure out where the object ends.
 60       assert(_sp->block_size(top_obj) == cast_to_oop(top_obj)->size(),
 61              "Block size and object size mismatch");
 62       top = top_obj + cast_to_oop(top_obj)->size();
 63     }
 64   } else {
 65     top = (_sp->toContiguousSpace())->top();
 66   }
 67   return top;
 68 }
 69 
 70 void DirtyCardToOopClosure::walk_mem_region(MemRegion mr,
 71                                             HeapWord* bottom,
 72                                             HeapWord* top) {
 73   // Note that this assumption won't hold if we have a concurrent
 74   // collector in this space, which may have freed up objects after
 75   // they were dirtied and before the stop-the-world GC that is
 76   // examining cards here.
 77   assert(bottom < top, "ought to be at least one obj on a dirty card.");
 78 
 79   walk_mem_region_with_cl(mr, bottom, top, _cl);
 80 }
 81 
 82 // We get called with "mr" representing the dirty region
 83 // that we want to process. Because of imprecise marking,
 84 // we may need to extend the incoming "mr" to the right,
 85 // and scan more. However, because we may already have
 86 // scanned some of that extended region, we may need to
 87 // trim its right-end back some so we do not scan what
 88 // we (or another worker thread) may already have scanned
 89 // or planning to scan.
 90 void DirtyCardToOopClosure::do_MemRegion(MemRegion mr) {
 91   HeapWord* bottom = mr.start();
 92   HeapWord* last = mr.last();
 93   HeapWord* top = mr.end();
 94   HeapWord* bottom_obj;
 95   HeapWord* top_obj;
 96 
 97   assert(_last_bottom == nullptr || top <= _last_bottom,
 98          "Not decreasing");
 99   NOT_PRODUCT(_last_bottom = mr.start());
100 
101   bottom_obj = _sp->block_start(bottom);
102   top_obj    = _sp->block_start(last);
103 
104   assert(bottom_obj <= bottom, "just checking");
105   assert(top_obj    <= top,    "just checking");
106 
107   // Given what we think is the top of the memory region and
108   // the start of the object at the top, get the actual
109   // value of the top.
110   top = get_actual_top(top, top_obj);
111 
112   // If the previous call did some part of this region, don't redo.
113   if (_min_done != nullptr && _min_done < top) {
114     top = _min_done;
115   }
116 
117   // Top may have been reset, and in fact may be below bottom,
118   // e.g. the dirty card region is entirely in a now free object
119   // -- something that could happen with a concurrent sweeper.
120   bottom = MIN2(bottom, top);
121   MemRegion extended_mr = MemRegion(bottom, top);
122   assert(bottom <= top &&
123          (_min_done == nullptr || top <= _min_done),
124          "overlap!");
125 
126   // Walk the region if it is not empty; otherwise there is nothing to do.
127   if (!extended_mr.is_empty()) {
128     walk_mem_region(extended_mr, bottom_obj, top);
129   }
130 
131   _min_done = bottom;
132 }
133 
134 void DirtyCardToOopClosure::walk_mem_region_with_cl(MemRegion mr,
135                                                     HeapWord* bottom,
136                                                     HeapWord* top,
137                                                     OopIterateClosure* cl) {
138   bottom += cast_to_oop(bottom)->oop_iterate_size(cl, mr);
139   if (bottom < top) {
140     HeapWord* next_obj = bottom + cast_to_oop(bottom)->size();
141     while (next_obj < top) {
142       /* Bottom lies entirely below top, so we can call the */
143       /* non-memRegion version of oop_iterate below. */
144       cast_to_oop(bottom)->oop_iterate(cl);
145       bottom = next_obj;
146       next_obj = bottom + cast_to_oop(bottom)->size();
147     }
148     /* Last object. */
149     cast_to_oop(bottom)->oop_iterate(cl, mr);
150   }
151 }
152 
153 void Space::initialize(MemRegion mr,
154                        bool clear_space,
155                        bool mangle_space) {
156   HeapWord* bottom = mr.start();
157   HeapWord* end    = mr.end();
158   assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end),
159          "invalid space boundaries");
160   set_bottom(bottom);
161   set_end(end);
162   if (clear_space) clear(mangle_space);
163 }
164 
165 void Space::clear(bool mangle_space) {
166   if (ZapUnusedHeapArea && mangle_space) {
167     mangle_unused_area();
168   }
169 }
170 
171 ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(nullptr) {
172   _mangler = new GenSpaceMangler(this);
173 }
174 
175 ContiguousSpace::~ContiguousSpace() {
176   delete _mangler;
177 }
178 
179 void ContiguousSpace::initialize(MemRegion mr,
180                                  bool clear_space,
181                                  bool mangle_space)
182 {
183   CompactibleSpace::initialize(mr, clear_space, mangle_space);
184 }
185 
186 void ContiguousSpace::clear(bool mangle_space) {
187   set_top(bottom());
188   set_saved_mark();
189   CompactibleSpace::clear(mangle_space);
190 }
191 
192 bool ContiguousSpace::is_free_block(const HeapWord* p) const {
193   return p >= _top;
194 }
195 
196 #if INCLUDE_SERIALGC
197 void TenuredSpace::clear(bool mangle_space) {
198   ContiguousSpace::clear(mangle_space);
199   _offsets.initialize_threshold();
200 }
201 
202 void TenuredSpace::set_bottom(HeapWord* new_bottom) {
203   Space::set_bottom(new_bottom);
204   _offsets.set_bottom(new_bottom);
205 }
206 
207 void TenuredSpace::set_end(HeapWord* new_end) {
208   // Space should not advertise an increase in size
209   // until after the underlying offset table has been enlarged.
210   _offsets.resize(pointer_delta(new_end, bottom()));
211   Space::set_end(new_end);
212 }
213 #endif // INCLUDE_SERIALGC
214 
215 #ifndef PRODUCT
216 
217 void ContiguousSpace::set_top_for_allocations(HeapWord* v) {
218   mangler()->set_top_for_allocations(v);
219 }
220 void ContiguousSpace::set_top_for_allocations() {
221   mangler()->set_top_for_allocations(top());
222 }
223 void ContiguousSpace::check_mangled_unused_area(HeapWord* limit) {
224   mangler()->check_mangled_unused_area(limit);
225 }
226 
227 void ContiguousSpace::check_mangled_unused_area_complete() {
228   mangler()->check_mangled_unused_area_complete();
229 }
230 
231 // Mangled only the unused space that has not previously
232 // been mangled and that has not been allocated since being
233 // mangled.
234 void ContiguousSpace::mangle_unused_area() {
235   mangler()->mangle_unused_area();
236 }
237 void ContiguousSpace::mangle_unused_area_complete() {
238   mangler()->mangle_unused_area_complete();
239 }
240 #endif  // NOT_PRODUCT
241 
242 void CompactibleSpace::initialize(MemRegion mr,
243                                   bool clear_space,
244                                   bool mangle_space) {
245   Space::initialize(mr, clear_space, mangle_space);
246   set_compaction_top(bottom());
247   _next_compaction_space = nullptr;
248 }
249 
250 void CompactibleSpace::clear(bool mangle_space) {
251   Space::clear(mangle_space);
252   _compaction_top = bottom();
253 }
254 
255 HeapWord* CompactibleSpace::forward(oop q, size_t size,
256                                     CompactPoint* cp, HeapWord* compact_top, SlidingForwarding* const forwarding) {
257   // q is alive
258   // First check if we should switch compaction space
259   assert(this == cp->space, "'this' should be current compaction space.");
260   size_t compaction_max_size = pointer_delta(end(), compact_top);
261   while (size > compaction_max_size) {
262     // switch to next compaction space
263     cp->space->set_compaction_top(compact_top);
264     cp->space = cp->space->next_compaction_space();
265     if (cp->space == nullptr) {
266       cp->gen = GenCollectedHeap::heap()->young_gen();
267       assert(cp->gen != nullptr, "compaction must succeed");
268       cp->space = cp->gen->first_compaction_space();
269       assert(cp->space != nullptr, "generation must have a first compaction space");
270     }
271     compact_top = cp->space->bottom();
272     cp->space->set_compaction_top(compact_top);
273     cp->space->initialize_threshold();
274     compaction_max_size = pointer_delta(cp->space->end(), compact_top);
275   }
276 
277   // store the forwarding pointer into the mark word
278   if (cast_from_oop<HeapWord*>(q) != compact_top) {
279     forwarding->forward_to(q, cast_to_oop(compact_top));
280     assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
281   } else {
282     // if the object isn't moving we can just set the mark to the default
283     // mark and handle it specially later on.
284     q->init_mark();
285     assert(!q->is_forwarded(), "should not be forwarded");
286   }
287 
288   compact_top += size;
289 
290   // We need to update the offset table so that the beginnings of objects can be
291   // found during scavenge.  Note that we are updating the offset table based on
292   // where the object will be once the compaction phase finishes.
293   cp->space->alloc_block(compact_top - size, compact_top);
294   return compact_top;
295 }
296 
297 #if INCLUDE_SERIALGC
298 
299 void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
300   // Compute the new addresses for the live objects and store it in the mark
301   // Used by universe::mark_sweep_phase2()
302 
303   // We're sure to be here before any objects are compacted into this
304   // space, so this is a good time to initialize this:
305   set_compaction_top(bottom());
306 
307   if (cp->space == nullptr) {
308     assert(cp->gen != nullptr, "need a generation");
309     assert(cp->gen->first_compaction_space() == this, "just checking");
310     cp->space = cp->gen->first_compaction_space();
311     cp->space->initialize_threshold();
312     cp->space->set_compaction_top(cp->space->bottom());
313   }
314 
315   HeapWord* compact_top = cp->space->compaction_top(); // This is where we are currently compacting to.
316 
317   DeadSpacer dead_spacer(this);
318 
319   HeapWord*  end_of_live = bottom();  // One byte beyond the last byte of the last live object.
320   HeapWord*  first_dead = nullptr; // The first dead object.
321 
322   const intx interval = PrefetchScanIntervalInBytes;
323 
324   HeapWord* cur_obj = bottom();
325   HeapWord* scan_limit = top();
326 
327   SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding();
328   while (cur_obj < scan_limit) {
329     if (cast_to_oop(cur_obj)->is_gc_marked()) {
330       // prefetch beyond cur_obj
331       Prefetch::write(cur_obj, interval);
332       size_t size = cast_to_oop(cur_obj)->size();
333       compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top, forwarding);
334       cur_obj += size;
335       end_of_live = cur_obj;
336     } else {
337       // run over all the contiguous dead objects
338       HeapWord* end = cur_obj;
339       do {
340         // prefetch beyond end
341         Prefetch::write(end, interval);
342         end += cast_to_oop(end)->size();
343       } while (end < scan_limit && !cast_to_oop(end)->is_gc_marked());
344 
345       // see if we might want to pretend this object is alive so that
346       // we don't have to compact quite as often.
347       if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) {
348         oop obj = cast_to_oop(cur_obj);
349         compact_top = cp->space->forward(obj, obj->size(), cp, compact_top, forwarding);
350         end_of_live = end;
351       } else {
352         // otherwise, it really is a free region.
353 
354         // cur_obj is a pointer to a dead object. Use this dead memory to store a pointer to the next live object.
355         *(HeapWord**)cur_obj = end;
356 
357         // see if this is the first dead region.
358         if (first_dead == nullptr) {
359           first_dead = cur_obj;
360         }
361       }
362 
363       // move on to the next object
364       cur_obj = end;
365     }
366   }
367 
368   assert(cur_obj == scan_limit, "just checking");
369   _end_of_live = end_of_live;
370   if (first_dead != nullptr) {
371     _first_dead = first_dead;
372   } else {
373     _first_dead = end_of_live;
374   }
375 
376   // save the compaction_top of the compaction space.
377   cp->space->set_compaction_top(compact_top);
378 }
379 
380 void CompactibleSpace::adjust_pointers() {
381   // Check first is there is any work to do.
382   if (used() == 0) {
383     return;   // Nothing to do.
384   }
385 
386   // adjust all the interior pointers to point at the new locations of objects
387   // Used by MarkSweep::mark_sweep_phase3()
388 
389   HeapWord* cur_obj = bottom();
390   HeapWord* const end_of_live = _end_of_live;  // Established by prepare_for_compaction().
391   HeapWord* const first_dead = _first_dead;    // Established by prepare_for_compaction().
392   const SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding();
393 
394   assert(first_dead <= end_of_live, "Stands to reason, no?");
395 
396   const intx interval = PrefetchScanIntervalInBytes;
397 
398   debug_only(HeapWord* prev_obj = nullptr);
399   while (cur_obj < end_of_live) {
400     Prefetch::write(cur_obj, interval);
401     if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) {
402       // cur_obj is alive
403       // point all the oops to the new location
404       size_t size = MarkSweep::adjust_pointers(forwarding, cast_to_oop(cur_obj));
405       debug_only(prev_obj = cur_obj);
406       cur_obj += size;
407     } else {
408       debug_only(prev_obj = cur_obj);
409       // cur_obj is not a live object, instead it points at the next live object
410       cur_obj = *(HeapWord**)cur_obj;
411       assert(cur_obj > prev_obj, "we should be moving forward through memory, cur_obj: " PTR_FORMAT ", prev_obj: " PTR_FORMAT, p2i(cur_obj), p2i(prev_obj));
412     }
413   }
414 
415   assert(cur_obj == end_of_live, "just checking");
416 }
417 
418 void CompactibleSpace::compact() {
419   // Copy all live objects to their new location
420   // Used by MarkSweep::mark_sweep_phase4()
421 
422   verify_up_to_first_dead(this);
423 
424   HeapWord* const start = bottom();
425   HeapWord* const end_of_live = _end_of_live;
426 
427   assert(_first_dead <= end_of_live, "Invariant. _first_dead: " PTR_FORMAT " <= end_of_live: " PTR_FORMAT, p2i(_first_dead), p2i(end_of_live));
428   if (_first_dead == end_of_live && (start == end_of_live || !cast_to_oop(start)->is_gc_marked())) {
429     // Nothing to compact. The space is either empty or all live object should be left in place.
430     clear_empty_region(this);
431     return;
432   }
433 
434   const intx scan_interval = PrefetchScanIntervalInBytes;
435   const intx copy_interval = PrefetchCopyIntervalInBytes;
436 
437   assert(start < end_of_live, "bottom: " PTR_FORMAT " should be < end_of_live: " PTR_FORMAT, p2i(start), p2i(end_of_live));
438   HeapWord* cur_obj = start;
439   if (_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) {
440     // All object before _first_dead can be skipped. They should not be moved.
441     // A pointer to the first live object is stored at the memory location for _first_dead.
442     cur_obj = *(HeapWord**)(_first_dead);
443   }
444 
445   const SlidingForwarding* const forwarding = GenCollectedHeap::heap()->forwarding();
446 
447   debug_only(HeapWord* prev_obj = nullptr);
448   while (cur_obj < end_of_live) {
449     if (!cast_to_oop(cur_obj)->is_forwarded()) {
450       debug_only(prev_obj = cur_obj);
451       // The first word of the dead object contains a pointer to the next live object or end of space.
452       cur_obj = *(HeapWord**)cur_obj;
453       assert(cur_obj > prev_obj, "we should be moving forward through memory");
454     } else {
455       // prefetch beyond q
456       Prefetch::read(cur_obj, scan_interval);
457 
458       // size and destination
459       size_t size = cast_to_oop(cur_obj)->size();
460       HeapWord* compaction_top = cast_from_oop<HeapWord*>(forwarding->forwardee(cast_to_oop(cur_obj)));
461 
462       // prefetch beyond compaction_top
463       Prefetch::write(compaction_top, copy_interval);
464 
465       // copy object and reinit its mark
466       assert(cur_obj != compaction_top, "everything in this pass should be moving");
467       Copy::aligned_conjoint_words(cur_obj, compaction_top, size);
468       oop new_obj = cast_to_oop(compaction_top);
469 
470       ContinuationGCSupport::transform_stack_chunk(new_obj);
471 
472       new_obj->init_mark();
473       assert(new_obj->klass() != nullptr, "should have a class");
474 
475       debug_only(prev_obj = cur_obj);
476       cur_obj += size;
477     }
478   }
479 
480   clear_empty_region(this);
481 }
482 
483 #endif // INCLUDE_SERIALGC
484 
485 void Space::print_short() const { print_short_on(tty); }
486 
487 void Space::print_short_on(outputStream* st) const {
488   st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K,
489               (int) ((double) used() * 100 / capacity()));
490 }
491 
492 void Space::print() const { print_on(tty); }
493 
494 void Space::print_on(outputStream* st) const {
495   print_short_on(st);
496   st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ")",
497                 p2i(bottom()), p2i(end()));
498 }
499 
500 void ContiguousSpace::print_on(outputStream* st) const {
501   print_short_on(st);
502   st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
503                 p2i(bottom()), p2i(top()), p2i(end()));
504 }
505 
506 #if INCLUDE_SERIALGC
507 void TenuredSpace::print_on(outputStream* st) const {
508   print_short_on(st);
509   st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", "
510                 PTR_FORMAT ", " PTR_FORMAT ")",
511               p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end()));
512 }
513 #endif
514 
515 void ContiguousSpace::verify() const {
516   HeapWord* p = bottom();
517   HeapWord* t = top();
518   HeapWord* prev_p = nullptr;
519   while (p < t) {
520     oopDesc::verify(cast_to_oop(p));
521     prev_p = p;
522     p += cast_to_oop(p)->size();
523   }
524   guarantee(p == top(), "end of last object must match end of space");
525   if (top() != end()) {
526     guarantee(top() == block_start_const(end()-1) &&
527               top() == block_start_const(top()),
528               "top should be start of unallocated block, if it exists");
529   }
530 }
531 
532 void Space::oop_iterate(OopIterateClosure* blk) {
533   ObjectToOopClosure blk2(blk);
534   object_iterate(&blk2);
535 }
536 
537 bool Space::obj_is_alive(const HeapWord* p) const {
538   assert (block_is_obj(p), "The address should point to an object");
539   return true;
540 }
541 
542 void ContiguousSpace::oop_iterate(OopIterateClosure* blk) {
543   if (is_empty()) return;
544   HeapWord* obj_addr = bottom();
545   HeapWord* t = top();
546   // Could call objects iterate, but this is easier.
547   while (obj_addr < t) {
548     obj_addr += cast_to_oop(obj_addr)->oop_iterate_size(blk);
549   }
550 }
551 
552 void ContiguousSpace::object_iterate(ObjectClosure* blk) {
553   if (is_empty()) return;
554   object_iterate_from(bottom(), blk);
555 }
556 
557 void ContiguousSpace::object_iterate_from(HeapWord* mark, ObjectClosure* blk) {
558   while (mark < top()) {
559     blk->do_object(cast_to_oop(mark));
560     mark += cast_to_oop(mark)->size();
561   }
562 }
563 
564 // Very general, slow implementation.
565 HeapWord* ContiguousSpace::block_start_const(const void* p) const {
566   assert(MemRegion(bottom(), end()).contains(p),
567          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
568          p2i(p), p2i(bottom()), p2i(end()));
569   if (p >= top()) {
570     return top();
571   } else {
572     HeapWord* last = bottom();
573     HeapWord* cur = last;
574     while (cur <= p) {
575       last = cur;
576       cur += cast_to_oop(cur)->size();
577     }
578     assert(oopDesc::is_oop(cast_to_oop(last)), PTR_FORMAT " should be an object start", p2i(last));
579     return last;
580   }
581 }
582 
583 size_t ContiguousSpace::block_size(const HeapWord* p) const {
584   assert(MemRegion(bottom(), end()).contains(p),
585          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
586          p2i(p), p2i(bottom()), p2i(end()));
587   HeapWord* current_top = top();
588   assert(p <= current_top,
589          "p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT,
590          p2i(p), p2i(current_top));
591   assert(p == current_top || oopDesc::is_oop(cast_to_oop(p)),
592          "p (" PTR_FORMAT ") is not a block start - "
593          "current_top: " PTR_FORMAT ", is_oop: %s",
594          p2i(p), p2i(current_top), BOOL_TO_STR(oopDesc::is_oop(cast_to_oop(p))));
595   if (p < current_top) {
596     return cast_to_oop(p)->size();
597   } else {
598     assert(p == current_top, "just checking");
599     return pointer_delta(end(), (HeapWord*) p);
600   }
601 }
602 
603 // This version requires locking.
604 inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
605   assert(Heap_lock->owned_by_self() ||
606          (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()),
607          "not locked");
608   HeapWord* obj = top();
609   if (pointer_delta(end(), obj) >= size) {
610     HeapWord* new_top = obj + size;
611     set_top(new_top);
612     assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
613     return obj;
614   } else {
615     return nullptr;
616   }
617 }
618 
619 // This version is lock-free.
620 inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) {
621   do {
622     HeapWord* obj = top();
623     if (pointer_delta(end(), obj) >= size) {
624       HeapWord* new_top = obj + size;
625       HeapWord* result = Atomic::cmpxchg(top_addr(), obj, new_top);
626       // result can be one of two:
627       //  the old top value: the exchange succeeded
628       //  otherwise: the new value of the top is returned.
629       if (result == obj) {
630         assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
631         return obj;
632       }
633     } else {
634       return nullptr;
635     }
636   } while (true);
637 }
638 
639 // Requires locking.
640 HeapWord* ContiguousSpace::allocate(size_t size) {
641   return allocate_impl(size);
642 }
643 
644 // Lock-free.
645 HeapWord* ContiguousSpace::par_allocate(size_t size) {
646   return par_allocate_impl(size);
647 }
648 
649 #if INCLUDE_SERIALGC
650 void TenuredSpace::initialize_threshold() {
651   _offsets.initialize_threshold();
652 }
653 
654 void TenuredSpace::alloc_block(HeapWord* start, HeapWord* end) {
655   _offsets.alloc_block(start, end);
656 }
657 
658 TenuredSpace::TenuredSpace(BlockOffsetSharedArray* sharedOffsetArray,
659                            MemRegion mr) :
660   _offsets(sharedOffsetArray, mr),
661   _par_alloc_lock(Mutex::safepoint, "TenuredSpaceParAlloc_lock", true)
662 {
663   _offsets.set_contig_space(this);
664   initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle);
665 }
666 
667 #define OBJ_SAMPLE_INTERVAL 0
668 #define BLOCK_SAMPLE_INTERVAL 100
669 
670 void TenuredSpace::verify() const {
671   HeapWord* p = bottom();
672   HeapWord* prev_p = nullptr;
673   int objs = 0;
674   int blocks = 0;
675 
676   if (VerifyObjectStartArray) {
677     _offsets.verify();
678   }
679 
680   while (p < top()) {
681     size_t size = cast_to_oop(p)->size();
682     // For a sampling of objects in the space, find it using the
683     // block offset table.
684     if (blocks == BLOCK_SAMPLE_INTERVAL) {
685       guarantee(p == block_start_const(p + (size/2)),
686                 "check offset computation");
687       blocks = 0;
688     } else {
689       blocks++;
690     }
691 
692     if (objs == OBJ_SAMPLE_INTERVAL) {
693       oopDesc::verify(cast_to_oop(p));
694       objs = 0;
695     } else {
696       objs++;
697     }
698     prev_p = p;
699     p += size;
700   }
701   guarantee(p == top(), "end of last object must match end of space");
702 }
703 
704 
705 size_t TenuredSpace::allowed_dead_ratio() const {
706   return MarkSweepDeadRatio;
707 }
708 #endif // INCLUDE_SERIALGC