1 /* 2 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP 27 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP 28 29 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 30 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahPacer.inline.hpp" 33 #include "runtime/atomic.hpp" 34 35 HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest &req, size_t alignment_in_bytes) { 36 shenandoah_assert_heaplocked_or_safepoint(); 37 assert(req.is_lab_alloc(), "allocate_aligned() only applies to LAB allocations"); 38 assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); 39 assert(is_old(), "aligned allocations are only taken from OLD regions to support PLABs"); 40 assert(is_aligned(alignment_in_bytes, HeapWordSize), "Expect hea word alignment"); 41 42 HeapWord* orig_top = top(); 43 size_t alignment_in_words = alignment_in_bytes / HeapWordSize; 44 45 // unalignment_words is the amount by which current top() exceeds the desired alignment point. We subtract this amount 46 // from alignment_in_words to determine padding required to next alignment point. 47 48 HeapWord* aligned_obj = (HeapWord*) align_up(orig_top, alignment_in_bytes); 49 size_t pad_words = aligned_obj - orig_top; 50 if ((pad_words > 0) && (pad_words < ShenandoahHeap::min_fill_size())) { 51 pad_words += alignment_in_words; 52 aligned_obj += alignment_in_words; 53 } 54 55 if (pointer_delta(end(), aligned_obj) < size) { 56 // Shrink size to fit within available space and align it 57 size = pointer_delta(end(), aligned_obj); 58 size = align_down(size, alignment_in_words); 59 } 60 61 // Both originally requested size and adjusted size must be properly aligned 62 assert (is_aligned(size, alignment_in_words), "Size must be multiple of alignment constraint"); 63 if (size >= req.min_size()) { 64 // Even if req.min_size() may not be a multiple of card size, we know that size is. 65 if (pad_words > 0) { 66 assert(pad_words >= ShenandoahHeap::min_fill_size(), "pad_words expanded above to meet size constraint"); 67 ShenandoahHeap::fill_with_object(orig_top, pad_words); 68 ShenandoahHeap::heap()->card_scan()->register_object(orig_top); 69 } 70 71 make_regular_allocation(req.affiliation()); 72 adjust_alloc_metadata(req.type(), size); 73 74 HeapWord* new_top = aligned_obj + size; 75 assert(new_top <= end(), "PLAB cannot span end of heap region"); 76 set_top(new_top); 77 req.set_actual_size(size); 78 req.set_waste(pad_words); 79 assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); 80 assert(is_aligned(aligned_obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(aligned_obj)); 81 return aligned_obj; 82 } else { 83 // The aligned size that fits in this region is smaller than min_size, so don't align top and don't allocate. Return failure. 84 return nullptr; 85 } 86 } 87 88 HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest req) { 89 shenandoah_assert_heaplocked_or_safepoint(); 90 assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); 91 92 HeapWord* obj = top(); 93 if (pointer_delta(end(), obj) >= size) { 94 make_regular_allocation(req.affiliation()); 95 adjust_alloc_metadata(req.type(), size); 96 97 HeapWord* new_top = obj + size; 98 set_top(new_top); 99 100 assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); 101 assert(is_object_aligned(obj), "obj is not aligned: " PTR_FORMAT, p2i(obj)); 102 103 return obj; 104 } else { 105 return nullptr; 106 } 107 } 108 109 inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest::Type type, size_t size) { 110 switch (type) { 111 case ShenandoahAllocRequest::_alloc_shared: 112 case ShenandoahAllocRequest::_alloc_shared_gc: 113 // Counted implicitly by tlab/gclab allocs 114 break; 115 case ShenandoahAllocRequest::_alloc_tlab: 116 _tlab_allocs += size; 117 break; 118 case ShenandoahAllocRequest::_alloc_gclab: 119 _gclab_allocs += size; 120 break; 121 case ShenandoahAllocRequest::_alloc_plab: 122 _plab_allocs += size; 123 break; 124 default: 125 ShouldNotReachHere(); 126 } 127 } 128 129 inline void ShenandoahHeapRegion::increase_live_data_alloc_words(size_t s) { 130 internal_increase_live_data(s); 131 } 132 133 inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) { 134 internal_increase_live_data(s); 135 if (ShenandoahPacing) { 136 ShenandoahHeap::heap()->pacer()->report_mark(s); 137 } 138 } 139 140 inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { 141 size_t new_live_data = Atomic::add(&_live_data, s, memory_order_relaxed); 142 } 143 144 inline void ShenandoahHeapRegion::clear_live_data() { 145 Atomic::store(&_live_data, (size_t)0); 146 } 147 148 inline size_t ShenandoahHeapRegion::get_live_data_words() const { 149 return Atomic::load(&_live_data); 150 } 151 152 inline size_t ShenandoahHeapRegion::get_live_data_bytes() const { 153 return get_live_data_words() * HeapWordSize; 154 } 155 156 inline bool ShenandoahHeapRegion::has_live() const { 157 return get_live_data_words() != 0; 158 } 159 160 inline size_t ShenandoahHeapRegion::garbage() const { 161 assert(used() >= get_live_data_bytes(), 162 "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT, 163 get_live_data_bytes(), used()); 164 165 size_t result = used() - get_live_data_bytes(); 166 return result; 167 } 168 169 inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const { 170 assert(get_top_before_promote() != nullptr, "top before promote should not equal null"); 171 size_t used_before_promote = byte_size(bottom(), get_top_before_promote()); 172 assert(used_before_promote >= get_live_data_bytes(), 173 "Live Data must be a subset of used before promotion live: " SIZE_FORMAT " used: " SIZE_FORMAT, 174 get_live_data_bytes(), used_before_promote); 175 size_t result = used_before_promote - get_live_data_bytes(); 176 return result; 177 178 } 179 180 inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const { 181 HeapWord* watermark = Atomic::load_acquire(&_update_watermark); 182 assert(bottom() <= watermark && watermark <= top(), "within bounds"); 183 return watermark; 184 } 185 186 inline void ShenandoahHeapRegion::set_update_watermark(HeapWord* w) { 187 assert(bottom() <= w && w <= top(), "within bounds"); 188 Atomic::release_store(&_update_watermark, w); 189 } 190 191 // Fast version that avoids synchronization, only to be used at safepoints. 192 inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) { 193 assert(bottom() <= w && w <= top(), "within bounds"); 194 assert(SafepointSynchronize::is_at_safepoint(), "Should be at Shenandoah safepoint"); 195 _update_watermark = w; 196 } 197 198 inline ShenandoahAffiliation ShenandoahHeapRegion::affiliation() const { 199 return ShenandoahHeap::heap()->region_affiliation(this); 200 } 201 202 inline const char* ShenandoahHeapRegion::affiliation_name() const { 203 return shenandoah_affiliation_name(affiliation()); 204 } 205 206 inline bool ShenandoahHeapRegion::is_young() const { 207 return affiliation() == YOUNG_GENERATION; 208 } 209 210 inline bool ShenandoahHeapRegion::is_old() const { 211 return affiliation() == OLD_GENERATION; 212 } 213 214 inline bool ShenandoahHeapRegion::is_affiliated() const { 215 return affiliation() != FREE; 216 } 217 218 inline void ShenandoahHeapRegion::save_top_before_promote() { 219 _top_before_promoted = _top; 220 } 221 222 inline void ShenandoahHeapRegion::restore_top_before_promote() { 223 _top = _top_before_promoted; 224 _top_before_promoted = nullptr; 225 } 226 227 228 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP