1 /*
  2  * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
  3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP
 27 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP
 28 
 29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 30 
 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 32 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
 33 #include "runtime/atomic.hpp"
 34 
 35 HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest &req, size_t alignment_in_bytes) {
 36   shenandoah_assert_heaplocked_or_safepoint();
 37   assert(req.is_lab_alloc(), "allocate_aligned() only applies to LAB allocations");
 38   assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);
 39   assert(is_old(), "aligned allocations are only taken from OLD regions to support PLABs");
 40   assert(is_aligned(alignment_in_bytes, HeapWordSize), "Expect hea word alignment");
 41 
 42   HeapWord* orig_top = top();
 43   size_t alignment_in_words = alignment_in_bytes / HeapWordSize;
 44 
 45   // unalignment_words is the amount by which current top() exceeds the desired alignment point.  We subtract this amount
 46   // from alignment_in_words to determine padding required to next alignment point.
 47 
 48   HeapWord* aligned_obj = (HeapWord*) align_up(orig_top, alignment_in_bytes);
 49   size_t pad_words = aligned_obj - orig_top;
 50   if ((pad_words > 0) && (pad_words < ShenandoahHeap::min_fill_size())) {
 51     pad_words += alignment_in_words;
 52     aligned_obj += alignment_in_words;
 53   }
 54 
 55   if (pointer_delta(end(), aligned_obj) < size) {
 56     // Shrink size to fit within available space and align it
 57     size = pointer_delta(end(), aligned_obj);
 58     size = align_down(size, alignment_in_words);
 59   }
 60 
 61   // Both originally requested size and adjusted size must be properly aligned
 62   assert (is_aligned(size, alignment_in_words), "Size must be multiple of alignment constraint");
 63   if (size >= req.min_size()) {
 64     // Even if req.min_size() may not be a multiple of card size, we know that size is.
 65     if (pad_words > 0) {
 66       assert(pad_words >= ShenandoahHeap::min_fill_size(), "pad_words expanded above to meet size constraint");
 67       ShenandoahHeap::fill_with_object(orig_top, pad_words);
 68       ShenandoahHeap::heap()->card_scan()->register_object(orig_top);
 69     }
 70 
 71     make_regular_allocation(req.affiliation());
 72     adjust_alloc_metadata(req.type(), size);
 73 
 74     HeapWord* new_top = aligned_obj + size;
 75     assert(new_top <= end(), "PLAB cannot span end of heap region");
 76     set_top(new_top);
 77     req.set_actual_size(size);
 78     req.set_waste(pad_words);
 79     assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top));
 80     assert(is_aligned(aligned_obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(aligned_obj));
 81     return aligned_obj;
 82   } else {
 83     // The aligned size that fits in this region is smaller than min_size, so don't align top and don't allocate.  Return failure.
 84     return nullptr;
 85   }
 86 }
 87 
 88 HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest req) {
 89   shenandoah_assert_heaplocked_or_safepoint();
 90   assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);
 91 
 92   HeapWord* obj = top();
 93   if (pointer_delta(end(), obj) >= size) {
 94     make_regular_allocation(req.affiliation());
 95     adjust_alloc_metadata(req.type(), size);
 96 
 97     HeapWord* new_top = obj + size;
 98     set_top(new_top);
 99 
100     assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top));
101     assert(is_object_aligned(obj),     "obj is not aligned: "       PTR_FORMAT, p2i(obj));
102 
103     return obj;
104   } else {
105     return nullptr;
106   }
107 }
108 
109 inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest::Type type, size_t size) {
110   switch (type) {
111     case ShenandoahAllocRequest::_alloc_shared:
112     case ShenandoahAllocRequest::_alloc_shared_gc:
113       // Counted implicitly by tlab/gclab allocs
114       break;
115     case ShenandoahAllocRequest::_alloc_tlab:
116       _tlab_allocs += size;
117       break;
118     case ShenandoahAllocRequest::_alloc_gclab:
119       _gclab_allocs += size;
120       break;
121     case ShenandoahAllocRequest::_alloc_plab:
122       _plab_allocs += size;
123       break;
124     default:
125       ShouldNotReachHere();
126   }
127 }
128 
129 inline void ShenandoahHeapRegion::increase_live_data_alloc_words(size_t s) {
130   internal_increase_live_data(s);
131 }
132 
133 inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) {
134   internal_increase_live_data(s);
135   if (ShenandoahPacing) {
136     ShenandoahHeap::heap()->pacer()->report_mark(s);
137   }
138 }
139 
140 inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) {
141   size_t new_live_data = Atomic::add(&_live_data, s, memory_order_relaxed);
142 }
143 
144 inline void ShenandoahHeapRegion::clear_live_data() {
145   Atomic::store(&_live_data, (size_t)0);
146 }
147 
148 inline size_t ShenandoahHeapRegion::get_live_data_words() const {
149   return Atomic::load(&_live_data);
150 }
151 
152 inline size_t ShenandoahHeapRegion::get_live_data_bytes() const {
153   return get_live_data_words() * HeapWordSize;
154 }
155 
156 inline bool ShenandoahHeapRegion::has_live() const {
157   return get_live_data_words() != 0;
158 }
159 
160 inline size_t ShenandoahHeapRegion::garbage() const {
161   assert(used() >= get_live_data_bytes(),
162          "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
163          get_live_data_bytes(), used());
164 
165   size_t result = used() - get_live_data_bytes();
166   return result;
167 }
168 
169 inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const {
170   assert(get_top_before_promote() != nullptr, "top before promote should not equal null");
171   size_t used_before_promote = byte_size(bottom(), get_top_before_promote());
172   assert(used_before_promote >= get_live_data_bytes(),
173          "Live Data must be a subset of used before promotion live: " SIZE_FORMAT " used: " SIZE_FORMAT,
174          get_live_data_bytes(), used_before_promote);
175   size_t result = used_before_promote - get_live_data_bytes();
176   return result;
177 
178 }
179 
180 inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const {
181   HeapWord* watermark = Atomic::load_acquire(&_update_watermark);
182   assert(bottom() <= watermark && watermark <= top(), "within bounds");
183   return watermark;
184 }
185 
186 inline void ShenandoahHeapRegion::set_update_watermark(HeapWord* w) {
187   assert(bottom() <= w && w <= top(), "within bounds");
188   Atomic::release_store(&_update_watermark, w);
189 }
190 
191 // Fast version that avoids synchronization, only to be used at safepoints.
192 inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) {
193   assert(bottom() <= w && w <= top(), "within bounds");
194   assert(SafepointSynchronize::is_at_safepoint(), "Should be at Shenandoah safepoint");
195   _update_watermark = w;
196 }
197 
198 inline ShenandoahAffiliation ShenandoahHeapRegion::affiliation() const {
199   return ShenandoahHeap::heap()->region_affiliation(this);
200 }
201 
202 inline const char* ShenandoahHeapRegion::affiliation_name() const {
203   return shenandoah_affiliation_name(affiliation());
204 }
205 
206 inline bool ShenandoahHeapRegion::is_young() const {
207   return affiliation() == YOUNG_GENERATION;
208 }
209 
210 inline bool ShenandoahHeapRegion::is_old() const {
211   return affiliation() == OLD_GENERATION;
212 }
213 
214 inline bool ShenandoahHeapRegion::is_affiliated() const {
215   return affiliation() != FREE;
216 }
217 
218 inline void ShenandoahHeapRegion::save_top_before_promote() {
219   _top_before_promoted = _top;
220 }
221 
222 inline void ShenandoahHeapRegion::restore_top_before_promote() {
223   _top = _top_before_promoted;
224   _top_before_promoted = nullptr;
225  }
226 
227 
228 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP