1 /*
  2  * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/cardTable.hpp"
 27 #include "gc/shared/collectedHeap.hpp"
 28 #include "gc/shared/gcLogPrecious.hpp"
 29 #include "gc/shared/gc_globals.hpp"
 30 #include "gc/shared/space.inline.hpp"
 31 #include "logging/log.hpp"
 32 #include "memory/virtualspace.hpp"
 33 #include "runtime/init.hpp"
 34 #include "runtime/java.hpp"
 35 #include "runtime/os.hpp"
 36 #include "services/memTracker.hpp"
 37 #include "utilities/align.hpp"
 38 #if INCLUDE_PARALLELGC
 39 #include "gc/parallel/objectStartArray.hpp"
 40 #endif
 41 
 42 uint CardTable::_card_shift = 0;
 43 uint CardTable::_card_size = 0;
 44 uint CardTable::_card_size_in_words = 0;
 45 
 46 void CardTable::initialize_card_size() {
 47   assert(UseG1GC || UseParallelGC || UseSerialGC || UseShenandoahGC,
 48          "Initialize card size should only be called by card based collectors.");
 49 
 50   _card_size = GCCardSizeInBytes;
 51   _card_shift = log2i_exact(_card_size);
 52   _card_size_in_words = _card_size / sizeof(HeapWord);
 53 
 54   // Set blockOffsetTable size based on card table entry size
 55   BOTConstants::initialize_bot_size(_card_shift);
 56 
 57 #if INCLUDE_PARALLELGC
 58   // Set ObjectStartArray block size based on card table entry size
 59   ObjectStartArray::initialize_block_size(_card_shift);
 60 #endif
 61 
 62   log_info_p(gc, init)("CardTable entry size: " UINT32_FORMAT,  _card_size);
 63 }
 64 
 65 size_t CardTable::compute_byte_map_size(size_t num_bytes) {
 66   assert(_page_size != 0, "uninitialized, check declaration order");
 67   const size_t granularity = os::vm_allocation_granularity();
 68   return align_up(num_bytes, MAX2(_page_size, granularity));
 69 }
 70 
 71 CardTable::CardTable(MemRegion whole_heap) :
 72   _whole_heap(whole_heap),
 73   _page_size(os::vm_page_size()),
 74   _byte_map_size(0),
 75   _byte_map(nullptr),
 76   _byte_map_base(nullptr),
 77   _guard_region()
 78 {
 79   assert((uintptr_t(_whole_heap.start())  & (_card_size - 1))  == 0, "heap must start at card boundary");
 80   assert((uintptr_t(_whole_heap.end()) & (_card_size - 1))  == 0, "heap must end at card boundary");
 81 }
 82 
 83 void CardTable::initialize(void* region0_start, void* region1_start) {
 84   size_t num_cards = cards_required(_whole_heap.word_size());
 85 
 86   // each card takes 1 byte; + 1 for the guard card
 87   size_t num_bytes = num_cards + 1;
 88   _byte_map_size = compute_byte_map_size(num_bytes);
 89 
 90   HeapWord* low_bound  = _whole_heap.start();
 91   HeapWord* high_bound = _whole_heap.end();
 92 
 93   const size_t rs_align = _page_size == os::vm_page_size() ? 0 :
 94     MAX2(_page_size, os::vm_allocation_granularity());
 95   ReservedSpace heap_rs(_byte_map_size, rs_align, _page_size);
 96 
 97   MemTracker::record_virtual_memory_type((address)heap_rs.base(), mtGC);
 98 
 99   os::trace_page_sizes("Card Table", num_bytes, num_bytes,
100                        _page_size, heap_rs.base(), heap_rs.size());
101   if (!heap_rs.is_reserved()) {
102     vm_exit_during_initialization("Could not reserve enough space for the "
103                                   "card marking array");
104   }
105 
106   // The assembler store_check code will do an unsigned shift of the oop,
107   // then add it to _byte_map_base, i.e.
108   //
109   //   _byte_map = _byte_map_base + (uintptr_t(low_bound) >> card_shift)
110   _byte_map = (CardValue*) heap_rs.base();
111   _byte_map_base = _byte_map - (uintptr_t(low_bound) >> _card_shift);
112   assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map");
113   assert(byte_for(high_bound-1) <= &_byte_map[last_valid_index()], "Checking end of map");
114 
115   CardValue* guard_card = &_byte_map[num_cards];
116   assert(is_aligned(guard_card, _page_size), "must be on its own OS page");
117   _guard_region = MemRegion((HeapWord*)guard_card, _page_size);
118 
119   initialize_covered_region(region0_start, region1_start);
120 
121   log_trace(gc, barrier)("CardTable::CardTable: ");
122   log_trace(gc, barrier)("    &_byte_map[0]: " PTR_FORMAT "  &_byte_map[last_valid_index()]: " PTR_FORMAT,
123                          p2i(&_byte_map[0]), p2i(&_byte_map[last_valid_index()]));
124   log_trace(gc, barrier)("    _byte_map_base: " PTR_FORMAT, p2i(_byte_map_base));
125 }
126 
127 MemRegion CardTable::committed_for(const MemRegion mr) const {
128   HeapWord* addr_l = (HeapWord*)align_down(byte_for(mr.start()), _page_size);
129   HeapWord* addr_r = mr.is_empty()
130                    ? addr_l
131                    : (HeapWord*)align_up(byte_after(mr.last()), _page_size);
132 
133   if (mr.start() == _covered[0].start()) {
134     // In case the card for gen-boundary is not page-size aligned, the crossing page belongs to _covered[1].
135     addr_r = MIN2(addr_r, (HeapWord*)align_down(byte_for(_covered[1].start()), _page_size));
136   }
137 
138   return MemRegion(addr_l, addr_r);
139 }
140 
141 void CardTable::initialize_covered_region(void* region0_start, void* region1_start) {
142   assert(_whole_heap.start() == region0_start, "precondition");
143   assert(region0_start < region1_start, "precondition");
144 
145   assert(_covered[0].start() == nullptr, "precondition");
146   assert(_covered[1].start() == nullptr, "precondition");
147 
148   _covered[0] = MemRegion((HeapWord*)region0_start, (size_t)0);
149   _covered[1] = MemRegion((HeapWord*)region1_start, (size_t)0);
150 }
151 
152 void CardTable::resize_covered_region(MemRegion new_region) {
153   assert(UseSerialGC || UseParallelGC, "only these two collectors");
154   assert(_whole_heap.contains(new_region),
155          "attempt to cover area not in reserved area");
156   assert(_covered[0].start() != nullptr, "precondition");
157   assert(_covered[1].start() != nullptr, "precondition");
158 
159   int idx = new_region.start() == _whole_heap.start() ? 0 : 1;
160 
161   // We don't allow changes to the start of a region, only the end.
162   assert(_covered[idx].start() == new_region.start(), "inv");
163 
164   MemRegion old_committed = committed_for(_covered[idx]);
165 
166   _covered[idx] = new_region;
167 
168   MemRegion new_committed = committed_for(new_region);
169 
170   if (new_committed.word_size() == old_committed.word_size()) {
171     return;
172   }
173 
174   if (new_committed.word_size() > old_committed.word_size()) {
175     // Expand.
176     MemRegion delta = MemRegion(old_committed.end(),
177                                 new_committed.word_size() - old_committed.word_size());
178 
179     os::commit_memory_or_exit((char*)delta.start(),
180                               delta.byte_size(),
181                               _page_size,
182                               !ExecMem,
183                               "card table expansion");
184 
185     memset(delta.start(), clean_card, delta.byte_size());
186   } else {
187     // Shrink.
188     MemRegion delta = MemRegion(new_committed.end(),
189                                 old_committed.word_size() - new_committed.word_size());
190     bool res = os::uncommit_memory((char*)delta.start(),
191                                    delta.byte_size());
192     assert(res, "uncommit should succeed");
193   }
194 
195   log_trace(gc, barrier)("CardTable::resize_covered_region: ");
196   log_trace(gc, barrier)("    _covered[%d].start(): " PTR_FORMAT " _covered[%d].last(): " PTR_FORMAT,
197                          idx, p2i(_covered[idx].start()), idx, p2i(_covered[idx].last()));
198   log_trace(gc, barrier)("    committed_start: " PTR_FORMAT "  committed_last: " PTR_FORMAT,
199                          p2i(new_committed.start()), p2i(new_committed.last()));
200   log_trace(gc, barrier)("    byte_for(start): " PTR_FORMAT "  byte_for(last): " PTR_FORMAT,
201                          p2i(byte_for(_covered[idx].start())),  p2i(byte_for(_covered[idx].last())));
202   log_trace(gc, barrier)("    addr_for(start): " PTR_FORMAT "  addr_for(last): " PTR_FORMAT,
203                          p2i(addr_for((CardValue*) new_committed.start())),  p2i(addr_for((CardValue*) new_committed.last())));
204 
205 #ifdef ASSERT
206   // Touch the last card of the covered region to show that it
207   // is committed (or SEGV).
208   if (is_init_completed()) {
209     (void) (*(volatile CardValue*)byte_for(_covered[idx].last()));
210   }
211 #endif
212 }
213 
214 // Note that these versions are precise!  The scanning code has to handle the
215 // fact that the write barrier may be either precise or imprecise.
216 void CardTable::dirty_MemRegion(MemRegion mr) {
217   assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
218   assert(align_up  (mr.end(),   HeapWordSize) == mr.end(),   "Unaligned end"  );
219   CardValue* cur  = byte_for(mr.start());
220   CardValue* last = byte_after(mr.last());
221   while (cur < last) {
222     *cur = dirty_card;
223     cur++;
224   }
225 }
226 
227 void CardTable::clear_MemRegion(MemRegion mr) {
228   // Be conservative: only clean cards entirely contained within the
229   // region.
230   CardValue* cur;
231   if (mr.start() == _whole_heap.start()) {
232     cur = byte_for(mr.start());
233   } else {
234     assert(mr.start() > _whole_heap.start(), "mr is not covered.");
235     cur = byte_after(mr.start() - 1);
236   }
237   CardValue* last = byte_after(mr.last());
238   memset(cur, clean_card, pointer_delta(last, cur, sizeof(CardValue)));
239 }
240 
241 uintx CardTable::ct_max_alignment_constraint() {
242   // Calculate maximum alignment using GCCardSizeInBytes as card_size hasn't been set yet
243   return GCCardSizeInBytes * os::vm_page_size();
244 }
245 
246 void CardTable::invalidate(MemRegion mr) {
247   assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
248   assert(align_up  (mr.end(),   HeapWordSize) == mr.end(),   "Unaligned end"  );
249   for (int i = 0; i < max_covered_regions; i++) {
250     MemRegion mri = mr.intersection(_covered[i]);
251     if (!mri.is_empty()) dirty_MemRegion(mri);
252   }
253 }
254 
255 #ifndef PRODUCT
256 void CardTable::verify_region(MemRegion mr, CardValue val, bool val_equals) {
257   CardValue* start    = byte_for(mr.start());
258   CardValue* end      = byte_for(mr.last());
259   bool failures = false;
260   for (CardValue* curr = start; curr <= end; ++curr) {
261     CardValue curr_val = *curr;
262     bool failed = (val_equals) ? (curr_val != val) : (curr_val == val);
263     if (failed) {
264       if (!failures) {
265         log_error(gc, verify)("== CT verification failed: [" PTR_FORMAT "," PTR_FORMAT "]", p2i(start), p2i(end));
266         log_error(gc, verify)("==   %sexpecting value: %d", (val_equals) ? "" : "not ", val);
267         failures = true;
268       }
269       log_error(gc, verify)("==   card " PTR_FORMAT " [" PTR_FORMAT "," PTR_FORMAT "], val: %d",
270                             p2i(curr), p2i(addr_for(curr)),
271                             p2i((HeapWord*) (((size_t) addr_for(curr)) + _card_size)),
272                             (int) curr_val);
273     }
274   }
275   guarantee(!failures, "there should not have been any failures");
276 }
277 
278 void CardTable::verify_not_dirty_region(MemRegion mr) {
279   verify_region(mr, dirty_card, false /* val_equals */);
280 }
281 
282 void CardTable::verify_dirty_region(MemRegion mr) {
283   verify_region(mr, dirty_card, true /* val_equals */);
284 }
285 #endif
286 
287 void CardTable::print_on(outputStream* st) const {
288   st->print_cr("Card table byte_map: [" PTR_FORMAT "," PTR_FORMAT "] _byte_map_base: " PTR_FORMAT,
289                p2i(_byte_map), p2i(_byte_map + _byte_map_size), p2i(_byte_map_base));
290 }