1 /*
2 * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/cardTable.hpp"
27 #include "gc/shared/collectedHeap.hpp"
28 #include "gc/shared/gcLogPrecious.hpp"
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/space.inline.hpp"
31 #include "logging/log.hpp"
32 #include "memory/virtualspace.hpp"
33 #include "runtime/init.hpp"
34 #include "runtime/java.hpp"
35 #include "runtime/os.hpp"
36 #include "services/memTracker.hpp"
37 #include "utilities/align.hpp"
38 #if INCLUDE_PARALLELGC
39 #include "gc/parallel/objectStartArray.hpp"
40 #endif
41
42 uint CardTable::_card_shift = 0;
43 uint CardTable::_card_size = 0;
44 uint CardTable::_card_size_in_words = 0;
45
46 void CardTable::initialize_card_size() {
47 assert(UseG1GC || UseParallelGC || UseSerialGC || UseShenandoahGC,
48 "Initialize card size should only be called by card based collectors.");
49
50 _card_size = GCCardSizeInBytes;
51 _card_shift = log2i_exact(_card_size);
52 _card_size_in_words = _card_size / sizeof(HeapWord);
53
54 // Set blockOffsetTable size based on card table entry size
55 BOTConstants::initialize_bot_size(_card_shift);
56
57 #if INCLUDE_PARALLELGC
58 // Set ObjectStartArray block size based on card table entry size
59 ObjectStartArray::initialize_block_size(_card_shift);
60 #endif
61
62 log_info_p(gc, init)("CardTable entry size: " UINT32_FORMAT, _card_size);
63 }
64
65 size_t CardTable::compute_byte_map_size(size_t num_bytes) {
66 assert(_page_size != 0, "uninitialized, check declaration order");
67 const size_t granularity = os::vm_allocation_granularity();
68 return align_up(num_bytes, MAX2(_page_size, granularity));
69 }
70
71 CardTable::CardTable(MemRegion whole_heap) :
72 _whole_heap(whole_heap),
73 _page_size(os::vm_page_size()),
74 _byte_map_size(0),
75 _byte_map(nullptr),
76 _byte_map_base(nullptr),
77 _guard_region()
78 {
79 assert((uintptr_t(_whole_heap.start()) & (_card_size - 1)) == 0, "heap must start at card boundary");
80 assert((uintptr_t(_whole_heap.end()) & (_card_size - 1)) == 0, "heap must end at card boundary");
81 }
82
83 void CardTable::initialize(void* region0_start, void* region1_start) {
84 size_t num_cards = cards_required(_whole_heap.word_size());
85
86 // each card takes 1 byte; + 1 for the guard card
87 size_t num_bytes = num_cards + 1;
88 _byte_map_size = compute_byte_map_size(num_bytes);
89
90 HeapWord* low_bound = _whole_heap.start();
91 HeapWord* high_bound = _whole_heap.end();
92
93 const size_t rs_align = _page_size == os::vm_page_size() ? 0 :
94 MAX2(_page_size, os::vm_allocation_granularity());
95 ReservedSpace heap_rs(_byte_map_size, rs_align, _page_size);
96
97 MemTracker::record_virtual_memory_type((address)heap_rs.base(), mtGC);
98
99 os::trace_page_sizes("Card Table", num_bytes, num_bytes,
100 _page_size, heap_rs.base(), heap_rs.size());
101 if (!heap_rs.is_reserved()) {
102 vm_exit_during_initialization("Could not reserve enough space for the "
103 "card marking array");
104 }
105
106 // The assembler store_check code will do an unsigned shift of the oop,
107 // then add it to _byte_map_base, i.e.
108 //
109 // _byte_map = _byte_map_base + (uintptr_t(low_bound) >> card_shift)
110 _byte_map = (CardValue*) heap_rs.base();
111 _byte_map_base = _byte_map - (uintptr_t(low_bound) >> _card_shift);
112 assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map");
113 assert(byte_for(high_bound-1) <= &_byte_map[last_valid_index()], "Checking end of map");
114
115 CardValue* guard_card = &_byte_map[num_cards];
116 assert(is_aligned(guard_card, _page_size), "must be on its own OS page");
117 _guard_region = MemRegion((HeapWord*)guard_card, _page_size);
118
119 initialize_covered_region(region0_start, region1_start);
120
121 log_trace(gc, barrier)("CardTable::CardTable: ");
122 log_trace(gc, barrier)(" &_byte_map[0]: " PTR_FORMAT " &_byte_map[last_valid_index()]: " PTR_FORMAT,
123 p2i(&_byte_map[0]), p2i(&_byte_map[last_valid_index()]));
124 log_trace(gc, barrier)(" _byte_map_base: " PTR_FORMAT, p2i(_byte_map_base));
125 }
126
127 MemRegion CardTable::committed_for(const MemRegion mr) const {
128 HeapWord* addr_l = (HeapWord*)align_down(byte_for(mr.start()), _page_size);
129 HeapWord* addr_r = mr.is_empty()
130 ? addr_l
131 : (HeapWord*)align_up(byte_after(mr.last()), _page_size);
132
133 if (mr.start() == _covered[0].start()) {
134 // In case the card for gen-boundary is not page-size aligned, the crossing page belongs to _covered[1].
135 addr_r = MIN2(addr_r, (HeapWord*)align_down(byte_for(_covered[1].start()), _page_size));
136 }
137
138 return MemRegion(addr_l, addr_r);
139 }
140
141 void CardTable::initialize_covered_region(void* region0_start, void* region1_start) {
142 assert(_whole_heap.start() == region0_start, "precondition");
143 assert(region0_start < region1_start, "precondition");
144
145 assert(_covered[0].start() == nullptr, "precondition");
146 assert(_covered[1].start() == nullptr, "precondition");
147
148 _covered[0] = MemRegion((HeapWord*)region0_start, (size_t)0);
149 _covered[1] = MemRegion((HeapWord*)region1_start, (size_t)0);
150 }
151
152 void CardTable::resize_covered_region(MemRegion new_region) {
153 assert(UseSerialGC || UseParallelGC, "only these two collectors");
154 assert(_whole_heap.contains(new_region),
155 "attempt to cover area not in reserved area");
156 assert(_covered[0].start() != nullptr, "precondition");
157 assert(_covered[1].start() != nullptr, "precondition");
158
159 int idx = new_region.start() == _whole_heap.start() ? 0 : 1;
160
161 // We don't allow changes to the start of a region, only the end.
162 assert(_covered[idx].start() == new_region.start(), "inv");
163
164 MemRegion old_committed = committed_for(_covered[idx]);
165
166 _covered[idx] = new_region;
167
168 MemRegion new_committed = committed_for(new_region);
169
170 if (new_committed.word_size() == old_committed.word_size()) {
171 return;
172 }
173
174 if (new_committed.word_size() > old_committed.word_size()) {
175 // Expand.
176 MemRegion delta = MemRegion(old_committed.end(),
177 new_committed.word_size() - old_committed.word_size());
178
179 os::commit_memory_or_exit((char*)delta.start(),
180 delta.byte_size(),
181 _page_size,
182 !ExecMem,
183 "card table expansion");
184
185 memset(delta.start(), clean_card, delta.byte_size());
186 } else {
187 // Shrink.
188 MemRegion delta = MemRegion(new_committed.end(),
189 old_committed.word_size() - new_committed.word_size());
190 bool res = os::uncommit_memory((char*)delta.start(),
191 delta.byte_size());
192 assert(res, "uncommit should succeed");
193 }
194
195 log_trace(gc, barrier)("CardTable::resize_covered_region: ");
196 log_trace(gc, barrier)(" _covered[%d].start(): " PTR_FORMAT " _covered[%d].last(): " PTR_FORMAT,
197 idx, p2i(_covered[idx].start()), idx, p2i(_covered[idx].last()));
198 log_trace(gc, barrier)(" committed_start: " PTR_FORMAT " committed_last: " PTR_FORMAT,
199 p2i(new_committed.start()), p2i(new_committed.last()));
200 log_trace(gc, barrier)(" byte_for(start): " PTR_FORMAT " byte_for(last): " PTR_FORMAT,
201 p2i(byte_for(_covered[idx].start())), p2i(byte_for(_covered[idx].last())));
202 log_trace(gc, barrier)(" addr_for(start): " PTR_FORMAT " addr_for(last): " PTR_FORMAT,
203 p2i(addr_for((CardValue*) new_committed.start())), p2i(addr_for((CardValue*) new_committed.last())));
204
205 #ifdef ASSERT
206 // Touch the last card of the covered region to show that it
207 // is committed (or SEGV).
208 if (is_init_completed()) {
209 (void) (*(volatile CardValue*)byte_for(_covered[idx].last()));
210 }
211 #endif
212 }
213
214 // Note that these versions are precise! The scanning code has to handle the
215 // fact that the write barrier may be either precise or imprecise.
216 void CardTable::dirty_MemRegion(MemRegion mr) {
217 assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
218 assert(align_up (mr.end(), HeapWordSize) == mr.end(), "Unaligned end" );
219 CardValue* cur = byte_for(mr.start());
220 CardValue* last = byte_after(mr.last());
221 while (cur < last) {
222 *cur = dirty_card;
223 cur++;
224 }
225 }
226
227 void CardTable::clear_MemRegion(MemRegion mr) {
228 // Be conservative: only clean cards entirely contained within the
229 // region.
230 CardValue* cur;
231 if (mr.start() == _whole_heap.start()) {
232 cur = byte_for(mr.start());
233 } else {
234 assert(mr.start() > _whole_heap.start(), "mr is not covered.");
235 cur = byte_after(mr.start() - 1);
236 }
237 CardValue* last = byte_after(mr.last());
238 memset(cur, clean_card, pointer_delta(last, cur, sizeof(CardValue)));
239 }
240
241 uintx CardTable::ct_max_alignment_constraint() {
242 // Calculate maximum alignment using GCCardSizeInBytes as card_size hasn't been set yet
243 return GCCardSizeInBytes * os::vm_page_size();
244 }
245
246 void CardTable::invalidate(MemRegion mr) {
247 assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
248 assert(align_up (mr.end(), HeapWordSize) == mr.end(), "Unaligned end" );
249 for (int i = 0; i < max_covered_regions; i++) {
250 MemRegion mri = mr.intersection(_covered[i]);
251 if (!mri.is_empty()) dirty_MemRegion(mri);
252 }
253 }
254
255 #ifndef PRODUCT
256 void CardTable::verify_region(MemRegion mr, CardValue val, bool val_equals) {
257 CardValue* start = byte_for(mr.start());
258 CardValue* end = byte_for(mr.last());
259 bool failures = false;
260 for (CardValue* curr = start; curr <= end; ++curr) {
261 CardValue curr_val = *curr;
262 bool failed = (val_equals) ? (curr_val != val) : (curr_val == val);
263 if (failed) {
264 if (!failures) {
265 log_error(gc, verify)("== CT verification failed: [" PTR_FORMAT "," PTR_FORMAT "]", p2i(start), p2i(end));
266 log_error(gc, verify)("== %sexpecting value: %d", (val_equals) ? "" : "not ", val);
267 failures = true;
268 }
269 log_error(gc, verify)("== card " PTR_FORMAT " [" PTR_FORMAT "," PTR_FORMAT "], val: %d",
270 p2i(curr), p2i(addr_for(curr)),
271 p2i((HeapWord*) (((size_t) addr_for(curr)) + _card_size)),
272 (int) curr_val);
273 }
274 }
275 guarantee(!failures, "there should not have been any failures");
276 }
277
278 void CardTable::verify_not_dirty_region(MemRegion mr) {
279 verify_region(mr, dirty_card, false /* val_equals */);
280 }
281
282 void CardTable::verify_dirty_region(MemRegion mr) {
283 verify_region(mr, dirty_card, true /* val_equals */);
284 }
285 #endif
286
287 void CardTable::print_on(outputStream* st) const {
288 st->print_cr("Card table byte_map: [" PTR_FORMAT "," PTR_FORMAT "] _byte_map_base: " PTR_FORMAT,
289 p2i(_byte_map), p2i(_byte_map + _byte_map_size), p2i(_byte_map_base));
290 }