1 /*
2 * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/cardTable.hpp"
27 #include "gc/shared/collectedHeap.hpp"
28 #include "gc/shared/gcLogPrecious.hpp"
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/space.inline.hpp"
31 #include "logging/log.hpp"
32 #include "memory/virtualspace.hpp"
33 #include "nmt/memTracker.hpp"
34 #include "runtime/init.hpp"
35 #include "runtime/java.hpp"
36 #include "runtime/os.hpp"
37 #include "utilities/align.hpp"
38 #if INCLUDE_PARALLELGC
39 #include "gc/parallel/objectStartArray.hpp"
40 #endif
41
42 uint CardTable::_card_shift = 0;
43 uint CardTable::_card_size = 0;
44 uint CardTable::_card_size_in_words = 0;
45
46 void CardTable::initialize_card_size() {
47 assert(UseG1GC || UseParallelGC || UseSerialGC || UseShenandoahGC,
48 "Initialize card size should only be called by card based collectors.");
49
50 _card_size = GCCardSizeInBytes;
51 _card_shift = log2i_exact(_card_size);
52 _card_size_in_words = _card_size / sizeof(HeapWord);
53
54 log_info_p(gc, init)("CardTable entry size: " UINT32_FORMAT, _card_size);
55 }
56
57 size_t CardTable::compute_byte_map_size(size_t num_bytes) {
58 assert(_page_size != 0, "uninitialized, check declaration order");
59 const size_t granularity = os::vm_allocation_granularity();
60 return align_up(num_bytes, MAX2(_page_size, granularity));
61 }
62
63 CardTable::CardTable(MemRegion whole_heap) :
64 _whole_heap(whole_heap),
65 _page_size(os::vm_page_size()),
66 _byte_map_size(0),
67 _byte_map(nullptr),
68 _byte_map_base(nullptr)
69 {
70 assert((uintptr_t(_whole_heap.start()) & (_card_size - 1)) == 0, "heap must start at card boundary");
71 assert((uintptr_t(_whole_heap.end()) & (_card_size - 1)) == 0, "heap must end at card boundary");
72 }
73
74 void CardTable::initialize(void* region0_start, void* region1_start) {
75 size_t num_cards = cards_required(_whole_heap.word_size());
76
77 size_t num_bytes = num_cards * sizeof(CardValue);
78 _byte_map_size = compute_byte_map_size(num_bytes);
79
80 HeapWord* low_bound = _whole_heap.start();
81 HeapWord* high_bound = _whole_heap.end();
82
83 const size_t rs_align = _page_size == os::vm_page_size() ? 0 :
84 MAX2(_page_size, os::vm_allocation_granularity());
85 ReservedSpace heap_rs(_byte_map_size, rs_align, _page_size);
86
87 MemTracker::record_virtual_memory_type((address)heap_rs.base(), mtGC);
88
89 os::trace_page_sizes("Card Table", num_bytes, num_bytes,
90 heap_rs.base(), heap_rs.size(), _page_size);
91 if (!heap_rs.is_reserved()) {
92 vm_exit_during_initialization("Could not reserve enough space for the "
93 "card marking array");
94 }
95
96 // The assembler store_check code will do an unsigned shift of the oop,
97 // then add it to _byte_map_base, i.e.
98 //
99 // _byte_map = _byte_map_base + (uintptr_t(low_bound) >> card_shift)
100 _byte_map = (CardValue*) heap_rs.base();
101 _byte_map_base = _byte_map - (uintptr_t(low_bound) >> _card_shift);
102 assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map");
103 assert(byte_for(high_bound-1) <= &_byte_map[last_valid_index()], "Checking end of map");
104
105 initialize_covered_region(region0_start, region1_start);
106
107 log_trace(gc, barrier)("CardTable::CardTable: ");
108 log_trace(gc, barrier)(" &_byte_map[0]: " PTR_FORMAT " &_byte_map[last_valid_index()]: " PTR_FORMAT,
109 p2i(&_byte_map[0]), p2i(&_byte_map[last_valid_index()]));
110 log_trace(gc, barrier)(" _byte_map_base: " PTR_FORMAT, p2i(_byte_map_base));
111 }
112
113 MemRegion CardTable::committed_for(const MemRegion mr) const {
114 HeapWord* addr_l = (HeapWord*)align_down(byte_for(mr.start()), _page_size);
115 HeapWord* addr_r = mr.is_empty()
116 ? addr_l
117 : (HeapWord*)align_up(byte_after(mr.last()), _page_size);
118
119 if (mr.start() == _covered[0].start()) {
120 // In case the card for gen-boundary is not page-size aligned, the crossing page belongs to _covered[1].
121 addr_r = MIN2(addr_r, (HeapWord*)align_down(byte_for(_covered[1].start()), _page_size));
122 }
123
124 return MemRegion(addr_l, addr_r);
125 }
126
127 void CardTable::initialize_covered_region(void* region0_start, void* region1_start) {
128 assert(_whole_heap.start() == region0_start, "precondition");
129 assert(region0_start < region1_start, "precondition");
130
131 assert(_covered[0].start() == nullptr, "precondition");
132 assert(_covered[1].start() == nullptr, "precondition");
133
134 _covered[0] = MemRegion((HeapWord*)region0_start, (size_t)0);
135 _covered[1] = MemRegion((HeapWord*)region1_start, (size_t)0);
136 }
137
138 void CardTable::resize_covered_region(MemRegion new_region) {
139 assert(UseSerialGC || UseParallelGC, "only these two collectors");
140 assert(_whole_heap.contains(new_region),
141 "attempt to cover area not in reserved area");
142 assert(_covered[0].start() != nullptr, "precondition");
143 assert(_covered[1].start() != nullptr, "precondition");
144
145 int idx = new_region.start() == _whole_heap.start() ? 0 : 1;
146
147 // We don't allow changes to the start of a region, only the end.
148 assert(_covered[idx].start() == new_region.start(), "inv");
149
150 MemRegion old_committed = committed_for(_covered[idx]);
151
152 _covered[idx] = new_region;
153
154 MemRegion new_committed = committed_for(new_region);
155
156 if (new_committed.word_size() == old_committed.word_size()) {
157 return;
158 }
159
160 if (new_committed.word_size() > old_committed.word_size()) {
161 // Expand.
162 MemRegion delta = MemRegion(old_committed.end(),
163 new_committed.word_size() - old_committed.word_size());
164
165 os::commit_memory_or_exit((char*)delta.start(),
166 delta.byte_size(),
167 _page_size,
168 !ExecMem,
169 "card table expansion");
170
171 memset(delta.start(), clean_card, delta.byte_size());
172 } else {
173 // Shrink.
174 MemRegion delta = MemRegion(new_committed.end(),
175 old_committed.word_size() - new_committed.word_size());
176 bool res = os::uncommit_memory((char*)delta.start(),
177 delta.byte_size());
178 assert(res, "uncommit should succeed");
179 }
180
181 log_trace(gc, barrier)("CardTable::resize_covered_region: ");
182 log_trace(gc, barrier)(" _covered[%d].start(): " PTR_FORMAT " _covered[%d].last(): " PTR_FORMAT,
183 idx, p2i(_covered[idx].start()), idx, p2i(_covered[idx].last()));
184 log_trace(gc, barrier)(" committed_start: " PTR_FORMAT " committed_last: " PTR_FORMAT,
185 p2i(new_committed.start()), p2i(new_committed.last()));
186 log_trace(gc, barrier)(" byte_for(start): " PTR_FORMAT " byte_for(last): " PTR_FORMAT,
187 p2i(byte_for(_covered[idx].start())), p2i(byte_for(_covered[idx].last())));
188 log_trace(gc, barrier)(" addr_for(start): " PTR_FORMAT " addr_for(last): " PTR_FORMAT,
189 p2i(addr_for((CardValue*) new_committed.start())), p2i(addr_for((CardValue*) new_committed.last())));
190
191 #ifdef ASSERT
192 // Touch the last card of the covered region to show that it
193 // is committed (or SEGV).
194 if (is_init_completed()) {
195 (void) (*(volatile CardValue*)byte_for(_covered[idx].last()));
196 }
197 #endif
198 }
199
200 // Note that these versions are precise! The scanning code has to handle the
201 // fact that the write barrier may be either precise or imprecise.
202 void CardTable::dirty_MemRegion(MemRegion mr) {
203 assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
204 assert(align_up (mr.end(), HeapWordSize) == mr.end(), "Unaligned end" );
205 CardValue* cur = byte_for(mr.start());
206 CardValue* last = byte_after(mr.last());
207 memset(cur, dirty_card, pointer_delta(last, cur, sizeof(CardValue)));
208 }
209
210 void CardTable::clear_MemRegion(MemRegion mr) {
211 // Be conservative: only clean cards entirely contained within the
212 // region.
213 CardValue* cur;
214 if (mr.start() == _whole_heap.start()) {
215 cur = byte_for(mr.start());
216 } else {
217 assert(mr.start() > _whole_heap.start(), "mr is not covered.");
218 cur = byte_after(mr.start() - 1);
219 }
220 CardValue* last = byte_after(mr.last());
221 memset(cur, clean_card, pointer_delta(last, cur, sizeof(CardValue)));
222 }
223
224 uintx CardTable::ct_max_alignment_constraint() {
225 // Calculate maximum alignment using GCCardSizeInBytes as card_size hasn't been set yet
226 return GCCardSizeInBytes * os::vm_page_size();
227 }
228
229 void CardTable::invalidate(MemRegion mr) {
230 assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start");
231 assert(align_up (mr.end(), HeapWordSize) == mr.end(), "Unaligned end" );
232 for (int i = 0; i < max_covered_regions; i++) {
233 MemRegion mri = mr.intersection(_covered[i]);
234 if (!mri.is_empty()) dirty_MemRegion(mri);
235 }
236 }
237
238 #ifndef PRODUCT
239 void CardTable::verify_region(MemRegion mr, CardValue val, bool val_equals) {
240 CardValue* start = byte_for(mr.start());
241 CardValue* end = byte_for(mr.last());
242 bool failures = false;
243 for (CardValue* curr = start; curr <= end; ++curr) {
244 CardValue curr_val = *curr;
245 bool failed = (val_equals) ? (curr_val != val) : (curr_val == val);
246 if (failed) {
247 if (!failures) {
248 log_error(gc, verify)("== CT verification failed: [" PTR_FORMAT "," PTR_FORMAT "]", p2i(start), p2i(end));
249 log_error(gc, verify)("== %sexpecting value: %d", (val_equals) ? "" : "not ", val);
250 failures = true;
251 }
252 log_error(gc, verify)("== card " PTR_FORMAT " [" PTR_FORMAT "," PTR_FORMAT "], val: %d",
253 p2i(curr), p2i(addr_for(curr)),
254 p2i((HeapWord*) (((size_t) addr_for(curr)) + _card_size)),
255 (int) curr_val);
256 }
257 }
258 guarantee(!failures, "there should not have been any failures");
259 }
260
261 void CardTable::verify_not_dirty_region(MemRegion mr) {
262 verify_region(mr, dirty_card, false /* val_equals */);
263 }
264
265 void CardTable::verify_dirty_region(MemRegion mr) {
266 verify_region(mr, dirty_card, true /* val_equals */);
267 }
268 #endif
269
270 void CardTable::print_on(outputStream* st) const {
271 st->print_cr("Card table byte_map: [" PTR_FORMAT "," PTR_FORMAT "] _byte_map_base: " PTR_FORMAT,
272 p2i(_byte_map), p2i(_byte_map + _byte_map_size), p2i(_byte_map_base));
273 }