1 /*
2 * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "gc/shared/gcLogPrecious.hpp"
25 #include "gc/z/zAddress.inline.hpp"
26 #include "gc/z/zCollectedHeap.hpp"
27 #include "gc/z/zForwarding.inline.hpp"
28 #include "gc/z/zPage.inline.hpp"
29 #include "gc/z/zStat.hpp"
30 #include "gc/z/zUtils.inline.hpp"
31 #include "logging/log.hpp"
32 #include "utilities/align.hpp"
33
34 //
35 // Reference count states:
36 //
37 // * If the reference count is zero, it will never change again.
38 //
39 // * If the reference count is positive, it can be both retained
40 // (increased) and released (decreased).
41 //
42 // * If the reference count is negative, is can only be released
43 // (increased). A negative reference count means that one or more
44 // threads are waiting for one or more other threads to release
45 // their references.
46 //
47 // The reference lock is used for waiting until the reference
48 // count has become zero (released) or negative one (claimed).
49 //
50
51 bool ZForwarding::claim() {
52 return _claimed.compare_set(false, true);
53 }
54
55 void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) {
56 _page->log_msg(" In-place reloc start - relocated to: " PTR_FORMAT, untype(relocated_watermark));
57
58 _in_place = true;
59
60 // Support for ZHeap::is_in checks of from-space objects
61 // in a page that is in-place relocating
62 _in_place_thread.store_relaxed(Thread::current());
63 _in_place_top_at_start = _page->top();
64 }
65
66 void ZForwarding::in_place_relocation_finish() {
67 assert(_in_place, "Must be an in-place relocated page");
68
69 _page->log_msg(" In-place reloc finish - top at start: " PTR_FORMAT, untype(_in_place_top_at_start));
70
71 if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) {
72 // Only do this for non-promoted pages, that still need to reset live map.
73 // Done with iterating over the "from-page" view, so can now drop the _livemap.
74 _page->reset_livemap();
75 }
76
77 // Disable relaxed ZHeap::is_in checks
78 _in_place_thread.store_relaxed(nullptr);
79 }
80
81 bool ZForwarding::in_place_relocation_is_below_top_at_start(zoffset offset) const {
82 // Only the relocating thread is allowed to know about the old relocation top.
83 return _in_place_thread.load_relaxed() == Thread::current() && offset < _in_place_top_at_start;
84 }
85
86 bool ZForwarding::retain_page(ZRelocateQueue* queue) {
87 for (;;) {
88 const int32_t ref_count = _ref_count.load_acquire();
89
90 if (ref_count == 0) {
91 // Released
92 return false;
93 }
94
95 if (ref_count < 0) {
96 // Claimed
97 queue->add_and_wait(this);
98
99 // Released
100 return false;
101 }
102
103 if (_ref_count.compare_set(ref_count, ref_count + 1)) {
104 // Retained
105 return true;
106 }
107 }
108 }
109
110 void ZForwarding::in_place_relocation_claim_page() {
111 for (;;) {
112 const int32_t ref_count = _ref_count.load_relaxed();
113 assert(ref_count > 0, "Invalid state");
114
115 // Invert reference count
116 if (!_ref_count.compare_set(ref_count, -ref_count)) {
117 continue;
118 }
119
120 // If the previous reference count was 1, then we just changed it to -1,
121 // and we have now claimed the page. Otherwise we wait until it is claimed.
122 if (ref_count != 1) {
123 ZLocker<ZConditionLock> locker(&_ref_lock);
124 while (_ref_count.load_acquire() != -1) {
125 _ref_lock.wait();
126 }
127 }
128
129 // Done
130 break;
131 }
132 }
133
134 void ZForwarding::release_page() {
135 for (;;) {
136 const int32_t ref_count = _ref_count.load_relaxed();
137 assert(ref_count != 0, "Invalid state");
138
139 if (ref_count > 0) {
140 // Decrement reference count
141 if (!_ref_count.compare_set(ref_count, ref_count - 1)) {
142 continue;
143 }
144
145 // If the previous reference count was 1, then we just decremented
146 // it to 0 and we should signal that the page is now released.
147 if (ref_count == 1) {
148 // Notify released
149 ZLocker<ZConditionLock> locker(&_ref_lock);
150 _ref_lock.notify_all();
151 }
152 } else {
153 // Increment reference count
154 if (!_ref_count.compare_set(ref_count, ref_count + 1)) {
155 continue;
156 }
157
158 // If the previous reference count was -2 or -1, then we just incremented it
159 // to -1 or 0, and we should signal the that page is now claimed or released.
160 if (ref_count == -2 || ref_count == -1) {
161 // Notify claimed or released
162 ZLocker<ZConditionLock> locker(&_ref_lock);
163 _ref_lock.notify_all();
164 }
165 }
166
167 return;
168 }
169 }
170
171 ZPage* ZForwarding::detach_page() {
172 // Wait until released
173 if (_ref_count.load_acquire() != 0) {
174 ZLocker<ZConditionLock> locker(&_ref_lock);
175 while (_ref_count.load_acquire() != 0) {
176 _ref_lock.wait();
177 }
178 }
179
180 return _page;
181 }
182
183 ZPage* ZForwarding::page() {
184 assert(_ref_count.load_relaxed() != 0, "The page has been released/detached");
185 return _page;
186 }
187
188 void ZForwarding::mark_done() {
189 _done.store_relaxed(true);
190 }
191
192 bool ZForwarding::is_done() const {
193 return _done.load_relaxed();
194 }
195
196 //
197 // The relocated_remembered_fields are used when the old generation
198 // collection is relocating objects, concurrently with the young
199 // generation collection's remembered set scanning for the marking.
200 //
201 // When the OC is relocating objects, the old remembered set bits
202 // for the from-space objects need to be moved over to the to-space
203 // objects.
204 //
205 // The YC doesn't want to wait for the OC, so it eagerly helps relocating
206 // objects with remembered set bits, so that it can perform marking on the
207 // to-space copy of the object fields that are associated with the remembered
208 // set bits.
209 //
210 // This requires some synchronization between the OC and YC, and this is
211 // mainly done via the _relocated_remembered_fields_state in each ZForwarding.
212 // The values corresponds to:
213 //
214 // none: Starting state - neither OC nor YC has stated their intentions
215 // published: The OC has completed relocating all objects, and published an array
216 // of all to-space fields that should have a remembered set entry.
217 // reject: The OC relocation of the page happened concurrently with the YC
218 // remset scanning. Two situations:
219 // a) The page had not been released yet: The YC eagerly relocated and
220 // scanned the to-space objects with remset entries.
221 // b) The page had been released: The YC accepts the array published in
222 // (published).
223 // accept: The YC found that the forwarding/page had already been relocated when
224 // the YC started.
225 //
226 // Central to this logic is the ZRemembered::scan_forwarding function, where
227 // the YC tries to "retain" the forwarding/page. If it succeeds it means that
228 // the OC has not finished (or maybe not even started) the relocation of all objects.
229 //
230 // When the YC manages to retaining the page it will bring the state from:
231 // none -> reject - Started collecting remembered set info
232 // published -> reject - Rejected the OC's remembered set info
233 // reject -> reject - An earlier YC had already handled the remembered set info
234 // accept -> - Invalid state - will not happen
235 //
236 // When the YC fails to retain the page the state transitions are:
237 // none -> x - The page was relocated before the YC started
238 // published -> x - The OC completed relocation before YC visited this forwarding.
239 // The YC will use the remembered set info collected by the OC.
240 // reject -> x - A previous YC has already handled the remembered set info
241 // accept -> x - See above
242 //
243 // x is:
244 // reject - if the relocation finished while the current YC was running
245 // accept - if the relocation finished before the current YC started
246 //
247 // Note the subtlety that even though the relocation could released the page
248 // and made it non-retainable, the relocation code might not have gotten to
249 // the point where the page is removed from the page table. It could also be
250 // the case that the relocated page became in-place relocated, and we therefore
251 // shouldn't be scanning it this YC.
252 //
253 // The (reject) state is the "dangerous" state, where both OC and YC work on
254 // the same forwarding/page somewhat concurrently. While (accept) denotes that
255 // that the entire relocation of a page (including freeing/reusing it) was
256 // completed before the current YC started.
257 //
258 // After all remset entries of relocated objects have been scanned, the code
259 // proceeds to visit all pages in the page table, to scan all pages not part
260 // of the OC relocation set. Pages with virtual addresses that doesn't match
261 // any of the once in the OC relocation set will be visited. Pages with
262 // virtual address that *do* have a corresponding forwarding entry has two
263 // cases:
264 //
265 // a) The forwarding entry is marked with (reject). This means that the
266 // corresponding page is guaranteed to be one that has been relocated by the
267 // current OC during the active YC. Any remset entry is guaranteed to have
268 // already been scanned by the scan_forwarding code.
269 //
270 // b) The forwarding entry is marked with (accept). This means that the page was
271 // *not* created by the OC relocation during this YC, which means that the
272 // page must be scanned.
273 //
274
275 void ZForwarding::relocated_remembered_fields_after_relocate() {
276 assert(from_age() == ZPageAge::old, "Only old pages have remsets");
277
278 _relocated_remembered_fields_publish_young_seqnum = ZGeneration::young()->seqnum();
279
280 if (ZGeneration::young()->is_phase_mark()) {
281 relocated_remembered_fields_publish();
282 }
283 }
284
285 void ZForwarding::relocated_remembered_fields_publish() {
286 // The OC has relocated all objects and collected all fields that
287 // used to have remembered set entries. Now publish the fields to
288 // the YC.
289
290 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::published);
291
292 // none: OK to publish
293 // published: Not possible - this operation makes this transition
294 // reject: YC started scanning the "from" page concurrently and rejects the fields
295 // the OC collected.
296 // accept: YC accepted the fields published by this function - not possible
297 // because they weren't published before the CAS above
298
299 if (res == ZPublishState::none) {
300 // fields were successfully published
301 log_debug(gc, remset)("Forwarding remset published : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
302
303 return;
304 }
305
306 log_debug(gc, remset)("Forwarding remset discarded : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
307
308 // reject: YC scans the remset concurrently
309 // accept: YC accepted published remset - not possible, we just atomically published it
310 // YC failed to retain page - not possible, since the current page is retainable
311 assert(res == ZPublishState::reject, "Unexpected value");
312
313 // YC has rejected the stored values and will (or have already) find them them itself
314 _relocated_remembered_fields_array.clear_and_deallocate();
315 }
316
317 void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() {
318 // Invariant: The page is being retained
319 assert(ZGeneration::young()->is_phase_mark(), "Only called when");
320
321 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::reject);
322
323 // none: OC has not completed relocation
324 // published: OC has completed and published all relocated remembered fields
325 // reject: A previous YC has already handled the field
326 // accept: A previous YC has determined that there's no concurrency between
327 // OC relocation and YC remembered fields scanning - not possible
328 // since the page has been retained (still being relocated) and
329 // we are in the process of scanning fields
330
331 if (res == ZPublishState::none) {
332 // Successfully notified and rejected any collected data from the OC
333 log_debug(gc, remset)("Forwarding remset eager : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
334
335 return;
336 }
337
338 if (res == ZPublishState::published) {
339 // OC relocation already collected and published fields
340
341 // Still notify concurrent scanning and reject the collected data from the OC
342 const ZPublishState res2 = _relocated_remembered_fields_state.compare_exchange(ZPublishState::published, ZPublishState::reject);
343 assert(res2 == ZPublishState::published, "Should not fail");
344
345 log_debug(gc, remset)("Forwarding remset eager and reject: " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
346
347 // The YC rejected the publish fields and is responsible for the array
348 // Eagerly deallocate the memory
349 _relocated_remembered_fields_array.clear_and_deallocate();
350 return;
351 }
352
353 log_debug(gc, remset)("Forwarding remset redundant : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
354
355 // Previous YC already handled the remembered fields
356 assert(res == ZPublishState::reject, "Unexpected value");
357 }
358
359 bool ZForwarding::relocated_remembered_fields_published_contains(volatile zpointer* p) {
360 for (volatile zpointer* const elem : _relocated_remembered_fields_array) {
361 if (elem == p) {
362 return true;
363 }
364 }
365
366 return false;
367 }
368
369 void ZForwarding::verify() const {
370 guarantee(_ref_count.load_relaxed() != 0, "Invalid reference count");
371 guarantee(_page != nullptr, "Invalid page");
372
373 uint32_t live_objects = 0;
374 size_t live_bytes = 0;
375
376 for (ZForwardingCursor i = 0; i < _entries.length(); i++) {
377 const ZForwardingEntry entry = at(&i);
378 if (!entry.populated()) {
379 // Skip empty entries
380 continue;
381 }
382
383 // Check from index
384 guarantee(entry.from_index() < _page->object_max_count(), "Invalid from index");
385
386 // Check for duplicates
387 for (ZForwardingCursor j = i + 1; j < _entries.length(); j++) {
388 const ZForwardingEntry other = at(&j);
389 if (!other.populated()) {
390 // Skip empty entries
391 continue;
392 }
393
394 guarantee(entry.from_index() != other.from_index(), "Duplicate from");
395 guarantee(entry.to_offset() != other.to_offset(), "Duplicate to");
396 }
397
398 const zaddress to_addr = ZOffset::address(to_zoffset(entry.to_offset()));
399 const size_t size = ZUtils::object_size(to_addr);
400 const size_t aligned_size = align_up(size, _page->object_alignment());
401 live_bytes += aligned_size;
402 live_objects++;
403 }
404
405 // Verify number of live objects and bytes
406 _page->verify_live(live_objects, live_bytes, _in_place);
407 }