1 /*
2 * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "gc/shared/gc_globals.hpp"
25 #include "gc/shared/gcLogPrecious.hpp"
26 #include "gc/z/zAddress.inline.hpp"
27 #include "gc/z/zCollectedHeap.hpp"
28 #include "gc/z/zForwarding.inline.hpp"
29 #include "gc/z/zPage.inline.hpp"
30 #include "gc/z/zStat.hpp"
31 #include "gc/z/zUtils.inline.hpp"
32 #include "logging/log.hpp"
33 #include "utilities/align.hpp"
34
35 //
36 // Reference count states:
37 //
38 // * If the reference count is zero, it will never change again.
39 //
40 // * If the reference count is positive, it can be both retained
41 // (increased) and released (decreased).
42 //
43 // * If the reference count is negative, is can only be released
44 // (increased). A negative reference count means that one or more
45 // threads are waiting for one or more other threads to release
46 // their references.
47 //
48 // The reference lock is used for waiting until the reference
49 // count has become zero (released) or negative one (claimed).
50 //
51
52 bool ZForwarding::claim() {
53 return _claimed.compare_set(false, true);
54 }
55
56 void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) {
57 _page->log_msg(" In-place reloc start - relocated to: " PTR_FORMAT, untype(relocated_watermark));
58
59 _in_place = true;
60
61 // Support for ZHeap::is_in checks of from-space objects
62 // in a page that is in-place relocating
63 _in_place_thread.store_relaxed(Thread::current());
64 _in_place_top_at_start = _page->top();
65 }
66
67 void ZForwarding::in_place_relocation_finish() {
68 assert(_in_place, "Must be an in-place relocated page");
69
70 _page->log_msg(" In-place reloc finish - top at start: " PTR_FORMAT, untype(_in_place_top_at_start));
71
72 if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) {
73 // Only do this for non-promoted pages, that still need to reset live map.
74 // Done with iterating over the "from-page" view, so can now drop the _livemap.
75 _page->reset_livemap();
76 }
77
78 // Disable relaxed ZHeap::is_in checks
79 _in_place_thread.store_relaxed(nullptr);
80 }
81
82 bool ZForwarding::in_place_relocation_is_below_top_at_start(zoffset offset) const {
83 // Only the relocating thread is allowed to know about the old relocation top.
84 return _in_place_thread.load_relaxed() == Thread::current() && offset < _in_place_top_at_start;
85 }
86
87 bool ZForwarding::retain_page(ZRelocateQueue* queue) {
88 for (;;) {
89 const int32_t ref_count = _ref_count.load_acquire();
90
91 if (ref_count == 0) {
92 // Released
93 return false;
94 }
95
96 if (ref_count < 0) {
97 // Claimed
98 queue->add_and_wait(this);
99
100 // Released
101 return false;
102 }
103
104 if (_ref_count.compare_set(ref_count, ref_count + 1)) {
105 // Retained
106 return true;
107 }
108 }
109 }
110
111 void ZForwarding::in_place_relocation_claim_page() {
112 for (;;) {
113 const int32_t ref_count = _ref_count.load_relaxed();
114 assert(ref_count > 0, "Invalid state");
115
116 // Invert reference count
117 if (!_ref_count.compare_set(ref_count, -ref_count)) {
118 continue;
119 }
120
121 // If the previous reference count was 1, then we just changed it to -1,
122 // and we have now claimed the page. Otherwise we wait until it is claimed.
123 if (ref_count != 1) {
124 ZLocker<ZConditionLock> locker(&_ref_lock);
125 while (_ref_count.load_acquire() != -1) {
126 _ref_lock.wait();
127 }
128 }
129
130 // Done
131 break;
132 }
133 }
134
135 void ZForwarding::release_page() {
136 for (;;) {
137 const int32_t ref_count = _ref_count.load_relaxed();
138 assert(ref_count != 0, "Invalid state");
139
140 if (ref_count > 0) {
141 // Decrement reference count
142 if (!_ref_count.compare_set(ref_count, ref_count - 1)) {
143 continue;
144 }
145
146 // If the previous reference count was 1, then we just decremented
147 // it to 0 and we should signal that the page is now released.
148 if (ref_count == 1) {
149 // Notify released
150 ZLocker<ZConditionLock> locker(&_ref_lock);
151 _ref_lock.notify_all();
152 }
153 } else {
154 // Increment reference count
155 if (!_ref_count.compare_set(ref_count, ref_count + 1)) {
156 continue;
157 }
158
159 // If the previous reference count was -2 or -1, then we just incremented it
160 // to -1 or 0, and we should signal the that page is now claimed or released.
161 if (ref_count == -2 || ref_count == -1) {
162 // Notify claimed or released
163 ZLocker<ZConditionLock> locker(&_ref_lock);
164 _ref_lock.notify_all();
165 }
166 }
167
168 return;
169 }
170 }
171
172 ZPage* ZForwarding::detach_page() {
173 // Wait until released
174 if (_ref_count.load_acquire() != 0) {
175 ZLocker<ZConditionLock> locker(&_ref_lock);
176 while (_ref_count.load_acquire() != 0) {
177 _ref_lock.wait();
178 }
179 }
180
181 return _page;
182 }
183
184 ZPage* ZForwarding::page() {
185 assert(_ref_count.load_relaxed() != 0, "The page has been released/detached");
186 return _page;
187 }
188
189 void ZForwarding::mark_done() {
190 _done.store_relaxed(true);
191 }
192
193 bool ZForwarding::is_done() const {
194 return _done.load_relaxed();
195 }
196
197 //
198 // The relocated_remembered_fields are used when the old generation
199 // collection is relocating objects, concurrently with the young
200 // generation collection's remembered set scanning for the marking.
201 //
202 // When the OC is relocating objects, the old remembered set bits
203 // for the from-space objects need to be moved over to the to-space
204 // objects.
205 //
206 // The YC doesn't want to wait for the OC, so it eagerly helps relocating
207 // objects with remembered set bits, so that it can perform marking on the
208 // to-space copy of the object fields that are associated with the remembered
209 // set bits.
210 //
211 // This requires some synchronization between the OC and YC, and this is
212 // mainly done via the _relocated_remembered_fields_state in each ZForwarding.
213 // The values corresponds to:
214 //
215 // none: Starting state - neither OC nor YC has stated their intentions
216 // published: The OC has completed relocating all objects, and published an array
217 // of all to-space fields that should have a remembered set entry.
218 // reject: The OC relocation of the page happened concurrently with the YC
219 // remset scanning. Two situations:
220 // a) The page had not been released yet: The YC eagerly relocated and
221 // scanned the to-space objects with remset entries.
222 // b) The page had been released: The YC accepts the array published in
223 // (published).
224 // accept: The YC found that the forwarding/page had already been relocated when
225 // the YC started.
226 //
227 // Central to this logic is the ZRemembered::scan_forwarding function, where
228 // the YC tries to "retain" the forwarding/page. If it succeeds it means that
229 // the OC has not finished (or maybe not even started) the relocation of all objects.
230 //
231 // When the YC manages to retaining the page it will bring the state from:
232 // none -> reject - Started collecting remembered set info
233 // published -> reject - Rejected the OC's remembered set info
234 // reject -> reject - An earlier YC had already handled the remembered set info
235 // accept -> - Invalid state - will not happen
236 //
237 // When the YC fails to retain the page the state transitions are:
238 // none -> x - The page was relocated before the YC started
239 // published -> x - The OC completed relocation before YC visited this forwarding.
240 // The YC will use the remembered set info collected by the OC.
241 // reject -> x - A previous YC has already handled the remembered set info
242 // accept -> x - See above
243 //
244 // x is:
245 // reject - if the relocation finished while the current YC was running
246 // accept - if the relocation finished before the current YC started
247 //
248 // Note the subtlety that even though the relocation could released the page
249 // and made it non-retainable, the relocation code might not have gotten to
250 // the point where the page is removed from the page table. It could also be
251 // the case that the relocated page became in-place relocated, and we therefore
252 // shouldn't be scanning it this YC.
253 //
254 // The (reject) state is the "dangerous" state, where both OC and YC work on
255 // the same forwarding/page somewhat concurrently. While (accept) denotes that
256 // that the entire relocation of a page (including freeing/reusing it) was
257 // completed before the current YC started.
258 //
259 // After all remset entries of relocated objects have been scanned, the code
260 // proceeds to visit all pages in the page table, to scan all pages not part
261 // of the OC relocation set. Pages with virtual addresses that doesn't match
262 // any of the once in the OC relocation set will be visited. Pages with
263 // virtual address that *do* have a corresponding forwarding entry has two
264 // cases:
265 //
266 // a) The forwarding entry is marked with (reject). This means that the
267 // corresponding page is guaranteed to be one that has been relocated by the
268 // current OC during the active YC. Any remset entry is guaranteed to have
269 // already been scanned by the scan_forwarding code.
270 //
271 // b) The forwarding entry is marked with (accept). This means that the page was
272 // *not* created by the OC relocation during this YC, which means that the
273 // page must be scanned.
274 //
275
276 void ZForwarding::relocated_remembered_fields_after_relocate() {
277 assert(from_age() == ZPageAge::old, "Only old pages have remsets");
278
279 _relocated_remembered_fields_publish_young_seqnum = ZGeneration::young()->seqnum();
280
281 if (ZGeneration::young()->is_phase_mark()) {
282 relocated_remembered_fields_publish();
283 }
284 }
285
286 void ZForwarding::relocated_remembered_fields_publish() {
287 // The OC has relocated all objects and collected all fields that
288 // used to have remembered set entries. Now publish the fields to
289 // the YC.
290
291 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::published);
292
293 // none: OK to publish
294 // published: Not possible - this operation makes this transition
295 // reject: YC started scanning the "from" page concurrently and rejects the fields
296 // the OC collected.
297 // accept: YC accepted the fields published by this function - not possible
298 // because they weren't published before the CAS above
299
300 if (res == ZPublishState::none) {
301 // fields were successfully published
302 log_debug(gc, remset)("Forwarding remset published : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
303
304 return;
305 }
306
307 log_debug(gc, remset)("Forwarding remset discarded : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
308
309 // reject: YC scans the remset concurrently
310 // accept: YC accepted published remset - not possible, we just atomically published it
311 // YC failed to retain page - not possible, since the current page is retainable
312 assert(res == ZPublishState::reject, "Unexpected value");
313
314 // YC has rejected the stored values and will (or have already) find them them itself
315 _relocated_remembered_fields_array.clear_and_deallocate();
316 }
317
318 void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() {
319 // Invariant: The page is being retained
320 assert(ZGeneration::young()->is_phase_mark(), "Only called when");
321
322 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::reject);
323
324 // none: OC has not completed relocation
325 // published: OC has completed and published all relocated remembered fields
326 // reject: A previous YC has already handled the field
327 // accept: A previous YC has determined that there's no concurrency between
328 // OC relocation and YC remembered fields scanning - not possible
329 // since the page has been retained (still being relocated) and
330 // we are in the process of scanning fields
331
332 if (res == ZPublishState::none) {
333 // Successfully notified and rejected any collected data from the OC
334 log_debug(gc, remset)("Forwarding remset eager : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
335
336 return;
337 }
338
339 if (res == ZPublishState::published) {
340 // OC relocation already collected and published fields
341
342 // Still notify concurrent scanning and reject the collected data from the OC
343 const ZPublishState res2 = _relocated_remembered_fields_state.compare_exchange(ZPublishState::published, ZPublishState::reject);
344 assert(res2 == ZPublishState::published, "Should not fail");
345
346 log_debug(gc, remset)("Forwarding remset eager and reject: " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
347
348 // The YC rejected the publish fields and is responsible for the array
349 // Eagerly deallocate the memory
350 _relocated_remembered_fields_array.clear_and_deallocate();
351 return;
352 }
353
354 log_debug(gc, remset)("Forwarding remset redundant : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
355
356 // Previous YC already handled the remembered fields
357 assert(res == ZPublishState::reject, "Unexpected value");
358 }
359
360 bool ZForwarding::relocated_remembered_fields_published_contains(volatile zpointer* p) {
361 for (volatile zpointer* const elem : _relocated_remembered_fields_array) {
362 if (elem == p) {
363 return true;
364 }
365 }
366
367 return false;
368 }
369
370 void ZForwarding::verify() const {
371 guarantee(_ref_count.load_relaxed() != 0, "Invalid reference count");
372 guarantee(_page != nullptr, "Invalid page");
373
374 uint32_t live_objects = 0;
375 size_t live_bytes = 0;
376 uint32_t no_move_expand_count = 0;
377
378 for (ZForwardingCursor i = 0; i < _entries.length(); i++) {
379 const ZForwardingEntry entry = at(&i);
380 if (!entry.populated()) {
381 // Skip empty entries
382 continue;
383 }
384
385 // Check from index
386 guarantee(entry.from_index() < _page->object_max_count(), "Invalid from index");
387
388 // Check for duplicates
389 for (ZForwardingCursor j = i + 1; j < _entries.length(); j++) {
390 const ZForwardingEntry other = at(&j);
391 if (!other.populated()) {
392 // Skip empty entries
393 continue;
394 }
395
396 guarantee(entry.from_index() != other.from_index(), "Duplicate from");
397 guarantee(entry.to_offset() != other.to_offset(), "Duplicate to");
398 }
399
400 // Read size from the TO object — always safe after relocation.
401 const zaddress to_addr = ZOffset::address(to_zoffset(entry.to_offset()));
402 const size_t size = ZUtils::object_size(to_addr);
403 const size_t aligned_size = align_up(size, _page->object_alignment());
404 live_bytes += aligned_size;
405 live_objects++;
406
407 // Detect non-moving in-place objects (from_offset == to_offset): they are counted in
408 // will_expand_objects if they had is_hashed_not_expanded() at mark time, but they did not
409 // actually expand during relocation (size = old_size). Track these so verify_live() can
410 // subtract them from the expected live_bytes adjustment.
411 if (_in_place && UseCompactObjectHeaders) {
412 const zoffset from_offset = start() + (entry.from_index() << object_alignment_shift());
413 if (to_zoffset(entry.to_offset()) == from_offset) {
414 const oop to_obj = to_oop(to_addr);
415 if (to_obj->mark().is_hashed_not_expanded() && to_obj->klass()->expand_for_hash(to_obj, to_obj->mark())) {
416 no_move_expand_count++;
417 }
418 }
419 }
420 }
421
422 // Verify number of live objects and bytes. live_bytes is computed from TO-space sizes;
423 // verify_live() adjusts for compact hash expansion using the will_expand and no_move counts.
424 _page->verify_live(live_objects, live_bytes, no_move_expand_count, _in_place);
425 }