1 /* 2 * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "gc/shared/gc_globals.hpp" 25 #include "gc/shared/gcLogPrecious.hpp" 26 #include "gc/z/zAddress.inline.hpp" 27 #include "gc/z/zCollectedHeap.hpp" 28 #include "gc/z/zForwarding.inline.hpp" 29 #include "gc/z/zPage.inline.hpp" 30 #include "gc/z/zStat.hpp" 31 #include "gc/z/zUtils.inline.hpp" 32 #include "logging/log.hpp" 33 #include "utilities/align.hpp" 34 35 // 36 // Reference count states: 37 // 38 // * If the reference count is zero, it will never change again. 39 // 40 // * If the reference count is positive, it can be both retained 41 // (increased) and released (decreased). 42 // 43 // * If the reference count is negative, is can only be released 44 // (increased). A negative reference count means that one or more 45 // threads are waiting for one or more other threads to release 46 // their references. 47 // 48 // The reference lock is used for waiting until the reference 49 // count has become zero (released) or negative one (claimed). 50 // 51 52 bool ZForwarding::claim() { 53 return _claimed.compare_set(false, true); 54 } 55 56 void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) { 57 _page->log_msg(" In-place reloc start - relocated to: " PTR_FORMAT, untype(relocated_watermark)); 58 59 _in_place = true; 60 61 // Support for ZHeap::is_in checks of from-space objects 62 // in a page that is in-place relocating 63 _in_place_thread.store_relaxed(Thread::current()); 64 _in_place_top_at_start = _page->top(); 65 } 66 67 void ZForwarding::in_place_relocation_finish() { 68 assert(_in_place, "Must be an in-place relocated page"); 69 70 _page->log_msg(" In-place reloc finish - top at start: " PTR_FORMAT, untype(_in_place_top_at_start)); 71 72 if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) { 73 // Only do this for non-promoted pages, that still need to reset live map. 74 // Done with iterating over the "from-page" view, so can now drop the _livemap. 75 _page->reset_livemap(); 76 } 77 78 // Disable relaxed ZHeap::is_in checks 79 _in_place_thread.store_relaxed(nullptr); 80 } 81 82 bool ZForwarding::in_place_relocation_is_below_top_at_start(zoffset offset) const { 83 // Only the relocating thread is allowed to know about the old relocation top. 84 return _in_place_thread.load_relaxed() == Thread::current() && offset < _in_place_top_at_start; 85 } 86 87 bool ZForwarding::retain_page(ZRelocateQueue* queue) { 88 for (;;) { 89 const int32_t ref_count = _ref_count.load_acquire(); 90 91 if (ref_count == 0) { 92 // Released 93 return false; 94 } 95 96 if (ref_count < 0) { 97 // Claimed 98 queue->add_and_wait(this); 99 100 // Released 101 return false; 102 } 103 104 if (_ref_count.compare_set(ref_count, ref_count + 1)) { 105 // Retained 106 return true; 107 } 108 } 109 } 110 111 void ZForwarding::in_place_relocation_claim_page() { 112 for (;;) { 113 const int32_t ref_count = _ref_count.load_relaxed(); 114 assert(ref_count > 0, "Invalid state"); 115 116 // Invert reference count 117 if (!_ref_count.compare_set(ref_count, -ref_count)) { 118 continue; 119 } 120 121 // If the previous reference count was 1, then we just changed it to -1, 122 // and we have now claimed the page. Otherwise we wait until it is claimed. 123 if (ref_count != 1) { 124 ZLocker<ZConditionLock> locker(&_ref_lock); 125 while (_ref_count.load_acquire() != -1) { 126 _ref_lock.wait(); 127 } 128 } 129 130 // Done 131 break; 132 } 133 } 134 135 void ZForwarding::release_page() { 136 for (;;) { 137 const int32_t ref_count = _ref_count.load_relaxed(); 138 assert(ref_count != 0, "Invalid state"); 139 140 if (ref_count > 0) { 141 // Decrement reference count 142 if (!_ref_count.compare_set(ref_count, ref_count - 1)) { 143 continue; 144 } 145 146 // If the previous reference count was 1, then we just decremented 147 // it to 0 and we should signal that the page is now released. 148 if (ref_count == 1) { 149 // Notify released 150 ZLocker<ZConditionLock> locker(&_ref_lock); 151 _ref_lock.notify_all(); 152 } 153 } else { 154 // Increment reference count 155 if (!_ref_count.compare_set(ref_count, ref_count + 1)) { 156 continue; 157 } 158 159 // If the previous reference count was -2 or -1, then we just incremented it 160 // to -1 or 0, and we should signal the that page is now claimed or released. 161 if (ref_count == -2 || ref_count == -1) { 162 // Notify claimed or released 163 ZLocker<ZConditionLock> locker(&_ref_lock); 164 _ref_lock.notify_all(); 165 } 166 } 167 168 return; 169 } 170 } 171 172 ZPage* ZForwarding::detach_page() { 173 // Wait until released 174 if (_ref_count.load_acquire() != 0) { 175 ZLocker<ZConditionLock> locker(&_ref_lock); 176 while (_ref_count.load_acquire() != 0) { 177 _ref_lock.wait(); 178 } 179 } 180 181 return _page; 182 } 183 184 ZPage* ZForwarding::page() { 185 assert(_ref_count.load_relaxed() != 0, "The page has been released/detached"); 186 return _page; 187 } 188 189 void ZForwarding::mark_done() { 190 _done.store_relaxed(true); 191 } 192 193 bool ZForwarding::is_done() const { 194 return _done.load_relaxed(); 195 } 196 197 // 198 // The relocated_remembered_fields are used when the old generation 199 // collection is relocating objects, concurrently with the young 200 // generation collection's remembered set scanning for the marking. 201 // 202 // When the OC is relocating objects, the old remembered set bits 203 // for the from-space objects need to be moved over to the to-space 204 // objects. 205 // 206 // The YC doesn't want to wait for the OC, so it eagerly helps relocating 207 // objects with remembered set bits, so that it can perform marking on the 208 // to-space copy of the object fields that are associated with the remembered 209 // set bits. 210 // 211 // This requires some synchronization between the OC and YC, and this is 212 // mainly done via the _relocated_remembered_fields_state in each ZForwarding. 213 // The values corresponds to: 214 // 215 // none: Starting state - neither OC nor YC has stated their intentions 216 // published: The OC has completed relocating all objects, and published an array 217 // of all to-space fields that should have a remembered set entry. 218 // reject: The OC relocation of the page happened concurrently with the YC 219 // remset scanning. Two situations: 220 // a) The page had not been released yet: The YC eagerly relocated and 221 // scanned the to-space objects with remset entries. 222 // b) The page had been released: The YC accepts the array published in 223 // (published). 224 // accept: The YC found that the forwarding/page had already been relocated when 225 // the YC started. 226 // 227 // Central to this logic is the ZRemembered::scan_forwarding function, where 228 // the YC tries to "retain" the forwarding/page. If it succeeds it means that 229 // the OC has not finished (or maybe not even started) the relocation of all objects. 230 // 231 // When the YC manages to retaining the page it will bring the state from: 232 // none -> reject - Started collecting remembered set info 233 // published -> reject - Rejected the OC's remembered set info 234 // reject -> reject - An earlier YC had already handled the remembered set info 235 // accept -> - Invalid state - will not happen 236 // 237 // When the YC fails to retain the page the state transitions are: 238 // none -> x - The page was relocated before the YC started 239 // published -> x - The OC completed relocation before YC visited this forwarding. 240 // The YC will use the remembered set info collected by the OC. 241 // reject -> x - A previous YC has already handled the remembered set info 242 // accept -> x - See above 243 // 244 // x is: 245 // reject - if the relocation finished while the current YC was running 246 // accept - if the relocation finished before the current YC started 247 // 248 // Note the subtlety that even though the relocation could released the page 249 // and made it non-retainable, the relocation code might not have gotten to 250 // the point where the page is removed from the page table. It could also be 251 // the case that the relocated page became in-place relocated, and we therefore 252 // shouldn't be scanning it this YC. 253 // 254 // The (reject) state is the "dangerous" state, where both OC and YC work on 255 // the same forwarding/page somewhat concurrently. While (accept) denotes that 256 // that the entire relocation of a page (including freeing/reusing it) was 257 // completed before the current YC started. 258 // 259 // After all remset entries of relocated objects have been scanned, the code 260 // proceeds to visit all pages in the page table, to scan all pages not part 261 // of the OC relocation set. Pages with virtual addresses that doesn't match 262 // any of the once in the OC relocation set will be visited. Pages with 263 // virtual address that *do* have a corresponding forwarding entry has two 264 // cases: 265 // 266 // a) The forwarding entry is marked with (reject). This means that the 267 // corresponding page is guaranteed to be one that has been relocated by the 268 // current OC during the active YC. Any remset entry is guaranteed to have 269 // already been scanned by the scan_forwarding code. 270 // 271 // b) The forwarding entry is marked with (accept). This means that the page was 272 // *not* created by the OC relocation during this YC, which means that the 273 // page must be scanned. 274 // 275 276 void ZForwarding::relocated_remembered_fields_after_relocate() { 277 assert(from_age() == ZPageAge::old, "Only old pages have remsets"); 278 279 _relocated_remembered_fields_publish_young_seqnum = ZGeneration::young()->seqnum(); 280 281 if (ZGeneration::young()->is_phase_mark()) { 282 relocated_remembered_fields_publish(); 283 } 284 } 285 286 void ZForwarding::relocated_remembered_fields_publish() { 287 // The OC has relocated all objects and collected all fields that 288 // used to have remembered set entries. Now publish the fields to 289 // the YC. 290 291 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::published); 292 293 // none: OK to publish 294 // published: Not possible - this operation makes this transition 295 // reject: YC started scanning the "from" page concurrently and rejects the fields 296 // the OC collected. 297 // accept: YC accepted the fields published by this function - not possible 298 // because they weren't published before the CAS above 299 300 if (res == ZPublishState::none) { 301 // fields were successfully published 302 log_debug(gc, remset)("Forwarding remset published : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end())); 303 304 return; 305 } 306 307 log_debug(gc, remset)("Forwarding remset discarded : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end())); 308 309 // reject: YC scans the remset concurrently 310 // accept: YC accepted published remset - not possible, we just atomically published it 311 // YC failed to retain page - not possible, since the current page is retainable 312 assert(res == ZPublishState::reject, "Unexpected value"); 313 314 // YC has rejected the stored values and will (or have already) find them them itself 315 _relocated_remembered_fields_array.clear_and_deallocate(); 316 } 317 318 void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() { 319 // Invariant: The page is being retained 320 assert(ZGeneration::young()->is_phase_mark(), "Only called when"); 321 322 const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::reject); 323 324 // none: OC has not completed relocation 325 // published: OC has completed and published all relocated remembered fields 326 // reject: A previous YC has already handled the field 327 // accept: A previous YC has determined that there's no concurrency between 328 // OC relocation and YC remembered fields scanning - not possible 329 // since the page has been retained (still being relocated) and 330 // we are in the process of scanning fields 331 332 if (res == ZPublishState::none) { 333 // Successfully notified and rejected any collected data from the OC 334 log_debug(gc, remset)("Forwarding remset eager : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end())); 335 336 return; 337 } 338 339 if (res == ZPublishState::published) { 340 // OC relocation already collected and published fields 341 342 // Still notify concurrent scanning and reject the collected data from the OC 343 const ZPublishState res2 = _relocated_remembered_fields_state.compare_exchange(ZPublishState::published, ZPublishState::reject); 344 assert(res2 == ZPublishState::published, "Should not fail"); 345 346 log_debug(gc, remset)("Forwarding remset eager and reject: " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end())); 347 348 // The YC rejected the publish fields and is responsible for the array 349 // Eagerly deallocate the memory 350 _relocated_remembered_fields_array.clear_and_deallocate(); 351 return; 352 } 353 354 log_debug(gc, remset)("Forwarding remset redundant : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end())); 355 356 // Previous YC already handled the remembered fields 357 assert(res == ZPublishState::reject, "Unexpected value"); 358 } 359 360 bool ZForwarding::relocated_remembered_fields_published_contains(volatile zpointer* p) { 361 for (volatile zpointer* const elem : _relocated_remembered_fields_array) { 362 if (elem == p) { 363 return true; 364 } 365 } 366 367 return false; 368 } 369 370 void ZForwarding::verify() const { 371 guarantee(_ref_count.load_relaxed() != 0, "Invalid reference count"); 372 guarantee(_page != nullptr, "Invalid page"); 373 374 uint32_t live_objects = 0; 375 size_t live_bytes = 0; 376 uint32_t no_move_expand_count = 0; 377 378 for (ZForwardingCursor i = 0; i < _entries.length(); i++) { 379 const ZForwardingEntry entry = at(&i); 380 if (!entry.populated()) { 381 // Skip empty entries 382 continue; 383 } 384 385 // Check from index 386 guarantee(entry.from_index() < _page->object_max_count(), "Invalid from index"); 387 388 // Check for duplicates 389 for (ZForwardingCursor j = i + 1; j < _entries.length(); j++) { 390 const ZForwardingEntry other = at(&j); 391 if (!other.populated()) { 392 // Skip empty entries 393 continue; 394 } 395 396 guarantee(entry.from_index() != other.from_index(), "Duplicate from"); 397 guarantee(entry.to_offset() != other.to_offset(), "Duplicate to"); 398 } 399 400 // Read size from the TO object — always safe after relocation. 401 const zaddress to_addr = ZOffset::address(to_zoffset(entry.to_offset())); 402 const size_t size = ZUtils::object_size(to_addr); 403 const size_t aligned_size = align_up(size, _page->object_alignment()); 404 live_bytes += aligned_size; 405 live_objects++; 406 407 // Detect non-moving in-place objects (from_offset == to_offset): they are counted in 408 // will_expand_objects if they had is_hashed_not_expanded() at mark time, but they did not 409 // actually expand during relocation (size = old_size). Track these so verify_live() can 410 // subtract them from the expected live_bytes adjustment. 411 if (_in_place && UseCompactObjectHeaders) { 412 const zoffset from_offset = start() + (entry.from_index() << object_alignment_shift()); 413 if (to_zoffset(entry.to_offset()) == from_offset) { 414 const oop to_obj = to_oop(to_addr); 415 if (to_obj->mark().is_hashed_not_expanded() && to_obj->klass()->expand_for_hash(to_obj, to_obj->mark())) { 416 no_move_expand_count++; 417 } 418 } 419 } 420 } 421 422 // Verify number of live objects and bytes. live_bytes is computed from TO-space sizes; 423 // verify_live() adjusts for compact hash expansion using the will_expand and no_move counts. 424 _page->verify_live(live_objects, live_bytes, no_move_expand_count, _in_place); 425 } --- EOF ---