1 /*
2 * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
26
27 #include "gc/z/zPage.hpp"
28
29 #include "gc/z/zAddress.inline.hpp"
30 #include "gc/z/zGeneration.inline.hpp"
31 #include "gc/z/zGlobals.hpp"
32 #include "gc/z/zLiveMap.inline.hpp"
33 #include "gc/z/zRememberedSet.inline.hpp"
34 #include "gc/z/zVirtualMemory.inline.hpp"
35 #include "logging/logStream.hpp"
36 #include "runtime/atomicAccess.hpp"
37 #include "runtime/os.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 inline const char* ZPage::type_to_string() const {
42 switch (type()) {
43 case ZPageType::small:
44 return "Small";
45
46 case ZPageType::medium:
47 return "Medium";
48
49 case ZPageType::large:
50 return "Large";
51
52 default:
53 fatal("Unexpected page type");
54 }
55 }
56
57 inline uint32_t ZPage::object_max_count() const {
58 switch (type()) {
59 case ZPageType::large:
60 // A large page can only contain a single
61 // object aligned to the start of the page.
62 return 1;
63
64 default:
65 return checked_cast<uint32_t>(size() >> object_alignment_shift());
66 }
67 }
68
69 inline size_t ZPage::object_alignment_shift() const {
70 switch (type()) {
71 case ZPageType::small:
72 return (size_t)ZObjectAlignmentSmallShift;
73
74 case ZPageType::medium:
75 return (size_t)ZObjectAlignmentMediumShift;
76
77 case ZPageType::large:
78 return (size_t)ZObjectAlignmentLargeShift;
79
80 default:
81 fatal("Unexpected page type");
82 return 0;
83 }
84 }
85
86 inline size_t ZPage::object_alignment() const {
87 switch (type()) {
88 case ZPageType::small:
89 return (size_t)ZObjectAlignmentSmall;
90
91 case ZPageType::medium:
92 return (size_t)ZObjectAlignmentMedium;
93
94 case ZPageType::large:
95 return (size_t)ZObjectAlignmentLarge;
96
97 default:
98 fatal("Unexpected page type");
99 return 0;
100 }
101 }
102
103 inline ZPageType ZPage::type() const {
104 return _type;
105 }
106
107 inline bool ZPage::is_small() const {
108 return _type == ZPageType::small;
109 }
110
111 inline bool ZPage::is_medium() const {
112 return _type == ZPageType::medium;
113 }
114
115 inline bool ZPage::is_large() const {
116 return _type == ZPageType::large;
117 }
118
119 inline ZGenerationId ZPage::generation_id() const {
120 return _generation_id;
121 }
122
123 inline bool ZPage::is_young() const {
124 return _generation_id == ZGenerationId::young;
125 }
126
127 inline bool ZPage::is_old() const {
128 return _generation_id == ZGenerationId::old;
129 }
130
131 inline zoffset ZPage::start() const {
132 return _virtual.start();
133 }
134
135 inline zoffset_end ZPage::end() const {
136 return _virtual.end();
137 }
138
139 inline size_t ZPage::size() const {
140 return _virtual.size();
141 }
142
143 inline zoffset_end ZPage::top() const {
144 return _top;
145 }
146
147 inline size_t ZPage::remaining() const {
148 return end() - top();
149 }
150
151 inline size_t ZPage::used() const {
152 return top() - start();
153 }
154
155 inline const ZVirtualMemory& ZPage::virtual_memory() const {
156 return _virtual;
157 }
158
159 inline uint32_t ZPage::single_partition_id() const {
160 assert(!is_multi_partition(), "Don't fetch single partition id if page is multi-partition");
161 return _single_partition_id;
162 }
163
164 inline bool ZPage::is_multi_partition() const {
165 return _multi_partition_tracker != nullptr;
166 }
167
168 inline ZMultiPartitionTracker* ZPage::multi_partition_tracker() const {
169 return _multi_partition_tracker;
170 }
171
172 inline ZPageAge ZPage::age() const {
173 return _age;
174 }
175
176 inline uint32_t ZPage::seqnum() const {
177 return _seqnum;
178 }
179
180 inline bool ZPage::is_allocating() const {
181 return _seqnum == generation()->seqnum();
182 }
183
184 inline bool ZPage::is_relocatable() const {
185 return _seqnum < generation()->seqnum();
186 }
187
188 inline bool ZPage::is_in(zoffset offset) const {
189 return offset >= start() && offset < top();
190 }
191
192 inline bool ZPage::is_in(zaddress addr) const {
193 const zoffset offset = ZAddress::offset(addr);
194 return is_in(offset);
195 }
196
197 inline uintptr_t ZPage::local_offset(zoffset offset) const {
198 assert(ZHeap::heap()->is_in_page_relaxed(this, ZOffset::address(offset)),
199 "Invalid offset " PTR_FORMAT " page [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
200 untype(offset), untype(start()), untype(top()), untype(end()));
201 return offset - start();
202 }
203
204 inline uintptr_t ZPage::local_offset(zoffset_end offset) const {
205 assert(offset <= end(), "Wrong offset");
206 return offset - start();
207 }
208
209 inline uintptr_t ZPage::local_offset(zaddress addr) const {
210 const zoffset offset = ZAddress::offset(addr);
211 return local_offset(offset);
212 }
213
214 inline uintptr_t ZPage::local_offset(zaddress_unsafe addr) const {
215 const zoffset offset = ZAddress::offset(addr);
216 return local_offset(offset);
217 }
218
219 inline zoffset ZPage::global_offset(uintptr_t local_offset) const {
220 return start() + local_offset;
221 }
222
223 inline bool ZPage::is_marked() const {
224 assert(is_relocatable(), "Invalid page state");
225 return _livemap.is_marked(_generation_id);
226 }
227
228 inline BitMap::idx_t ZPage::bit_index(zaddress addr) const {
229 return (local_offset(addr) >> object_alignment_shift()) * 2;
230 }
231
232 inline zoffset ZPage::offset_from_bit_index(BitMap::idx_t index) const {
233 const uintptr_t l_offset = ((index / 2) << object_alignment_shift());
234 return start() + l_offset;
235 }
236
237 inline oop ZPage::object_from_bit_index(BitMap::idx_t index) const {
238 const zoffset offset = offset_from_bit_index(index);
239 return to_oop(ZOffset::address(offset));
240 }
241
242 inline bool ZPage::is_live_bit_set(zaddress addr) const {
243 assert(is_relocatable(), "Invalid page state");
244 const BitMap::idx_t index = bit_index(addr);
245 return _livemap.get(_generation_id, index);
246 }
247
248 inline bool ZPage::is_strong_bit_set(zaddress addr) const {
249 assert(is_relocatable(), "Invalid page state");
250 const BitMap::idx_t index = bit_index(addr);
251 return _livemap.get(_generation_id, index + 1);
252 }
253
254 inline bool ZPage::is_object_live(zaddress addr) const {
255 return is_allocating() || is_live_bit_set(addr);
256 }
257
258 inline bool ZPage::is_object_strongly_live(zaddress addr) const {
259 return is_allocating() || is_strong_bit_set(addr);
260 }
261
262 inline bool ZPage::is_object_marked_live(zaddress addr) const {
263 // This function is only used by the marking code and therefore has stronger
264 // asserts that are not always valid to ask when checking for liveness.
265 assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
266 assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
267
268 return is_object_live(addr);
269 }
270
271 inline bool ZPage::is_object_marked_strong(zaddress addr) const {
272 // This function is only used by the marking code and therefore has stronger
273 // asserts that are not always valid to ask when checking for liveness.
274 assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
275 assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
276
277 return is_object_strongly_live(addr);
278 }
279
280 inline bool ZPage::is_object_marked(zaddress addr, bool finalizable) const {
281 return finalizable ? is_object_marked_live(addr) : is_object_marked_strong(addr);
282 }
283
284 inline bool ZPage::mark_object(zaddress addr, bool finalizable, bool& inc_live) {
285 assert(is_relocatable(), "Invalid page state");
286 assert(is_in(addr), "Invalid address");
287
288 // Verify oop
289 assert_is_oop(addr);
290
291 // Set mark bit
292 const BitMap::idx_t index = bit_index(addr);
293 return _livemap.set(_generation_id, index, finalizable, inc_live);
294 }
295
296 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
297 _livemap.inc_live(objects, bytes);
298 }
299
300 inline void ZPage::inc_will_expand(uint32_t objects) {
301 _livemap.inc_will_expand(objects);
302 }
303
304 #define assert_zpage_mark_state() \
305 do { \
306 assert(is_marked(), "Should be marked"); \
307 assert(!is_young() || !ZGeneration::young()->is_phase_mark(), "Wrong phase"); \
308 assert(!is_old() || !ZGeneration::old()->is_phase_mark(), "Wrong phase"); \
309 } while (0)
310
311 inline uint32_t ZPage::live_objects() const {
312 assert_zpage_mark_state();
313
314 return _livemap.live_objects();
315 }
316
317 inline size_t ZPage::live_bytes() const {
318 assert_zpage_mark_state();
319
320 return _livemap.live_bytes();
321 }
322
323 template <typename Function>
324 inline void ZPage::object_iterate(Function function) {
325 auto do_bit = [&](BitMap::idx_t index) -> bool {
326 const oop obj = object_from_bit_index(index);
327
328 // Apply function
329 function(obj);
330
331 return true;
332 };
333
334 _livemap.iterate(_generation_id, do_bit);
335 }
336
337 inline void ZPage::remember(volatile zpointer* p) {
338 const zaddress addr = to_zaddress((uintptr_t)p);
339 const uintptr_t l_offset = local_offset(addr);
340 _remembered_set.set_current(l_offset);
341 }
342
343 inline void ZPage::clear_remset_bit_non_par_current(uintptr_t l_offset) {
344 _remembered_set.unset_non_par_current(l_offset);
345 }
346
347 inline void ZPage::clear_remset_range_non_par_current(uintptr_t l_offset, size_t size) {
348 _remembered_set.unset_range_non_par_current(l_offset, size);
349 }
350
351 inline ZBitMap::ReverseIterator ZPage::remset_reverse_iterator_previous() {
352 return _remembered_set.iterator_reverse_previous();
353 }
354
355 inline BitMap::Iterator ZPage::remset_iterator_limited_current(uintptr_t l_offset, size_t size) {
356 return _remembered_set.iterator_limited_current(l_offset, size);
357 }
358
359 inline BitMap::Iterator ZPage::remset_iterator_limited_previous(uintptr_t l_offset, size_t size) {
360 return _remembered_set.iterator_limited_previous(l_offset, size);
361 }
362
363 inline bool ZPage::is_remembered(volatile zpointer* p) {
364 const zaddress addr = to_zaddress((uintptr_t)p);
365 const uintptr_t l_offset = local_offset(addr);
366 return _remembered_set.at_current(l_offset);
367 }
368
369 inline bool ZPage::was_remembered(volatile zpointer* p) {
370 const zaddress addr = to_zaddress((uintptr_t)p);
371 const uintptr_t l_offset = local_offset(addr);
372 return _remembered_set.at_previous(l_offset);
373 }
374
375 inline zaddress_unsafe ZPage::find_base_unsafe(volatile zpointer* p) {
376 if (is_large()) {
377 return ZOffset::address_unsafe(start());
378 }
379
380 // Note: when thinking about excluding looking at the index corresponding to
381 // the field address p, it's important to note that for medium pages both p
382 // and it's associated base could map to the same index.
383 const BitMap::idx_t index = bit_index(zaddress(uintptr_t(p)));
384 const BitMap::idx_t base_index = _livemap.find_base_bit(index);
385 if (base_index == BitMap::idx_t(-1)) {
386 return zaddress_unsafe::null;
387 } else {
388 return ZOffset::address_unsafe(offset_from_bit_index(base_index));
389 }
390 }
391
392 inline zaddress_unsafe ZPage::find_base(volatile zpointer* p) {
393 assert_zpage_mark_state();
394
395 return find_base_unsafe(p);
396 }
397
398 template <typename Function>
399 inline void ZPage::oops_do_remembered(Function function) {
400 _remembered_set.iterate_previous([&](uintptr_t local_offset) {
401 const zoffset offset = start() + local_offset;
402 const zaddress addr = ZOffset::address(offset);
403
404 function((volatile zpointer*)addr);
405 });
406 }
407
408 template <typename Function>
409 inline void ZPage::oops_do_remembered_in_live(Function function) {
410 assert(!is_allocating(), "Must have liveness information");
411 assert(!ZGeneration::old()->is_phase_mark(), "Must have liveness information");
412 assert(is_marked(), "Must have liveness information");
413
414 ZRememberedSetContainingInLiveIterator iter(this);
415 for (ZRememberedSetContaining containing; iter.next(&containing);) {
416 function((volatile zpointer*)containing._field_addr);
417 }
418
419 iter.print_statistics();
420 }
421
422 template <typename Function>
423 inline void ZPage::oops_do_current_remembered(Function function) {
424 _remembered_set.iterate_current([&](uintptr_t local_offset) {
425 const zoffset offset = start() + local_offset;
426 const zaddress addr = ZOffset::address(offset);
427
428 function((volatile zpointer*)addr);
429 });
430 }
431
432 inline zaddress ZPage::alloc_object(size_t size) {
433 assert(is_allocating(), "Invalid state");
434
435 const size_t aligned_size = align_up(size, object_alignment());
436 const zoffset_end addr = top();
437
438 zoffset_end new_top;
439
440 if (!to_zoffset_end(&new_top, addr, aligned_size)) {
441 // Next top would be outside of the heap - bail
442 return zaddress::null;
443 }
444
445 if (new_top > end()) {
446 // Not enough space left in the page
447 return zaddress::null;
448 }
449
450 _top = new_top;
451
452 return ZOffset::address(to_zoffset(addr));
453 }
454
455 inline zaddress ZPage::alloc_object_atomic(size_t size) {
456 assert(is_allocating(), "Invalid state");
457
458 const size_t aligned_size = align_up(size, object_alignment());
459 zoffset_end addr = top();
460
461 for (;;) {
462 zoffset_end new_top;
463
464 if (!to_zoffset_end(&new_top, addr, aligned_size)) {
465 // Next top would be outside of the heap - bail
466 return zaddress::null;
467 }
468
469 if (new_top > end()) {
470 // Not enough space left
471 return zaddress::null;
472 }
473
474 const zoffset_end prev_top = AtomicAccess::cmpxchg(&_top, addr, new_top);
475 if (prev_top == addr) {
476 // Success
477 return ZOffset::address(to_zoffset(addr));
478 }
479
480 // Retry
481 addr = prev_top;
482 }
483 }
484
485 inline bool ZPage::undo_alloc_object(zaddress addr, size_t size) {
486 assert(is_allocating(), "Invalid state");
487
488 const zoffset offset = ZAddress::offset(addr);
489 const size_t aligned_size = align_up(size, object_alignment());
490 const zoffset_end old_top = top();
491 const zoffset_end new_top = old_top - aligned_size;
492
493 if (new_top != offset) {
494 // Failed to undo allocation, not the last allocated object
495 return false;
496 }
497
498 _top = new_top;
499
500 // Success
501 return true;
502 }
503
504 inline bool ZPage::undo_alloc_object_atomic(zaddress addr, size_t size) {
505 assert(is_allocating(), "Invalid state");
506
507 const zoffset offset = ZAddress::offset(addr);
508 const size_t aligned_size = align_up(size, object_alignment());
509 zoffset_end old_top = top();
510
511 for (;;) {
512 const zoffset_end new_top = old_top - aligned_size;
513 if (new_top != offset) {
514 // Failed to undo allocation, not the last allocated object
515 return false;
516 }
517
518 const zoffset_end prev_top = AtomicAccess::cmpxchg(&_top, old_top, new_top);
519 if (prev_top == old_top) {
520 // Success
521 return true;
522 }
523
524 // Retry
525 old_top = prev_top;
526 }
527 }
528
529 inline void ZPage::log_msg(const char* msg_format, ...) const {
530 LogTarget(Trace, gc, page) target;
531 if (target.is_enabled()) {
532 va_list argp;
533 va_start(argp, msg_format);
534 LogStream stream(target);
535 print_on_msg(&stream, err_msg(FormatBufferDummy(), msg_format, argp));
536 va_end(argp);
537 }
538 }
539
540 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP