1 /*
2 * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
26
27 #include "gc/z/zPage.hpp"
28
29 #include "gc/z/zAddress.inline.hpp"
30 #include "gc/z/zGeneration.inline.hpp"
31 #include "gc/z/zGlobals.hpp"
32 #include "gc/z/zLiveMap.inline.hpp"
33 #include "gc/z/zRememberedSet.inline.hpp"
34 #include "gc/z/zVirtualMemory.inline.hpp"
35 #include "logging/logStream.hpp"
36 #include "runtime/atomicAccess.hpp"
37 #include "runtime/os.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 inline const char* ZPage::type_to_string() const {
42 switch (type()) {
43 case ZPageType::small:
44 return "Small";
45
46 case ZPageType::medium:
47 return "Medium";
48
49 case ZPageType::large:
50 return "Large";
51
52 default:
53 fatal("Unexpected page type");
54 }
55 }
56
57 inline uint32_t ZPage::object_max_count() const {
58 switch (type()) {
59 case ZPageType::large:
60 // A large page can only contain a single
61 // object aligned to the start of the page.
62 return 1;
63
64 default:
65 return checked_cast<uint32_t>(size() >> object_alignment_shift());
66 }
67 }
68
69 inline size_t ZPage::object_alignment_shift() const {
70 switch (type()) {
71 case ZPageType::small:
72 return (size_t)ZObjectAlignmentSmallShift;
73
74 case ZPageType::medium:
75 return (size_t)ZObjectAlignmentMediumShift;
76
77 case ZPageType::large:
78 return (size_t)ZObjectAlignmentLargeShift;
79
80 default:
81 fatal("Unexpected page type");
82 return 0;
83 }
84 }
85
86 inline size_t ZPage::object_alignment() const {
87 switch (type()) {
88 case ZPageType::small:
89 return (size_t)ZObjectAlignmentSmall;
90
91 case ZPageType::medium:
92 return (size_t)ZObjectAlignmentMedium;
93
94 case ZPageType::large:
95 return (size_t)ZObjectAlignmentLarge;
96
97 default:
98 fatal("Unexpected page type");
99 return 0;
100 }
101 }
102
103 inline ZPageType ZPage::type() const {
104 return _type;
105 }
106
107 inline bool ZPage::is_small() const {
108 return _type == ZPageType::small;
109 }
110
111 inline bool ZPage::is_medium() const {
112 return _type == ZPageType::medium;
113 }
114
115 inline bool ZPage::is_large() const {
116 return _type == ZPageType::large;
117 }
118
119 inline ZGenerationId ZPage::generation_id() const {
120 return _generation_id;
121 }
122
123 inline bool ZPage::is_young() const {
124 return _generation_id == ZGenerationId::young;
125 }
126
127 inline bool ZPage::is_old() const {
128 return _generation_id == ZGenerationId::old;
129 }
130
131 inline zoffset ZPage::start() const {
132 return _virtual.start();
133 }
134
135 inline zoffset_end ZPage::end() const {
136 return _virtual.end();
137 }
138
139 inline size_t ZPage::size() const {
140 return _virtual.size();
141 }
142
143 inline zoffset_end ZPage::top() const {
144 return _top;
145 }
146
147 inline size_t ZPage::remaining() const {
148 return end() - top();
149 }
150
151 inline size_t ZPage::used() const {
152 return top() - start();
153 }
154
155 inline const ZVirtualMemory& ZPage::virtual_memory() const {
156 return _virtual;
157 }
158
159 inline uint32_t ZPage::single_partition_id() const {
160 assert(!is_multi_partition(), "Don't fetch single partition id if page is multi-partition");
161 return _single_partition_id;
162 }
163
164 inline bool ZPage::is_multi_partition() const {
165 return _multi_partition_tracker != nullptr;
166 }
167
168 inline ZMultiPartitionTracker* ZPage::multi_partition_tracker() const {
169 return _multi_partition_tracker;
170 }
171
172 inline ZPageAge ZPage::age() const {
173 return _age;
174 }
175
176 inline uint32_t ZPage::seqnum() const {
177 return _seqnum;
178 }
179
180 inline bool ZPage::is_allocating() const {
181 return _seqnum == generation()->seqnum();
182 }
183
184 inline bool ZPage::is_relocatable() const {
185 return _seqnum < generation()->seqnum();
186 }
187
188 inline bool ZPage::is_in(zoffset offset) const {
189 return offset >= start() && offset < top();
190 }
191
192 inline bool ZPage::is_in(zaddress addr) const {
193 const zoffset offset = ZAddress::offset(addr);
194 return is_in(offset);
195 }
196
197 inline uintptr_t ZPage::local_offset(zoffset offset) const {
198 assert(ZHeap::heap()->is_in_page_relaxed(this, ZOffset::address(offset)),
199 "Invalid offset " PTR_FORMAT " page [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
200 untype(offset), untype(start()), untype(top()), untype(end()));
201 return offset - start();
202 }
203
204 inline uintptr_t ZPage::local_offset(zoffset_end offset) const {
205 assert(offset <= end(), "Wrong offset");
206 return offset - start();
207 }
208
209 inline uintptr_t ZPage::local_offset(zaddress addr) const {
210 const zoffset offset = ZAddress::offset(addr);
211 return local_offset(offset);
212 }
213
214 inline uintptr_t ZPage::local_offset(zaddress_unsafe addr) const {
215 const zoffset offset = ZAddress::offset(addr);
216 return local_offset(offset);
217 }
218
219 inline zoffset ZPage::global_offset(uintptr_t local_offset) const {
220 return start() + local_offset;
221 }
222
223 inline bool ZPage::is_marked() const {
224 assert(is_relocatable(), "Invalid page state");
225 return _livemap.is_marked(_generation_id);
226 }
227
228 inline BitMap::idx_t ZPage::bit_index(zaddress addr) const {
229 return (local_offset(addr) >> object_alignment_shift()) * 2;
230 }
231
232 inline zoffset ZPage::offset_from_bit_index(BitMap::idx_t index) const {
233 const uintptr_t l_offset = ((index / 2) << object_alignment_shift());
234 return start() + l_offset;
235 }
236
237 inline oop ZPage::object_from_bit_index(BitMap::idx_t index) const {
238 const zoffset offset = offset_from_bit_index(index);
239 return to_oop(ZOffset::address(offset));
240 }
241
242 inline bool ZPage::is_live_bit_set(zaddress addr) const {
243 assert(is_relocatable(), "Invalid page state");
244 const BitMap::idx_t index = bit_index(addr);
245 return _livemap.get(_generation_id, index);
246 }
247
248 inline bool ZPage::is_strong_bit_set(zaddress addr) const {
249 assert(is_relocatable(), "Invalid page state");
250 const BitMap::idx_t index = bit_index(addr);
251 return _livemap.get(_generation_id, index + 1);
252 }
253
254 inline bool ZPage::is_object_live(zaddress addr) const {
255 return is_allocating() || is_live_bit_set(addr);
256 }
257
258 inline bool ZPage::is_object_strongly_live(zaddress addr) const {
259 return is_allocating() || is_strong_bit_set(addr);
260 }
261
262 inline bool ZPage::is_object_marked_live(zaddress addr) const {
263 // This function is only used by the marking code and therefore has stronger
264 // asserts that are not always valid to ask when checking for liveness.
265 assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
266 assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
267
268 return is_object_live(addr);
269 }
270
271 inline bool ZPage::is_object_marked_strong(zaddress addr) const {
272 // This function is only used by the marking code and therefore has stronger
273 // asserts that are not always valid to ask when checking for liveness.
274 assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
275 assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
276
277 return is_object_strongly_live(addr);
278 }
279
280 inline bool ZPage::is_object_marked(zaddress addr, bool finalizable) const {
281 return finalizable ? is_object_marked_live(addr) : is_object_marked_strong(addr);
282 }
283
284 inline bool ZPage::mark_object(zaddress addr, bool finalizable, bool& inc_live) {
285 assert(is_relocatable(), "Invalid page state");
286 assert(is_in(addr), "Invalid address");
287
288 // Verify oop
289 assert_is_oop(addr);
290
291 // Set mark bit
292 const BitMap::idx_t index = bit_index(addr);
293 return _livemap.set(_generation_id, index, finalizable, inc_live);
294 }
295
296 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
297 _livemap.inc_live(objects, bytes);
298 }
299
300 #define assert_zpage_mark_state() \
301 do { \
302 assert(is_marked(), "Should be marked"); \
303 assert(!is_young() || !ZGeneration::young()->is_phase_mark(), "Wrong phase"); \
304 assert(!is_old() || !ZGeneration::old()->is_phase_mark(), "Wrong phase"); \
305 } while (0)
306
307 inline uint32_t ZPage::live_objects() const {
308 assert_zpage_mark_state();
309
310 return _livemap.live_objects();
311 }
312
313 inline size_t ZPage::live_bytes() const {
314 assert_zpage_mark_state();
315
316 return _livemap.live_bytes();
317 }
318
319 template <typename Function>
320 inline void ZPage::object_iterate(Function function) {
321 auto do_bit = [&](BitMap::idx_t index) -> bool {
322 const oop obj = object_from_bit_index(index);
323
324 // Apply function
325 function(obj);
326
327 return true;
328 };
329
330 _livemap.iterate(_generation_id, do_bit);
331 }
332
333 inline void ZPage::remember(volatile zpointer* p) {
334 const zaddress addr = to_zaddress((uintptr_t)p);
335 const uintptr_t l_offset = local_offset(addr);
336 _remembered_set.set_current(l_offset);
337 }
338
339 inline void ZPage::clear_remset_bit_non_par_current(uintptr_t l_offset) {
340 _remembered_set.unset_non_par_current(l_offset);
341 }
342
343 inline void ZPage::clear_remset_range_non_par_current(uintptr_t l_offset, size_t size) {
344 _remembered_set.unset_range_non_par_current(l_offset, size);
345 }
346
347 inline ZBitMap::ReverseIterator ZPage::remset_reverse_iterator_previous() {
348 return _remembered_set.iterator_reverse_previous();
349 }
350
351 inline BitMap::Iterator ZPage::remset_iterator_limited_current(uintptr_t l_offset, size_t size) {
352 return _remembered_set.iterator_limited_current(l_offset, size);
353 }
354
355 inline BitMap::Iterator ZPage::remset_iterator_limited_previous(uintptr_t l_offset, size_t size) {
356 return _remembered_set.iterator_limited_previous(l_offset, size);
357 }
358
359 inline bool ZPage::is_remembered(volatile zpointer* p) {
360 const zaddress addr = to_zaddress((uintptr_t)p);
361 const uintptr_t l_offset = local_offset(addr);
362 return _remembered_set.at_current(l_offset);
363 }
364
365 inline bool ZPage::was_remembered(volatile zpointer* p) {
366 const zaddress addr = to_zaddress((uintptr_t)p);
367 const uintptr_t l_offset = local_offset(addr);
368 return _remembered_set.at_previous(l_offset);
369 }
370
371 inline zaddress_unsafe ZPage::find_base_unsafe(volatile zpointer* p) {
372 if (is_large()) {
373 return ZOffset::address_unsafe(start());
374 }
375
376 // Note: when thinking about excluding looking at the index corresponding to
377 // the field address p, it's important to note that for medium pages both p
378 // and it's associated base could map to the same index.
379 const BitMap::idx_t index = bit_index(zaddress(uintptr_t(p)));
380 const BitMap::idx_t base_index = _livemap.find_base_bit(index);
381 if (base_index == BitMap::idx_t(-1)) {
382 return zaddress_unsafe::null;
383 } else {
384 return ZOffset::address_unsafe(offset_from_bit_index(base_index));
385 }
386 }
387
388 inline zaddress_unsafe ZPage::find_base(volatile zpointer* p) {
389 assert_zpage_mark_state();
390
391 return find_base_unsafe(p);
392 }
393
394 template <typename Function>
395 inline void ZPage::oops_do_remembered(Function function) {
396 _remembered_set.iterate_previous([&](uintptr_t local_offset) {
397 const zoffset offset = start() + local_offset;
398 const zaddress addr = ZOffset::address(offset);
399
400 function((volatile zpointer*)addr);
401 });
402 }
403
404 template <typename Function>
405 inline void ZPage::oops_do_remembered_in_live(Function function) {
406 assert(!is_allocating(), "Must have liveness information");
407 assert(!ZGeneration::old()->is_phase_mark(), "Must have liveness information");
408 assert(is_marked(), "Must have liveness information");
409
410 ZRememberedSetContainingInLiveIterator iter(this);
411 for (ZRememberedSetContaining containing; iter.next(&containing);) {
412 function((volatile zpointer*)containing._field_addr);
413 }
414
415 iter.print_statistics();
416 }
417
418 template <typename Function>
419 inline void ZPage::oops_do_current_remembered(Function function) {
420 _remembered_set.iterate_current([&](uintptr_t local_offset) {
421 const zoffset offset = start() + local_offset;
422 const zaddress addr = ZOffset::address(offset);
423
424 function((volatile zpointer*)addr);
425 });
426 }
427
428 inline zaddress ZPage::alloc_object(size_t size) {
429 assert(is_allocating(), "Invalid state");
430
431 const size_t aligned_size = align_up(size, object_alignment());
432 const zoffset_end addr = top();
433
434 zoffset_end new_top;
435
436 if (!to_zoffset_end(&new_top, addr, aligned_size)) {
437 // Next top would be outside of the heap - bail
438 return zaddress::null;
439 }
440
441 if (new_top > end()) {
442 // Not enough space left in the page
443 return zaddress::null;
444 }
445
446 _top = new_top;
447
448 return ZOffset::address(to_zoffset(addr));
449 }
450
451 inline zaddress ZPage::alloc_object_atomic(size_t size) {
452 assert(is_allocating(), "Invalid state");
453
454 const size_t aligned_size = align_up(size, object_alignment());
455 zoffset_end addr = top();
456
457 for (;;) {
458 zoffset_end new_top;
459
460 if (!to_zoffset_end(&new_top, addr, aligned_size)) {
461 // Next top would be outside of the heap - bail
462 return zaddress::null;
463 }
464
465 if (new_top > end()) {
466 // Not enough space left
467 return zaddress::null;
468 }
469
470 const zoffset_end prev_top = AtomicAccess::cmpxchg(&_top, addr, new_top);
471 if (prev_top == addr) {
472 // Success
473 return ZOffset::address(to_zoffset(addr));
474 }
475
476 // Retry
477 addr = prev_top;
478 }
479 }
480
481 inline bool ZPage::undo_alloc_object(zaddress addr, size_t size) {
482 assert(is_allocating(), "Invalid state");
483
484 const zoffset offset = ZAddress::offset(addr);
485 const size_t aligned_size = align_up(size, object_alignment());
486 const zoffset_end old_top = top();
487 const zoffset_end new_top = old_top - aligned_size;
488
489 if (new_top != offset) {
490 // Failed to undo allocation, not the last allocated object
491 return false;
492 }
493
494 _top = new_top;
495
496 // Success
497 return true;
498 }
499
500 inline bool ZPage::undo_alloc_object_atomic(zaddress addr, size_t size) {
501 assert(is_allocating(), "Invalid state");
502
503 const zoffset offset = ZAddress::offset(addr);
504 const size_t aligned_size = align_up(size, object_alignment());
505 zoffset_end old_top = top();
506
507 for (;;) {
508 const zoffset_end new_top = old_top - aligned_size;
509 if (new_top != offset) {
510 // Failed to undo allocation, not the last allocated object
511 return false;
512 }
513
514 const zoffset_end prev_top = AtomicAccess::cmpxchg(&_top, old_top, new_top);
515 if (prev_top == old_top) {
516 // Success
517 return true;
518 }
519
520 // Retry
521 old_top = prev_top;
522 }
523 }
524
525 inline void ZPage::log_msg(const char* msg_format, ...) const {
526 LogTarget(Trace, gc, page) target;
527 if (target.is_enabled()) {
528 va_list argp;
529 va_start(argp, msg_format);
530 LogStream stream(target);
531 print_on_msg(&stream, err_msg(FormatBufferDummy(), msg_format, argp));
532 va_end(argp);
533 }
534 }
535
536 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP