1 /*
2 * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25 #ifndef SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP
26 #define SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP
27
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/slidingForwarding.hpp"
30 #include "oops/markWord.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "utilities/macros.hpp"
33
34 inline bool SlidingForwarding::is_forwarded(oop obj) {
35 return obj->is_forwarded();
36 }
37
38 inline bool SlidingForwarding::is_not_forwarded(oop obj) {
39 return !obj->is_forwarded();
40 }
41
42 size_t SlidingForwarding::biased_region_index_containing(HeapWord* addr) {
43 return (uintptr_t)addr >> _region_size_bytes_shift;
44 }
45
46 uintptr_t SlidingForwarding::encode_forwarding(HeapWord* from, HeapWord* to) {
47 static_assert(NUM_TARGET_REGIONS == 2, "Only implemented for this amount");
48
49 size_t from_reg_idx = biased_region_index_containing(from);
50 HeapWord* to_region_base = (HeapWord*)((uintptr_t)to & _region_mask);
51
52 HeapWord** base = &_biased_bases[0][from_reg_idx];
53 uintptr_t alternate = 0;
54 if (*base == to_region_base) {
55 // Primary is good
56 } else if (*base == UNUSED_BASE) {
57 // Primary is free
58 *base = to_region_base;
59 } else {
60 base = &_biased_bases[1][from_reg_idx];
61 if (*base == to_region_base) {
62 // Alternate is good
63 } else if (*base == UNUSED_BASE) {
64 // Alternate is free
65 *base = to_region_base;
66 } else {
67 // Both primary and alternate are not fitting
68 // This happens only in the following rare situations:
69 // - In Serial GC, sometimes when compact-top switches spaces, because the
70 // region boudaries are virtual and objects can cross regions
71 // - In G1 serial compaction, because tails of various compaction chains
72 // are distributed across the remainders of already compacted regions.
73 return (1 << FALLBACK_SHIFT) | markWord::marked_value;
74 }
75 alternate = 1;
76 }
77
78 size_t offset = pointer_delta(to, to_region_base);
79 assert(offset < _region_size_words, "Offset should be within the region. from: " PTR_FORMAT
80 ", to: " PTR_FORMAT ", to_region_base: " PTR_FORMAT ", offset: " SIZE_FORMAT,
81 p2i(from), p2i(to), p2i(to_region_base), offset);
82
83 uintptr_t encoded = (offset << OFFSET_BITS_SHIFT) |
84 (alternate << ALT_REGION_SHIFT) |
85 markWord::marked_value;
86
87 assert(to == decode_forwarding(from, encoded), "must be reversible");
88 assert((encoded & ~MARK_LOWER_HALF_MASK) == 0, "must encode to lowest 32 bits");
89 return encoded;
90 }
91
92 HeapWord* SlidingForwarding::decode_forwarding(HeapWord* from, uintptr_t encoded) {
93 assert((encoded & markWord::lock_mask_in_place) == markWord::marked_value, "must be marked as forwarded");
94 assert((encoded & FALLBACK_MASK) == 0, "must not be fallback-forwarded");
95 assert((encoded & ~MARK_LOWER_HALF_MASK) == 0, "must decode from lowest 32 bits");
96 size_t alternate = (encoded >> ALT_REGION_SHIFT) & right_n_bits(ALT_REGION_BITS);
97 assert(alternate < NUM_TARGET_REGIONS, "Sanity");
98 uintptr_t offset = (encoded >> OFFSET_BITS_SHIFT);
99
100 size_t from_idx = biased_region_index_containing(from);
101 HeapWord* base = _biased_bases[alternate][from_idx];
102 assert(base != UNUSED_BASE, "must not be unused base");
103 HeapWord* decoded = base + offset;
104 assert(decoded >= _heap_start,
105 "Address must be above heap start. encoded: " INTPTR_FORMAT ", alt_region: " SIZE_FORMAT ", base: " PTR_FORMAT,
106 encoded, alternate, p2i(base));
107
108 return decoded;
109 }
110
111 inline void SlidingForwarding::forward_to_impl(oop from, oop to) {
112 assert(_bases_table != nullptr, "call begin() before forwarding");
113
114 markWord from_header = from->mark();
115 if (from_header.has_displaced_mark_helper()) {
116 from_header = from_header.displaced_mark_helper();
117 }
118
119 HeapWord* from_hw = cast_from_oop<HeapWord*>(from);
120 HeapWord* to_hw = cast_from_oop<HeapWord*>(to);
121 uintptr_t encoded = encode_forwarding(from_hw, to_hw);
122 markWord new_header = markWord((from_header.value() & ~MARK_LOWER_HALF_MASK) | encoded);
123 from->set_mark(new_header);
124
125 if ((encoded & FALLBACK_MASK) != 0) {
126 fallback_forward_to(from_hw, to_hw);
127 }
128 }
129
130 template <bool ALT_FWD>
131 inline void SlidingForwarding::forward_to(oop obj, oop fwd) {
132 #ifdef _LP64
133 if (ALT_FWD) {
134 assert(_bases_table != nullptr, "expect sliding forwarding initialized");
135 forward_to_impl(obj, fwd);
136 assert(forwardee<ALT_FWD>(obj) == fwd, "must be forwarded to correct forwardee");
137 } else
138 #endif
139 {
140 obj->forward_to(fwd);
141 }
142 }
143
144 inline oop SlidingForwarding::forwardee_impl(oop from) {
145 assert(_bases_table != nullptr, "call begin() before asking for forwarding");
146
147 markWord header = from->mark();
148 HeapWord* from_hw = cast_from_oop<HeapWord*>(from);
149 if ((header.value() & FALLBACK_MASK) != 0) {
150 HeapWord* to = fallback_forwardee(from_hw);
151 return cast_to_oop(to);
152 }
153 uintptr_t encoded = header.value() & MARK_LOWER_HALF_MASK;
154 HeapWord* to = decode_forwarding(from_hw, encoded);
155 return cast_to_oop(to);
156 }
157
158 template <bool ALT_FWD>
159 inline oop SlidingForwarding::forwardee(oop obj) {
160 #ifdef _LP64
161 if (ALT_FWD) {
162 assert(_bases_table != nullptr, "expect sliding forwarding initialized");
163 return forwardee_impl(obj);
164 } else
165 #endif
166 {
167 return obj->forwardee();
168 }
169 }
170
171 #endif // SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP