11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29
30 #include "gc/shenandoah/shenandoahAsserts.hpp"
31 #include "oops/markWord.hpp"
32 #include "runtime/javaThread.hpp"
33
34 inline oop ShenandoahForwarding::get_forwardee_raw(oop obj) {
35 shenandoah_assert_in_heap_bounds(nullptr, obj);
36 return get_forwardee_raw_unchecked(obj);
37 }
38
39 inline oop ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
40 // JVMTI and JFR code use mark words for marking objects for their needs.
41 // On this path, we can encounter the "marked" object, but with null
42 // fwdptr. That object is still not forwarded, and we need to return
43 // the object itself.
44 markWord mark = obj->mark();
45 if (mark.is_marked()) {
46 HeapWord* fwdptr = (HeapWord*) mark.clear_lock_bits().to_pointer();
47 if (fwdptr != nullptr) {
48 return cast_to_oop(fwdptr);
49 }
50 }
51 // Self-forwarded (evacuation failure): the object stays put; the
52 // self-fwd bit is set alongside normal lock bits.
53 return obj;
54 }
55
56 inline oop ShenandoahForwarding::get_forwardee_mutator(oop obj) {
57 // Same as above, but mutator thread cannot ever see null forwardee.
58 shenandoah_assert_correct(nullptr, obj);
59 assert(Thread::current()->is_Java_thread(), "Must be a mutator thread");
60
61 markWord mark = obj->mark();
62 if (mark.is_marked()) {
63 HeapWord* fwdptr = (HeapWord*) mark.clear_lock_bits().to_pointer();
64 assert(fwdptr != nullptr, "Forwarding pointer is never null here");
65 return cast_to_oop(fwdptr);
66 }
67 // Self-forwarded or not forwarded: return the object itself.
68 return obj;
69 }
70
71 inline oop ShenandoahForwarding::get_forwardee(oop obj) {
72 shenandoah_assert_correct(nullptr, obj);
73 return get_forwardee_raw_unchecked(obj);
74 }
75
76 inline bool ShenandoahForwarding::is_forwarded(oop obj) {
77 return obj->mark().is_forwarded();
78 }
79
80 inline bool ShenandoahForwarding::is_self_forwarded(oop obj) {
81 return obj->mark().is_self_forwarded();
82 }
83
84 inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
85 markWord old_mark = obj->mark();
86 if (old_mark.is_marked()) {
87 return cast_to_oop(old_mark.clear_lock_bits().to_pointer());
88 }
89 if (old_mark.is_self_forwarded()) {
90 // Another thread lost the evacuation race; the object stays put.
91 return obj;
92 }
93
94 markWord new_mark = markWord::encode_pointer_as_mark(update);
95 markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
96 if (prev_mark == old_mark) {
97 return update;
98 }
99 // Concurrent writers on a cset object's mark can only be other evacuation
100 // threads installing forwarding (real or self). Mutators cannot reach the
101 // mark of a not-yet-forwarded cset object: LRB + stack watermark barriers
102 // redirect all reference uses before a Java-level operation can touch it.
103 // So the only possible failure modes are a regular forwardee (marked) or
104 // a self-forward (possibly with mutator lock/hash mods layered on top
105 // after the self-forward became visible).
106 if (prev_mark.is_marked()) {
107 return cast_to_oop(prev_mark.clear_lock_bits().to_pointer());
108 }
109 assert(prev_mark.is_self_forwarded(),
110 "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
111 prev_mark.value());
112 return obj;
113 }
114
115 inline oop ShenandoahForwarding::try_forward_to_self(oop obj, markWord old_mark) {
116 assert(!old_mark.is_forwarded(),
117 "caller must pass a non-forwarded mark: old=" INTPTR_FORMAT, old_mark.value());
118 markWord new_mark = old_mark.set_self_forwarded();
119 markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
120 if (prev_mark == old_mark) {
121 // We installed the self-forward.
122 return nullptr;
123 }
124 // Same invariant as in try_update_forwardee: the only races on a
125 // cset object's mark come from other evac threads installing forwarding.
126 if (prev_mark.is_marked()) {
127 return cast_to_oop(prev_mark.clear_lock_bits().to_pointer());
128 }
129 assert(prev_mark.is_self_forwarded(),
130 "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
131 prev_mark.value());
132 return obj;
133 }
134
135 inline Klass* ShenandoahForwarding::klass(oop obj) {
136 if (UseCompactObjectHeaders) {
137 markWord mark = obj->mark();
138 if (mark.is_marked()) {
139 oop fwd = cast_to_oop(mark.clear_lock_bits().to_pointer());
140 mark = fwd->mark();
141 }
142 return mark.klass();
143 } else {
144 return obj->klass();
145 }
146 }
147
148 inline size_t ShenandoahForwarding::size(oop obj) {
149 return obj->size_given_klass(klass(obj));
150 }
151
152 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
|
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29
30 #include "gc/shenandoah/shenandoahAsserts.hpp"
31 #include "oops/klass.hpp"
32 #include "oops/markWord.hpp"
33 #include "runtime/javaThread.hpp"
34
35 inline oop ShenandoahForwarding::get_forwardee_raw(oop obj) {
36 shenandoah_assert_in_heap_bounds(nullptr, obj);
37 return get_forwardee_raw_unchecked(obj);
38 }
39
40 static HeapWord* to_forwardee(markWord mark) {
41 return reinterpret_cast<HeapWord*>(mark.value() & ~(markWord::lock_mask_in_place | markWord::self_fwd_bit_in_place));
42 }
43
44 inline bool ShenandoahForwarding::has_forwardee(markWord m) {
45 // Lock bits == marked_value (0b11): the upper bits encode a forwardee
46 // pointer. Matches normal-forwarded (0b011) and forward-expanded (0b111);
47 // excludes self-forwarded (0b100, 0b101, 0b110).
48 return (m.value() & markWord::lock_mask_in_place) == markWord::marked_value;
49 }
50
51 inline oop ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
52 // JVMTI and JFR code use mark words for marking objects for their needs.
53 // On this path, we can encounter the "marked" object, but with null
54 // fwdptr. That object is still not forwarded, and we need to return
55 // the object itself.
56 markWord mark = obj->mark();
57 if (has_forwardee(mark)) {
58 HeapWord* fwdptr = to_forwardee(mark);
59 if (fwdptr != nullptr) {
60 return cast_to_oop(fwdptr);
61 }
62 }
63 // Self-forwarded (evacuation failure): the object stays put; the
64 // self-fwd bit is set alongside normal lock bits.
65 return obj;
66 }
67
68 inline oop ShenandoahForwarding::get_forwardee_mutator(oop obj) {
69 // Same as above, but mutator thread cannot ever see null forwardee.
70 shenandoah_assert_correct(nullptr, obj);
71 assert(Thread::current()->is_Java_thread(), "Must be a mutator thread");
72
73 markWord mark = obj->mark();
74 if (has_forwardee(mark)) {
75 HeapWord* fwdptr = to_forwardee(mark);
76 assert(fwdptr != nullptr, "Forwarding pointer is never null here");
77 return cast_to_oop(fwdptr);
78 }
79 // Self-forwarded or not forwarded: return the object itself.
80 return obj;
81 }
82
83 inline oop ShenandoahForwarding::get_forwardee(oop obj) {
84 shenandoah_assert_correct(nullptr, obj);
85 return get_forwardee_raw_unchecked(obj);
86 }
87
88 inline bool ShenandoahForwarding::is_forwarded(oop obj) {
89 return obj->mark().is_forwarded();
90 }
91
92 inline bool ShenandoahForwarding::is_self_forwarded(oop obj) {
93 return obj->mark().is_self_forwarded();
94 }
95
96 inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
97 markWord old_mark = obj->mark();
98 if (has_forwardee(old_mark)) {
99 return cast_to_oop(to_forwardee(old_mark));
100 }
101 if (old_mark.is_self_forwarded()) {
102 // Another thread lost the evacuation race; the object stays put.
103 return obj;
104 }
105
106 markWord new_mark = markWord::encode_pointer_as_mark(update);
107 if (UseCompactObjectHeaders && old_mark.is_hashed_not_expanded()) {
108 new_mark = markWord(new_mark.value() | FWDED_HASH_TRANSITION);
109 }
110 markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
111 if (prev_mark == old_mark) {
112 return update;
113 }
114 // Concurrent writers on a cset object's mark can only be other evacuation
115 // threads installing forwarding (real or self). Mutators cannot reach the
116 // mark of a not-yet-forwarded cset object: LRB + stack watermark barriers
117 // redirect all reference uses before a Java-level operation can touch it.
118 // So the only possible failure modes are a regular forwardee (marked) or
119 // a self-forward (possibly with mutator lock/hash mods layered on top
120 // after the self-forward became visible).
121 if (has_forwardee(prev_mark)) {
122 return cast_to_oop(to_forwardee(prev_mark));
123 }
124 assert(prev_mark.is_self_forwarded(),
125 "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
126 prev_mark.value());
127 return obj;
128 }
129
130 inline oop ShenandoahForwarding::try_forward_to_self(oop obj, markWord old_mark) {
131 assert(!old_mark.is_forwarded(),
132 "caller must pass a non-forwarded mark: old=" INTPTR_FORMAT, old_mark.value());
133 markWord new_mark = old_mark.set_self_forwarded();
134 markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
135 if (prev_mark == old_mark) {
136 // We installed the self-forward.
137 return nullptr;
138 }
139 // Same invariant as in try_update_forwardee: the only races on a
140 // cset object's mark come from other evac threads installing forwarding.
141 if (has_forwardee(prev_mark)) {
142 return cast_to_oop(to_forwardee(prev_mark));
143 }
144 assert(prev_mark.is_self_forwarded(),
145 "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
146 prev_mark.value());
147 return obj;
148 }
149
150 inline Klass* ShenandoahForwarding::klass(oop obj) {
151 if (UseCompactObjectHeaders) {
152 markWord mark = obj->mark();
153 if (has_forwardee(mark)) {
154 oop fwd = cast_to_oop(to_forwardee(mark));
155 mark = fwd->mark();
156 }
157 return mark.klass();
158 } else {
159 return obj->klass();
160 }
161 }
162
163 inline size_t ShenandoahForwarding::size(oop obj) {
164 markWord mark = obj->mark();
165 if (has_forwardee(mark)) {
166 oop fwd = cast_to_oop(to_forwardee(mark));
167 markWord fwd_mark = fwd->mark();
168 Klass* klass = UseCompactObjectHeaders ? fwd_mark.klass() : fwd->klass();
169 size_t size = fwd->base_size_given_klass(fwd_mark, klass);
170 if (UseCompactObjectHeaders) {
171 if ((mark.value() & FWDED_HASH_TRANSITION) != FWDED_HASH_TRANSITION) {
172 if (fwd_mark.is_expanded() && klass->expand_for_hash(fwd, fwd_mark)) {
173 size = align_object_size(size + 1);
174 }
175 }
176 }
177 return size;
178 } else {
179 Klass* klass = UseCompactObjectHeaders ? mark.klass() : obj->klass();
180 return obj->size_given_mark_and_klass(mark, klass);
181 }
182 }
183
184 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
|