1 /*
2 * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.hpp"
31 #include "gc/z/zAddress.inline.hpp"
32 #include "gc/z/zHeap.hpp"
33 #include "gc/z/zNMethod.hpp"
34 #include "gc/z/zUtils.inline.hpp"
35 #include "oops/inlineKlass.inline.hpp"
36 #include "oops/objArrayOop.hpp"
37 #include "utilities/copy.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/globalDefinitions.hpp"
40
41 template <DecoratorSet decorators, typename BarrierSetT>
42 template <DecoratorSet expected>
43 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
44 if ((decorators & expected) == 0) {
45 fatal("Using unsupported access decorators");
46 }
47 }
48
49 template <DecoratorSet decorators, typename BarrierSetT>
50 template <DecoratorSet expected>
51 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
52 if ((decorators & expected) != 0) {
53 fatal("Using unsupported access decorators");
54 }
55 }
56
57 template <DecoratorSet decorators, typename BarrierSetT>
58 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
59 ShouldNotReachHere();
60 }
61
62 template <DecoratorSet decorators, typename BarrierSetT>
63 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
64 assert(base != nullptr, "Invalid base");
65 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
66 }
67
68 template <DecoratorSet decorators, typename BarrierSetT>
69 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
70 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
71
72 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
73 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
74 // Load barriers on strong oop refs don't keep objects alive
75 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
76 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
77 return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
78 } else {
79 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
80 return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
81 }
82 } else {
83 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
84 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
85 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
86 return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
87 } else {
88 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
89 return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
90 }
91 }
92 }
93
94 template <DecoratorSet decorators, typename BarrierSetT>
95 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
96 verify_decorators_present<ON_UNKNOWN_OOP_REF>();
97
98 const DecoratorSet decorators_known_strength =
99 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
100
101 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
102 if (decorators_known_strength & ON_STRONG_OOP_REF) {
103 // Load barriers on strong oop refs don't keep objects alive
104 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
105 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
106 return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
107 } else {
108 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
109 return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
110 }
111 } else {
112 if (decorators_known_strength & ON_STRONG_OOP_REF) {
113 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
114 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
115 return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
116 } else {
117 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
118 return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
119 }
120 }
121 }
122
123 inline zpointer ZBarrierSet::store_good(oop obj) {
124 assert(ZPointerStoreGoodMask != 0, "sanity");
125
126 const zaddress addr = to_zaddress(obj);
127 return ZAddress::store_good(addr);
128 }
129
130 template <DecoratorSet decorators, typename BarrierSetT>
131 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
132 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
133 ZBarrierSet::store_barrier_on_heap_oop_field(p, true /* heal */);
134 } else {
135 assert(false, "Should not be used on uninitialized memory");
136 }
137 }
138
139 template <DecoratorSet decorators, typename BarrierSetT>
140 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
141 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
142 ZBarrierSet::store_barrier_on_heap_oop_field(p, false /* heal */);
143 }
144 }
145
146 template <DecoratorSet decorators, typename BarrierSetT>
147 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
148 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
149 ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(p);
150 }
151 }
152
153 template <DecoratorSet decorators, typename BarrierSetT>
154 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
155 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
156 ZBarrierSet::store_barrier_on_native_oop_field(p, true /* heal */);
157 } else {
158 assert(false, "Should not be used on uninitialized memory");
159 }
160 }
161
162 template <DecoratorSet decorators, typename BarrierSetT>
163 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
164 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
165 ZBarrierSet::store_barrier_on_native_oop_field(p, false /* heal */);
166 }
167 }
168
169 //
170 // In heap
171 //
172 template <DecoratorSet decorators, typename BarrierSetT>
173 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
174 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
175
176 const zpointer o = Raw::load_in_heap(p);
177 assert_is_valid(o);
178
179 return to_oop(load_barrier(p, o));
180 }
181
182 template <DecoratorSet decorators, typename BarrierSetT>
183 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
184 zpointer* const p = field_addr(base, offset);
185
186 const zpointer o = Raw::load_in_heap(p);
187 assert_is_valid(o);
188
189 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
190 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
191 }
192
193 return to_oop(load_barrier(p, o));
194 }
195
196 template <DecoratorSet decorators>
197 bool is_store_barrier_no_keep_alive() {
198 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
199 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
200 }
201
202 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
203 return true;
204 }
205
206 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
207 return true;
208 }
209
210 template <DecoratorSet decorators>
211 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
212 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
213 return is_store_barrier_no_keep_alive<decorators>();
214 }
215
216 const DecoratorSet decorators_known_strength =
217 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
218
219 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
220 return (decorators & AS_NO_KEEPALIVE) != 0;
221 }
222
223 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
224 return true;
225 }
226
227 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
228 return true;
229 }
230
231 template <DecoratorSet decorators, typename BarrierSetT>
232 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
233 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
234
235 if (is_store_barrier_no_keep_alive<decorators>()) {
236 no_keep_alive_store_barrier_heap(p);
237 } else {
238 store_barrier_heap_without_healing(p);
239 }
240
241 Raw::store_in_heap(p, store_good(value));
242 }
243
244 template <DecoratorSet decorators, typename BarrierSetT>
245 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
246 zpointer* const p = field_addr(base, offset);
247
248 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
249 no_keep_alive_store_barrier_heap(p);
250 } else {
251 store_barrier_heap_without_healing(p);
252 }
253
254 Raw::store_in_heap(p, store_good(value));
255 }
256
257 template <DecoratorSet decorators, typename BarrierSetT>
258 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
259 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
260
261 if (!is_store_barrier_no_keep_alive<decorators>()) {
262 store_barrier_native_without_healing(p);
263 }
264
265 Raw::store(p, store_good(value));
266 }
267
268 template <DecoratorSet decorators, typename BarrierSetT>
269 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
270 verify_decorators_present<ON_STRONG_OOP_REF>();
271 verify_decorators_absent<AS_NO_KEEPALIVE>();
272
273 store_barrier_heap_with_healing(p);
274
275 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
276 assert_is_valid(o);
277
278 return to_oop(ZPointer::uncolor_store_good(o));
279 }
280
281 template <DecoratorSet decorators, typename BarrierSetT>
282 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
283 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
284 verify_decorators_absent<AS_NO_KEEPALIVE>();
285
286 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
287 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
288 // with the motivation that if you're doing Unsafe operations on a Reference.referent
289 // field, then you're on your own anyway.
290 zpointer* const p = field_addr(base, offset);
291
292 store_barrier_heap_with_healing(p);
293
294 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
295 assert_is_valid(o);
296
297 return to_oop(ZPointer::uncolor_store_good(o));
298 }
299
300 template <DecoratorSet decorators, typename BarrierSetT>
301 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
302 verify_decorators_present<ON_STRONG_OOP_REF>();
303 verify_decorators_absent<AS_NO_KEEPALIVE>();
304
305 store_barrier_heap_with_healing(p);
306
307 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
308 assert_is_valid(o);
309
310 return to_oop(ZPointer::uncolor_store_good(o));
311 }
312
313 template <DecoratorSet decorators, typename BarrierSetT>
314 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
315 verify_decorators_present<ON_STRONG_OOP_REF>();
316 verify_decorators_absent<AS_NO_KEEPALIVE>();
317
318 zpointer* const p = field_addr(base, offset);
319
320 store_barrier_heap_with_healing(p);
321
322 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
323 assert_is_valid(o);
324
325 return to_oop(ZPointer::uncolor_store_good(o));
326 }
327
328 template <DecoratorSet decorators, typename BarrierSetT>
329 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
330 store_barrier_heap_without_healing(dst);
331
332 return ZBarrierSet::load_barrier_on_oop_field(src);
333 }
334
335 template <DecoratorSet decorators, typename BarrierSetT>
336 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
337 const zaddress obj = oop_copy_one_barriers(dst, src);
338
339 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
340 return OopCopyResult::failed_check_null;
341 }
342
343 AtomicAccess::store(dst, ZAddress::store_good(obj));
344
345 return OopCopyResult::ok;
346 }
347
348 template <DecoratorSet decorators, typename BarrierSetT>
349 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_clear_one(zpointer* dst) {
350 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value) {
351 return OopCopyResult::failed_check_null;
352 }
353
354 // Store barrier
355 store_barrier_heap_without_healing(dst);
356
357 // Store colored null
358 AtomicAccess::store(dst, color_null());
359
360 return OopCopyResult::ok;
361 }
362
363 template <DecoratorSet decorators, typename BarrierSetT>
364 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
365 const zaddress obj = oop_copy_one_barriers(dst, src);
366
367 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
368 return OopCopyResult::failed_check_null;
369 }
370
371 if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
372 // Check cast failed
373 return OopCopyResult::failed_check_class_cast;
374 }
375
376 AtomicAccess::store(dst, ZAddress::store_good(obj));
377
378 return OopCopyResult::ok;
379 }
380
381 template <DecoratorSet decorators, typename BarrierSetT>
382 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
383 // Check cast and copy each elements
384 for (const zpointer* const end = src + length; src < end; src++, dst++) {
385 const OopCopyResult result = oop_copy_one_check_cast(dst, src, dst_klass);
386 if (result != OopCopyResult::ok) {
387 return result;
388 }
389 }
390
391 return OopCopyResult::ok;
392 }
393
394 template <DecoratorSet decorators, typename BarrierSetT>
395 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
396 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
397
398 if (is_disjoint || src > dst) {
399 for (const zpointer* const end = src + length; src < end; src++, dst++) {
400 const OopCopyResult result = oop_copy_one(dst, src);
401 if (result != OopCopyResult::ok) {
402 return result;
403 }
404 }
405
406 return OopCopyResult::ok;
407 }
408
409 if (src < dst) {
410 const zpointer* const end = src;
411 src += length - 1;
412 dst += length - 1;
413 for ( ; src >= end; src--, dst--) {
414 const OopCopyResult result = oop_copy_one(dst, src);
415 if (result != OopCopyResult::ok) {
416 return result;
417 }
418 }
419
420 return OopCopyResult::ok;
421 }
422
423 // src and dst are the same; nothing to do
424 return OopCopyResult::ok;
425 }
426
427 template <DecoratorSet decorators, typename BarrierSetT>
428 inline OopCopyResult ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
429 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
430 size_t length) {
431 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
432 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
433
434 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
435 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
436 return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
437 } else {
438 return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
439 }
440 }
441
442 template <DecoratorSet decorators, typename BarrierSetT>
443 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
444 check_is_valid_zaddress(src);
445 check_is_valid_zaddress(dst);
446 precond(src->klass() == dst->klass());
447
448 clone_obj(to_zaddress(src), to_zaddress(dst), ZUtils::words_to_bytes(size));
449 }
450
451 static inline void copy_primitive_payload(const void* src, const void* dst, const size_t payload_size_bytes, size_t& copied_bytes) {
452 if (payload_size_bytes == 0) {
453 return;
454 }
455
456 void* src_payload = (void*)(address(src) + copied_bytes);
457 void* dst_payload = (void*)(address(dst) + copied_bytes);
458 Copy::copy_value_content(src_payload, dst_payload, payload_size_bytes);
459 copied_bytes += payload_size_bytes;
460 }
461
462 static inline void clear_primitive_payload(const void* dst, const size_t payload_size_bytes, size_t& copied_bytes) {
463 if (payload_size_bytes == 0) {
464 return;
465 }
466
467 void* dst_payload = (void*)(address(dst) + copied_bytes);
468 Copy::fill_to_memory_atomic(dst_payload, payload_size_bytes);
469 copied_bytes += payload_size_bytes;
470 }
471
472 template <DecoratorSet decorators, typename BarrierSetT>
473 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(const ValuePayload& src, const ValuePayload& dst) {
474 precond(src.klass() == dst.klass());
475
476 const LayoutKind lk = LayoutKindHelper::get_copy_layout(src.layout_kind(), dst.layout_kind());
477 const InlineKlass* md = src.klass();
478 if (md->contains_oops()) {
479 assert(!LayoutKindHelper::is_atomic_flat(lk) ||
480 (md->nonstatic_oop_map_count() == 1 &&
481 md->layout_size_in_bytes(lk) == sizeof(zpointer)),
482 "ZGC can only handle atomic flat values with a single oop");
483
484 // Iterate over each oop map, performing:
485 // 1) possibly raw copy for any primitive payload before each map
486 // 2) load and store barrier for each oop
487 // 3) possibly raw copy for any primitive payload trailer
488
489 // addr() points at the payload start, the oop map offset are relative to
490 // the object header, adjust address to account for this discrepancy.
491 const address src_addr = src.addr();
492 const address dst_addr = dst.addr();
493 const address oop_map_adjusted_src_addr = src_addr - md->payload_offset();
494 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
495 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
496 size_t size_in_bytes = md->layout_size_in_bytes(lk);
497 size_t copied_bytes = 0;
498 while (map != end) {
499 zpointer* src_p = (zpointer*)(oop_map_adjusted_src_addr + map->offset());
500 const uintptr_t oop_offset = uintptr_t(src_p) - uintptr_t(src_addr);
501 zpointer* dst_p = (zpointer*)(uintptr_t(dst_addr) + oop_offset);
502
503 // Copy any leading primitive payload before every cluster of oops
504 assert(copied_bytes < oop_offset || copied_bytes == oop_offset, "Negative sized leading payload segment");
505 copy_primitive_payload(src_addr, dst_addr, oop_offset - copied_bytes, copied_bytes);
506
507 // Copy a cluster of oops
508 for (const zpointer* const src_end = src_p + map->count(); src_p < src_end; src_p++, dst_p++) {
509 oop_copy_one(dst_p, src_p);
510 copied_bytes += sizeof(zpointer);
511 }
512 map++;
513 }
514
515 // Copy trailing primitive payload after potential oops
516 assert(copied_bytes < size_in_bytes || copied_bytes == size_in_bytes, "Negative sized trailing payload segment");
517 copy_primitive_payload(src_addr, dst_addr, size_in_bytes - copied_bytes, copied_bytes);
518 } else {
519 Raw::value_copy_in_heap(src, dst);
520 }
521 }
522
523 template <DecoratorSet decorators, typename BarrierSetT>
524 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_store_null_in_heap(const ValuePayload& dst) {
525 const LayoutKind lk = dst.layout_kind();
526 assert(!LayoutKindHelper::is_null_free_flat(lk), "Cannot store null in null free layout");
527 const InlineKlass* md = dst.klass();
528
529 if (md->contains_oops()) {
530 assert(!LayoutKindHelper::is_atomic_flat(lk) ||
531 (md->nonstatic_oop_map_count() == 1 &&
532 md->layout_size_in_bytes(lk) == sizeof(zpointer)),
533 "ZGC can only handle atomic flat values with a single oop");
534
535 // Iterate over each oop map, performing:
536 // 1) possibly raw clear for any primitive payload before each map
537 // 2) store barrier and clear for each oop
538 // 3) possibly raw clear for any primitive payload trailer
539
540 // addr() points at the payload start, the oop map offset are relative to
541 // the object header, adjust address to account for this discrepancy.
542 const address dst_addr = dst.addr();
543 const address oop_map_adjusted_dst_addr = dst_addr - md->payload_offset();
544 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
545 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
546 size_t size_in_bytes = md->layout_size_in_bytes(lk);
547 size_t copied_bytes = 0;
548 while (map != end) {
549 zpointer* dst_p = (zpointer*)(oop_map_adjusted_dst_addr + map->offset());
550 const uintptr_t oop_offset = uintptr_t(dst_p) - uintptr_t(dst_addr);
551
552 // Clear any leading primitive payload before every cluster of oops
553 assert(copied_bytes < oop_offset || copied_bytes == oop_offset, "Negative sized leading payload segment");
554 clear_primitive_payload(dst_addr, oop_offset - copied_bytes, copied_bytes);
555
556 // Clear a cluster of oops
557 for (const zpointer* const dst_end = dst_p + map->count(); dst_p < dst_end; dst_p++) {
558 oop_clear_one(dst_p);
559 copied_bytes += sizeof(zpointer);
560 }
561 map++;
562 }
563
564 // Clear trailing primitive payload after potential oops
565 assert(copied_bytes < size_in_bytes || copied_bytes == size_in_bytes, "Negative sized trailing payload segment");
566 clear_primitive_payload(dst_addr, size_in_bytes - copied_bytes, copied_bytes);
567 } else {
568 Raw::value_store_null(dst);
569 }
570 }
571
572 //
573 // Not in heap
574 //
575 template <DecoratorSet decorators, typename BarrierSetT>
576 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
577 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
578
579 const zpointer o = Raw::template load<zpointer>(p);
580 assert_is_valid(o);
581 return to_oop(load_barrier(p, o));
582 }
583
584 template <DecoratorSet decorators, typename BarrierSetT>
585 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
586 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
587
588 return oop_load_not_in_heap((zpointer*)p);
589 }
590
591 template <DecoratorSet decorators, typename BarrierSetT>
592 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
593 verify_decorators_present<ON_STRONG_OOP_REF>();
594 verify_decorators_absent<AS_NO_KEEPALIVE>();
595
596 store_barrier_native_with_healing(p);
597
598 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
599 assert_is_valid(o);
600
601 return to_oop(ZPointer::uncolor_store_good(o));
602 }
603
604 template <DecoratorSet decorators, typename BarrierSetT>
605 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
606 verify_decorators_present<ON_STRONG_OOP_REF>();
607 verify_decorators_absent<AS_NO_KEEPALIVE>();
608
609 store_barrier_native_with_healing(p);
610
611 const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
612 assert_is_valid(o);
613
614 return to_oop(ZPointer::uncolor_store_good(o));
615 }
616
617 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP