1 /*
2 * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.inline.hpp"
31 #include "gc/z/zBarrier.inline.hpp"
32 #include "gc/z/zIterator.inline.hpp"
33 #include "gc/z/zNMethod.hpp"
34 #include "memory/iterator.inline.hpp"
35 #include "utilities/debug.hpp"
36
37 template <DecoratorSet decorators, typename BarrierSetT>
38 template <DecoratorSet expected>
39 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
40 if ((decorators & expected) == 0) {
41 fatal("Using unsupported access decorators");
42 }
43 }
44
45 template <DecoratorSet decorators, typename BarrierSetT>
46 template <DecoratorSet expected>
47 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
48 if ((decorators & expected) != 0) {
49 fatal("Using unsupported access decorators");
50 }
51 }
52
53 template <DecoratorSet decorators, typename BarrierSetT>
54 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
55 ShouldNotReachHere();
312 verify_decorators_absent<AS_NO_KEEPALIVE>();
313
314 zpointer* const p = field_addr(base, offset);
315
316 store_barrier_heap_with_healing(p);
317
318 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
319 assert_is_valid(o);
320
321 return to_oop(ZPointer::uncolor_store_good(o));
322 }
323
324 template <DecoratorSet decorators, typename BarrierSetT>
325 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
326 store_barrier_heap_without_healing(dst);
327
328 return ZBarrier::load_barrier_on_oop_field(src);
329 }
330
331 template <DecoratorSet decorators, typename BarrierSetT>
332 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
333 const zaddress obj = oop_copy_one_barriers(dst, src);
334
335 Atomic::store(dst, ZAddress::store_good(obj));
336 }
337
338 template <DecoratorSet decorators, typename BarrierSetT>
339 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
340 const zaddress obj = oop_copy_one_barriers(dst, src);
341
342 if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
343 // Check cast failed
344 return false;
345 }
346
347 Atomic::store(dst, ZAddress::store_good(obj));
348
349 return true;
350 }
351
352
353 template <DecoratorSet decorators, typename BarrierSetT>
354 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
355 // Check cast and copy each elements
356 for (const zpointer* const end = src + length; src < end; src++, dst++) {
357 if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
358 // Check cast failed
359 return false;
360 }
361 }
362
363 return true;
364 }
365
366 template <DecoratorSet decorators, typename BarrierSetT>
367 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
368 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
369
370 if (is_disjoint || src > dst) {
371 for (const zpointer* const end = src + length; src < end; src++, dst++) {
372 oop_copy_one(dst, src);
373 }
374 return true;
375 }
376
377 if (src < dst) {
378 const zpointer* const end = src;
379 src += length - 1;
380 dst += length - 1;
381 for ( ; src >= end; src--, dst--) {
382 oop_copy_one(dst, src);
383 }
384 return true;
385 }
386
387 // src and dst are the same; nothing to do
388 return true;
389 }
390
391 template <DecoratorSet decorators, typename BarrierSetT>
392 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
393 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
394 size_t length) {
395 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
396 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
397
398 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
399 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
400 return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
401 }
402
403 return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
404 }
405
406 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
407 public:
408 virtual void do_oop(oop* p_) {
409 volatile zpointer* const p = (volatile zpointer*)p_;
410 const zpointer ptr = ZBarrier::load_atomic(p);
411 const zaddress addr = ZPointer::uncolor(ptr);
412 Atomic::store(p, ZAddress::store_good(addr));
413 }
414
415 virtual void do_oop(narrowOop* p) {
416 ShouldNotReachHere();
417 }
418 };
419
420 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
421 public:
422 virtual void do_oop(oop* p) {
423 ZBarrier::load_barrier_on_oop_field((zpointer*)p);
440 // and arraycopy sequence, so the performance of this runtime call
441 // does not matter for object arrays.
442 clone_obj_array(objArrayOop(src), objArrayOop(dst));
443 return;
444 }
445
446 // Fix the oops
447 ZLoadBarrierOopClosure cl;
448 ZIterator::oop_iterate(src, &cl);
449
450 // Clone the object
451 Raw::clone_in_heap(src, dst, size);
452
453 assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
454
455 // Color store good before handing out
456 ZColorStoreGoodOopClosure cl_sg;
457 ZIterator::oop_iterate(dst, &cl_sg);
458 }
459
460 //
461 // Not in heap
462 //
463 template <DecoratorSet decorators, typename BarrierSetT>
464 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
465 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
466
467 const zpointer o = Raw::template load<zpointer>(p);
468 assert_is_valid(o);
469 return to_oop(load_barrier(p, o));
470 }
471
472 template <DecoratorSet decorators, typename BarrierSetT>
473 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
474 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
475
476 return oop_load_not_in_heap((zpointer*)p);
477 }
478
479 template <DecoratorSet decorators, typename BarrierSetT>
|
1 /*
2 * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zAddress.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29
30 #include "gc/shared/accessBarrierSupport.inline.hpp"
31 #include "gc/z/zAddress.inline.hpp"
32 #include "gc/z/zBarrier.inline.hpp"
33 #include "gc/z/zIterator.inline.hpp"
34 #include "gc/z/zNMethod.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "oops/inlineKlass.inline.hpp"
37 #include "utilities/debug.hpp"
38 #include "utilities/copy.hpp"
39
40 template <DecoratorSet decorators, typename BarrierSetT>
41 template <DecoratorSet expected>
42 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
43 if ((decorators & expected) == 0) {
44 fatal("Using unsupported access decorators");
45 }
46 }
47
48 template <DecoratorSet decorators, typename BarrierSetT>
49 template <DecoratorSet expected>
50 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
51 if ((decorators & expected) != 0) {
52 fatal("Using unsupported access decorators");
53 }
54 }
55
56 template <DecoratorSet decorators, typename BarrierSetT>
57 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
58 ShouldNotReachHere();
315 verify_decorators_absent<AS_NO_KEEPALIVE>();
316
317 zpointer* const p = field_addr(base, offset);
318
319 store_barrier_heap_with_healing(p);
320
321 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
322 assert_is_valid(o);
323
324 return to_oop(ZPointer::uncolor_store_good(o));
325 }
326
327 template <DecoratorSet decorators, typename BarrierSetT>
328 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
329 store_barrier_heap_without_healing(dst);
330
331 return ZBarrier::load_barrier_on_oop_field(src);
332 }
333
334 template <DecoratorSet decorators, typename BarrierSetT>
335 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
336 const zaddress obj = oop_copy_one_barriers(dst, src);
337
338 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
339 return oop_copy_check_null;
340 }
341
342 Atomic::store(dst, ZAddress::store_good(obj));
343 return oop_copy_check_ok;
344 }
345
346 template <DecoratorSet decorators, typename BarrierSetT>
347 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
348 const zaddress obj = oop_copy_one_barriers(dst, src);
349 const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
350
351 if (null_check && is_null(obj)) {
352 return oop_copy_check_null;
353 }
354 else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
355 // Check cast failed
356 return oop_copy_check_class_cast;
357 }
358
359 Atomic::store(dst, ZAddress::store_good(obj));
360
361 return oop_copy_check_ok;
362 }
363
364
365 template <DecoratorSet decorators, typename BarrierSetT>
366 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
367 // Check cast and copy each elements
368 OopCopyCheckStatus check_status = oop_copy_check_ok;
369 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
370 check_status = oop_copy_one_check_cast(dst, src, dst_klass);
371 }
372 return check_status;
373 }
374
375 template <DecoratorSet decorators, typename BarrierSetT>
376 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
377 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
378 OopCopyCheckStatus check_status = oop_copy_check_ok;
379 if (is_disjoint || src > dst) {
380 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
381 check_status = oop_copy_one(dst, src);
382 }
383 return check_status;
384 }
385
386 if (src < dst) {
387 const zpointer* const end = src;
388 src += length - 1;
389 dst += length - 1;
390 for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
391 check_status = oop_copy_one(dst, src);
392 }
393 return check_status;
394 }
395
396 // src and dst are the same; nothing to do
397 return check_status;
398 }
399
400 template <DecoratorSet decorators, typename BarrierSetT>
401 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
402 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
403 size_t length) {
404 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
405 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
406 OopCopyCheckStatus check_status;
407
408 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
409 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
410 check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
411 } else {
412 check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
413 }
414
415 switch (check_status) {
416 case oop_copy_check_ok:
417 return;
418 case oop_copy_check_class_cast:
419 throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
420 break;
421 case oop_copy_check_null:
422 throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
423 break;
424 default:
425 ShouldNotReachHere();
426 return;
427 }
428 }
429
430 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
431 public:
432 virtual void do_oop(oop* p_) {
433 volatile zpointer* const p = (volatile zpointer*)p_;
434 const zpointer ptr = ZBarrier::load_atomic(p);
435 const zaddress addr = ZPointer::uncolor(ptr);
436 Atomic::store(p, ZAddress::store_good(addr));
437 }
438
439 virtual void do_oop(narrowOop* p) {
440 ShouldNotReachHere();
441 }
442 };
443
444 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
445 public:
446 virtual void do_oop(oop* p) {
447 ZBarrier::load_barrier_on_oop_field((zpointer*)p);
464 // and arraycopy sequence, so the performance of this runtime call
465 // does not matter for object arrays.
466 clone_obj_array(objArrayOop(src), objArrayOop(dst));
467 return;
468 }
469
470 // Fix the oops
471 ZLoadBarrierOopClosure cl;
472 ZIterator::oop_iterate(src, &cl);
473
474 // Clone the object
475 Raw::clone_in_heap(src, dst, size);
476
477 assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
478
479 // Color store good before handing out
480 ZColorStoreGoodOopClosure cl_sg;
481 ZIterator::oop_iterate(dst, &cl_sg);
482 }
483
484 static inline void copy_primitive_payload(const void* src, const void* dst, const size_t payload_size_bytes, size_t& copied_bytes) {
485 if (payload_size_bytes == 0) {
486 return;
487 }
488 void* src_payload = (void*)(address(src) + copied_bytes);
489 void* dst_payload = (void*)(address(dst) + copied_bytes);
490 Copy::copy_value_content(src_payload, dst_payload, payload_size_bytes);
491 copied_bytes += payload_size_bytes;
492 }
493
494 template <DecoratorSet decorators, typename BarrierSetT>
495 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
496 if (md->contains_oops()) {
497 // Iterate over each oop map, performing:
498 // 1) possibly raw copy for any primitive payload before each map
499 // 2) load and store barrier for each oop
500 // 3) possibly raw copy for any primitive payload trailer
501
502 // src/dst may not be oops, need offset to adjust oop map offset
503 const address src_oop_addr_offset = ((address) src) - md->payload_offset();
504 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
505 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
506 size_t size_in_bytes = md->layout_size_in_bytes(lk);
507 size_t copied_bytes = 0;
508 while (map != end) {
509 zpointer *src_p = (zpointer*)(src_oop_addr_offset + map->offset());
510 const uintptr_t oop_offset = uintptr_t(src_p) - uintptr_t(src);
511 zpointer *dst_p = (zpointer*)(uintptr_t(dst) + oop_offset);
512
513 // Copy any leading primitive payload before every cluster of oops
514 assert(copied_bytes < oop_offset || copied_bytes == oop_offset, "Negative sized leading payload segment");
515 copy_primitive_payload(src, dst, oop_offset - copied_bytes, copied_bytes);
516
517 // Copy a cluster of oops
518 for (const zpointer* const src_end = src_p + map->count(); src_p < src_end; src_p++, dst_p++) {
519 oop_copy_one(dst_p, src_p);
520 copied_bytes += sizeof(zpointer);
521 }
522 map++;
523 }
524
525 // Copy trailing primitive payload after potential oops
526 assert(copied_bytes < size_in_bytes || copied_bytes == size_in_bytes, "Negative sized trailing payload segment");
527 copy_primitive_payload(src, dst, size_in_bytes - copied_bytes, copied_bytes);
528 } else {
529 Raw::value_copy_in_heap(src, dst, md, lk);
530 }
531 }
532
533 //
534 // Not in heap
535 //
536 template <DecoratorSet decorators, typename BarrierSetT>
537 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
538 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
539
540 const zpointer o = Raw::template load<zpointer>(p);
541 assert_is_valid(o);
542 return to_oop(load_barrier(p, o));
543 }
544
545 template <DecoratorSet decorators, typename BarrierSetT>
546 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
547 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
548
549 return oop_load_not_in_heap((zpointer*)p);
550 }
551
552 template <DecoratorSet decorators, typename BarrierSetT>
|