10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.inline.hpp"
31 #include "gc/z/zHeap.hpp"
32 #include "gc/z/zNMethod.hpp"
33 #include "oops/objArrayOop.hpp"
34 #include "utilities/debug.hpp"
35
36 template <DecoratorSet decorators, typename BarrierSetT>
37 template <DecoratorSet expected>
38 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
39 if ((decorators & expected) == 0) {
40 fatal("Using unsupported access decorators");
41 }
42 }
43
44 template <DecoratorSet decorators, typename BarrierSetT>
45 template <DecoratorSet expected>
46 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
47 if ((decorators & expected) != 0) {
48 fatal("Using unsupported access decorators");
49 }
50 }
51
52 template <DecoratorSet decorators, typename BarrierSetT>
53 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
311 verify_decorators_absent<AS_NO_KEEPALIVE>();
312
313 zpointer* const p = field_addr(base, offset);
314
315 store_barrier_heap_with_healing(p);
316
317 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
318 assert_is_valid(o);
319
320 return to_oop(ZPointer::uncolor_store_good(o));
321 }
322
323 template <DecoratorSet decorators, typename BarrierSetT>
324 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
325 store_barrier_heap_without_healing(dst);
326
327 return ZBarrierSet::load_barrier_on_oop_field(src);
328 }
329
330 template <DecoratorSet decorators, typename BarrierSetT>
331 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
332 const zaddress obj = oop_copy_one_barriers(dst, src);
333
334 AtomicAccess::store(dst, ZAddress::store_good(obj));
335 }
336
337 template <DecoratorSet decorators, typename BarrierSetT>
338 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
339 const zaddress obj = oop_copy_one_barriers(dst, src);
340
341 if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
342 // Check cast failed
343 return false;
344 }
345
346 AtomicAccess::store(dst, ZAddress::store_good(obj));
347
348 return true;
349 }
350
351 template <DecoratorSet decorators, typename BarrierSetT>
352 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
353 // Check cast and copy each elements
354 for (const zpointer* const end = src + length; src < end; src++, dst++) {
355 if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
356 // Check cast failed
357 return false;
358 }
359 }
360
361 return true;
362 }
363
364 template <DecoratorSet decorators, typename BarrierSetT>
365 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
366 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
367
368 if (is_disjoint || src > dst) {
369 for (const zpointer* const end = src + length; src < end; src++, dst++) {
370 oop_copy_one(dst, src);
371 }
372 return true;
373 }
374
375 if (src < dst) {
376 const zpointer* const end = src;
377 src += length - 1;
378 dst += length - 1;
379 for ( ; src >= end; src--, dst--) {
380 oop_copy_one(dst, src);
381 }
382 return true;
383 }
384
385 // src and dst are the same; nothing to do
386 return true;
387 }
388
389 template <DecoratorSet decorators, typename BarrierSetT>
390 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
391 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
392 size_t length) {
393 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
394 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
395
396 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
397 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
398 return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
399 }
400
401 return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
402 }
403
404 template <DecoratorSet decorators, typename BarrierSetT>
405 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
406 check_is_valid_zaddress(src);
407
408 if (dst->is_objArray()) {
409 // Cloning an object array is similar to performing array copy.
410 // If an array is large enough to have its allocation segmented,
411 // this operation might require GC barriers. However, the intrinsics
412 // for cloning arrays transform the clone to an optimized allocation
413 // and arraycopy sequence, so the performance of this runtime call
414 // does not matter for object arrays.
415 clone_obj_array(objArrayOop(src), objArrayOop(dst));
416 return;
417 }
418
419 // Fix the oops
420 ZBarrierSet::load_barrier_all(src, size);
421
422 // Clone the object
423 Raw::clone_in_heap(src, dst, size);
424
425 // Color store good before handing out
426 ZBarrierSet::color_store_good_all(dst, size);
427 }
428
429 //
430 // Not in heap
431 //
432 template <DecoratorSet decorators, typename BarrierSetT>
433 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
434 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
435
436 const zpointer o = Raw::template load<zpointer>(p);
437 assert_is_valid(o);
438 return to_oop(load_barrier(p, o));
439 }
440
441 template <DecoratorSet decorators, typename BarrierSetT>
442 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
443 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
444
445 return oop_load_not_in_heap((zpointer*)p);
446 }
447
448 template <DecoratorSet decorators, typename BarrierSetT>
|
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.hpp"
31 #include "gc/z/zAddress.inline.hpp"
32 #include "gc/z/zHeap.hpp"
33 #include "gc/z/zNMethod.hpp"
34 #include "oops/inlineKlass.inline.hpp"
35 #include "oops/objArrayOop.hpp"
36 #include "utilities/copy.hpp"
37 #include "utilities/debug.hpp"
38
39 template <DecoratorSet decorators, typename BarrierSetT>
40 template <DecoratorSet expected>
41 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
42 if ((decorators & expected) == 0) {
43 fatal("Using unsupported access decorators");
44 }
45 }
46
47 template <DecoratorSet decorators, typename BarrierSetT>
48 template <DecoratorSet expected>
49 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
50 if ((decorators & expected) != 0) {
51 fatal("Using unsupported access decorators");
52 }
53 }
54
55 template <DecoratorSet decorators, typename BarrierSetT>
56 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
314 verify_decorators_absent<AS_NO_KEEPALIVE>();
315
316 zpointer* const p = field_addr(base, offset);
317
318 store_barrier_heap_with_healing(p);
319
320 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
321 assert_is_valid(o);
322
323 return to_oop(ZPointer::uncolor_store_good(o));
324 }
325
326 template <DecoratorSet decorators, typename BarrierSetT>
327 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
328 store_barrier_heap_without_healing(dst);
329
330 return ZBarrierSet::load_barrier_on_oop_field(src);
331 }
332
333 template <DecoratorSet decorators, typename BarrierSetT>
334 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
335 const zaddress obj = oop_copy_one_barriers(dst, src);
336
337 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
338 return oop_copy_check_null;
339 }
340
341 AtomicAccess::store(dst, ZAddress::store_good(obj));
342 return oop_copy_check_ok;
343 }
344
345 template <DecoratorSet decorators, typename BarrierSetT>
346 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
347 const zaddress obj = oop_copy_one_barriers(dst, src);
348 const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
349
350 if (null_check && is_null(obj)) {
351 return oop_copy_check_null;
352 }
353 else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
354 // Check cast failed
355 return oop_copy_check_class_cast;
356 }
357
358 AtomicAccess::store(dst, ZAddress::store_good(obj));
359
360 return oop_copy_check_ok;
361 }
362
363 template <DecoratorSet decorators, typename BarrierSetT>
364 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
365 // Check cast and copy each elements
366 OopCopyCheckStatus check_status = oop_copy_check_ok;
367 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
368 check_status = oop_copy_one_check_cast(dst, src, dst_klass);
369 }
370 return check_status;
371 }
372
373 template <DecoratorSet decorators, typename BarrierSetT>
374 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
375 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
376 OopCopyCheckStatus check_status = oop_copy_check_ok;
377 if (is_disjoint || src > dst) {
378 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
379 check_status = oop_copy_one(dst, src);
380 }
381 return check_status;
382 }
383
384 if (src < dst) {
385 const zpointer* const end = src;
386 src += length - 1;
387 dst += length - 1;
388 for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
389 check_status = oop_copy_one(dst, src);
390 }
391 return check_status;
392 }
393
394 // src and dst are the same; nothing to do
395 return check_status;
396 }
397
398 template <DecoratorSet decorators, typename BarrierSetT>
399 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
400 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
401 size_t length) {
402 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
403 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
404 OopCopyCheckStatus check_status;
405
406 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
407 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
408 check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
409 } else {
410 check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
411 }
412
413 switch (check_status) {
414 case oop_copy_check_ok:
415 return;
416 case oop_copy_check_class_cast:
417 throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
418 break;
419 case oop_copy_check_null:
420 throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
421 break;
422 default:
423 ShouldNotReachHere();
424 return;
425 }
426 }
427
428 template <DecoratorSet decorators, typename BarrierSetT>
429 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
430 check_is_valid_zaddress(src);
431
432 if (dst->is_refArray()) {
433 // Cloning an object array is similar to performing array copy.
434 // If an array is large enough to have its allocation segmented,
435 // this operation might require GC barriers. However, the intrinsics
436 // for cloning arrays transform the clone to an optimized allocation
437 // and arraycopy sequence, so the performance of this runtime call
438 // does not matter for object arrays.
439 clone_obj_array(objArrayOop(src), objArrayOop(dst));
440 return;
441 }
442
443 // Fix the oops
444 ZBarrierSet::load_barrier_all(src, size);
445
446 // Clone the object
447 Raw::clone_in_heap(src, dst, size);
448
449 // Color store good before handing out
450 ZBarrierSet::color_store_good_all(dst, size);
451 }
452
453 static inline void copy_primitive_payload(const void* src, const void* dst, const size_t payload_size_bytes, size_t& copied_bytes) {
454 if (payload_size_bytes == 0) {
455 return;
456 }
457 void* src_payload = (void*)(address(src) + copied_bytes);
458 void* dst_payload = (void*)(address(dst) + copied_bytes);
459 Copy::copy_value_content(src_payload, dst_payload, payload_size_bytes);
460 copied_bytes += payload_size_bytes;
461 }
462
463 template <DecoratorSet decorators, typename BarrierSetT>
464 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
465 if (md->contains_oops()) {
466 // Iterate over each oop map, performing:
467 // 1) possibly raw copy for any primitive payload before each map
468 // 2) load and store barrier for each oop
469 // 3) possibly raw copy for any primitive payload trailer
470
471 // src/dst may not be oops, need offset to adjust oop map offset
472 const address src_oop_addr_offset = ((address) src) - md->payload_offset();
473 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
474 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
475 size_t size_in_bytes = md->layout_size_in_bytes(lk);
476 size_t copied_bytes = 0;
477 while (map != end) {
478 zpointer *src_p = (zpointer*)(src_oop_addr_offset + map->offset());
479 const uintptr_t oop_offset = uintptr_t(src_p) - uintptr_t(src);
480 zpointer *dst_p = (zpointer*)(uintptr_t(dst) + oop_offset);
481
482 // Copy any leading primitive payload before every cluster of oops
483 assert(copied_bytes < oop_offset || copied_bytes == oop_offset, "Negative sized leading payload segment");
484 copy_primitive_payload(src, dst, oop_offset - copied_bytes, copied_bytes);
485
486 // Copy a cluster of oops
487 for (const zpointer* const src_end = src_p + map->count(); src_p < src_end; src_p++, dst_p++) {
488 oop_copy_one(dst_p, src_p);
489 copied_bytes += sizeof(zpointer);
490 }
491 map++;
492 }
493
494 // Copy trailing primitive payload after potential oops
495 assert(copied_bytes < size_in_bytes || copied_bytes == size_in_bytes, "Negative sized trailing payload segment");
496 copy_primitive_payload(src, dst, size_in_bytes - copied_bytes, copied_bytes);
497 } else {
498 Raw::value_copy_in_heap(src, dst, md, lk);
499 }
500 }
501
502 //
503 // Not in heap
504 //
505 template <DecoratorSet decorators, typename BarrierSetT>
506 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
507 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
508
509 const zpointer o = Raw::template load<zpointer>(p);
510 assert_is_valid(o);
511 return to_oop(load_barrier(p, o));
512 }
513
514 template <DecoratorSet decorators, typename BarrierSetT>
515 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
516 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
517
518 return oop_load_not_in_heap((zpointer*)p);
519 }
520
521 template <DecoratorSet decorators, typename BarrierSetT>
|