1 /* 2 * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/z/zThreadLocalData.hpp" 26 #include "gc/z/zObjArrayAllocator.hpp" 27 #include "gc/z/zUtils.inline.hpp" 28 #include "oops/arrayKlass.hpp" 29 #include "runtime/interfaceSupport.inline.hpp" 30 #include "utilities/debug.hpp" 31 32 ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread) 33 : ObjArrayAllocator(klass, word_size, length, do_zero, thread) {} 34 35 void ZObjArrayAllocator::yield_for_safepoint() const { 36 ThreadBlockInVM tbivm(JavaThread::cast(_thread)); 37 } 38 39 oop ZObjArrayAllocator::initialize(HeapWord* mem) const { 40 // ZGC specializes the initialization by performing segmented clearing 41 // to allow shorter time-to-safepoints. 42 43 if (!_do_zero) { 44 // No need for ZGC specialization 45 return ObjArrayAllocator::initialize(mem); 46 } 47 48 // A max segment size of 64K was chosen because microbenchmarking 49 // suggested that it offered a good trade-off between allocation 50 // time and time-to-safepoint 51 const size_t segment_max = ZUtils::bytes_to_words(64 * K); 52 const BasicType element_type = ArrayKlass::cast(_klass)->element_type(); 53 54 // Clear leading 32 bits, if necessary. 55 int base_offset = arrayOopDesc::base_offset_in_bytes(element_type); 56 if (!is_aligned(base_offset, HeapWordSize)) { 57 assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned"); 58 *reinterpret_cast<jint*>(reinterpret_cast<char*>(mem) + base_offset) = 0; 59 base_offset += BytesPerInt; 60 } 61 assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned"); 62 63 const size_t header = heap_word_size(base_offset); 64 const size_t payload_size = _word_size - header; 65 66 if (payload_size <= segment_max) { 67 // To small to use segmented clearing 68 return ObjArrayAllocator::initialize(mem); 69 } 70 71 // Segmented clearing 72 73 // The array is going to be exposed before it has been completely 74 // cleared, therefore we can't expose the header at the end of this 75 // function. Instead explicitly initialize it according to our needs. 76 77 // Signal to the ZIterator that this is an invisible root, by setting 78 // the mark word to "marked". Reset to prototype() after the clearing. 79 arrayOopDesc::set_mark(mem, markWord::prototype().set_marked()); 80 arrayOopDesc::release_set_klass(mem, _klass); 81 assert(_length >= 0, "length should be non-negative"); 82 arrayOopDesc::set_length(mem, _length); 83 84 // Keep the array alive across safepoints through an invisible 85 // root. Invisible roots are not visited by the heap iterator 86 // and the marking logic will not attempt to follow its elements. 87 // Relocation and remembered set code know how to dodge iterating 88 // over such objects. 89 ZThreadLocalData::set_invisible_root(_thread, (zaddress_unsafe*)&mem); 90 91 uint32_t old_seqnum_before = ZGeneration::old()->seqnum(); 92 uint32_t young_seqnum_before = ZGeneration::young()->seqnum(); 93 uintptr_t color_before = ZPointerStoreGoodMask; 94 auto gc_safepoint_happened = [&]() { 95 return old_seqnum_before != ZGeneration::old()->seqnum() || 96 young_seqnum_before != ZGeneration::young()->seqnum() || 97 color_before != ZPointerStoreGoodMask; 98 }; 99 100 bool seen_gc_safepoint = false; 101 102 auto initialize_memory = [&]() { 103 for (size_t processed = 0; processed < payload_size; processed += segment_max) { 104 // Clear segment 105 uintptr_t* const start = (uintptr_t*)(mem + header + processed); 106 const size_t remaining = payload_size - processed; 107 const size_t segment = MIN2(remaining, segment_max); 108 // Usually, the young marking code has the responsibility to color 109 // raw nulls, before they end up in the old generation. However, the 110 // invisible roots are hidden from the marking code, and therefore 111 // we must color the nulls already here in the initialization. The 112 // color we choose must be store bad for any subsequent stores, regardless 113 // of how many GC flips later it will arrive. That's why we OR in 11 114 // (ZPointerRememberedMask) in the remembered bits, similar to how 115 // forgotten old oops also have 11, for the very same reason. 116 // However, we opportunistically try to color without the 11 remembered 117 // bits, hoping to not get interrupted in the middle of a GC safepoint. 118 // Most of the time, we manage to do that, and can the avoid having GC 119 // barriers trigger slow paths for this. 120 const uintptr_t colored_null = seen_gc_safepoint ? (ZPointerStoreGoodMask | ZPointerRememberedMask) 121 : ZPointerStoreGoodMask; 122 const uintptr_t fill_value = is_reference_type(element_type) ? colored_null : 0; 123 ZUtils::fill(start, segment, fill_value); 124 125 // Safepoint 126 yield_for_safepoint(); 127 128 // Deal with safepoints 129 if (!seen_gc_safepoint && gc_safepoint_happened()) { 130 // The first time we observe a GC safepoint in the yield point, 131 // we have to restart processing with 11 remembered bits. 132 seen_gc_safepoint = true; 133 return false; 134 } 135 } 136 return true; 137 }; 138 139 if (!initialize_memory()) { 140 // Re-color with 11 remset bits if we got intercepted by a GC safepoint 141 const bool result = initialize_memory(); 142 assert(result, "Array initialization should always succeed the second time"); 143 } 144 145 mem_zap_end_padding(mem); 146 147 ZThreadLocalData::clear_invisible_root(_thread); 148 149 // Signal to the ZIterator that this is no longer an invisible root 150 oopDesc::release_set_mark(mem, markWord::prototype()); 151 152 return cast_to_oop(mem); 153 }