1 /*
  2  * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "gc/z/zThreadLocalData.hpp"
 25 #include "gc/z/zObjArrayAllocator.hpp"
 26 #include "gc/z/zUtils.inline.hpp"
 27 #include "oops/arrayKlass.hpp"
 28 #include "runtime/interfaceSupport.inline.hpp"
 29 #include "utilities/debug.hpp"
 30 
 31 ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread)
 32   : ObjArrayAllocator(klass, word_size, length, do_zero, thread) {}
 33 
 34 void ZObjArrayAllocator::yield_for_safepoint() const {
 35   ThreadBlockInVM tbivm(JavaThread::cast(_thread));
 36 }
 37 
 38 oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
 39   // ZGC specializes the initialization by performing segmented clearing
 40   // to allow shorter time-to-safepoints.
 41 
 42   if (!_do_zero) {
 43     // No need for ZGC specialization
 44     return ObjArrayAllocator::initialize(mem);
 45   }
 46 
 47   // A max segment size of 64K was chosen because microbenchmarking
 48   // suggested that it offered a good trade-off between allocation
 49   // time and time-to-safepoint
 50   const size_t segment_max = ZUtils::bytes_to_words(64 * K);
 51 
 52   if (_word_size <= segment_max) {
 53     // To small to use segmented clearing
 54     return ObjArrayAllocator::initialize(mem);
 55   }
 56 
 57   // Segmented clearing
 58 
 59   // The array is going to be exposed before it has been completely
 60   // cleared, therefore we can't expose the header at the end of this
 61   // function. Instead explicitly initialize it according to our needs.
 62 
 63   // Signal to the ZIterator that this is an invisible root, by setting
 64   // the mark word to "marked". Reset to prototype() after the clearing.
 65   if (UseCompactObjectHeaders) {
 66     oopDesc::release_set_mark(mem, _klass->prototype_header().set_marked());
 67   } else {
 68     arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
 69     arrayOopDesc::release_set_klass(mem, _klass);
 70   }
 71   assert(_length >= 0, "length should be non-negative");
 72   arrayOopDesc::set_length(mem, _length);
 73 
 74   // Keep the array alive across safepoints through an invisible
 75   // root. Invisible roots are not visited by the heap iterator
 76   // and the marking logic will not attempt to follow its elements.
 77   // Relocation and remembered set code know how to dodge iterating
 78   // over such objects.
 79   ZThreadLocalData::set_invisible_root(_thread, (zaddress_unsafe*)&mem);
 80 
 81   const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
 82   const size_t base_offset_in_bytes = (size_t)arrayOopDesc::base_offset_in_bytes(element_type);
 83   const size_t process_start_offset_in_bytes = align_up(base_offset_in_bytes, (size_t)BytesPerWord);
 84 
 85   if (process_start_offset_in_bytes != base_offset_in_bytes) {
 86     // initialize_memory can only fill word aligned memory,
 87     // fill the first 4 bytes here.
 88     assert(process_start_offset_in_bytes - base_offset_in_bytes == 4, "Must be 4-byte aligned");
 89     assert(!is_reference_type(element_type), "Only TypeArrays can be 4-byte aligned");
 90     *reinterpret_cast<int*>(reinterpret_cast<char*>(mem) + base_offset_in_bytes) = 0;
 91   }
 92 
 93   // Note: initialize_memory may clear padding bytes at the end
 94   const size_t process_start_offset = ZUtils::bytes_to_words(process_start_offset_in_bytes);
 95   const size_t process_size = _word_size - process_start_offset;
 96 
 97   uint32_t old_seqnum_before = ZGeneration::old()->seqnum();
 98   uint32_t young_seqnum_before = ZGeneration::young()->seqnum();
 99   uintptr_t color_before = ZPointerStoreGoodMask;
100   auto gc_safepoint_happened = [&]() {
101     return old_seqnum_before != ZGeneration::old()->seqnum() ||
102            young_seqnum_before != ZGeneration::young()->seqnum() ||
103            color_before != ZPointerStoreGoodMask;
104   };
105 
106   bool seen_gc_safepoint = false;
107 
108   auto initialize_memory = [&]() {
109     for (size_t processed = 0; processed < process_size; processed += segment_max) {
110       // Clear segment
111       uintptr_t* const start = (uintptr_t*)(mem + process_start_offset + processed);
112       const size_t remaining = process_size - processed;
113       const size_t segment = MIN2(remaining, segment_max);
114       // Usually, the young marking code has the responsibility to color
115       // raw nulls, before they end up in the old generation. However, the
116       // invisible roots are hidden from the marking code, and therefore
117       // we must color the nulls already here in the initialization. The
118       // color we choose must be store bad for any subsequent stores, regardless
119       // of how many GC flips later it will arrive. That's why we OR in 11
120       // (ZPointerRememberedMask) in the remembered bits, similar to how
121       // forgotten old oops also have 11, for the very same reason.
122       // However, we opportunistically try to color without the 11 remembered
123       // bits, hoping to not get interrupted in the middle of a GC safepoint.
124       // Most of the time, we manage to do that, and can the avoid having GC
125       // barriers trigger slow paths for this.
126       const uintptr_t colored_null = seen_gc_safepoint ? (ZPointerStoreGoodMask | ZPointerRememberedMask)
127                                                        : ZPointerStoreGoodMask;
128       const uintptr_t fill_value = is_reference_type(element_type) ? colored_null : 0;
129       ZUtils::fill(start, segment, fill_value);
130 
131       // Safepoint
132       yield_for_safepoint();
133 
134       // Deal with safepoints
135       if (is_reference_type(element_type) && !seen_gc_safepoint && gc_safepoint_happened()) {
136         // The first time we observe a GC safepoint in the yield point,
137         // we have to restart processing with 11 remembered bits.
138         seen_gc_safepoint = true;
139         return false;
140       }
141     }
142     return true;
143   };
144 
145   mem_zap_start_padding(mem);
146 
147   if (!initialize_memory()) {
148     // Re-color with 11 remset bits if we got intercepted by a GC safepoint
149     const bool result = initialize_memory();
150     assert(result, "Array initialization should always succeed the second time");
151   }
152 
153   mem_zap_end_padding(mem);
154 
155   ZThreadLocalData::clear_invisible_root(_thread);
156 
157   // Signal to the ZIterator that this is no longer an invisible root
158   if (UseCompactObjectHeaders) {
159     oopDesc::release_set_mark(mem, _klass->prototype_header());
160   } else {
161     oopDesc::release_set_mark(mem, markWord::prototype());
162   }
163 
164   return cast_to_oop(mem);
165 }