1 /*
2 * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "gc/z/zGeneration.inline.hpp"
25 #include "gc/z/zObjArrayAllocator.hpp"
26 #include "gc/z/zThreadLocalData.hpp"
27 #include "gc/z/zUtils.inline.hpp"
28 #include "oops/arrayKlass.hpp"
29 #include "runtime/arguments.hpp"
30 #include "runtime/interfaceSupport.inline.hpp"
31 #include "utilities/debug.hpp"
32 #include "utilities/globalDefinitions.hpp"
33
34 ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread)
35 : ObjArrayAllocator(klass, word_size, length, do_zero, thread) {}
36
37 void ZObjArrayAllocator::yield_for_safepoint() const {
38 ThreadBlockInVM tbivm(JavaThread::cast(_thread));
39 }
40
41 static bool is_oop_containing_flat_array(ArrayKlass* ak) {
42 return ak->is_flatArray_klass() &&
43 FlatArrayKlass::cast(ak)->contains_oops();
44 }
45
46 oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
47 // ZGC specializes the initialization by performing segmented clearing
48 // to allow shorter time-to-safepoints.
49
50 if (!_do_zero) {
51 // No need for ZGC specialization
52 return ObjArrayAllocator::initialize(mem);
53 }
54
55 // A max segment size of 64K was chosen because microbenchmarking
56 // suggested that it offered a good trade-off between allocation
57 // time and time-to-safepoint
58 const size_t segment_max = ZUtils::bytes_to_words(64 * K);
59
60 if (_word_size <= segment_max) {
61 // Too small to use segmented clearing
62 return ObjArrayAllocator::initialize(mem);
63 }
64
65 ArrayKlass* const ak = ArrayKlass::cast(_klass);
66
67 if (is_oop_containing_flat_array(ak)) {
68 // Flat arrays containing oops are not supported in ZGC without relying on
69 // internal-only features such as loose-consistency and null-restriction.
70 // A value object that contains an oop and a null-marker will always exceed
71 // 64 bits when using ZGC. As a result, such objects will not be flattened
72 // in practice due to the 64-bit atomicity limit.
73 //
74 // We only need to support flat arrays containing oops when/if value objects
75 // can be user-declared with loose-consistency and/or null-restriction.
76 return ObjArrayAllocator::initialize(mem);
77 }
78
79 const BasicType element_type = ak->element_type();
80
81 // Flat arrays containing oops are not supported and only contain primitives
82 // from here on out.
83 const bool is_oop_array = element_type != T_FLAT_ELEMENT &&
84 is_reference_type(element_type);
85
86 // Segmented clearing
87
88 // The array is going to be exposed before it has been completely
89 // cleared, therefore we can't expose the header at the end of this
90 // function. Instead explicitly initialize it according to our needs.
91
92 // Signal to the ZIterator that this is an invisible root, by setting
93 // the mark word to "marked". Reset to prototype() after the clearing.
94 if (UseCompactObjectHeaders) {
95 oopDesc::release_set_mark(mem, _klass->prototype_header().set_marked());
96 } else {
97 if (Arguments::is_valhalla_enabled()) {
98 arrayOopDesc::set_mark(mem, _klass->prototype_header().set_marked());
99 } else {
100 arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
101 }
102 arrayOopDesc::release_set_klass(mem, _klass);
103 }
104 assert(_length >= 0, "length should be non-negative");
105 arrayOopDesc::set_length(mem, _length);
106
107 // Keep the array alive across safepoints through an invisible
108 // root. Invisible roots are not visited by the heap iterator
109 // and the marking logic will not attempt to follow its elements.
110 // Relocation and remembered set code know how to dodge iterating
111 // over such objects.
112 ZThreadLocalData::set_invisible_root(_thread, (zaddress_unsafe*)&mem);
113
114 const size_t base_offset_in_bytes = (size_t)arrayOopDesc::base_offset_in_bytes(element_type);
115 const size_t process_start_offset_in_bytes = align_up(base_offset_in_bytes, (size_t)BytesPerWord);
116
117 if (process_start_offset_in_bytes != base_offset_in_bytes) {
118 // initialize_memory can only fill word aligned memory,
119 // fill the first 4 bytes here.
120 assert(process_start_offset_in_bytes - base_offset_in_bytes == 4, "Must be 4-byte aligned");
121 assert(!is_oop_array, "Only TypeArrays can be 4-byte aligned");
122 *reinterpret_cast<int*>(reinterpret_cast<char*>(mem) + base_offset_in_bytes) = 0;
123 }
124
125 // Note: initialize_memory may clear padding bytes at the end
126 const size_t process_start_offset = ZUtils::bytes_to_words(process_start_offset_in_bytes);
127 const size_t process_size = _word_size - process_start_offset;
128
129 uint32_t old_seqnum_before = ZGeneration::old()->seqnum();
130 uint32_t young_seqnum_before = ZGeneration::young()->seqnum();
131 uintptr_t color_before = ZPointerStoreGoodMask;
132 auto gc_safepoint_happened = [&]() {
133 return old_seqnum_before != ZGeneration::old()->seqnum() ||
134 young_seqnum_before != ZGeneration::young()->seqnum() ||
135 color_before != ZPointerStoreGoodMask;
136 };
137
138 bool seen_gc_safepoint = false;
139
140 auto initialize_memory = [&]() {
141 for (size_t processed = 0; processed < process_size; processed += segment_max) {
142 // Clear segment
143 uintptr_t* const start = (uintptr_t*)(mem + process_start_offset + processed);
144 const size_t remaining = process_size - processed;
145 const size_t segment = MIN2(remaining, segment_max);
146 // Usually, the young marking code has the responsibility to color
147 // raw nulls, before they end up in the old generation. However, the
148 // invisible roots are hidden from the marking code, and therefore
149 // we must color the nulls already here in the initialization. The
150 // color we choose must be store bad for any subsequent stores, regardless
151 // of how many GC flips later it will arrive. That's why we OR in 11
152 // (ZPointerRememberedMask) in the remembered bits, similar to how
153 // forgotten old oops also have 11, for the very same reason.
154 // However, we opportunistically try to color without the 11 remembered
155 // bits, hoping to not get interrupted in the middle of a GC safepoint.
156 // Most of the time, we manage to do that, and can the avoid having GC
157 // barriers trigger slow paths for this.
158 const uintptr_t colored_null = seen_gc_safepoint ? (ZPointerStoreGoodMask | ZPointerRememberedMask)
159 : ZPointerStoreGoodMask;
160 const uintptr_t fill_value = is_oop_array ? colored_null : 0;
161 ZUtils::fill(start, segment, fill_value);
162
163 // Safepoint
164 yield_for_safepoint();
165
166 // Deal with safepoints
167 if (is_oop_array && !seen_gc_safepoint && gc_safepoint_happened()) {
168 // The first time we observe a GC safepoint in the yield point,
169 // we have to restart processing with 11 remembered bits.
170 seen_gc_safepoint = true;
171 return false;
172 }
173 }
174 return true;
175 };
176
177 mem_zap_start_padding(mem);
178
179 if (!initialize_memory()) {
180 // Re-color with 11 remset bits if we got intercepted by a GC safepoint
181 const bool result = initialize_memory();
182 assert(result, "Array initialization should always succeed the second time");
183 }
184
185 mem_zap_end_padding(mem);
186
187 ZThreadLocalData::clear_invisible_root(_thread);
188
189 // Signal to the ZIterator that this is no longer an invisible root
190 if (UseCompactObjectHeaders || Arguments::is_valhalla_enabled()) {
191 oopDesc::release_set_mark(mem, _klass->prototype_header());
192 } else {
193 oopDesc::release_set_mark(mem, markWord::prototype());
194 }
195
196 return cast_to_oop(mem);
197 }