1 /*
2 * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zThreadLocalData.hpp"
26 #include "gc/z/zObjArrayAllocator.hpp"
27 #include "gc/z/zUtils.inline.hpp"
28 #include "oops/arrayKlass.hpp"
29 #include "runtime/interfaceSupport.inline.hpp"
30 #include "utilities/debug.hpp"
31
32 ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread)
33 : ObjArrayAllocator(klass, word_size, length, do_zero, thread) {}
34
35 void ZObjArrayAllocator::yield_for_safepoint() const {
36 ThreadBlockInVM tbivm(JavaThread::cast(_thread));
37 }
38
39 oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
40 // ZGC specializes the initialization by performing segmented clearing
41 // to allow shorter time-to-safepoints.
42
43 if (!_do_zero) {
44 // No need for ZGC specialization
45 return ObjArrayAllocator::initialize(mem);
46 }
47
48 // A max segment size of 64K was chosen because microbenchmarking
49 // suggested that it offered a good trade-off between allocation
50 // time and time-to-safepoint
51 const size_t segment_max = ZUtils::bytes_to_words(64 * K);
52 const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
53
54 // Clear leading 32 bits, if necessary.
55 int base_offset = arrayOopDesc::base_offset_in_bytes(element_type);
56 if (!is_aligned(base_offset, HeapWordSize)) {
57 assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned");
58 *reinterpret_cast<jint*>(reinterpret_cast<char*>(mem) + base_offset) = 0;
59 base_offset += BytesPerInt;
60 }
61 assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned");
62
63 const size_t header = heap_word_size(base_offset);
64 const size_t payload_size = _word_size - header;
65
66 if (payload_size <= segment_max) {
67 // To small to use segmented clearing
68 return ObjArrayAllocator::initialize(mem);
69 }
70
71 // Segmented clearing
72
73 // The array is going to be exposed before it has been completely
74 // cleared, therefore we can't expose the header at the end of this
75 // function. Instead explicitly initialize it according to our needs.
76
77 // Signal to the ZIterator that this is an invisible root, by setting
78 // the mark word to "marked". Reset to prototype() after the clearing.
79 if (UseCompactObjectHeaders) {
80 arrayOopDesc::release_set_mark(mem, _klass->prototype_header().set_marked());
81 } else {
82 arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
83 arrayOopDesc::release_set_klass(mem, _klass);
84 }
85 assert(_length >= 0, "length should be non-negative");
86 arrayOopDesc::set_length(mem, _length);
87
88 // Keep the array alive across safepoints through an invisible
89 // root. Invisible roots are not visited by the heap iterator
90 // and the marking logic will not attempt to follow its elements.
91 // Relocation and remembered set code know how to dodge iterating
92 // over such objects.
93 ZThreadLocalData::set_invisible_root(_thread, (zaddress_unsafe*)&mem);
94
95 uint32_t old_seqnum_before = ZGeneration::old()->seqnum();
96 uint32_t young_seqnum_before = ZGeneration::young()->seqnum();
97 uintptr_t color_before = ZPointerStoreGoodMask;
98 auto gc_safepoint_happened = [&]() {
99 return old_seqnum_before != ZGeneration::old()->seqnum() ||
100 young_seqnum_before != ZGeneration::young()->seqnum() ||
101 color_before != ZPointerStoreGoodMask;
102 };
103
104 bool seen_gc_safepoint = false;
105
106 auto initialize_memory = [&]() {
107 for (size_t processed = 0; processed < payload_size; processed += segment_max) {
108 // Clear segment
109 uintptr_t* const start = (uintptr_t*)(mem + header + processed);
110 const size_t remaining = payload_size - processed;
111 const size_t segment = MIN2(remaining, segment_max);
112 // Usually, the young marking code has the responsibility to color
113 // raw nulls, before they end up in the old generation. However, the
114 // invisible roots are hidden from the marking code, and therefore
115 // we must color the nulls already here in the initialization. The
116 // color we choose must be store bad for any subsequent stores, regardless
117 // of how many GC flips later it will arrive. That's why we OR in 11
118 // (ZPointerRememberedMask) in the remembered bits, similar to how
119 // forgotten old oops also have 11, for the very same reason.
120 // However, we opportunistically try to color without the 11 remembered
121 // bits, hoping to not get interrupted in the middle of a GC safepoint.
122 // Most of the time, we manage to do that, and can the avoid having GC
123 // barriers trigger slow paths for this.
124 const uintptr_t colored_null = seen_gc_safepoint ? (ZPointerStoreGoodMask | ZPointerRememberedMask)
125 : ZPointerStoreGoodMask;
126 const uintptr_t fill_value = is_reference_type(element_type) ? colored_null : 0;
127 ZUtils::fill(start, segment, fill_value);
128
129 // Safepoint
130 yield_for_safepoint();
131
132 // Deal with safepoints
133 if (!seen_gc_safepoint && gc_safepoint_happened()) {
134 // The first time we observe a GC safepoint in the yield point,
135 // we have to restart processing with 11 remembered bits.
136 seen_gc_safepoint = true;
137 return false;
138 }
139 }
140 return true;
141 };
142
143 if (!initialize_memory()) {
144 // Re-color with 11 remset bits if we got intercepted by a GC safepoint
145 const bool result = initialize_memory();
146 assert(result, "Array initialization should always succeed the second time");
147 }
148
149 ZThreadLocalData::clear_invisible_root(_thread);
150
151 // Signal to the ZIterator that this is no longer an invisible root
152 if (UseCompactObjectHeaders) {
153 oopDesc::release_set_mark(mem, _klass->prototype_header());
154 } else {
155 oopDesc::release_set_mark(mem, markWord::prototype());
156 }
157
158 return cast_to_oop(mem);
159 }