1 /*
2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "classfile/vmClasses.hpp"
28 #include "gc/shared/allocTracer.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "gc/shared/memAllocator.hpp"
31 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/universe.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/jvmtiExport.hpp"
37 #include "runtime/continuationJavaClasses.inline.hpp"
38 #include "runtime/handles.inline.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "services/lowMemoryDetector.hpp"
42 #include "utilities/align.hpp"
43 #include "utilities/copy.hpp"
44 #include "utilities/globalDefinitions.hpp"
45
46 class MemAllocator::Allocation: StackObj {
47 friend class MemAllocator;
48
49 const MemAllocator& _allocator;
50 JavaThread* _thread;
51 oop* _obj_ptr;
52 bool _overhead_limit_exceeded;
53 bool _allocated_outside_tlab;
54 size_t _allocated_tlab_size;
55 bool _tlab_end_reset_for_sample;
56
57 bool check_out_of_memory();
58 void verify_before();
59 void verify_after();
60 void notify_allocation();
61 void notify_allocation_jvmti_sampler();
62 void notify_allocation_low_memory_detector();
63 void notify_allocation_jfr_sampler();
64 void notify_allocation_dtrace_sampler();
65 #ifdef ASSERT
66 void check_for_valid_allocation_state() const;
67 #endif
68
69 class PreserveObj;
70
71 public:
72 Allocation(const MemAllocator& allocator, oop* obj_ptr)
73 : _allocator(allocator),
74 _thread(JavaThread::cast(allocator._thread)), // Do not use Allocation in non-JavaThreads.
75 _obj_ptr(obj_ptr),
76 _overhead_limit_exceeded(false),
77 _allocated_outside_tlab(false),
78 _allocated_tlab_size(0),
79 _tlab_end_reset_for_sample(false)
80 {
81 assert(Thread::current() == allocator._thread, "do not pass MemAllocator across threads");
82 verify_before();
83 }
84
85 ~Allocation() {
86 if (!check_out_of_memory()) {
87 notify_allocation();
88 }
89 }
90
91 oop obj() const { return *_obj_ptr; }
92 };
93
94 class MemAllocator::Allocation::PreserveObj: StackObj {
95 HandleMark _handle_mark;
96 Handle _handle;
97 oop* const _obj_ptr;
98
99 public:
100 PreserveObj(JavaThread* thread, oop* obj_ptr)
101 : _handle_mark(thread),
102 _handle(thread, *obj_ptr),
103 _obj_ptr(obj_ptr)
104 {
105 *obj_ptr = nullptr;
106 }
107
108 ~PreserveObj() {
109 *_obj_ptr = _handle();
110 }
111
112 oop operator()() const {
113 return _handle();
114 }
115 };
116
117 bool MemAllocator::Allocation::check_out_of_memory() {
118 JavaThread* THREAD = _thread; // For exception macros.
119 assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage");
120
121 if (obj() != nullptr) {
122 return false;
123 }
124
125 const char* message = _overhead_limit_exceeded ? "GC overhead limit exceeded" : "Java heap space";
126 if (!_thread->in_retryable_allocation()) {
127 // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
128 report_java_out_of_memory(message);
129
130 if (JvmtiExport::should_post_resource_exhausted()) {
131 JvmtiExport::post_resource_exhausted(
132 JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP,
133 message);
134 }
135 oop exception = _overhead_limit_exceeded ?
136 Universe::out_of_memory_error_gc_overhead_limit() :
137 Universe::out_of_memory_error_java_heap();
138 THROW_OOP_(exception, true);
139 } else {
140 THROW_OOP_(Universe::out_of_memory_error_retry(), true);
141 }
142 }
143
144 void MemAllocator::Allocation::verify_before() {
145 // Clear unhandled oops for memory allocation. Memory allocation might
146 // not take out a lock if from tlab, so clear here.
147 JavaThread* THREAD = _thread; // For exception macros.
148 assert(!HAS_PENDING_EXCEPTION, "Should not allocate with exception pending");
149 debug_only(check_for_valid_allocation_state());
150 assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed");
151 }
152
153 #ifdef ASSERT
154 void MemAllocator::Allocation::check_for_valid_allocation_state() const {
155 // How to choose between a pending exception and a potential
156 // OutOfMemoryError? Don't allow pending exceptions.
157 // This is a VM policy failure, so how do we exhaustively test it?
158 assert(!_thread->has_pending_exception(),
159 "shouldn't be allocating with pending exception");
160 // Allocation of an oop can always invoke a safepoint.
161 _thread->check_for_valid_safepoint_state();
162 }
163 #endif
164
165 void MemAllocator::Allocation::notify_allocation_jvmti_sampler() {
166 // support for JVMTI VMObjectAlloc event (no-op if not enabled)
167 JvmtiExport::vm_object_alloc_event_collector(obj());
168
169 if (!JvmtiExport::should_post_sampled_object_alloc()) {
170 // Sampling disabled
171 return;
172 }
173
174 if (!_allocated_outside_tlab && _allocated_tlab_size == 0 && !_tlab_end_reset_for_sample) {
175 // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB
176 // or expands it due to taking a sampler induced slow path.
177 return;
178 }
179
180 // If we want to be sampling, protect the allocated object with a Handle
181 // before doing the callback. The callback is done in the destructor of
182 // the JvmtiSampledObjectAllocEventCollector.
183 size_t bytes_since_last = 0;
184
185 {
186 PreserveObj obj_h(_thread, _obj_ptr);
187 JvmtiSampledObjectAllocEventCollector collector;
188 size_t size_in_bytes = _allocator._word_size * HeapWordSize;
189 ThreadLocalAllocBuffer& tlab = _thread->tlab();
190
191 if (!_allocated_outside_tlab) {
192 bytes_since_last = tlab.bytes_since_last_sample_point();
193 }
194
195 _thread->heap_sampler().check_for_sampling(obj_h(), size_in_bytes, bytes_since_last);
196 }
197
198 if (_tlab_end_reset_for_sample || _allocated_tlab_size != 0) {
199 // Tell tlab to forget bytes_since_last if we passed it to the heap sampler.
200 _thread->tlab().set_sample_end(bytes_since_last != 0);
201 }
202 }
203
204 void MemAllocator::Allocation::notify_allocation_low_memory_detector() {
205 // support low memory notifications (no-op if not enabled)
206 LowMemoryDetector::detect_low_memory_for_collected_pools();
207 }
208
209 void MemAllocator::Allocation::notify_allocation_jfr_sampler() {
210 HeapWord* mem = cast_from_oop<HeapWord*>(obj());
211 size_t size_in_bytes = _allocator._word_size * HeapWordSize;
212
213 if (_allocated_outside_tlab) {
214 AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread);
215 } else if (_allocated_tlab_size != 0) {
216 // TLAB was refilled
217 AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize,
218 size_in_bytes, _thread);
219 }
220 }
221
222 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() {
223 if (DTraceAllocProbes) {
224 // support for Dtrace object alloc event (no-op most of the time)
225 Klass* klass = obj()->klass();
226 size_t word_size = _allocator._word_size;
227 if (klass != nullptr && klass->name() != nullptr) {
228 SharedRuntime::dtrace_object_alloc(_thread, obj(), word_size);
229 }
230 }
231 }
232
233 void MemAllocator::Allocation::notify_allocation() {
234 notify_allocation_low_memory_detector();
235 notify_allocation_jfr_sampler();
236 notify_allocation_dtrace_sampler();
237 notify_allocation_jvmti_sampler();
238 }
239
240 HeapWord* MemAllocator::mem_allocate_outside_tlab(Allocation& allocation) const {
241 allocation._allocated_outside_tlab = true;
242 HeapWord* mem = Universe::heap()->mem_allocate(_word_size, &allocation._overhead_limit_exceeded);
243 if (mem == nullptr) {
244 return mem;
245 }
246
247 size_t size_in_bytes = _word_size * HeapWordSize;
248 _thread->incr_allocated_bytes(size_in_bytes);
249
250 return mem;
251 }
252
253 HeapWord* MemAllocator::mem_allocate_inside_tlab(Allocation& allocation) const {
254 assert(UseTLAB, "should use UseTLAB");
255
256 // Try allocating from an existing TLAB.
257 HeapWord* mem = mem_allocate_inside_tlab_fast();
258 if (mem != nullptr) {
259 return mem;
260 }
261
262 // Try refilling the TLAB and allocating the object in it.
263 return mem_allocate_inside_tlab_slow(allocation);
264 }
265
266 HeapWord* MemAllocator::mem_allocate_inside_tlab_fast() const {
267 return _thread->tlab().allocate(_word_size);
268 }
269
270 HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) const {
271 HeapWord* mem = nullptr;
272 ThreadLocalAllocBuffer& tlab = _thread->tlab();
273
274 if (JvmtiExport::should_post_sampled_object_alloc()) {
275 tlab.set_back_allocation_end();
276 mem = tlab.allocate(_word_size);
277
278 // We set back the allocation sample point to try to allocate this, reset it
279 // when done.
280 allocation._tlab_end_reset_for_sample = true;
281
282 if (mem != nullptr) {
283 return mem;
284 }
285 }
286
287 // Retain tlab and allocate object in shared space if
288 // the amount free in the tlab is too large to discard.
289 if (tlab.free() > tlab.refill_waste_limit()) {
290 tlab.record_slow_allocation(_word_size);
291 return nullptr;
292 }
293
294 // Discard tlab and allocate a new one.
295 // To minimize fragmentation, the last TLAB may be smaller than the rest.
296 size_t new_tlab_size = tlab.compute_size(_word_size);
297
298 tlab.retire_before_allocation();
299
300 if (new_tlab_size == 0) {
301 return nullptr;
302 }
303
304 // Allocate a new TLAB requesting new_tlab_size. Any size
305 // between minimal and new_tlab_size is accepted.
306 size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size);
307 mem = Universe::heap()->allocate_new_tlab(min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size);
308 if (mem == nullptr) {
309 assert(allocation._allocated_tlab_size == 0,
310 "Allocation failed, but actual size was updated. min: " SIZE_FORMAT
311 ", desired: " SIZE_FORMAT ", actual: " SIZE_FORMAT,
312 min_tlab_size, new_tlab_size, allocation._allocated_tlab_size);
313 return nullptr;
314 }
315 assert(allocation._allocated_tlab_size != 0, "Allocation succeeded but actual size not updated. mem at: "
316 PTR_FORMAT " min: " SIZE_FORMAT ", desired: " SIZE_FORMAT,
317 p2i(mem), min_tlab_size, new_tlab_size);
318
319 // ...and clear or zap just allocated TLAB, if needed.
320 if (ZeroTLAB) {
321 Copy::zero_to_words(mem, allocation._allocated_tlab_size);
322 } else if (ZapTLAB) {
323 // Skip mangling the space corresponding to the object header to
324 // ensure that the returned space is not considered parsable by
325 // any concurrent GC thread.
326 size_t hdr_size = oopDesc::header_size();
327 Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal);
328 }
329
330 tlab.fill(mem, mem + _word_size, allocation._allocated_tlab_size);
331 return mem;
332 }
333
334
335 HeapWord* MemAllocator::mem_allocate_slow(Allocation& allocation) const {
336 // Allocation of an oop can always invoke a safepoint.
337 debug_only(allocation._thread->check_for_valid_safepoint_state());
338
339 if (UseTLAB) {
340 // Try refilling the TLAB and allocating the object in it.
341 HeapWord* mem = mem_allocate_inside_tlab_slow(allocation);
342 if (mem != nullptr) {
343 return mem;
344 }
345 }
346
347 return mem_allocate_outside_tlab(allocation);
348 }
349
350 HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const {
351 if (UseTLAB) {
352 // Try allocating from an existing TLAB.
353 HeapWord* mem = mem_allocate_inside_tlab_fast();
354 if (mem != nullptr) {
355 return mem;
356 }
357 }
358
359 return mem_allocate_slow(allocation);
360 }
361
362 oop MemAllocator::allocate() const {
363 oop obj = nullptr;
364 {
365 Allocation allocation(*this, &obj);
366 HeapWord* mem = mem_allocate(allocation);
367 if (mem != nullptr) {
368 obj = initialize(mem);
369 } else {
370 // The unhandled oop detector will poison local variable obj,
371 // so reset it to null if mem is null.
372 obj = nullptr;
373 }
374 }
375 return obj;
376 }
377
378 void MemAllocator::mem_clear(HeapWord* mem) const {
379 assert(mem != nullptr, "cannot initialize null object");
380 const size_t hs = oopDesc::header_size();
381 assert(_word_size >= hs, "unexpected object size");
382 oopDesc::set_klass_gap(mem, 0);
383 Copy::fill_to_aligned_words(mem + hs, _word_size - hs);
384 }
385
386 oop MemAllocator::finish(HeapWord* mem) const {
387 assert(mem != nullptr, "null object pointer");
388 // May be bootstrapping
389 oopDesc::set_mark(mem, markWord::prototype());
390 // Need a release store to ensure array/class length, mark word, and
391 // object zeroing are visible before setting the klass non-null, for
392 // concurrent collectors.
393 oopDesc::release_set_klass(mem, _klass);
394 return cast_to_oop(mem);
395 }
396
397 oop ObjAllocator::initialize(HeapWord* mem) const {
398 mem_clear(mem);
399 return finish(mem);
400 }
401
402 oop ObjArrayAllocator::initialize(HeapWord* mem) const {
403 // Set array length before setting the _klass field because a
404 // non-null klass field indicates that the object is parsable by
405 // concurrent GC.
406 assert(_length >= 0, "length should be non-negative");
407 if (_do_zero) {
408 mem_clear(mem);
409 mem_zap_end_padding(mem);
410 }
411 arrayOopDesc::set_length(mem, _length);
412 return finish(mem);
413 }
414
415 #ifndef PRODUCT
416 void ObjArrayAllocator::mem_zap_end_padding(HeapWord* mem) const {
417 const size_t length_in_bytes = static_cast<size_t>(_length) << ArrayKlass::cast(_klass)->log2_element_size();
418 const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
419 const size_t base_offset_in_bytes = arrayOopDesc::base_offset_in_bytes(element_type);
420 const size_t size_in_bytes = _word_size * BytesPerWord;
421
422 const address obj_end = reinterpret_cast<address>(mem) + size_in_bytes;
423 const address base = reinterpret_cast<address>(mem) + base_offset_in_bytes;
424 const address elements_end = base + length_in_bytes;
425 assert(elements_end <= obj_end, "payload must fit in object");
426 if (elements_end < obj_end) {
427 const size_t padding_in_bytes = obj_end - elements_end;
428 Copy::fill_to_bytes(elements_end, padding_in_bytes, heapPaddingByteVal);
429 }
430 }
431 #endif
432
433 oop ClassAllocator::initialize(HeapWord* mem) const {
434 // Set oop_size field before setting the _klass field because a
435 // non-null _klass field indicates that the object is parsable by
436 // concurrent GC.
437 assert(_word_size > 0, "oop_size must be positive.");
438 mem_clear(mem);
439 java_lang_Class::set_oop_size(mem, _word_size);
440 return finish(mem);
441 }