1 /* 2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "classfile/javaClasses.hpp" 26 #include "classfile/vmClasses.hpp" 27 #include "gc/shared/allocTracer.hpp" 28 #include "gc/shared/collectedHeap.hpp" 29 #include "gc/shared/memAllocator.hpp" 30 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" 31 #include "gc/shared/tlab_globals.hpp" 32 #include "memory/universe.hpp" 33 #include "oops/arrayOop.hpp" 34 #include "oops/oop.inline.hpp" 35 #include "prims/jvmtiExport.hpp" 36 #include "runtime/continuationJavaClasses.inline.hpp" 37 #include "runtime/handles.inline.hpp" 38 #include "runtime/javaThread.hpp" 39 #include "runtime/sharedRuntime.hpp" 40 #include "services/lowMemoryDetector.hpp" 41 #include "utilities/align.hpp" 42 #include "utilities/copy.hpp" 43 #include "utilities/globalDefinitions.hpp" 44 45 class MemAllocator::Allocation: StackObj { 46 friend class MemAllocator; 47 48 const MemAllocator& _allocator; 49 JavaThread* _thread; 50 oop* _obj_ptr; 51 bool _overhead_limit_exceeded; 52 bool _allocated_outside_tlab; 53 size_t _allocated_tlab_size; 54 55 bool check_out_of_memory(); 56 void verify_before(); 57 void verify_after(); 58 void notify_allocation(); 59 void notify_allocation_jvmti_sampler(); 60 void notify_allocation_low_memory_detector(); 61 void notify_allocation_jfr_sampler(); 62 void notify_allocation_dtrace_sampler(); 63 #ifdef ASSERT 64 void check_for_valid_allocation_state() const; 65 #endif 66 67 class PreserveObj; 68 69 public: 70 Allocation(const MemAllocator& allocator, oop* obj_ptr) 71 : _allocator(allocator), 72 _thread(JavaThread::cast(allocator._thread)), // Do not use Allocation in non-JavaThreads. 73 _obj_ptr(obj_ptr), 74 _overhead_limit_exceeded(false), 75 _allocated_outside_tlab(false), 76 _allocated_tlab_size(0) 77 { 78 assert(Thread::current() == allocator._thread, "do not pass MemAllocator across threads"); 79 verify_before(); 80 } 81 82 ~Allocation() { 83 if (!check_out_of_memory()) { 84 notify_allocation(); 85 } 86 } 87 88 oop obj() const { return *_obj_ptr; } 89 }; 90 91 class MemAllocator::Allocation::PreserveObj: StackObj { 92 HandleMark _handle_mark; 93 Handle _handle; 94 oop* const _obj_ptr; 95 96 public: 97 PreserveObj(JavaThread* thread, oop* obj_ptr) 98 : _handle_mark(thread), 99 _handle(thread, *obj_ptr), 100 _obj_ptr(obj_ptr) 101 { 102 *obj_ptr = nullptr; 103 } 104 105 ~PreserveObj() { 106 *_obj_ptr = _handle(); 107 } 108 109 oop operator()() const { 110 return _handle(); 111 } 112 }; 113 114 bool MemAllocator::Allocation::check_out_of_memory() { 115 JavaThread* THREAD = _thread; // For exception macros. 116 assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage"); 117 118 if (obj() != nullptr) { 119 return false; 120 } 121 122 const char* message = _overhead_limit_exceeded ? "GC overhead limit exceeded" : "Java heap space"; 123 if (!_thread->is_in_internal_oome_mark()) { 124 // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support 125 report_java_out_of_memory(message); 126 if (JvmtiExport::should_post_resource_exhausted()) { 127 #ifdef CHECK_UNHANDLED_OOPS 128 // obj is null, no need to handle, but CheckUnhandledOops is not aware about null 129 THREAD->allow_unhandled_oop(_obj_ptr); 130 #endif // CHECK_UNHANDLED_OOPS 131 JvmtiExport::post_resource_exhausted( 132 JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, 133 message); 134 } 135 136 oop exception = _overhead_limit_exceeded ? 137 Universe::out_of_memory_error_gc_overhead_limit() : 138 Universe::out_of_memory_error_java_heap(); 139 THROW_OOP_(exception, true); 140 } else { 141 THROW_OOP_(Universe::out_of_memory_error_java_heap_without_backtrace(), true); 142 } 143 } 144 145 void MemAllocator::Allocation::verify_before() { 146 // Clear unhandled oops for memory allocation. Memory allocation might 147 // not take out a lock if from tlab, so clear here. 148 JavaThread* THREAD = _thread; // For exception macros. 149 assert(!HAS_PENDING_EXCEPTION, "Should not allocate with exception pending"); 150 DEBUG_ONLY(check_for_valid_allocation_state()); 151 assert(!Universe::heap()->is_stw_gc_active(), "Allocation during GC pause not allowed"); 152 } 153 154 #ifdef ASSERT 155 void MemAllocator::Allocation::check_for_valid_allocation_state() const { 156 // How to choose between a pending exception and a potential 157 // OutOfMemoryError? Don't allow pending exceptions. 158 // This is a VM policy failure, so how do we exhaustively test it? 159 assert(!_thread->has_pending_exception(), 160 "shouldn't be allocating with pending exception"); 161 // Allocation of an oop can always invoke a safepoint. 162 _thread->check_for_valid_safepoint_state(); 163 } 164 #endif 165 166 void MemAllocator::Allocation::notify_allocation_jvmti_sampler() { 167 // support for JVMTI VMObjectAlloc event (no-op if not enabled) 168 JvmtiExport::vm_object_alloc_event_collector(obj()); 169 170 if (!JvmtiExport::should_post_sampled_object_alloc()) { 171 // Sampling disabled 172 return; 173 } 174 175 ThreadHeapSampler& heap_sampler = _thread->heap_sampler(); 176 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 177 178 // Log sample decision 179 heap_sampler.log_sample_decision(tlab.top()); 180 181 if (heap_sampler.should_sample(tlab.top())) { 182 // If we want to be sampling, protect the allocated object with a Handle 183 // before doing the callback. The callback is done in the destructor of 184 // the JvmtiSampledObjectAllocEventCollector. 185 PreserveObj obj_h(_thread, _obj_ptr); 186 JvmtiSampledObjectAllocEventCollector collector; 187 188 // Perform the sampling 189 heap_sampler.sample(obj_h(), tlab.top()); 190 191 // Note that after this point all the TLAB can have been retired, and agent 192 // code can run and allocate, don't rely on earlier calculations involving 193 // the TLAB. 194 } 195 196 // Set a new sampling point in the TLAB if it fits in the current TLAB 197 const size_t words_until_sample = heap_sampler.bytes_until_sample(tlab.top()) / HeapWordSize; 198 if (words_until_sample <= tlab.free()) { 199 tlab.set_sampling_point(tlab.top() + words_until_sample); 200 } 201 } 202 203 void MemAllocator::Allocation::notify_allocation_low_memory_detector() { 204 // support low memory notifications (no-op if not enabled) 205 LowMemoryDetector::detect_low_memory_for_collected_pools(); 206 } 207 208 void MemAllocator::Allocation::notify_allocation_jfr_sampler() { 209 HeapWord* mem = cast_from_oop<HeapWord*>(obj()); 210 size_t size_in_bytes = _allocator._word_size * HeapWordSize; 211 212 if (_allocated_outside_tlab) { 213 AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread); 214 } else if (_allocated_tlab_size != 0) { 215 // TLAB was refilled 216 AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize, 217 size_in_bytes, _thread); 218 } 219 } 220 221 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() { 222 if (DTraceAllocProbes) { 223 // support for Dtrace object alloc event (no-op most of the time) 224 Klass* klass = obj()->klass(); 225 size_t word_size = _allocator._word_size; 226 if (klass != nullptr && klass->name() != nullptr) { 227 SharedRuntime::dtrace_object_alloc(_thread, obj(), word_size); 228 } 229 } 230 } 231 232 void MemAllocator::Allocation::notify_allocation() { 233 notify_allocation_low_memory_detector(); 234 notify_allocation_jfr_sampler(); 235 notify_allocation_dtrace_sampler(); 236 notify_allocation_jvmti_sampler(); 237 } 238 239 HeapWord* MemAllocator::mem_allocate_outside_tlab(Allocation& allocation) const { 240 allocation._allocated_outside_tlab = true; 241 HeapWord* mem = Universe::heap()->mem_allocate(_word_size, &allocation._overhead_limit_exceeded); 242 if (mem == nullptr) { 243 return mem; 244 } 245 246 size_t size_in_bytes = _word_size * HeapWordSize; 247 _thread->incr_allocated_bytes(size_in_bytes); 248 _thread->heap_sampler().inc_outside_tlab_bytes(size_in_bytes); 249 250 return mem; 251 } 252 253 HeapWord* MemAllocator::mem_allocate_inside_tlab_fast() const { 254 return _thread->tlab().allocate(_word_size); 255 } 256 257 HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) const { 258 HeapWord* mem = nullptr; 259 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 260 261 if (JvmtiExport::should_post_sampled_object_alloc()) { 262 // When sampling we artificially set the TLAB end to the sample point. 263 // When we hit that point it looks like the TLAB is full, but it's 264 // not necessarily the case. Set the real end and retry the allocation. 265 266 // Undo previous adjustment of end. 267 // Note that notify_allocation_jvmti_sampler will set a new sample point. 268 tlab.set_back_allocation_end(); 269 270 // Retry the TLAB allocation with the proper end 271 mem = tlab.allocate(_word_size); 272 273 if (mem != nullptr) { 274 return mem; 275 } 276 } 277 278 // Retain tlab and allocate object in shared space if 279 // the amount free in the tlab is too large to discard. 280 if (tlab.free() > tlab.refill_waste_limit()) { 281 tlab.record_slow_allocation(_word_size); 282 return nullptr; 283 } 284 285 // Discard tlab and allocate a new one. 286 287 // Record the amount wasted 288 tlab.record_refill_waste(); 289 290 // Retire the current TLAB 291 _thread->retire_tlab(); 292 293 // To minimize fragmentation, the last TLAB may be smaller than the rest. 294 size_t new_tlab_size = tlab.compute_size(_word_size); 295 296 if (new_tlab_size == 0) { 297 return nullptr; 298 } 299 300 // Allocate a new TLAB requesting new_tlab_size. Any size 301 // between minimal and new_tlab_size is accepted. 302 size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size); 303 mem = Universe::heap()->allocate_new_tlab(min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size); 304 if (mem == nullptr) { 305 assert(allocation._allocated_tlab_size == 0, 306 "Allocation failed, but actual size was updated. min: %zu" 307 ", desired: %zu, actual: %zu", 308 min_tlab_size, new_tlab_size, allocation._allocated_tlab_size); 309 return nullptr; 310 } 311 assert(allocation._allocated_tlab_size != 0, "Allocation succeeded but actual size not updated. mem at: " 312 PTR_FORMAT " min: %zu, desired: %zu", 313 p2i(mem), min_tlab_size, new_tlab_size); 314 315 // ...and clear or zap just allocated TLAB, if needed. 316 if (ZeroTLAB) { 317 Copy::zero_to_words(mem, allocation._allocated_tlab_size); 318 } else if (ZapTLAB) { 319 // Skip mangling the space corresponding to the object header to 320 // ensure that the returned space is not considered parsable by 321 // any concurrent GC thread. 322 size_t hdr_size = oopDesc::header_size(); 323 Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal); 324 } 325 326 _thread->fill_tlab(mem, _word_size, allocation._allocated_tlab_size); 327 328 return mem; 329 } 330 331 HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const { 332 if (UseTLAB) { 333 // Try allocating from an existing TLAB. 334 HeapWord* mem = mem_allocate_inside_tlab_fast(); 335 if (mem != nullptr) { 336 return mem; 337 } 338 } 339 340 // Allocation of an oop can always invoke a safepoint. 341 DEBUG_ONLY(allocation._thread->check_for_valid_safepoint_state()); 342 343 if (UseTLAB) { 344 // Try refilling the TLAB and allocating the object in it. 345 HeapWord* mem = mem_allocate_inside_tlab_slow(allocation); 346 if (mem != nullptr) { 347 return mem; 348 } 349 } 350 351 return mem_allocate_outside_tlab(allocation); 352 } 353 354 oop MemAllocator::allocate() const { 355 oop obj = nullptr; 356 { 357 Allocation allocation(*this, &obj); 358 HeapWord* mem = mem_allocate(allocation); 359 if (mem != nullptr) { 360 obj = initialize(mem); 361 } else { 362 // The unhandled oop detector will poison local variable obj, 363 // so reset it to null if mem is null. 364 obj = nullptr; 365 } 366 } 367 return obj; 368 } 369 370 void MemAllocator::mem_clear(HeapWord* mem) const { 371 assert(mem != nullptr, "cannot initialize null object"); 372 const size_t hs = oopDesc::header_size(); 373 assert(_word_size >= hs, "unexpected object size"); 374 if (oopDesc::has_klass_gap()) { 375 oopDesc::set_klass_gap(mem, 0); 376 } 377 Copy::fill_to_aligned_words(mem + hs, _word_size - hs); 378 } 379 380 oop MemAllocator::finish(HeapWord* mem) const { 381 assert(mem != nullptr, "null object pointer"); 382 // Need a release store to ensure array/class length, mark word, and 383 // object zeroing are visible before setting the klass non-null, for 384 // concurrent collectors. 385 if (UseCompactObjectHeaders) { 386 oopDesc::release_set_mark(mem, _klass->prototype_header()); 387 } else { 388 oopDesc::set_mark(mem, markWord::prototype()); 389 oopDesc::release_set_klass(mem, _klass); 390 } 391 return cast_to_oop(mem); 392 } 393 394 oop ObjAllocator::initialize(HeapWord* mem) const { 395 mem_clear(mem); 396 return finish(mem); 397 } 398 399 oop ObjArrayAllocator::initialize(HeapWord* mem) const { 400 // Set array length before setting the _klass field because a 401 // non-null klass field indicates that the object is parsable by 402 // concurrent GC. 403 assert(_length >= 0, "length should be non-negative"); 404 if (_do_zero) { 405 mem_clear(mem); 406 mem_zap_start_padding(mem); 407 mem_zap_end_padding(mem); 408 } 409 arrayOopDesc::set_length(mem, _length); 410 return finish(mem); 411 } 412 413 #ifndef PRODUCT 414 void ObjArrayAllocator::mem_zap_start_padding(HeapWord* mem) const { 415 const BasicType element_type = ArrayKlass::cast(_klass)->element_type(); 416 const size_t base_offset_in_bytes = arrayOopDesc::base_offset_in_bytes(element_type); 417 const size_t header_size_in_bytes = arrayOopDesc::header_size_in_bytes(); 418 419 const address base = reinterpret_cast<address>(mem) + base_offset_in_bytes; 420 const address header_end = reinterpret_cast<address>(mem) + header_size_in_bytes; 421 422 if (header_end < base) { 423 const size_t padding_in_bytes = base - header_end; 424 Copy::fill_to_bytes(header_end, padding_in_bytes, heapPaddingByteVal); 425 } 426 } 427 428 void ObjArrayAllocator::mem_zap_end_padding(HeapWord* mem) const { 429 const size_t length_in_bytes = static_cast<size_t>(_length) << ArrayKlass::cast(_klass)->log2_element_size(); 430 const BasicType element_type = ArrayKlass::cast(_klass)->element_type(); 431 const size_t base_offset_in_bytes = arrayOopDesc::base_offset_in_bytes(element_type); 432 const size_t size_in_bytes = _word_size * BytesPerWord; 433 434 const address obj_end = reinterpret_cast<address>(mem) + size_in_bytes; 435 const address base = reinterpret_cast<address>(mem) + base_offset_in_bytes; 436 const address elements_end = base + length_in_bytes; 437 assert(elements_end <= obj_end, "payload must fit in object"); 438 if (elements_end < obj_end) { 439 const size_t padding_in_bytes = obj_end - elements_end; 440 Copy::fill_to_bytes(elements_end, padding_in_bytes, heapPaddingByteVal); 441 } 442 } 443 #endif 444 445 oop ClassAllocator::initialize(HeapWord* mem) const { 446 // Set oop_size field before setting the _klass field because a 447 // non-null _klass field indicates that the object is parsable by 448 // concurrent GC. 449 assert(_word_size > 0, "oop_size must be positive."); 450 mem_clear(mem); 451 java_lang_Class::set_oop_size(mem, _word_size); 452 return finish(mem); 453 }