1 /* 2 * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/javaClasses.hpp" 27 #include "gc/shared/allocTracer.hpp" 28 #include "gc/shared/collectedHeap.hpp" 29 #include "gc/shared/memAllocator.hpp" 30 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" 31 #include "gc/shared/tlab_globals.hpp" 32 #include "memory/universe.hpp" 33 #include "oops/arrayOop.hpp" 34 #include "oops/oop.inline.hpp" 35 #include "prims/jvmtiExport.hpp" 36 #include "runtime/sharedRuntime.hpp" 37 #include "runtime/handles.inline.hpp" 38 #include "runtime/thread.inline.hpp" 39 #include "services/lowMemoryDetector.hpp" 40 #include "utilities/align.hpp" 41 #include "utilities/copy.hpp" 42 43 class MemAllocator::Allocation: StackObj { 44 friend class MemAllocator; 45 46 const MemAllocator& _allocator; 47 JavaThread* _thread; 48 oop* _obj_ptr; 49 bool _overhead_limit_exceeded; 50 bool _allocated_outside_tlab; 51 size_t _allocated_tlab_size; 52 bool _tlab_end_reset_for_sample; 53 54 bool check_out_of_memory(); 55 void verify_before(); 56 void verify_after(); 57 void notify_allocation(); 58 void notify_allocation_jvmti_sampler(); 59 void notify_allocation_low_memory_detector(); 60 void notify_allocation_jfr_sampler(); 61 void notify_allocation_dtrace_sampler(); 62 #ifdef ASSERT 63 void check_for_valid_allocation_state() const; 64 #endif 65 66 class PreserveObj; 67 68 public: 69 Allocation(const MemAllocator& allocator, oop* obj_ptr) 70 : _allocator(allocator), 71 _thread(JavaThread::current()), 72 _obj_ptr(obj_ptr), 73 _overhead_limit_exceeded(false), 74 _allocated_outside_tlab(false), 75 _allocated_tlab_size(0), 76 _tlab_end_reset_for_sample(false) 77 { 78 verify_before(); 79 } 80 81 ~Allocation() { 82 if (!check_out_of_memory()) { 83 notify_allocation(); 84 } 85 } 86 87 oop obj() const { return *_obj_ptr; } 88 }; 89 90 class MemAllocator::Allocation::PreserveObj: StackObj { 91 HandleMark _handle_mark; 92 Handle _handle; 93 oop* const _obj_ptr; 94 95 public: 96 PreserveObj(JavaThread* thread, oop* obj_ptr) 97 : _handle_mark(thread), 98 _handle(thread, *obj_ptr), 99 _obj_ptr(obj_ptr) 100 { 101 *obj_ptr = NULL; 102 } 103 104 ~PreserveObj() { 105 *_obj_ptr = _handle(); 106 } 107 108 oop operator()() const { 109 return _handle(); 110 } 111 }; 112 113 bool MemAllocator::Allocation::check_out_of_memory() { 114 JavaThread* THREAD = _thread; // For exception macros. 115 assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage"); 116 117 if (obj() != NULL) { 118 return false; 119 } 120 121 const char* message = _overhead_limit_exceeded ? "GC overhead limit exceeded" : "Java heap space"; 122 if (!_thread->in_retryable_allocation()) { 123 // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support 124 report_java_out_of_memory(message); 125 126 if (JvmtiExport::should_post_resource_exhausted()) { 127 JvmtiExport::post_resource_exhausted( 128 JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, 129 message); 130 } 131 oop exception = _overhead_limit_exceeded ? 132 Universe::out_of_memory_error_gc_overhead_limit() : 133 Universe::out_of_memory_error_java_heap(); 134 THROW_OOP_(exception, true); 135 } else { 136 THROW_OOP_(Universe::out_of_memory_error_retry(), true); 137 } 138 } 139 140 void MemAllocator::Allocation::verify_before() { 141 // Clear unhandled oops for memory allocation. Memory allocation might 142 // not take out a lock if from tlab, so clear here. 143 JavaThread* THREAD = _thread; // For exception macros. 144 assert(!HAS_PENDING_EXCEPTION, "Should not allocate with exception pending"); 145 debug_only(check_for_valid_allocation_state()); 146 assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); 147 } 148 149 #ifdef ASSERT 150 void MemAllocator::Allocation::check_for_valid_allocation_state() const { 151 // How to choose between a pending exception and a potential 152 // OutOfMemoryError? Don't allow pending exceptions. 153 // This is a VM policy failure, so how do we exhaustively test it? 154 assert(!_thread->has_pending_exception(), 155 "shouldn't be allocating with pending exception"); 156 // Allocation of an oop can always invoke a safepoint. 157 _thread->as_Java_thread()->check_for_valid_safepoint_state(); 158 } 159 #endif 160 161 void MemAllocator::Allocation::notify_allocation_jvmti_sampler() { 162 // support for JVMTI VMObjectAlloc event (no-op if not enabled) 163 JvmtiExport::vm_object_alloc_event_collector(obj()); 164 165 if (!JvmtiExport::should_post_sampled_object_alloc()) { 166 // Sampling disabled 167 return; 168 } 169 170 if (!_allocated_outside_tlab && _allocated_tlab_size == 0 && !_tlab_end_reset_for_sample) { 171 // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB 172 // or expands it due to taking a sampler induced slow path. 173 return; 174 } 175 176 // If we want to be sampling, protect the allocated object with a Handle 177 // before doing the callback. The callback is done in the destructor of 178 // the JvmtiSampledObjectAllocEventCollector. 179 size_t bytes_since_last = 0; 180 181 { 182 PreserveObj obj_h(_thread, _obj_ptr); 183 JvmtiSampledObjectAllocEventCollector collector; 184 size_t size_in_bytes = _allocator._word_size * HeapWordSize; 185 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 186 187 if (!_allocated_outside_tlab) { 188 bytes_since_last = tlab.bytes_since_last_sample_point(); 189 } 190 191 _thread->heap_sampler().check_for_sampling(obj_h(), size_in_bytes, bytes_since_last); 192 } 193 194 if (_tlab_end_reset_for_sample || _allocated_tlab_size != 0) { 195 // Tell tlab to forget bytes_since_last if we passed it to the heap sampler. 196 _thread->tlab().set_sample_end(bytes_since_last != 0); 197 } 198 } 199 200 void MemAllocator::Allocation::notify_allocation_low_memory_detector() { 201 // support low memory notifications (no-op if not enabled) 202 LowMemoryDetector::detect_low_memory_for_collected_pools(); 203 } 204 205 void MemAllocator::Allocation::notify_allocation_jfr_sampler() { 206 HeapWord* mem = cast_from_oop<HeapWord*>(obj()); 207 size_t size_in_bytes = _allocator._word_size * HeapWordSize; 208 209 if (_allocated_outside_tlab) { 210 AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread); 211 } else if (_allocated_tlab_size != 0) { 212 // TLAB was refilled 213 AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize, 214 size_in_bytes, _thread); 215 } 216 } 217 218 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() { 219 if (DTraceAllocProbes) { 220 // support for Dtrace object alloc event (no-op most of the time) 221 Klass* klass = obj()->klass(); 222 size_t word_size = _allocator._word_size; 223 if (klass != NULL && klass->name() != NULL) { 224 SharedRuntime::dtrace_object_alloc(obj(), (int)word_size); 225 } 226 } 227 } 228 229 void MemAllocator::Allocation::notify_allocation() { 230 notify_allocation_low_memory_detector(); 231 notify_allocation_jfr_sampler(); 232 notify_allocation_dtrace_sampler(); 233 notify_allocation_jvmti_sampler(); 234 } 235 236 HeapWord* MemAllocator::allocate_outside_tlab(Allocation& allocation) const { 237 allocation._allocated_outside_tlab = true; 238 HeapWord* mem = Universe::heap()->mem_allocate(_word_size, &allocation._overhead_limit_exceeded); 239 if (mem == NULL) { 240 return mem; 241 } 242 243 size_t size_in_bytes = _word_size * HeapWordSize; 244 _thread->incr_allocated_bytes(size_in_bytes); 245 246 return mem; 247 } 248 249 HeapWord* MemAllocator::allocate_inside_tlab(Allocation& allocation) const { 250 assert(UseTLAB, "should use UseTLAB"); 251 252 // Try allocating from an existing TLAB. 253 HeapWord* mem = _thread->tlab().allocate(_word_size); 254 if (mem != NULL) { 255 return mem; 256 } 257 258 // Try refilling the TLAB and allocating the object in it. 259 return allocate_inside_tlab_slow(allocation); 260 } 261 262 HeapWord* MemAllocator::allocate_inside_tlab_slow(Allocation& allocation) const { 263 HeapWord* mem = NULL; 264 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 265 266 if (JvmtiExport::should_post_sampled_object_alloc()) { 267 tlab.set_back_allocation_end(); 268 mem = tlab.allocate(_word_size); 269 270 // We set back the allocation sample point to try to allocate this, reset it 271 // when done. 272 allocation._tlab_end_reset_for_sample = true; 273 274 if (mem != NULL) { 275 return mem; 276 } 277 } 278 279 // Retain tlab and allocate object in shared space if 280 // the amount free in the tlab is too large to discard. 281 if (tlab.free() > tlab.refill_waste_limit()) { 282 tlab.record_slow_allocation(_word_size); 283 return NULL; 284 } 285 286 // Discard tlab and allocate a new one. 287 // To minimize fragmentation, the last TLAB may be smaller than the rest. 288 size_t new_tlab_size = tlab.compute_size(_word_size); 289 290 tlab.retire_before_allocation(); 291 292 if (new_tlab_size == 0) { 293 return NULL; 294 } 295 296 // Allocate a new TLAB requesting new_tlab_size. Any size 297 // between minimal and new_tlab_size is accepted. 298 size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size); 299 mem = Universe::heap()->allocate_new_tlab(min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size); 300 if (mem == NULL) { 301 assert(allocation._allocated_tlab_size == 0, 302 "Allocation failed, but actual size was updated. min: " SIZE_FORMAT 303 ", desired: " SIZE_FORMAT ", actual: " SIZE_FORMAT, 304 min_tlab_size, new_tlab_size, allocation._allocated_tlab_size); 305 return NULL; 306 } 307 assert(allocation._allocated_tlab_size != 0, "Allocation succeeded but actual size not updated. mem at: " 308 PTR_FORMAT " min: " SIZE_FORMAT ", desired: " SIZE_FORMAT, 309 p2i(mem), min_tlab_size, new_tlab_size); 310 311 if (ZeroTLAB) { 312 // ..and clear it. 313 Copy::zero_to_words(mem, allocation._allocated_tlab_size); 314 } else { 315 // ...and zap just allocated object. 316 #ifdef ASSERT 317 // Skip mangling the space corresponding to the object header to 318 // ensure that the returned space is not considered parsable by 319 // any concurrent GC thread. 320 size_t hdr_size = oopDesc::header_size(); 321 Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal); 322 #endif // ASSERT 323 } 324 325 tlab.fill(mem, mem + _word_size, allocation._allocated_tlab_size); 326 return mem; 327 } 328 329 HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const { 330 if (UseTLAB) { 331 HeapWord* result = allocate_inside_tlab(allocation); 332 if (result != NULL) { 333 return result; 334 } 335 } 336 337 return allocate_outside_tlab(allocation); 338 } 339 340 oop MemAllocator::allocate() const { 341 oop obj = NULL; 342 { 343 Allocation allocation(*this, &obj); 344 HeapWord* mem = mem_allocate(allocation); 345 if (mem != NULL) { 346 obj = initialize(mem); 347 } else { 348 // The unhandled oop detector will poison local variable obj, 349 // so reset it to NULL if mem is NULL. 350 obj = NULL; 351 } 352 } 353 return obj; 354 } 355 356 void MemAllocator::mem_clear(HeapWord* mem) const { 357 assert(mem != NULL, "cannot initialize NULL object"); 358 const size_t hs = oopDesc::header_size(); 359 assert(_word_size >= hs, "unexpected object size"); 360 if (!UseCompactObjectHeaders) { 361 oopDesc::set_klass_gap(mem, 0); 362 } 363 Copy::fill_to_aligned_words(mem + hs, _word_size - hs); 364 } 365 366 oop MemAllocator::finish(HeapWord* mem) const { 367 assert(mem != NULL, "NULL object pointer"); 368 if (UseBiasedLocking) { 369 oopDesc::set_mark(mem, _klass->prototype_header()); 370 } else if (UseCompactObjectHeaders) { 371 oopDesc::release_set_mark(mem, _klass->prototype_header()); 372 } else { 373 // May be bootstrapping 374 oopDesc::set_mark(mem, markWord::prototype()); 375 } 376 // Need a release store to ensure array/class length, mark word, and 377 // object zeroing are visible before setting the klass non-NULL, for 378 // concurrent collectors. 379 if (!UseCompactObjectHeaders) { 380 oopDesc::release_set_klass(mem, _klass); 381 } 382 return cast_to_oop(mem); 383 } 384 385 oop ObjAllocator::initialize(HeapWord* mem) const { 386 mem_clear(mem); 387 return finish(mem); 388 } 389 390 oop ObjArrayAllocator::initialize(HeapWord* mem) const { 391 // Set array length before setting the _klass field because a 392 // non-NULL klass field indicates that the object is parsable by 393 // concurrent GC. 394 assert(_length >= 0, "length should be non-negative"); 395 if (_do_zero) { 396 mem_clear(mem); 397 } 398 arrayOopDesc::set_length(mem, _length); 399 return finish(mem); 400 } 401 402 oop ClassAllocator::initialize(HeapWord* mem) const { 403 // Set oop_size field before setting the _klass field because a 404 // non-NULL _klass field indicates that the object is parsable by 405 // concurrent GC. 406 assert(_word_size > 0, "oop_size must be positive."); 407 mem_clear(mem); 408 java_lang_Class::set_oop_size(mem, (int)_word_size); 409 return finish(mem); 410 } --- EOF ---