1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/javaClasses.hpp"
 27 #include "classfile/vmClasses.hpp"
 28 #include "gc/shared/allocTracer.hpp"
 29 #include "gc/shared/collectedHeap.hpp"
 30 #include "gc/shared/memAllocator.hpp"
 31 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 32 #include "gc/shared/tlab_globals.hpp"
 33 #include "memory/universe.hpp"
 34 #include "oops/arrayOop.hpp"
 35 #include "oops/oop.inline.hpp"
 36 #include "prims/jvmtiExport.hpp"
 37 #include "runtime/continuationJavaClasses.inline.hpp"
 38 #include "runtime/handles.inline.hpp"
 39 #include "runtime/sharedRuntime.hpp"
 40 #include "runtime/javaThread.hpp"
 41 #include "services/lowMemoryDetector.hpp"
 42 #include "utilities/align.hpp"
 43 #include "utilities/copy.hpp"
 44 
 45 class MemAllocator::Allocation: StackObj {
 46   friend class MemAllocator;
 47 
 48   const MemAllocator& _allocator;
 49   JavaThread*         _thread;
 50   oop*                _obj_ptr;
 51   bool                _overhead_limit_exceeded;
 52   bool                _allocated_outside_tlab;
 53   size_t              _allocated_tlab_size;
 54   bool                _tlab_end_reset_for_sample;
 55 
 56   bool check_out_of_memory();
 57   void verify_before();
 58   void verify_after();
 59   void notify_allocation(JavaThread* thread);
 60   void notify_allocation_jvmti_sampler();
 61   void notify_allocation_low_memory_detector();
 62   void notify_allocation_jfr_sampler();
 63   void notify_allocation_dtrace_sampler(JavaThread* thread);
 64 #ifdef ASSERT
 65   void check_for_valid_allocation_state() const;
 66 #endif
 67 
 68   class PreserveObj;
 69 
 70 public:
 71   Allocation(const MemAllocator& allocator, oop* obj_ptr)
 72     : _allocator(allocator),
 73       _thread(JavaThread::current()),
 74       _obj_ptr(obj_ptr),
 75       _overhead_limit_exceeded(false),
 76       _allocated_outside_tlab(false),
 77       _allocated_tlab_size(0),
 78       _tlab_end_reset_for_sample(false)
 79   {
 80     verify_before();
 81   }
 82 
 83   ~Allocation() {
 84     if (!check_out_of_memory()) {
 85       notify_allocation(_thread);
 86     }
 87   }
 88 
 89   oop obj() const { return *_obj_ptr; }
 90 };
 91 
 92 class MemAllocator::Allocation::PreserveObj: StackObj {
 93   HandleMark _handle_mark;
 94   Handle     _handle;
 95   oop* const _obj_ptr;
 96 
 97 public:
 98   PreserveObj(JavaThread* thread, oop* obj_ptr)
 99     : _handle_mark(thread),
100       _handle(thread, *obj_ptr),
101       _obj_ptr(obj_ptr)
102   {
103     *obj_ptr = nullptr;
104   }
105 
106   ~PreserveObj() {
107     *_obj_ptr = _handle();
108   }
109 
110   oop operator()() const {
111     return _handle();
112   }
113 };
114 
115 bool MemAllocator::Allocation::check_out_of_memory() {
116   JavaThread* THREAD = _thread; // For exception macros.
117   assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage");
118 
119   if (obj() != nullptr) {
120     return false;
121   }
122 
123   const char* message = _overhead_limit_exceeded ? "GC overhead limit exceeded" : "Java heap space";
124   if (!_thread->in_retryable_allocation()) {
125     // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
126     report_java_out_of_memory(message);
127 
128     if (JvmtiExport::should_post_resource_exhausted()) {
129       JvmtiExport::post_resource_exhausted(
130         JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP,
131         message);
132     }
133     oop exception = _overhead_limit_exceeded ?
134         Universe::out_of_memory_error_gc_overhead_limit() :
135         Universe::out_of_memory_error_java_heap();
136     THROW_OOP_(exception, true);
137   } else {
138     THROW_OOP_(Universe::out_of_memory_error_retry(), true);
139   }
140 }
141 
142 void MemAllocator::Allocation::verify_before() {
143   // Clear unhandled oops for memory allocation.  Memory allocation might
144   // not take out a lock if from tlab, so clear here.
145   JavaThread* THREAD = _thread; // For exception macros.
146   assert(!HAS_PENDING_EXCEPTION, "Should not allocate with exception pending");
147   debug_only(check_for_valid_allocation_state());
148   assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed");
149 }
150 
151 #ifdef ASSERT
152 void MemAllocator::Allocation::check_for_valid_allocation_state() const {
153   // How to choose between a pending exception and a potential
154   // OutOfMemoryError?  Don't allow pending exceptions.
155   // This is a VM policy failure, so how do we exhaustively test it?
156   assert(!_thread->has_pending_exception(),
157          "shouldn't be allocating with pending exception");
158   // Allocation of an oop can always invoke a safepoint.
159   JavaThread::cast(_thread)->check_for_valid_safepoint_state();
160 }
161 #endif
162 
163 void MemAllocator::Allocation::notify_allocation_jvmti_sampler() {
164   // support for JVMTI VMObjectAlloc event (no-op if not enabled)
165   JvmtiExport::vm_object_alloc_event_collector(obj());
166 
167   if (!JvmtiExport::should_post_sampled_object_alloc()) {
168     // Sampling disabled
169     return;
170   }
171 
172   if (!_allocated_outside_tlab && _allocated_tlab_size == 0 && !_tlab_end_reset_for_sample) {
173     // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB
174     // or expands it due to taking a sampler induced slow path.
175     return;
176   }
177 
178   // If we want to be sampling, protect the allocated object with a Handle
179   // before doing the callback. The callback is done in the destructor of
180   // the JvmtiSampledObjectAllocEventCollector.
181   size_t bytes_since_last = 0;
182 
183   {
184     PreserveObj obj_h(_thread, _obj_ptr);
185     JvmtiSampledObjectAllocEventCollector collector;
186     size_t size_in_bytes = _allocator._word_size * HeapWordSize;
187     ThreadLocalAllocBuffer& tlab = _thread->tlab();
188 
189     if (!_allocated_outside_tlab) {
190       bytes_since_last = tlab.bytes_since_last_sample_point();
191     }
192 
193     _thread->heap_sampler().check_for_sampling(obj_h(), size_in_bytes, bytes_since_last);
194   }
195 
196   if (_tlab_end_reset_for_sample || _allocated_tlab_size != 0) {
197     // Tell tlab to forget bytes_since_last if we passed it to the heap sampler.
198     _thread->tlab().set_sample_end(bytes_since_last != 0);
199   }
200 }
201 
202 void MemAllocator::Allocation::notify_allocation_low_memory_detector() {
203   // support low memory notifications (no-op if not enabled)
204   LowMemoryDetector::detect_low_memory_for_collected_pools();
205 }
206 
207 void MemAllocator::Allocation::notify_allocation_jfr_sampler() {
208   HeapWord* mem = cast_from_oop<HeapWord*>(obj());
209   size_t size_in_bytes = _allocator._word_size * HeapWordSize;
210 
211   if (_allocated_outside_tlab) {
212     AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread);
213   } else if (_allocated_tlab_size != 0) {
214     // TLAB was refilled
215     AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize,
216                                              size_in_bytes, _thread);
217   }
218 }
219 
220 void MemAllocator::Allocation::notify_allocation_dtrace_sampler(JavaThread* thread) {
221   if (DTraceAllocProbes) {
222     // support for Dtrace object alloc event (no-op most of the time)
223     Klass* klass = obj()->klass();
224     size_t word_size = _allocator._word_size;
225     if (klass != nullptr && klass->name() != nullptr) {
226       SharedRuntime::dtrace_object_alloc(thread, obj(), word_size);
227     }
228   }
229 }
230 
231 void MemAllocator::Allocation::notify_allocation(JavaThread* thread) {
232   notify_allocation_low_memory_detector();
233   notify_allocation_jfr_sampler();
234   notify_allocation_dtrace_sampler(thread);
235   notify_allocation_jvmti_sampler();
236 }
237 
238 HeapWord* MemAllocator::mem_allocate_outside_tlab(Allocation& allocation) const {
239   allocation._allocated_outside_tlab = true;
240   HeapWord* mem = Universe::heap()->mem_allocate(_word_size, &allocation._overhead_limit_exceeded);
241   if (mem == nullptr) {
242     return mem;
243   }
244 
245   size_t size_in_bytes = _word_size * HeapWordSize;
246   _thread->incr_allocated_bytes(size_in_bytes);
247 
248   return mem;
249 }
250 
251 HeapWord* MemAllocator::mem_allocate_inside_tlab(Allocation& allocation) const {
252   assert(UseTLAB, "should use UseTLAB");
253 
254   // Try allocating from an existing TLAB.
255   HeapWord* mem = mem_allocate_inside_tlab_fast();
256   if (mem != nullptr) {
257     return mem;
258   }
259 
260   // Try refilling the TLAB and allocating the object in it.
261   return mem_allocate_inside_tlab_slow(allocation);
262 }
263 
264 HeapWord* MemAllocator::mem_allocate_inside_tlab_fast() const {
265   return _thread->tlab().allocate(_word_size);
266 }
267 
268 HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) const {
269   HeapWord* mem = nullptr;
270   ThreadLocalAllocBuffer& tlab = _thread->tlab();
271 
272   if (JvmtiExport::should_post_sampled_object_alloc()) {
273     tlab.set_back_allocation_end();
274     mem = tlab.allocate(_word_size);
275 
276     // We set back the allocation sample point to try to allocate this, reset it
277     // when done.
278     allocation._tlab_end_reset_for_sample = true;
279 
280     if (mem != nullptr) {
281       return mem;
282     }
283   }
284 
285   // Retain tlab and allocate object in shared space if
286   // the amount free in the tlab is too large to discard.
287   if (tlab.free() > tlab.refill_waste_limit()) {
288     tlab.record_slow_allocation(_word_size);
289     return nullptr;
290   }
291 
292   // Discard tlab and allocate a new one.
293   // To minimize fragmentation, the last TLAB may be smaller than the rest.
294   size_t new_tlab_size = tlab.compute_size(_word_size);
295 
296   tlab.retire_before_allocation();
297 
298   if (new_tlab_size == 0) {
299     return nullptr;
300   }
301 
302   // Allocate a new TLAB requesting new_tlab_size. Any size
303   // between minimal and new_tlab_size is accepted.
304   size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size);
305   mem = Universe::heap()->allocate_new_tlab(min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size);
306   if (mem == nullptr) {
307     assert(allocation._allocated_tlab_size == 0,
308            "Allocation failed, but actual size was updated. min: " SIZE_FORMAT
309            ", desired: " SIZE_FORMAT ", actual: " SIZE_FORMAT,
310            min_tlab_size, new_tlab_size, allocation._allocated_tlab_size);
311     return nullptr;
312   }
313   assert(allocation._allocated_tlab_size != 0, "Allocation succeeded but actual size not updated. mem at: "
314          PTR_FORMAT " min: " SIZE_FORMAT ", desired: " SIZE_FORMAT,
315          p2i(mem), min_tlab_size, new_tlab_size);
316 
317   if (ZeroTLAB) {
318     // ..and clear it.
319     Copy::zero_to_words(mem, allocation._allocated_tlab_size);
320   } else {
321     // ...and zap just allocated object.
322 #ifdef ASSERT
323     // Skip mangling the space corresponding to the object header to
324     // ensure that the returned space is not considered parsable by
325     // any concurrent GC thread.
326     size_t hdr_size = oopDesc::header_size();
327     Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal);
328 #endif // ASSERT
329   }
330 
331   tlab.fill(mem, mem + _word_size, allocation._allocated_tlab_size);
332   return mem;
333 }
334 
335 
336 HeapWord* MemAllocator::mem_allocate_slow(Allocation& allocation) const {
337   // Allocation of an oop can always invoke a safepoint.
338   debug_only(JavaThread::cast(_thread)->check_for_valid_safepoint_state());
339 
340   if (UseTLAB) {
341     // Try refilling the TLAB and allocating the object in it.
342     HeapWord* mem = mem_allocate_inside_tlab_slow(allocation);
343     if (mem != nullptr) {
344       return mem;
345     }
346   }
347 
348   return mem_allocate_outside_tlab(allocation);
349 }
350 
351 HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const {
352   if (UseTLAB) {
353     // Try allocating from an existing TLAB.
354     HeapWord* mem = mem_allocate_inside_tlab_fast();
355     if (mem != nullptr) {
356       return mem;
357     }
358   }
359 
360   return mem_allocate_slow(allocation);
361 }
362 
363 oop MemAllocator::allocate() const {
364   oop obj = nullptr;
365   {
366     Allocation allocation(*this, &obj);
367     HeapWord* mem = mem_allocate(allocation);
368     if (mem != nullptr) {
369       obj = initialize(mem);
370     } else {
371       // The unhandled oop detector will poison local variable obj,
372       // so reset it to null if mem is null.
373       obj = nullptr;
374     }
375   }
376   return obj;
377 }
378 
379 void MemAllocator::mem_clear(HeapWord* mem) const {
380   assert(mem != nullptr, "cannot initialize null object");
381   const size_t hs = oopDesc::header_size();
382   assert(_word_size >= hs, "unexpected object size");
383   oopDesc::set_klass_gap(mem, 0);


384   Copy::fill_to_aligned_words(mem + hs, _word_size - hs);
385 }
386 
387 oop MemAllocator::finish(HeapWord* mem) const {
388   assert(mem != nullptr, "null object pointer");
389   // May be bootstrapping
390   oopDesc::set_mark(mem, markWord::prototype());
391   // Need a release store to ensure array/class length, mark word, and
392   // object zeroing are visible before setting the klass non-null, for
393   // concurrent collectors.
394   oopDesc::release_set_klass(mem, _klass);





395   return cast_to_oop(mem);
396 }
397 
398 oop ObjAllocator::initialize(HeapWord* mem) const {
399   mem_clear(mem);
400   return finish(mem);
401 }
402 
403 oop ObjArrayAllocator::initialize(HeapWord* mem) const {
404   // Set array length before setting the _klass field because a
405   // non-null klass field indicates that the object is parsable by
406   // concurrent GC.
407   assert(_length >= 0, "length should be non-negative");
408   if (_do_zero) {
409     mem_clear(mem);
410   }
411   arrayOopDesc::set_length(mem, _length);
412   return finish(mem);
413 }
414 
415 oop ClassAllocator::initialize(HeapWord* mem) const {
416   // Set oop_size field before setting the _klass field because a
417   // non-null _klass field indicates that the object is parsable by
418   // concurrent GC.
419   assert(_word_size > 0, "oop_size must be positive.");
420   mem_clear(mem);
421   java_lang_Class::set_oop_size(mem, _word_size);
422   return finish(mem);
423 }
--- EOF ---