1 /*
  2  * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/javaClasses.inline.hpp"
 27 #include "jfr/jni/jfrJavaSupport.hpp"
 28 #include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp"
 29 #include "jfr/leakprofiler/leakProfiler.hpp"
 30 #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
 31 #include "jfr/recorder/checkpoint/jfrCheckpointWriter.hpp"
 32 #include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp"
 33 #include "jfr/recorder/checkpoint/types/jfrTypeSet.hpp"
 34 #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
 35 #include "jfr/recorder/jfrRecorder.hpp"
 36 #include "jfr/recorder/repository/jfrChunkWriter.hpp"
 37 #include "jfr/recorder/service/jfrOptionSet.hpp"
 38 #include "jfr/recorder/storage/jfrEpochStorage.inline.hpp"
 39 #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
 40 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
 41 #include "jfr/support/jfrKlassUnloading.hpp"
 42 #include "jfr/utilities/jfrBigEndian.hpp"
 43 #include "jfr/utilities/jfrIterator.hpp"
 44 #include "jfr/utilities/jfrLinkedList.inline.hpp"
 45 #include "jfr/utilities/jfrSignal.hpp"
 46 #include "jfr/utilities/jfrThreadIterator.hpp"
 47 #include "jfr/utilities/jfrTypes.hpp"
 48 #include "jfr/writers/jfrJavaEventWriter.hpp"
 49 #include "logging/log.hpp"
 50 #include "memory/iterator.hpp"
 51 #include "memory/resourceArea.hpp"
 52 #include "runtime/atomic.hpp"
 53 #include "runtime/handles.inline.hpp"
 54 #include "runtime/interfaceSupport.inline.hpp"
 55 #include "runtime/mutex.hpp"
 56 #include "runtime/safepoint.hpp"
 57 
 58 typedef JfrCheckpointManager::BufferPtr BufferPtr;
 59 
 60 static JfrSignal _new_checkpoint;
 61 static JfrCheckpointManager* _instance = NULL;
 62 
 63 JfrCheckpointManager& JfrCheckpointManager::instance() {
 64   return *_instance;
 65 }
 66 
 67 JfrCheckpointManager* JfrCheckpointManager::create(JfrChunkWriter& cw) {
 68   assert(_instance == NULL, "invariant");
 69   _instance = new JfrCheckpointManager(cw);
 70   return _instance;
 71 }
 72 
 73 void JfrCheckpointManager::destroy() {
 74   assert(_instance != NULL, "invariant");
 75   delete _instance;
 76   _instance = NULL;
 77 }
 78 
 79 JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) :
 80   _global_mspace(NULL),
 81   _thread_local_mspace(NULL),
 82   _chunkwriter(cw) {}
 83 
 84 JfrCheckpointManager::~JfrCheckpointManager() {
 85   JfrTraceIdLoadBarrier::destroy();
 86   JfrTypeManager::destroy();
 87   delete _global_mspace;
 88   delete _thread_local_mspace;
 89 }
 90 
 91 static const size_t global_buffer_prealloc_count = 2;
 92 static const size_t global_buffer_size = 512 * K;
 93 
 94 static const size_t thread_local_buffer_prealloc_count = 16;
 95 static const size_t thread_local_buffer_size = 128;
 96 
 97 bool JfrCheckpointManager::initialize() {
 98   assert(_global_mspace == NULL, "invariant");
 99   _global_mspace =  create_mspace<JfrCheckpointMspace, JfrCheckpointManager>(global_buffer_size, 0, 0, false, this); // post-pone preallocation
100   if (_global_mspace == NULL) {
101     return false;
102   }
103   // preallocate buffer count to each of the epoch live lists
104   for (size_t i = 0; i < global_buffer_prealloc_count * 2; ++i) {
105     Buffer* const buffer = mspace_allocate(global_buffer_size, _global_mspace);
106     _global_mspace->add_to_live_list(buffer, i % 2 == 0);
107   }
108   assert(_global_mspace->free_list_is_empty(), "invariant");
109 
110   assert(_thread_local_mspace == NULL, "invariant");
111   _thread_local_mspace = new JfrThreadLocalCheckpointMspace();
112   if (_thread_local_mspace == NULL || !_thread_local_mspace->initialize(thread_local_buffer_size,
113                                                                         JFR_MSPACE_UNLIMITED_CACHE_SIZE,
114                                                                         thread_local_buffer_prealloc_count)) {
115     return false;
116   }
117   return JfrTypeManager::initialize() && JfrTraceIdLoadBarrier::initialize();
118 }
119 
120 #ifdef ASSERT
121 static void assert_lease(const BufferPtr buffer) {
122   assert(buffer != NULL, "invariant");
123   assert(buffer->acquired_by_self(), "invariant");
124   assert(buffer->lease(), "invariant");
125 }
126 
127 static void assert_release(const BufferPtr buffer) {
128   assert(buffer != NULL, "invariant");
129   assert(buffer->lease(), "invariant");
130   assert(buffer->acquired_by_self(), "invariant");
131 }
132 
133 static void assert_retired(const BufferPtr buffer, Thread* thread) {
134   assert(buffer != NULL, "invariant");
135   assert(buffer->acquired_by(thread), "invariant");
136   assert(buffer->retired(), "invariant");
137 }
138 #endif // ASSERT
139 
140 void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) {
141   DEBUG_ONLY(assert_retired(buffer, thread);)
142   // nothing here at the moment
143 }
144 
145 BufferPtr JfrCheckpointManager::lease(Thread* thread, bool previous_epoch /* false */, size_t size /* 0 */) {
146   JfrCheckpointMspace* const mspace = instance()._global_mspace;
147   assert(mspace != NULL, "invariant");
148   static const size_t max_elem_size = mspace->min_element_size(); // min is max
149   BufferPtr buffer;
150   if (size <= max_elem_size) {
151     buffer = mspace_acquire_live(size, mspace, thread, previous_epoch);
152     if (buffer != NULL) {
153       buffer->set_lease();
154       DEBUG_ONLY(assert_lease(buffer);)
155       return buffer;
156     }
157   }
158   buffer = mspace_allocate_transient_lease_to_live_list(size, mspace, thread, previous_epoch);
159   DEBUG_ONLY(assert_lease(buffer);)
160   return buffer;
161 }
162 
163 const u1 thread_local_context = 1;
164 
165 static bool is_thread_local(JfrBuffer* buffer) {
166   assert(buffer != NULL, "invariant");
167   return buffer->context() == thread_local_context;
168 }
169 
170 static void retire(JfrBuffer* buffer) {
171   DEBUG_ONLY(assert_release(buffer);)
172   buffer->clear_lease();
173   buffer->set_retired();
174 }
175 
176 /*
177  * The buffer is effectively invalidated for the thread post-return,
178  * and the caller should take means to ensure that it is not referenced.
179  */
180 static void release(JfrBuffer* buffer) {
181   DEBUG_ONLY(assert_release(buffer);)
182   if (is_thread_local(buffer)) {
183     retire(buffer);
184   } else {
185     buffer->clear_lease();
186     buffer->release();
187   }
188 }
189 BufferPtr JfrCheckpointManager::acquire_thread_local(size_t size, Thread* thread) {
190   assert(thread != NULL, "invariant");
191   JfrBuffer* const buffer = instance()._thread_local_mspace->acquire(size, thread);
192   assert(buffer != NULL, "invariant");
193   assert(buffer->free_size() >= size, "invariant");
194   buffer->set_context(thread_local_context);
195   assert(is_thread_local(buffer), "invariant");
196   buffer->set_lease();
197   return buffer;
198 }
199 
200 BufferPtr JfrCheckpointManager::lease_thread_local(Thread* thread, size_t size /* 0 */) {
201   JfrBuffer* const buffer = acquire_thread_local(size, thread);
202   DEBUG_ONLY(assert_lease(buffer);)
203   return buffer;
204 }
205 
206 BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size) {
207   assert(old != NULL, "invariant");
208   return is_thread_local(old) ? acquire_thread_local(size, thread) :
209                                 lease(thread, instance()._global_mspace->in_previous_epoch_list(old), size);
210 }
211 
212 BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
213   assert(old != NULL, "invariant");
214   assert(old->lease(), "invariant");
215   if (0 == requested) {
216     // indicates a lease is being returned
217     release(old);
218     // signal completion of a new checkpoint
219     _new_checkpoint.signal();
220     return NULL;
221   }
222   BufferPtr new_buffer = lease(old, thread, used + requested);
223   assert(new_buffer != NULL, "invariant");
224   migrate_outstanding_writes(old, new_buffer, used, requested);
225   retire(old);
226   return new_buffer;
227 }
228 
229 // offsets into the JfrCheckpointEntry
230 static const juint starttime_offset = sizeof(jlong);
231 static const juint duration_offset = starttime_offset + sizeof(jlong);
232 static const juint checkpoint_type_offset = duration_offset + sizeof(jlong);
233 static const juint types_offset = checkpoint_type_offset + sizeof(juint);
234 static const juint payload_offset = types_offset + sizeof(juint);
235 
236 template <typename Return>
237 static Return read_data(const u1* data) {
238   return JfrBigEndian::read<Return>(data);
239 }
240 
241 static jlong total_size(const u1* data) {
242   return read_data<jlong>(data);
243 }
244 
245 static jlong starttime(const u1* data) {
246   return read_data<jlong>(data + starttime_offset);
247 }
248 
249 static jlong duration(const u1* data) {
250   return read_data<jlong>(data + duration_offset);
251 }
252 
253 static u1 checkpoint_type(const u1* data) {
254   return read_data<u1>(data + checkpoint_type_offset);
255 }
256 
257 static juint number_of_types(const u1* data) {
258   return read_data<juint>(data + types_offset);
259 }
260 
261 static void write_checkpoint_header(JfrChunkWriter& cw, int64_t delta_to_last_checkpoint, const u1* data) {
262   cw.reserve(sizeof(u4));
263   cw.write<u8>(EVENT_CHECKPOINT);
264   cw.write(starttime(data));
265   cw.write(duration(data));
266   cw.write(delta_to_last_checkpoint);
267   cw.write(checkpoint_type(data));
268   cw.write(number_of_types(data));
269 }
270 
271 static void write_checkpoint_content(JfrChunkWriter& cw, const u1* data, size_t size) {
272   assert(data != NULL, "invariant");
273   cw.write_unbuffered(data + payload_offset, size - sizeof(JfrCheckpointEntry));
274 }
275 
276 static size_t write_checkpoint_event(JfrChunkWriter& cw, const u1* data) {
277   assert(data != NULL, "invariant");
278   const int64_t event_begin = cw.current_offset();
279   const int64_t last_checkpoint_event = cw.last_checkpoint_offset();
280   const int64_t delta_to_last_checkpoint = last_checkpoint_event == 0 ? 0 : last_checkpoint_event - event_begin;
281   const int64_t checkpoint_size = total_size(data);
282   write_checkpoint_header(cw, delta_to_last_checkpoint, data);
283   write_checkpoint_content(cw, data, checkpoint_size);
284   const int64_t event_size = cw.current_offset() - event_begin;
285   cw.write_padded_at_offset<u4>(event_size, event_begin);
286   cw.set_last_checkpoint_offset(event_begin);
287   return (size_t)checkpoint_size;
288 }
289 
290 static size_t write_checkpoints(JfrChunkWriter& cw, const u1* data, size_t size) {
291   assert(cw.is_valid(), "invariant");
292   assert(data != NULL, "invariant");
293   assert(size > 0, "invariant");
294   const u1* const limit = data + size;
295   const u1* next = data;
296   size_t processed = 0;
297   while (next < limit) {
298     const size_t checkpoint_size = write_checkpoint_event(cw, next);
299     processed += checkpoint_size;
300     next += checkpoint_size;
301   }
302   assert(next == limit, "invariant");
303   return processed;
304 }
305 
306 template <typename T>
307 class CheckpointWriteOp {
308  private:
309   JfrChunkWriter& _writer;
310   size_t _processed;
311  public:
312   typedef T Type;
313   CheckpointWriteOp(JfrChunkWriter& writer) : _writer(writer), _processed(0) {}
314   bool write(Type* t, const u1* data, size_t size) {
315     _processed += write_checkpoints(_writer, data, size);
316     return true;
317   }
318   size_t processed() const { return _processed; }
319 };
320 
321 typedef CheckpointWriteOp<JfrCheckpointManager::Buffer> WriteOperation;
322 typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation;
323 typedef ReleaseWithExcisionOp<JfrCheckpointMspace, JfrCheckpointMspace::LiveList> ReleaseOperation;
324 typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> WriteReleaseOperation;
325 
326 void JfrCheckpointManager::begin_epoch_shift() {
327   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
328   JfrTraceIdEpoch::begin_epoch_shift();
329 }
330 
331 void JfrCheckpointManager::end_epoch_shift() {
332   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
333   debug_only(const u1 current_epoch = JfrTraceIdEpoch::current();)
334   JfrTraceIdEpoch::end_epoch_shift();
335   assert(current_epoch != JfrTraceIdEpoch::current(), "invariant");
336 }
337 
338 size_t JfrCheckpointManager::write() {
339   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current()));
340   WriteOperation wo(_chunkwriter);
341   MutexedWriteOperation mwo(wo);
342   _thread_local_mspace->iterate(mwo, true); // previous epoch list
343   assert(_global_mspace->free_list_is_empty(), "invariant");
344   ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true));
345   WriteReleaseOperation wro(&mwo, &ro);
346   process_live_list(wro, _global_mspace, true); // previous epoch list
347   return wo.processed();
348 }
349 
350 typedef DiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperation;
351 typedef CompositeOperation<DiscardOperation, ReleaseOperation> DiscardReleaseOperation;
352 
353 size_t JfrCheckpointManager::clear() {
354   JfrTraceIdLoadBarrier::clear();
355   clear_type_set();
356   DiscardOperation discard_operation(mutexed); // mutexed discard mode
357   _thread_local_mspace->iterate(discard_operation, true); // previous epoch list
358   ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true));
359   DiscardReleaseOperation discard_op(&discard_operation, &ro);
360   assert(_global_mspace->free_list_is_empty(), "invariant");
361   process_live_list(discard_op, _global_mspace, true); // previous epoch list
362   return discard_operation.elements();
363 }
364 
365 size_t JfrCheckpointManager::write_static_type_set(Thread* thread) {
366   assert(thread != NULL, "invariant");
367   JfrCheckpointWriter writer(true, thread, STATICS);
368   JfrTypeManager::write_static_types(writer);
369   return writer.used_size();
370 }
371 
372 size_t JfrCheckpointManager::write_threads(JavaThread* thread) {
373   assert(thread != NULL, "invariant");
374   // can safepoint here
375   ThreadInVMfromNative transition(thread);
376   ResourceMark rm(thread);
377   HandleMark hm(thread);
378   JfrCheckpointWriter writer(true, thread, THREADS);
379   JfrTypeManager::write_threads(writer);
380   return writer.used_size();
381 }
382 
383 size_t JfrCheckpointManager::write_static_type_set_and_threads() {
384   JavaThread* const thread = JavaThread::current();
385   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread));
386   write_static_type_set(thread);
387   write_threads(thread);
388   return write();
389 }
390 
391 void JfrCheckpointManager::on_rotation() {
392   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
393   JfrTypeManager::on_rotation();
394   notify_threads();
395 }
396 
397 void JfrCheckpointManager::clear_type_set() {
398   assert(!JfrRecorder::is_recording(), "invariant");
399   JavaThread* t = JavaThread::current();
400   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(t));
401   // can safepoint here
402   ThreadInVMfromNative transition(t);
403   MutexLocker cld_lock(ClassLoaderDataGraph_lock);
404   MutexLocker module_lock(Module_lock);
405   JfrTypeSet::clear();
406 }
407 
408 void JfrCheckpointManager::write_type_set() {
409   {
410     JavaThread* const thread = JavaThread::current();
411     DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread));
412     // can safepoint here
413     ThreadInVMfromNative transition(thread);
414     MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
415     MutexLocker module_lock(thread, Module_lock);
416     if (LeakProfiler::is_running()) {
417       JfrCheckpointWriter leakp_writer(true, thread);
418       JfrCheckpointWriter writer(true, thread);
419       JfrTypeSet::serialize(&writer, &leakp_writer, false, false);
420       ObjectSampleCheckpoint::on_type_set(leakp_writer);
421     } else {
422       JfrCheckpointWriter writer(true, thread);
423       JfrTypeSet::serialize(&writer, NULL, false, false);
424     }
425   }
426   write();
427 }
428 
429 void JfrCheckpointManager::on_unloading_classes() {
430   assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
431   JfrCheckpointWriter writer(Thread::current());
432   JfrTypeSet::on_unloading_classes(&writer);
433   if (LeakProfiler::is_running()) {
434     ObjectSampleCheckpoint::on_type_set_unload(writer);
435   }
436 }
437 
438 static size_t flush_type_set(Thread* thread) {
439   assert(thread != NULL, "invariant");
440   JfrCheckpointWriter writer(thread);
441   MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
442   MutexLocker module_lock(thread, Module_lock);
443   return JfrTypeSet::serialize(&writer, NULL, false, true);
444 }
445 
446 size_t JfrCheckpointManager::flush_type_set() {
447   size_t elements = 0;
448   if (JfrTraceIdEpoch::has_changed_tag_state()) {
449     Thread* const thread = Thread::current();
450     if (thread->is_Java_thread()) {
451       // can safepoint here
452       ThreadInVMfromNative transition(JavaThread::cast(thread));
453       elements = ::flush_type_set(thread);
454     } else {
455       elements = ::flush_type_set(thread);
456     }
457   }
458   if (_new_checkpoint.is_signaled_with_reset()) {
459     WriteOperation wo(_chunkwriter);
460     MutexedWriteOperation mwo(wo);
461     _thread_local_mspace->iterate(mwo); // current epoch list
462     assert(_global_mspace->live_list_is_nonempty(), "invariant");
463     process_live_list(mwo, _global_mspace); // current epoch list
464   }
465   return elements;
466 }
467 
468 void JfrCheckpointManager::create_thread_blob(Thread* thread) {
469   JfrTypeManager::create_thread_blob(thread);
470 }
471 
472 void JfrCheckpointManager::write_thread_checkpoint(Thread* thread) {
473   JfrTypeManager::write_thread_checkpoint(thread);
474 }
475 
476 class JfrNotifyClosure : public ThreadClosure {
477  public:
478   void do_thread(Thread* thread) {
479     assert(thread != NULL, "invariant");
480     assert_locked_or_safepoint(Threads_lock);
481     JfrJavaEventWriter::notify(JavaThread::cast(thread));
482   }
483 };
484 
485 void JfrCheckpointManager::notify_threads() {
486   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
487   JfrNotifyClosure tc;
488   JfrJavaThreadIterator iter;
489   while (iter.has_next()) {
490     tc.do_thread(iter.next());
491   }
492 }