1 /*
  2  * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/javaClasses.inline.hpp"
 27 #include "jfr/jni/jfrJavaSupport.hpp"
 28 #include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp"
 29 #include "jfr/leakprofiler/leakProfiler.hpp"
 30 #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
 31 #include "jfr/recorder/checkpoint/jfrCheckpointWriter.hpp"
 32 #include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp"
 33 #include "jfr/recorder/checkpoint/types/jfrTypeSet.hpp"
 34 #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
 35 #include "jfr/recorder/jfrRecorder.hpp"
 36 #include "jfr/recorder/repository/jfrChunkWriter.hpp"
 37 #include "jfr/recorder/service/jfrOptionSet.hpp"
 38 #include "jfr/recorder/storage/jfrEpochStorage.inline.hpp"
 39 #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
 40 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
 41 #include "jfr/support/jfrKlassUnloading.hpp"
 42 #include "jfr/support/jfrThreadLocal.hpp"
 43 #include "jfr/utilities/jfrBigEndian.hpp"
 44 #include "jfr/utilities/jfrIterator.hpp"
 45 #include "jfr/utilities/jfrLinkedList.inline.hpp"
 46 #include "jfr/utilities/jfrSignal.hpp"
 47 #include "jfr/utilities/jfrThreadIterator.hpp"
 48 #include "jfr/writers/jfrJavaEventWriter.hpp"
 49 #include "logging/log.hpp"
 50 #include "memory/iterator.hpp"
 51 #include "memory/resourceArea.hpp"
 52 #include "runtime/atomic.hpp"
 53 #include "runtime/handles.inline.hpp"
 54 #include "runtime/interfaceSupport.inline.hpp"
 55 #include "runtime/mutex.hpp"
 56 #include "runtime/safepoint.hpp"
 57 
 58 typedef JfrCheckpointManager::BufferPtr BufferPtr;
 59 
 60 static JfrSignal _new_checkpoint;
 61 static JfrCheckpointManager* _instance = NULL;
 62 
 63 JfrCheckpointManager& JfrCheckpointManager::instance() {
 64   return *_instance;
 65 }
 66 
 67 JfrCheckpointManager* JfrCheckpointManager::create(JfrChunkWriter& cw) {
 68   assert(_instance == NULL, "invariant");
 69   _instance = new JfrCheckpointManager(cw);
 70   return _instance;
 71 }
 72 
 73 void JfrCheckpointManager::destroy() {
 74   assert(_instance != NULL, "invariant");
 75   delete _instance;
 76   _instance = NULL;
 77 }
 78 
 79 JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) :
 80   _global_mspace(NULL),
 81   _java_thread_local_mspace(NULL),
 82   _chunkwriter(cw) {}
 83 
 84 JfrCheckpointManager::~JfrCheckpointManager() {
 85   JfrTraceIdLoadBarrier::destroy();
 86   JfrTypeManager::destroy();
 87   delete _global_mspace;
 88   delete _java_thread_local_mspace;
 89 }
 90 
 91 static const size_t global_buffer_prealloc_count = 2;
 92 static const size_t global_buffer_size = 512 * K;
 93 
 94 static const size_t thread_local_buffer_prealloc_count = 32;
 95 static const size_t thread_local_buffer_size = 2 * K;
 96 
 97 bool JfrCheckpointManager::initialize() {
 98   assert(_global_mspace == NULL, "invariant");
 99   _global_mspace =  create_mspace<JfrCheckpointMspace, JfrCheckpointManager>(global_buffer_size, 0, 0, false, this); // post-pone preallocation
100   if (_global_mspace == NULL) {
101     return false;
102   }
103   // preallocate buffer count to each of the epoch live lists
104   for (size_t i = 0; i < global_buffer_prealloc_count * 2; ++i) {
105     Buffer* const buffer = mspace_allocate(global_buffer_size, _global_mspace);
106     _global_mspace->add_to_live_list(buffer, i % 2 == 0);
107   }
108   assert(_global_mspace->free_list_is_empty(), "invariant");
109 
110   assert(_java_thread_local_mspace == NULL, "invariant");
111   _java_thread_local_mspace = new JfrThreadLocalCheckpointMspace();
112   if (_java_thread_local_mspace == NULL || !_java_thread_local_mspace->initialize(thread_local_buffer_size,
113                                                                                   JFR_MSPACE_UNLIMITED_CACHE_SIZE,
114                                                                                   thread_local_buffer_prealloc_count)) {
115     return false;
116   }
117   return JfrTypeManager::initialize() && JfrTraceIdLoadBarrier::initialize();
118 }
119 
120 #ifdef ASSERT
121 static void assert_lease(const BufferPtr buffer) {
122   assert(buffer != NULL, "invariant");
123   assert(buffer->acquired_by_self(), "invariant");
124   assert(buffer->lease(), "invariant");
125 }
126 
127 static void assert_release(const BufferPtr buffer) {
128   assert(buffer != NULL, "invariant");
129   assert(buffer->lease(), "invariant");
130   assert(buffer->acquired_by_self(), "invariant");
131 }
132 
133 static void assert_retired(const BufferPtr buffer, Thread* thread) {
134   assert(buffer != NULL, "invariant");
135   assert(buffer->acquired_by(thread), "invariant");
136   assert(buffer->retired(), "invariant");
137 }
138 #endif // ASSERT
139 
140 void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) {
141   DEBUG_ONLY(assert_retired(buffer, thread);)
142   // nothing here at the moment
143 }
144 
145 BufferPtr JfrCheckpointManager::lease(Thread* thread, bool previous_epoch /* false */, size_t size /* 0 */) {
146   JfrCheckpointMspace* const mspace = instance()._global_mspace;
147   assert(mspace != NULL, "invariant");
148   static const size_t max_elem_size = mspace->min_element_size(); // min is max
149   BufferPtr buffer;
150   if (size <= max_elem_size) {
151     buffer = mspace_acquire_live(size, mspace, thread, previous_epoch);
152     if (buffer != NULL) {
153       buffer->set_lease();
154       DEBUG_ONLY(assert_lease(buffer);)
155       return buffer;
156     }
157   }
158   buffer = mspace_allocate_transient_lease_to_live_list(size, mspace, thread, previous_epoch);
159   DEBUG_ONLY(assert_lease(buffer);)
160   return buffer;
161 }
162 
163 const u1 thread_local_context = 1;
164 
165 static bool is_thread_local(const JfrBuffer* buffer) {
166   assert(buffer != NULL, "invariant");
167   return buffer->context() == thread_local_context;
168 }
169 
170 static void retire(JfrBuffer* buffer) {
171   DEBUG_ONLY(assert_release(buffer);)
172   buffer->clear_lease();
173   buffer->set_retired();
174 }
175 
176 /*
177  * The buffer is effectively invalidated for the thread post-return,
178  * and the caller should take means to ensure that it is not referenced.
179  */
180 static void release(JfrBuffer* buffer) {
181   DEBUG_ONLY(assert_release(buffer);)
182   if (!is_thread_local(buffer)) {
183     buffer->clear_lease();
184     buffer->release();
185   }
186 }
187 
188 BufferPtr JfrCheckpointManager::get_thread_local(Thread* thread) {
189   assert(thread != NULL, "invariant");
190   assert(thread->is_Java_thread(), "invariant");
191   return JfrTraceIdEpoch::epoch() ? thread->jfr_thread_local()->_checkpoint_buffer_epoch_1 :
192                                     thread->jfr_thread_local()->_checkpoint_buffer_epoch_0;
193 }
194 
195 void JfrCheckpointManager::set_thread_local(Thread* thread, BufferPtr buffer) {
196   assert(thread != NULL, "invariant");
197   assert(thread->is_Java_thread(), "invariant");
198   if (JfrTraceIdEpoch::epoch()) {
199     thread->jfr_thread_local()->_checkpoint_buffer_epoch_1 = buffer;
200   } else {
201     thread->jfr_thread_local()->_checkpoint_buffer_epoch_0 = buffer;
202   }
203 }
204 
205 BufferPtr JfrCheckpointManager::acquire_thread_local(size_t size, Thread* thread) {
206   assert(thread != NULL, "invariant");
207   assert(thread->is_Java_thread(), "invariant");
208   JfrBuffer* const buffer = instance()._java_thread_local_mspace->acquire(size, thread);
209   assert(buffer != NULL, "invariant");
210   assert(buffer->free_size() >= size, "invariant");
211   buffer->set_context(thread_local_context);
212   assert(is_thread_local(buffer), "invariant");
213   buffer->set_lease();
214   set_thread_local(thread, buffer);
215   return buffer;
216 }
217 
218 BufferPtr JfrCheckpointManager::lease_thread_local(Thread* thread, size_t size /* 0 */) {
219   JfrBuffer* buffer = get_thread_local(thread);
220   if (buffer == NULL) {
221     buffer = acquire_thread_local(size, thread);
222   } else if (buffer->free_size() < size) {
223     retire(buffer);
224     buffer = acquire_thread_local(size, thread);
225   }
226   DEBUG_ONLY(assert_lease(buffer);)
227   assert(buffer->free_size() >= size, "invariant");
228   assert(get_thread_local(thread) == buffer, "invariant");
229   return buffer;
230 }
231 
232 BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size) {
233   assert(old != NULL, "invariant");
234   return is_thread_local(old) ? acquire_thread_local(size, thread) :
235                                 lease(thread, instance()._global_mspace->in_previous_epoch_list(old), size);
236 }
237 
238 BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
239   assert(old != NULL, "invariant");
240   assert(old->lease(), "invariant");
241   if (0 == requested) {
242     // indicates a lease is being returned
243     release(old);
244     // signal completion of a new checkpoint
245     _new_checkpoint.signal();
246     return NULL;
247   }
248   BufferPtr new_buffer = lease(old, thread, used + requested);
249   assert(new_buffer != NULL, "invariant");
250   migrate_outstanding_writes(old, new_buffer, used, requested);
251   retire(old);
252   return new_buffer;
253 }
254 
255 // offsets into the JfrCheckpointEntry
256 static const juint starttime_offset = sizeof(jlong);
257 static const juint duration_offset = starttime_offset + sizeof(jlong);
258 static const juint checkpoint_type_offset = duration_offset + sizeof(jlong);
259 static const juint types_offset = checkpoint_type_offset + sizeof(juint);
260 static const juint payload_offset = types_offset + sizeof(juint);
261 
262 template <typename Return>
263 static Return read_data(const u1* data) {
264   return JfrBigEndian::read<Return>(data);
265 }
266 
267 static jlong total_size(const u1* data) {
268   return read_data<jlong>(data);
269 }
270 
271 static jlong starttime(const u1* data) {
272   return read_data<jlong>(data + starttime_offset);
273 }
274 
275 static jlong duration(const u1* data) {
276   return read_data<jlong>(data + duration_offset);
277 }
278 
279 static u1 checkpoint_type(const u1* data) {
280   return read_data<u1>(data + checkpoint_type_offset);
281 }
282 
283 static juint number_of_types(const u1* data) {
284   return read_data<juint>(data + types_offset);
285 }
286 
287 static void write_checkpoint_header(JfrChunkWriter& cw, int64_t delta_to_last_checkpoint, const u1* data) {
288   cw.reserve(sizeof(u4));
289   cw.write<u8>(EVENT_CHECKPOINT);
290   cw.write(starttime(data));
291   cw.write(duration(data));
292   cw.write(delta_to_last_checkpoint);
293   cw.write(checkpoint_type(data));
294   cw.write(number_of_types(data));
295 }
296 
297 static void write_checkpoint_content(JfrChunkWriter& cw, const u1* data, size_t size) {
298   assert(data != NULL, "invariant");
299   cw.write_unbuffered(data + payload_offset, size - sizeof(JfrCheckpointEntry));
300 }
301 
302 static size_t write_checkpoint_event(JfrChunkWriter& cw, const u1* data) {
303   assert(data != NULL, "invariant");
304   const int64_t event_begin = cw.current_offset();
305   const int64_t last_checkpoint_event = cw.last_checkpoint_offset();
306   const int64_t delta_to_last_checkpoint = last_checkpoint_event == 0 ? 0 : last_checkpoint_event - event_begin;
307   const int64_t checkpoint_size = total_size(data);
308   write_checkpoint_header(cw, delta_to_last_checkpoint, data);
309   write_checkpoint_content(cw, data, checkpoint_size);
310   const int64_t event_size = cw.current_offset() - event_begin;
311   cw.write_padded_at_offset<u4>(event_size, event_begin);
312   cw.set_last_checkpoint_offset(event_begin);
313   return (size_t)checkpoint_size;
314 }
315 
316 static size_t write_checkpoints(JfrChunkWriter& cw, const u1* data, size_t size) {
317   assert(cw.is_valid(), "invariant");
318   assert(data != NULL, "invariant");
319   assert(size > 0, "invariant");
320   const u1* const limit = data + size;
321   const u1* next = data;
322   size_t processed = 0;
323   while (next < limit) {
324     const size_t checkpoint_size = write_checkpoint_event(cw, next);
325     processed += checkpoint_size;
326     next += checkpoint_size;
327   }
328   assert(next == limit, "invariant");
329   return processed;
330 }
331 
332 template <typename T>
333 class CheckpointWriteOp {
334  private:
335   JfrChunkWriter& _writer;
336   size_t _processed;
337  public:
338   typedef T Type;
339   CheckpointWriteOp(JfrChunkWriter& writer) : _writer(writer), _processed(0) {}
340   bool write(Type* t, const u1* data, size_t size) {
341     _processed += write_checkpoints(_writer, data, size);
342     return true;
343   }
344   size_t processed() const { return _processed; }
345 };
346 
347 typedef CheckpointWriteOp<JfrCheckpointManager::Buffer> WriteOperation;
348 typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation;
349 typedef ReleaseWithExcisionOp<JfrCheckpointMspace, JfrCheckpointMspace::LiveList> ReleaseOperation;
350 typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> WriteReleaseOperation;
351 
352 void JfrCheckpointManager::begin_epoch_shift() {
353   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
354   JfrTraceIdEpoch::begin_epoch_shift();
355 }
356 
357 void JfrCheckpointManager::end_epoch_shift() {
358   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
359   debug_only(const u1 current_epoch = JfrTraceIdEpoch::current();)
360   JfrTraceIdEpoch::end_epoch_shift();
361   assert(current_epoch != JfrTraceIdEpoch::current(), "invariant");
362 }
363 
364 size_t JfrCheckpointManager::write() {
365   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current()));
366   WriteOperation wo(_chunkwriter);
367   MutexedWriteOperation mwo(wo);
368   _java_thread_local_mspace->iterate(mwo, true); // previous epoch list
369   assert(_global_mspace->free_list_is_empty(), "invariant");
370   ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true));
371   WriteReleaseOperation wro(&mwo, &ro);
372   process_live_list(wro, _global_mspace, true); // previous epoch list
373   return wo.processed();
374 }
375 
376 typedef DiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperation;
377 typedef CompositeOperation<DiscardOperation, ReleaseOperation> DiscardReleaseOperation;
378 
379 size_t JfrCheckpointManager::clear() {
380   JfrTraceIdLoadBarrier::clear();
381   clear_type_set();
382   DiscardOperation discard_operation(mutexed); // mutexed discard mode
383   _java_thread_local_mspace->iterate(discard_operation, true); // previous epoch list
384   ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true));
385   DiscardReleaseOperation discard_op(&discard_operation, &ro);
386   assert(_global_mspace->free_list_is_empty(), "invariant");
387   process_live_list(discard_op, _global_mspace, true); // previous epoch list
388   return discard_operation.elements();
389 }
390 
391 size_t JfrCheckpointManager::write_static_type_set(Thread* thread) {
392   assert(thread != NULL, "invariant");
393   JfrCheckpointWriter writer(true, thread, STATICS);
394   JfrTypeManager::write_static_types(writer);
395   return writer.used_size();
396 }
397 
398 size_t JfrCheckpointManager::write_threads(JavaThread* thread) {
399   assert(thread != NULL, "invariant");
400   // can safepoint here
401   ThreadInVMfromNative transition(thread);
402   ResourceMark rm(thread);
403   HandleMark hm(thread);
404   JfrCheckpointWriter writer(true, thread, THREADS);
405   JfrTypeManager::write_threads(writer);
406   return writer.used_size();
407 }
408 
409 size_t JfrCheckpointManager::write_static_type_set_and_threads() {
410   JavaThread* const thread = JavaThread::current();
411   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread));
412   write_static_type_set(thread);
413   write_threads(thread);
414   return write();
415 }
416 
417 void JfrCheckpointManager::on_rotation() {
418   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
419   JfrTypeManager::on_rotation();
420   notify_threads();
421 }
422 
423 void JfrCheckpointManager::clear_type_set() {
424   assert(!JfrRecorder::is_recording(), "invariant");
425   JavaThread* t = JavaThread::current();
426   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(t));
427   // can safepoint here
428   ThreadInVMfromNative transition(t);
429   MutexLocker cld_lock(ClassLoaderDataGraph_lock);
430   MutexLocker module_lock(Module_lock);
431   JfrTypeSet::clear();
432 }
433 
434 void JfrCheckpointManager::write_type_set() {
435   {
436     JavaThread* const thread = JavaThread::current();
437     DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread));
438     // can safepoint here
439     ThreadInVMfromNative transition(thread);
440     MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
441     MutexLocker module_lock(thread, Module_lock);
442     if (LeakProfiler::is_running()) {
443       JfrCheckpointWriter leakp_writer(true, thread);
444       JfrCheckpointWriter writer(true, thread);
445       JfrTypeSet::serialize(&writer, &leakp_writer, false, false);
446       ObjectSampleCheckpoint::on_type_set(leakp_writer);
447     } else {
448       JfrCheckpointWriter writer(true, thread);
449       JfrTypeSet::serialize(&writer, NULL, false, false);
450     }
451   }
452   write();
453 }
454 
455 void JfrCheckpointManager::on_unloading_classes() {
456   assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
457   JfrCheckpointWriter writer(Thread::current());
458   JfrTypeSet::on_unloading_classes(&writer);
459   if (LeakProfiler::is_running()) {
460     ObjectSampleCheckpoint::on_type_set_unload(writer);
461   }
462 }
463 
464 static size_t flush_type_set(Thread* thread) {
465   assert(thread != NULL, "invariant");
466   JfrCheckpointWriter writer(thread);
467   MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
468   MutexLocker module_lock(thread, Module_lock);
469   return JfrTypeSet::serialize(&writer, NULL, false, true);
470 }
471 
472 size_t JfrCheckpointManager::flush_type_set() {
473   size_t elements = 0;
474   if (JfrTraceIdEpoch::has_changed_tag_state()) {
475     Thread* const thread = Thread::current();
476     if (thread->is_Java_thread()) {
477       // can safepoint here
478       ThreadInVMfromNative transition(JavaThread::cast(thread));
479       elements = ::flush_type_set(thread);
480     } else {
481       elements = ::flush_type_set(thread);
482     }
483   }
484   if (_new_checkpoint.is_signaled_with_reset()) {
485     WriteOperation wo(_chunkwriter);
486     MutexedWriteOperation mwo(wo);
487     _java_thread_local_mspace->iterate(mwo); // current epoch list
488     assert(_global_mspace->live_list_is_nonempty(), "invariant");
489     process_live_list(mwo, _global_mspace); // current epoch list
490   }
491   return elements;
492 }
493 
494 JfrBlobHandle JfrCheckpointManager::create_thread_blob(JavaThread* jt, traceid tid, oop vthread) {
495   assert(jt != NULL, "invariant");
496   assert(Thread::current() == jt, "invariant");
497   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(jt));
498   return JfrTypeManager::create_thread_blob(jt, tid, vthread);
499 }
500 
501 void JfrCheckpointManager::write_checkpoint(Thread* thread, traceid tid, oop vthread) {
502   JfrTypeManager::write_checkpoint(thread, tid, vthread);
503 }
504 
505 class JfrNotifyClosure : public ThreadClosure {
506  public:
507   void do_thread(Thread* thread) {
508     assert(thread != NULL, "invariant");
509     assert_locked_or_safepoint(Threads_lock);
510     JfrJavaEventWriter::notify(JavaThread::cast(thread));
511   }
512 };
513 
514 void JfrCheckpointManager::notify_threads() {
515   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
516   JfrNotifyClosure tc;
517   JfrJavaThreadIterator iter;
518   while (iter.has_next()) {
519     tc.do_thread(iter.next());
520   }
521 }