1 /*
   2  * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "classfile/vmSymbols.hpp"
  26 #include "gc/shared/collectedHeap.hpp"
  27 #include "jfr/jfrEvents.hpp"
  28 #include "logging/log.hpp"
  29 #include "logging/logStream.hpp"
  30 #include "memory/allocation.inline.hpp"
  31 #include "memory/padded.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/markWord.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "runtime/atomic.hpp"
  37 #include "runtime/basicLock.inline.hpp"
  38 #include "runtime/frame.inline.hpp"
  39 #include "runtime/globals.hpp"
  40 #include "runtime/handles.inline.hpp"
  41 #include "runtime/handshake.hpp"
  42 #include "runtime/interfaceSupport.inline.hpp"
  43 #include "runtime/javaThread.hpp"
  44 #include "runtime/lightweightSynchronizer.hpp"
  45 #include "runtime/lockStack.inline.hpp"
  46 #include "runtime/mutexLocker.hpp"
  47 #include "runtime/objectMonitor.inline.hpp"
  48 #include "runtime/os.inline.hpp"
  49 #include "runtime/osThread.hpp"
  50 #include "runtime/safepointMechanism.inline.hpp"
  51 #include "runtime/safepointVerifiers.hpp"
  52 #include "runtime/sharedRuntime.hpp"
  53 #include "runtime/stubRoutines.hpp"
  54 #include "runtime/synchronizer.inline.hpp"
  55 #include "runtime/threads.hpp"
  56 #include "runtime/timer.hpp"
  57 #include "runtime/trimNativeHeap.hpp"
  58 #include "runtime/vframe.hpp"
  59 #include "runtime/vmThread.hpp"
  60 #include "utilities/align.hpp"
  61 #include "utilities/dtrace.hpp"
  62 #include "utilities/events.hpp"
  63 #include "utilities/globalCounter.inline.hpp"
  64 #include "utilities/globalDefinitions.hpp"
  65 #include "utilities/linkedlist.hpp"
  66 #include "utilities/preserveException.hpp"
  67 
  68 class ObjectMonitorDeflationLogging;
  69 
  70 void MonitorList::add(ObjectMonitor* m) {
  71   ObjectMonitor* head;
  72   do {
  73     head = Atomic::load(&_head);
  74     m->set_next_om(head);
  75   } while (Atomic::cmpxchg(&_head, head, m) != head);
  76 
  77   size_t count = Atomic::add(&_count, 1u, memory_order_relaxed);
  78   size_t old_max;
  79   do {
  80     old_max = Atomic::load(&_max);
  81     if (count <= old_max) {
  82       break;
  83     }
  84   } while (Atomic::cmpxchg(&_max, old_max, count, memory_order_relaxed) != old_max);
  85 }
  86 
  87 size_t MonitorList::count() const {
  88   return Atomic::load(&_count);
  89 }
  90 
  91 size_t MonitorList::max() const {
  92   return Atomic::load(&_max);
  93 }
  94 
  95 class ObjectMonitorDeflationSafepointer : public StackObj {
  96   JavaThread* const                    _current;
  97   ObjectMonitorDeflationLogging* const _log;
  98 
  99 public:
 100   ObjectMonitorDeflationSafepointer(JavaThread* current, ObjectMonitorDeflationLogging* log)
 101     : _current(current), _log(log) {}
 102 
 103   void block_for_safepoint(const char* op_name, const char* count_name, size_t counter);
 104 };
 105 
 106 // Walk the in-use list and unlink deflated ObjectMonitors.
 107 // Returns the number of unlinked ObjectMonitors.
 108 size_t MonitorList::unlink_deflated(size_t deflated_count,
 109                                     GrowableArray<ObjectMonitor*>* unlinked_list,
 110                                     ObjectMonitorDeflationSafepointer* safepointer) {
 111   size_t unlinked_count = 0;
 112   ObjectMonitor* prev = nullptr;
 113   ObjectMonitor* m = Atomic::load_acquire(&_head);
 114 
 115   while (m != nullptr) {
 116     if (m->is_being_async_deflated()) {
 117       // Find next live ObjectMonitor. Batch up the unlinkable monitors, so we can
 118       // modify the list once per batch. The batch starts at "m".
 119       size_t unlinked_batch = 0;
 120       ObjectMonitor* next = m;
 121       // Look for at most MonitorUnlinkBatch monitors, or the number of
 122       // deflated and not unlinked monitors, whatever comes first.
 123       assert(deflated_count >= unlinked_count, "Sanity: underflow");
 124       size_t unlinked_batch_limit = MIN2<size_t>(deflated_count - unlinked_count, MonitorUnlinkBatch);
 125       do {
 126         ObjectMonitor* next_next = next->next_om();
 127         unlinked_batch++;
 128         unlinked_list->append(next);
 129         next = next_next;
 130         if (unlinked_batch >= unlinked_batch_limit) {
 131           // Reached the max batch, so bail out of the gathering loop.
 132           break;
 133         }
 134         if (prev == nullptr && Atomic::load(&_head) != m) {
 135           // Current batch used to be at head, but it is not at head anymore.
 136           // Bail out and figure out where we currently are. This avoids long
 137           // walks searching for new prev during unlink under heavy list inserts.
 138           break;
 139         }
 140       } while (next != nullptr && next->is_being_async_deflated());
 141 
 142       // Unlink the found batch.
 143       if (prev == nullptr) {
 144         // The current batch is the first batch, so there is a chance that it starts at head.
 145         // Optimistically assume no inserts happened, and try to unlink the entire batch from the head.
 146         ObjectMonitor* prev_head = Atomic::cmpxchg(&_head, m, next);
 147         if (prev_head != m) {
 148           // Something must have updated the head. Figure out the actual prev for this batch.
 149           for (ObjectMonitor* n = prev_head; n != m; n = n->next_om()) {
 150             prev = n;
 151           }
 152           assert(prev != nullptr, "Should have found the prev for the current batch");
 153           prev->set_next_om(next);
 154         }
 155       } else {
 156         // The current batch is preceded by another batch. This guarantees the current batch
 157         // does not start at head. Unlink the entire current batch without updating the head.
 158         assert(Atomic::load(&_head) != m, "Sanity");
 159         prev->set_next_om(next);
 160       }
 161 
 162       unlinked_count += unlinked_batch;
 163       if (unlinked_count >= deflated_count) {
 164         // Reached the max so bail out of the searching loop.
 165         // There should be no more deflated monitors left.
 166         break;
 167       }
 168       m = next;
 169     } else {
 170       prev = m;
 171       m = m->next_om();
 172     }
 173 
 174     // Must check for a safepoint/handshake and honor it.
 175     safepointer->block_for_safepoint("unlinking", "unlinked_count", unlinked_count);
 176   }
 177 
 178 #ifdef ASSERT
 179   // Invariant: the code above should unlink all deflated monitors.
 180   // The code that runs after this unlinking does not expect deflated monitors.
 181   // Notably, attempting to deflate the already deflated monitor would break.
 182   {
 183     ObjectMonitor* m = Atomic::load_acquire(&_head);
 184     while (m != nullptr) {
 185       assert(!m->is_being_async_deflated(), "All deflated monitors should be unlinked");
 186       m = m->next_om();
 187     }
 188   }
 189 #endif
 190 
 191   Atomic::sub(&_count, unlinked_count);
 192   return unlinked_count;
 193 }
 194 
 195 MonitorList::Iterator MonitorList::iterator() const {
 196   return Iterator(Atomic::load_acquire(&_head));
 197 }
 198 
 199 ObjectMonitor* MonitorList::Iterator::next() {
 200   ObjectMonitor* current = _current;
 201   _current = current->next_om();
 202   return current;
 203 }
 204 
 205 // The "core" versions of monitor enter and exit reside in this file.
 206 // The interpreter and compilers contain specialized transliterated
 207 // variants of the enter-exit fast-path operations.  See c2_MacroAssembler_x86.cpp
 208 // fast_lock(...) for instance.  If you make changes here, make sure to modify the
 209 // interpreter, and both C1 and C2 fast-path inline locking code emission.
 210 //
 211 // -----------------------------------------------------------------------------
 212 
 213 #ifdef DTRACE_ENABLED
 214 
 215 // Only bother with this argument setup if dtrace is available
 216 // TODO-FIXME: probes should not fire when caller is _blocked.  assert() accordingly.
 217 
 218 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread)                           \
 219   char* bytes = nullptr;                                                      \
 220   int len = 0;                                                             \
 221   jlong jtid = SharedRuntime::get_java_tid(thread);                        \
 222   Symbol* klassname = obj->klass()->name();                                \
 223   if (klassname != nullptr) {                                                 \
 224     bytes = (char*)klassname->bytes();                                     \
 225     len = klassname->utf8_length();                                        \
 226   }
 227 
 228 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis)            \
 229   {                                                                        \
 230     if (DTraceMonitorProbes) {                                             \
 231       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
 232       HOTSPOT_MONITOR_WAIT(jtid,                                           \
 233                            (uintptr_t)(monitor), bytes, len, (millis));    \
 234     }                                                                      \
 235   }
 236 
 237 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
 238 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
 239 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
 240 
 241 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread)                  \
 242   {                                                                        \
 243     if (DTraceMonitorProbes) {                                             \
 244       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
 245       HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */             \
 246                                     (uintptr_t)(monitor), bytes, len);     \
 247     }                                                                      \
 248   }
 249 
 250 #else //  ndef DTRACE_ENABLED
 251 
 252 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon)    {;}
 253 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon)          {;}
 254 
 255 #endif // ndef DTRACE_ENABLED
 256 
 257 // This exists only as a workaround of dtrace bug 6254741
 258 static int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, JavaThread* thr) {
 259   DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
 260   return 0;
 261 }
 262 
 263 static constexpr size_t inflation_lock_count() {
 264   return 256;
 265 }
 266 
 267 // Static storage for an array of PlatformMutex.
 268 alignas(PlatformMutex) static uint8_t _inflation_locks[inflation_lock_count()][sizeof(PlatformMutex)];
 269 
 270 static inline PlatformMutex* inflation_lock(size_t index) {
 271   return reinterpret_cast<PlatformMutex*>(_inflation_locks[index]);
 272 }
 273 
 274 void ObjectSynchronizer::initialize() {
 275   for (size_t i = 0; i < inflation_lock_count(); i++) {
 276     ::new(static_cast<void*>(inflation_lock(i))) PlatformMutex();
 277   }
 278   // Start the ceiling with the estimate for one thread.
 279   set_in_use_list_ceiling(AvgMonitorsPerThreadEstimate);
 280 
 281   // Start the timer for deflations, so it does not trigger immediately.
 282   _last_async_deflation_time_ns = os::javaTimeNanos();
 283 
 284   if (LockingMode == LM_LIGHTWEIGHT) {
 285     LightweightSynchronizer::initialize();
 286   }
 287 }
 288 
 289 MonitorList ObjectSynchronizer::_in_use_list;
 290 // monitors_used_above_threshold() policy is as follows:
 291 //
 292 // The ratio of the current _in_use_list count to the ceiling is used
 293 // to determine if we are above MonitorUsedDeflationThreshold and need
 294 // to do an async monitor deflation cycle. The ceiling is increased by
 295 // AvgMonitorsPerThreadEstimate when a thread is added to the system
 296 // and is decreased by AvgMonitorsPerThreadEstimate when a thread is
 297 // removed from the system.
 298 //
 299 // Note: If the _in_use_list max exceeds the ceiling, then
 300 // monitors_used_above_threshold() will use the in_use_list max instead
 301 // of the thread count derived ceiling because we have used more
 302 // ObjectMonitors than the estimated average.
 303 //
 304 // Note: If deflate_idle_monitors() has NoAsyncDeflationProgressMax
 305 // no-progress async monitor deflation cycles in a row, then the ceiling
 306 // is adjusted upwards by monitors_used_above_threshold().
 307 //
 308 // Start the ceiling with the estimate for one thread in initialize()
 309 // which is called after cmd line options are processed.
 310 static size_t _in_use_list_ceiling = 0;
 311 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
 312 bool volatile ObjectSynchronizer::_is_final_audit = false;
 313 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
 314 static uintx _no_progress_cnt = 0;
 315 static bool _no_progress_skip_increment = false;
 316 
 317 // =====================> Quick functions
 318 
 319 // The quick_* forms are special fast-path variants used to improve
 320 // performance.  In the simplest case, a "quick_*" implementation could
 321 // simply return false, in which case the caller will perform the necessary
 322 // state transitions and call the slow-path form.
 323 // The fast-path is designed to handle frequently arising cases in an efficient
 324 // manner and is just a degenerate "optimistic" variant of the slow-path.
 325 // returns true  -- to indicate the call was satisfied.
 326 // returns false -- to indicate the call needs the services of the slow-path.
 327 // A no-loitering ordinance is in effect for code in the quick_* family
 328 // operators: safepoints or indefinite blocking (blocking that might span a
 329 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
 330 // entry.
 331 //
 332 // Consider: An interesting optimization is to have the JIT recognize the
 333 // following common idiom:
 334 //   synchronized (someobj) { .... ; notify(); }
 335 // That is, we find a notify() or notifyAll() call that immediately precedes
 336 // the monitorexit operation.  In that case the JIT could fuse the operations
 337 // into a single notifyAndExit() runtime primitive.
 338 
 339 bool ObjectSynchronizer::quick_notify(oopDesc* obj, JavaThread* current, bool all) {
 340   assert(current->thread_state() == _thread_in_Java, "invariant");
 341   NoSafepointVerifier nsv;
 342   if (obj == nullptr) return false;  // slow-path for invalid obj
 343   const markWord mark = obj->mark();
 344 
 345   if (LockingMode == LM_LIGHTWEIGHT) {
 346     if (mark.is_fast_locked() && current->lock_stack().contains(cast_to_oop(obj))) {
 347       // Degenerate notify
 348       // fast-locked by caller so by definition the implied waitset is empty.
 349       return true;
 350     }
 351   } else if (LockingMode == LM_LEGACY) {
 352     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
 353       // Degenerate notify
 354       // stack-locked by caller so by definition the implied waitset is empty.
 355       return true;
 356     }
 357   }
 358 
 359   if (mark.has_monitor()) {
 360     ObjectMonitor* const mon = read_monitor(current, obj, mark);
 361     if (LockingMode == LM_LIGHTWEIGHT && mon == nullptr) {
 362       // Racing with inflation/deflation go slow path
 363       return false;
 364     }
 365     assert(mon->object() == oop(obj), "invariant");
 366     if (!mon->has_owner(current)) return false;  // slow-path for IMS exception
 367 
 368     if (mon->first_waiter() != nullptr) {
 369       // We have one or more waiters. Since this is an inflated monitor
 370       // that we own, we quickly notify them here and now, avoiding the slow-path.
 371       if (all) {
 372         mon->quick_notifyAll(current);
 373       } else {
 374         mon->quick_notify(current);
 375       }
 376     }
 377     return true;
 378   }
 379 
 380   // other IMS exception states take the slow-path
 381   return false;
 382 }
 383 
 384 static bool useHeavyMonitors() {
 385 #if defined(X86) || defined(AARCH64) || defined(PPC64) || defined(RISCV64) || defined(S390)
 386   return LockingMode == LM_MONITOR;
 387 #else
 388   return false;
 389 #endif
 390 }
 391 
 392 // The LockNode emitted directly at the synchronization site would have
 393 // been too big if it were to have included support for the cases of inflated
 394 // recursive enter and exit, so they go here instead.
 395 // Note that we can't safely call AsyncPrintJavaStack() from within
 396 // quick_enter() as our thread state remains _in_Java.
 397 
 398 bool ObjectSynchronizer::quick_enter_legacy(oop obj, BasicLock* lock, JavaThread* current) {
 399   assert(current->thread_state() == _thread_in_Java, "invariant");
 400 
 401   if (useHeavyMonitors()) {
 402     return false;  // Slow path
 403   }
 404 
 405   assert(LockingMode == LM_LEGACY, "legacy mode below");
 406 
 407   const markWord mark = obj->mark();
 408 
 409   if (mark.has_monitor()) {
 410 
 411     ObjectMonitor* const m = read_monitor(mark);
 412     // An async deflation or GC can race us before we manage to make
 413     // the ObjectMonitor busy by setting the owner below. If we detect
 414     // that race we just bail out to the slow-path here.
 415     if (m->object_peek() == nullptr) {
 416       return false;
 417     }
 418 
 419     // Lock contention and Transactional Lock Elision (TLE) diagnostics
 420     // and observability
 421     // Case: light contention possibly amenable to TLE
 422     // Case: TLE inimical operations such as nested/recursive synchronization
 423 
 424     if (m->has_owner(current)) {
 425       m->increment_recursions(current);
 426       current->inc_held_monitor_count();
 427       return true;
 428     }
 429 
 430     // This Java Monitor is inflated so obj's header will never be
 431     // displaced to this thread's BasicLock. Make the displaced header
 432     // non-null so this BasicLock is not seen as recursive nor as
 433     // being locked. We do this unconditionally so that this thread's
 434     // BasicLock cannot be mis-interpreted by any stack walkers. For
 435     // performance reasons, stack walkers generally first check for
 436     // stack-locking in the object's header, the second check is for
 437     // recursive stack-locking in the displaced header in the BasicLock,
 438     // and last are the inflated Java Monitor (ObjectMonitor) checks.
 439     lock->set_displaced_header(markWord::unused_mark());
 440 
 441     if (!m->has_owner() && m->try_set_owner(current)) {
 442       assert(m->recursions() == 0, "invariant");
 443       current->inc_held_monitor_count();
 444       return true;
 445     }
 446   }
 447 
 448   // Note that we could inflate in quick_enter.
 449   // This is likely a useful optimization
 450   // Critically, in quick_enter() we must not:
 451   // -- block indefinitely, or
 452   // -- reach a safepoint
 453 
 454   return false;        // revert to slow-path
 455 }
 456 
 457 // Handle notifications when synchronizing on value based classes
 458 void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread* locking_thread) {
 459   assert(locking_thread == Thread::current() || locking_thread->is_obj_deopt_suspend(), "must be");
 460   frame last_frame = locking_thread->last_frame();
 461   bool bcp_was_adjusted = false;
 462   // Don't decrement bcp if it points to the frame's first instruction.  This happens when
 463   // handle_sync_on_value_based_class() is called because of a synchronized method.  There
 464   // is no actual monitorenter instruction in the byte code in this case.
 465   if (last_frame.is_interpreted_frame() &&
 466       (last_frame.interpreter_frame_method()->code_base() < last_frame.interpreter_frame_bcp())) {
 467     // adjust bcp to point back to monitorenter so that we print the correct line numbers
 468     last_frame.interpreter_frame_set_bcp(last_frame.interpreter_frame_bcp() - 1);
 469     bcp_was_adjusted = true;
 470   }
 471 
 472   if (DiagnoseSyncOnValueBasedClasses == FATAL_EXIT) {
 473     ResourceMark rm;
 474     stringStream ss;
 475     locking_thread->print_active_stack_on(&ss);
 476     char* base = (char*)strstr(ss.base(), "at");
 477     char* newline = (char*)strchr(ss.base(), '\n');
 478     if (newline != nullptr) {
 479       *newline = '\0';
 480     }
 481     fatal("Synchronizing on object " INTPTR_FORMAT " of klass %s %s", p2i(obj()), obj->klass()->external_name(), base);
 482   } else {
 483     assert(DiagnoseSyncOnValueBasedClasses == LOG_WARNING, "invalid value for DiagnoseSyncOnValueBasedClasses");
 484     ResourceMark rm;
 485     Log(valuebasedclasses) vblog;
 486 
 487     vblog.info("Synchronizing on object " INTPTR_FORMAT " of klass %s", p2i(obj()), obj->klass()->external_name());
 488     if (locking_thread->has_last_Java_frame()) {
 489       LogStream info_stream(vblog.info());
 490       locking_thread->print_active_stack_on(&info_stream);
 491     } else {
 492       vblog.info("Cannot find the last Java frame");
 493     }
 494 
 495     EventSyncOnValueBasedClass event;
 496     if (event.should_commit()) {
 497       event.set_valueBasedClass(obj->klass());
 498       event.commit();
 499     }
 500   }
 501 
 502   if (bcp_was_adjusted) {
 503     last_frame.interpreter_frame_set_bcp(last_frame.interpreter_frame_bcp() + 1);
 504   }
 505 }
 506 
 507 // -----------------------------------------------------------------------------
 508 // Monitor Enter/Exit
 509 
 510 void ObjectSynchronizer::enter_for(Handle obj, BasicLock* lock, JavaThread* locking_thread) {
 511   // When called with locking_thread != Thread::current() some mechanism must synchronize
 512   // the locking_thread with respect to the current thread. Currently only used when
 513   // deoptimizing and re-locking locks. See Deoptimization::relock_objects
 514   assert(locking_thread == Thread::current() || locking_thread->is_obj_deopt_suspend(), "must be");
 515 
 516   if (LockingMode == LM_LIGHTWEIGHT) {
 517     return LightweightSynchronizer::enter_for(obj, lock, locking_thread);
 518   }
 519 
 520   if (!enter_fast_impl(obj, lock, locking_thread)) {
 521     // Inflated ObjectMonitor::enter_for is required
 522 
 523     // An async deflation can race after the inflate_for() call and before
 524     // enter_for() can make the ObjectMonitor busy. enter_for() returns false
 525     // if we have lost the race to async deflation and we simply try again.
 526     while (true) {
 527       ObjectMonitor* monitor = inflate_for(locking_thread, obj(), inflate_cause_monitor_enter);
 528       if (monitor->enter_for(locking_thread)) {
 529         return;
 530       }
 531       assert(monitor->is_being_async_deflated(), "must be");
 532     }
 533   }
 534 }
 535 
 536 void ObjectSynchronizer::enter_legacy(Handle obj, BasicLock* lock, JavaThread* current) {
 537   if (!enter_fast_impl(obj, lock, current)) {
 538     // Inflated ObjectMonitor::enter is required
 539 
 540     // An async deflation can race after the inflate() call and before
 541     // enter() can make the ObjectMonitor busy. enter() returns false if
 542     // we have lost the race to async deflation and we simply try again.
 543     while (true) {
 544       ObjectMonitor* monitor = inflate(current, obj(), inflate_cause_monitor_enter);
 545       if (monitor->enter(current)) {
 546         return;
 547       }
 548     }
 549   }
 550 }
 551 
 552 // The interpreter and compiler assembly code tries to lock using the fast path
 553 // of this algorithm. Make sure to update that code if the following function is
 554 // changed. The implementation is extremely sensitive to race condition. Be careful.
 555 bool ObjectSynchronizer::enter_fast_impl(Handle obj, BasicLock* lock, JavaThread* locking_thread) {
 556   assert(LockingMode != LM_LIGHTWEIGHT, "Use LightweightSynchronizer");
 557 
 558   if (obj->klass()->is_value_based()) {
 559     handle_sync_on_value_based_class(obj, locking_thread);
 560   }
 561 
 562   locking_thread->inc_held_monitor_count();
 563 
 564   if (!useHeavyMonitors()) {
 565     if (LockingMode == LM_LEGACY) {
 566       markWord mark = obj->mark();
 567       if (mark.is_unlocked()) {
 568         // Anticipate successful CAS -- the ST of the displaced mark must
 569         // be visible <= the ST performed by the CAS.
 570         lock->set_displaced_header(mark);
 571         if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
 572           return true;
 573         }
 574       } else if (mark.has_locker() &&
 575                  locking_thread->is_lock_owned((address) mark.locker())) {
 576         assert(lock != mark.locker(), "must not re-lock the same lock");
 577         assert(lock != (BasicLock*) obj->mark().value(), "don't relock with same BasicLock");
 578         lock->set_displaced_header(markWord::from_pointer(nullptr));
 579         return true;
 580       }
 581 
 582       // The object header will never be displaced to this lock,
 583       // so it does not matter what the value is, except that it
 584       // must be non-zero to avoid looking like a re-entrant lock,
 585       // and must not look locked either.
 586       lock->set_displaced_header(markWord::unused_mark());
 587 
 588       // Failed to fast lock.
 589       return false;
 590     }
 591   } else if (VerifyHeavyMonitors) {
 592     guarantee((obj->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
 593   }
 594 
 595   return false;
 596 }
 597 
 598 void ObjectSynchronizer::exit_legacy(oop object, BasicLock* lock, JavaThread* current) {
 599   assert(LockingMode != LM_LIGHTWEIGHT, "Use LightweightSynchronizer");
 600 
 601   if (!useHeavyMonitors()) {
 602     markWord mark = object->mark();
 603     if (LockingMode == LM_LEGACY) {
 604       markWord dhw = lock->displaced_header();
 605       if (dhw.value() == 0) {
 606         // If the displaced header is null, then this exit matches up with
 607         // a recursive enter. No real work to do here except for diagnostics.
 608 #ifndef PRODUCT
 609         if (mark != markWord::INFLATING()) {
 610           // Only do diagnostics if we are not racing an inflation. Simply
 611           // exiting a recursive enter of a Java Monitor that is being
 612           // inflated is safe; see the has_monitor() comment below.
 613           assert(!mark.is_unlocked(), "invariant");
 614           assert(!mark.has_locker() ||
 615                  current->is_lock_owned((address)mark.locker()), "invariant");
 616           if (mark.has_monitor()) {
 617             // The BasicLock's displaced_header is marked as a recursive
 618             // enter and we have an inflated Java Monitor (ObjectMonitor).
 619             // This is a special case where the Java Monitor was inflated
 620             // after this thread entered the stack-lock recursively. When a
 621             // Java Monitor is inflated, we cannot safely walk the Java
 622             // Monitor owner's stack and update the BasicLocks because a
 623             // Java Monitor can be asynchronously inflated by a thread that
 624             // does not own the Java Monitor.
 625             ObjectMonitor* m = read_monitor(mark);
 626             assert(m->object()->mark() == mark, "invariant");
 627             assert(m->is_entered(current), "invariant");
 628           }
 629         }
 630 #endif
 631         return;
 632       }
 633 
 634       if (mark == markWord::from_pointer(lock)) {
 635         // If the object is stack-locked by the current thread, try to
 636         // swing the displaced header from the BasicLock back to the mark.
 637         assert(dhw.is_neutral(), "invariant");
 638         if (object->cas_set_mark(dhw, mark) == mark) {
 639           return;
 640         }
 641       }
 642     }
 643   } else if (VerifyHeavyMonitors) {
 644     guarantee((object->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
 645   }
 646 
 647   // We have to take the slow-path of possible inflation and then exit.
 648   // The ObjectMonitor* can't be async deflated until ownership is
 649   // dropped inside exit() and the ObjectMonitor* must be !is_busy().
 650   ObjectMonitor* monitor = inflate(current, object, inflate_cause_vm_internal);
 651   assert(!monitor->has_anonymous_owner(), "must not be");
 652   monitor->exit(current);
 653 }
 654 
 655 // -----------------------------------------------------------------------------
 656 // JNI locks on java objects
 657 // NOTE: must use heavy weight monitor to handle jni monitor enter
 658 void ObjectSynchronizer::jni_enter(Handle obj, JavaThread* current) {
 659   // Top native frames in the stack will not be seen if we attempt
 660   // preemption, since we start walking from the last Java anchor.
 661   NoPreemptMark npm(current);
 662 
 663   if (obj->klass()->is_value_based()) {
 664     handle_sync_on_value_based_class(obj, current);
 665   }
 666 
 667   // the current locking is from JNI instead of Java code
 668   current->set_current_pending_monitor_is_from_java(false);
 669   // An async deflation can race after the inflate() call and before
 670   // enter() can make the ObjectMonitor busy. enter() returns false if
 671   // we have lost the race to async deflation and we simply try again.
 672   while (true) {
 673     ObjectMonitor* monitor;
 674     bool entered;
 675     if (LockingMode == LM_LIGHTWEIGHT) {
 676       BasicLock lock;
 677       entered = LightweightSynchronizer::inflate_and_enter(obj(), &lock, inflate_cause_jni_enter, current, current) != nullptr;
 678     } else {
 679       monitor = inflate(current, obj(), inflate_cause_jni_enter);
 680       entered = monitor->enter(current);
 681     }
 682 
 683     if (entered) {
 684       current->inc_held_monitor_count(1, true);
 685       break;
 686     }
 687   }
 688   current->set_current_pending_monitor_is_from_java(true);
 689 }
 690 
 691 // NOTE: must use heavy weight monitor to handle jni monitor exit
 692 void ObjectSynchronizer::jni_exit(oop obj, TRAPS) {
 693   JavaThread* current = THREAD;
 694 
 695   ObjectMonitor* monitor;
 696   if (LockingMode == LM_LIGHTWEIGHT) {
 697     monitor = LightweightSynchronizer::inflate_locked_or_imse(obj, inflate_cause_jni_exit, CHECK);
 698   } else {
 699     // The ObjectMonitor* can't be async deflated until ownership is
 700     // dropped inside exit() and the ObjectMonitor* must be !is_busy().
 701     monitor = inflate(current, obj, inflate_cause_jni_exit);
 702   }
 703   // If this thread has locked the object, exit the monitor. We
 704   // intentionally do not use CHECK on check_owner because we must exit the
 705   // monitor even if an exception was already pending.
 706   if (monitor->check_owner(THREAD)) {
 707     monitor->exit(current);
 708     current->dec_held_monitor_count(1, true);
 709   }
 710 }
 711 
 712 // -----------------------------------------------------------------------------
 713 // Internal VM locks on java objects
 714 // standard constructor, allows locking failures
 715 ObjectLocker::ObjectLocker(Handle obj, TRAPS) : _thread(THREAD), _obj(obj),
 716   _npm(_thread, _thread->at_preemptable_init() /* ignore_mark */), _skip_exit(false) {
 717   assert(!_thread->preempting(), "");
 718 
 719   _thread->check_for_valid_safepoint_state();
 720 
 721   if (_obj() != nullptr) {
 722     ObjectSynchronizer::enter(_obj, &_lock, _thread);
 723 
 724     if (_thread->preempting()) {
 725       // If preemption was cancelled we acquired the monitor after freezing
 726       // the frames. Redoing the vm call laterĀ in thaw will require us to
 727       // release it since the call should look like the original one. We
 728       // do it in ~ObjectLocker to reduce the window of time we hold the
 729       // monitor since we can't do anything useful with it now, and would
 730       // otherwise just force other vthreads to preempt in case they try
 731       // to acquire this monitor.
 732       _skip_exit = !_thread->preemption_cancelled();
 733       _thread->set_pending_preempted_exception();
 734     }
 735   }
 736 }
 737 
 738 ObjectLocker::~ObjectLocker() {
 739   if (_obj() != nullptr && !_skip_exit) {
 740     ObjectSynchronizer::exit(_obj(), &_lock, _thread);
 741   }
 742 }
 743 
 744 void ObjectLocker::wait_uninterruptibly(TRAPS) {
 745   ObjectSynchronizer::waitUninterruptibly(_obj, 0, _thread);
 746   if (_thread->preempting()) {
 747     _skip_exit = true;
 748     _thread->set_pending_preempted_exception();
 749   }
 750 }
 751 
 752 // -----------------------------------------------------------------------------
 753 //  Wait/Notify/NotifyAll
 754 // NOTE: must use heavy weight monitor to handle wait()
 755 
 756 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
 757   JavaThread* current = THREAD;
 758   if (millis < 0) {
 759     THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
 760   }
 761 
 762   ObjectMonitor* monitor;
 763   if (LockingMode == LM_LIGHTWEIGHT) {
 764     monitor = LightweightSynchronizer::inflate_locked_or_imse(obj(), inflate_cause_wait, CHECK_0);
 765   } else {
 766     // The ObjectMonitor* can't be async deflated because the _waiters
 767     // field is incremented before ownership is dropped and decremented
 768     // after ownership is regained.
 769     monitor = inflate(current, obj(), inflate_cause_wait);
 770   }
 771 
 772   DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), current, millis);
 773   monitor->wait(millis, true, THREAD); // Not CHECK as we need following code
 774 
 775   // This dummy call is in place to get around dtrace bug 6254741.  Once
 776   // that's fixed we can uncomment the following line, remove the call
 777   // and change this function back into a "void" func.
 778   // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
 779   int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
 780   return ret_code;
 781 }
 782 
 783 void ObjectSynchronizer::waitUninterruptibly(Handle obj, jlong millis, TRAPS) {
 784   assert(millis >= 0, "timeout value is negative");
 785 
 786   ObjectMonitor* monitor;
 787   if (LockingMode == LM_LIGHTWEIGHT) {
 788     monitor = LightweightSynchronizer::inflate_locked_or_imse(obj(), inflate_cause_wait, CHECK);
 789   } else {
 790     monitor = inflate(THREAD, obj(), inflate_cause_wait);
 791   }
 792   monitor->wait(millis, false, THREAD);
 793 }
 794 
 795 
 796 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
 797   JavaThread* current = THREAD;
 798 
 799   markWord mark = obj->mark();
 800   if (LockingMode == LM_LIGHTWEIGHT) {
 801     if ((mark.is_fast_locked() && current->lock_stack().contains(obj()))) {
 802       // Not inflated so there can't be any waiters to notify.
 803       return;
 804     }
 805   } else if (LockingMode == LM_LEGACY) {
 806     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
 807       // Not inflated so there can't be any waiters to notify.
 808       return;
 809     }
 810   }
 811 
 812   ObjectMonitor* monitor;
 813   if (LockingMode == LM_LIGHTWEIGHT) {
 814     monitor = LightweightSynchronizer::inflate_locked_or_imse(obj(), inflate_cause_notify, CHECK);
 815   } else {
 816     // The ObjectMonitor* can't be async deflated until ownership is
 817     // dropped by the calling thread.
 818     monitor = inflate(current, obj(), inflate_cause_notify);
 819   }
 820   monitor->notify(CHECK);
 821 }
 822 
 823 // NOTE: see comment of notify()
 824 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
 825   JavaThread* current = THREAD;
 826 
 827   markWord mark = obj->mark();
 828   if (LockingMode == LM_LIGHTWEIGHT) {
 829     if ((mark.is_fast_locked() && current->lock_stack().contains(obj()))) {
 830       // Not inflated so there can't be any waiters to notify.
 831       return;
 832     }
 833   } else if (LockingMode == LM_LEGACY) {
 834     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
 835       // Not inflated so there can't be any waiters to notify.
 836       return;
 837     }
 838   }
 839 
 840   ObjectMonitor* monitor;
 841   if (LockingMode == LM_LIGHTWEIGHT) {
 842     monitor = LightweightSynchronizer::inflate_locked_or_imse(obj(), inflate_cause_notify, CHECK);
 843   } else {
 844     // The ObjectMonitor* can't be async deflated until ownership is
 845     // dropped by the calling thread.
 846     monitor = inflate(current, obj(), inflate_cause_notify);
 847   }
 848   monitor->notifyAll(CHECK);
 849 }
 850 
 851 // -----------------------------------------------------------------------------
 852 // Hash Code handling
 853 
 854 struct SharedGlobals {
 855   char         _pad_prefix[OM_CACHE_LINE_SIZE];
 856   // This is a highly shared mostly-read variable.
 857   // To avoid false-sharing it needs to be the sole occupant of a cache line.
 858   volatile int stw_random;
 859   DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int));
 860   // Hot RW variable -- Sequester to avoid false-sharing
 861   volatile int hc_sequence;
 862   DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
 863 };
 864 
 865 static SharedGlobals GVars;
 866 
 867 static markWord read_stable_mark(oop obj) {
 868   markWord mark = obj->mark_acquire();
 869   if (!mark.is_being_inflated() || LockingMode == LM_LIGHTWEIGHT) {
 870     // New lightweight locking does not use the markWord::INFLATING() protocol.
 871     return mark;       // normal fast-path return
 872   }
 873 
 874   int its = 0;
 875   for (;;) {
 876     markWord mark = obj->mark_acquire();
 877     if (!mark.is_being_inflated()) {
 878       return mark;    // normal fast-path return
 879     }
 880 
 881     // The object is being inflated by some other thread.
 882     // The caller of read_stable_mark() must wait for inflation to complete.
 883     // Avoid live-lock.
 884 
 885     ++its;
 886     if (its > 10000 || !os::is_MP()) {
 887       if (its & 1) {
 888         os::naked_yield();
 889       } else {
 890         // Note that the following code attenuates the livelock problem but is not
 891         // a complete remedy.  A more complete solution would require that the inflating
 892         // thread hold the associated inflation lock.  The following code simply restricts
 893         // the number of spinners to at most one.  We'll have N-2 threads blocked
 894         // on the inflationlock, 1 thread holding the inflation lock and using
 895         // a yield/park strategy, and 1 thread in the midst of inflation.
 896         // A more refined approach would be to change the encoding of INFLATING
 897         // to allow encapsulation of a native thread pointer.  Threads waiting for
 898         // inflation to complete would use CAS to push themselves onto a singly linked
 899         // list rooted at the markword.  Once enqueued, they'd loop, checking a per-thread flag
 900         // and calling park().  When inflation was complete the thread that accomplished inflation
 901         // would detach the list and set the markword to inflated with a single CAS and
 902         // then for each thread on the list, set the flag and unpark() the thread.
 903 
 904         // Index into the lock array based on the current object address.
 905         static_assert(is_power_of_2(inflation_lock_count()), "must be");
 906         size_t ix = (cast_from_oop<intptr_t>(obj) >> 5) & (inflation_lock_count() - 1);
 907         int YieldThenBlock = 0;
 908         assert(ix < inflation_lock_count(), "invariant");
 909         inflation_lock(ix)->lock();
 910         while (obj->mark_acquire() == markWord::INFLATING()) {
 911           // Beware: naked_yield() is advisory and has almost no effect on some platforms
 912           // so we periodically call current->_ParkEvent->park(1).
 913           // We use a mixed spin/yield/block mechanism.
 914           if ((YieldThenBlock++) >= 16) {
 915             Thread::current()->_ParkEvent->park(1);
 916           } else {
 917             os::naked_yield();
 918           }
 919         }
 920         inflation_lock(ix)->unlock();
 921       }
 922     } else {
 923       SpinPause();       // SMP-polite spinning
 924     }
 925   }
 926 }
 927 
 928 // hashCode() generation :
 929 //
 930 // Possibilities:
 931 // * MD5Digest of {obj,stw_random}
 932 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
 933 // * A DES- or AES-style SBox[] mechanism
 934 // * One of the Phi-based schemes, such as:
 935 //   2654435761 = 2^32 * Phi (golden ratio)
 936 //   HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
 937 // * A variation of Marsaglia's shift-xor RNG scheme.
 938 // * (obj ^ stw_random) is appealing, but can result
 939 //   in undesirable regularity in the hashCode values of adjacent objects
 940 //   (objects allocated back-to-back, in particular).  This could potentially
 941 //   result in hashtable collisions and reduced hashtable efficiency.
 942 //   There are simple ways to "diffuse" the middle address bits over the
 943 //   generated hashCode values:
 944 
 945 static intptr_t get_next_hash(Thread* current, oop obj) {
 946   intptr_t value = 0;
 947   if (hashCode == 0) {
 948     // This form uses global Park-Miller RNG.
 949     // On MP system we'll have lots of RW access to a global, so the
 950     // mechanism induces lots of coherency traffic.
 951     value = os::random();
 952   } else if (hashCode == 1) {
 953     // This variation has the property of being stable (idempotent)
 954     // between STW operations.  This can be useful in some of the 1-0
 955     // synchronization schemes.
 956     intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
 957     value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
 958   } else if (hashCode == 2) {
 959     value = 1;            // for sensitivity testing
 960   } else if (hashCode == 3) {
 961     value = ++GVars.hc_sequence;
 962   } else if (hashCode == 4) {
 963     value = cast_from_oop<intptr_t>(obj);
 964   } else {
 965     // Marsaglia's xor-shift scheme with thread-specific state
 966     // This is probably the best overall implementation -- we'll
 967     // likely make this the default in future releases.
 968     unsigned t = current->_hashStateX;
 969     t ^= (t << 11);
 970     current->_hashStateX = current->_hashStateY;
 971     current->_hashStateY = current->_hashStateZ;
 972     current->_hashStateZ = current->_hashStateW;
 973     unsigned v = current->_hashStateW;
 974     v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
 975     current->_hashStateW = v;
 976     value = v;
 977   }
 978 
 979   value &= markWord::hash_mask;
 980   if (value == 0) value = 0xBAD;
 981   assert(value != markWord::no_hash, "invariant");
 982   return value;
 983 }
 984 
 985 static intptr_t install_hash_code(Thread* current, oop obj) {
 986   assert(UseObjectMonitorTable && LockingMode == LM_LIGHTWEIGHT, "must be");
 987 
 988   markWord mark = obj->mark_acquire();
 989   for (;;) {
 990     intptr_t hash = mark.hash();
 991     if (hash != 0) {
 992       return hash;
 993     }
 994 
 995     hash = get_next_hash(current, obj);
 996     const markWord old_mark = mark;
 997     const markWord new_mark = old_mark.copy_set_hash(hash);
 998 
 999     mark = obj->cas_set_mark(new_mark, old_mark);
1000     if (old_mark == mark) {
1001       return hash;
1002     }
1003   }
1004 }
1005 
1006 intptr_t ObjectSynchronizer::FastHashCode(Thread* current, oop obj) {
1007   if (UseObjectMonitorTable) {
1008     // Since the monitor isn't in the object header, the hash can simply be
1009     // installed in the object header.
1010     return install_hash_code(current, obj);
1011   }
1012 
1013   while (true) {
1014     ObjectMonitor* monitor = nullptr;
1015     markWord temp, test;
1016     intptr_t hash;
1017     markWord mark = read_stable_mark(obj);
1018     if (VerifyHeavyMonitors) {
1019       assert(LockingMode == LM_MONITOR, "+VerifyHeavyMonitors requires LockingMode == 0 (LM_MONITOR)");
1020       guarantee((obj->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
1021     }
1022     if (mark.is_unlocked() || (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked())) {
1023       hash = mark.hash();
1024       if (hash != 0) {                     // if it has a hash, just return it
1025         return hash;
1026       }
1027       hash = get_next_hash(current, obj);  // get a new hash
1028       temp = mark.copy_set_hash(hash);     // merge the hash into header
1029                                            // try to install the hash
1030       test = obj->cas_set_mark(temp, mark);
1031       if (test == mark) {                  // if the hash was installed, return it
1032         return hash;
1033       }
1034       if (LockingMode == LM_LIGHTWEIGHT) {
1035         // CAS failed, retry
1036         continue;
1037       }
1038       // Failed to install the hash. It could be that another thread
1039       // installed the hash just before our attempt or inflation has
1040       // occurred or... so we fall thru to inflate the monitor for
1041       // stability and then install the hash.
1042     } else if (mark.has_monitor()) {
1043       monitor = mark.monitor();
1044       temp = monitor->header();
1045       assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1046       hash = temp.hash();
1047       if (hash != 0) {
1048         // It has a hash.
1049 
1050         // Separate load of dmw/header above from the loads in
1051         // is_being_async_deflated().
1052 
1053         // dmw/header and _contentions may get written by different threads.
1054         // Make sure to observe them in the same order when having several observers.
1055         OrderAccess::loadload_for_IRIW();
1056 
1057         if (monitor->is_being_async_deflated()) {
1058           // But we can't safely use the hash if we detect that async
1059           // deflation has occurred. So we attempt to restore the
1060           // header/dmw to the object's header so that we only retry
1061           // once if the deflater thread happens to be slow.
1062           monitor->install_displaced_markword_in_object(obj);
1063           continue;
1064         }
1065         return hash;
1066       }
1067       // Fall thru so we only have one place that installs the hash in
1068       // the ObjectMonitor.
1069     } else if (LockingMode == LM_LEGACY && mark.has_locker()
1070                && current->is_Java_thread()
1071                && JavaThread::cast(current)->is_lock_owned((address)mark.locker())) {
1072       // This is a stack-lock owned by the calling thread so fetch the
1073       // displaced markWord from the BasicLock on the stack.
1074       temp = mark.displaced_mark_helper();
1075       assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1076       hash = temp.hash();
1077       if (hash != 0) {                  // if it has a hash, just return it
1078         return hash;
1079       }
1080       // WARNING:
1081       // The displaced header in the BasicLock on a thread's stack
1082       // is strictly immutable. It CANNOT be changed in ANY cases.
1083       // So we have to inflate the stack-lock into an ObjectMonitor
1084       // even if the current thread owns the lock. The BasicLock on
1085       // a thread's stack can be asynchronously read by other threads
1086       // during an inflate() call so any change to that stack memory
1087       // may not propagate to other threads correctly.
1088     }
1089 
1090     // Inflate the monitor to set the hash.
1091 
1092     // There's no need to inflate if the mark has already got a monitor.
1093     // NOTE: an async deflation can race after we get the monitor and
1094     // before we can update the ObjectMonitor's header with the hash
1095     // value below.
1096     monitor = mark.has_monitor() ? mark.monitor() : inflate(current, obj, inflate_cause_hash_code);
1097     // Load ObjectMonitor's header/dmw field and see if it has a hash.
1098     mark = monitor->header();
1099     assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1100     hash = mark.hash();
1101     if (hash == 0) {                       // if it does not have a hash
1102       hash = get_next_hash(current, obj);  // get a new hash
1103       temp = mark.copy_set_hash(hash)   ;  // merge the hash into header
1104       assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1105       uintptr_t v = Atomic::cmpxchg(monitor->metadata_addr(), mark.value(), temp.value());
1106       test = markWord(v);
1107       if (test != mark) {
1108         // The attempt to update the ObjectMonitor's header/dmw field
1109         // did not work. This can happen if another thread managed to
1110         // merge in the hash just before our cmpxchg().
1111         // If we add any new usages of the header/dmw field, this code
1112         // will need to be updated.
1113         hash = test.hash();
1114         assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1115         assert(hash != 0, "should only have lost the race to a thread that set a non-zero hash");
1116       }
1117       if (monitor->is_being_async_deflated() && !UseObjectMonitorTable) {
1118         // If we detect that async deflation has occurred, then we
1119         // attempt to restore the header/dmw to the object's header
1120         // so that we only retry once if the deflater thread happens
1121         // to be slow.
1122         monitor->install_displaced_markword_in_object(obj);
1123         continue;
1124       }
1125     }
1126     // We finally get the hash.
1127     return hash;
1128   }
1129 }
1130 
1131 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* current,
1132                                                    Handle h_obj) {
1133   assert(current == JavaThread::current(), "Can only be called on current thread");
1134   oop obj = h_obj();
1135 
1136   markWord mark = read_stable_mark(obj);
1137 
1138   if (LockingMode == LM_LEGACY && mark.has_locker()) {
1139     // stack-locked case, header points into owner's stack
1140     return current->is_lock_owned((address)mark.locker());
1141   }
1142 
1143   if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
1144     // fast-locking case, see if lock is in current's lock stack
1145     return current->lock_stack().contains(h_obj());
1146   }
1147 
1148   while (LockingMode == LM_LIGHTWEIGHT && mark.has_monitor()) {
1149     ObjectMonitor* monitor = read_monitor(current, obj, mark);
1150     if (monitor != nullptr) {
1151       return monitor->is_entered(current) != 0;
1152     }
1153     // Racing with inflation/deflation, retry
1154     mark = obj->mark_acquire();
1155 
1156     if (mark.is_fast_locked()) {
1157       // Some other thread fast_locked, current could not have held the lock
1158       return false;
1159     }
1160   }
1161 
1162   if (LockingMode != LM_LIGHTWEIGHT && mark.has_monitor()) {
1163     // Inflated monitor so header points to ObjectMonitor (tagged pointer).
1164     // The first stage of async deflation does not affect any field
1165     // used by this comparison so the ObjectMonitor* is usable here.
1166     ObjectMonitor* monitor = read_monitor(mark);
1167     return monitor->is_entered(current) != 0;
1168   }
1169   // Unlocked case, header in place
1170   assert(mark.is_unlocked(), "sanity check");
1171   return false;
1172 }
1173 
1174 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1175   oop obj = h_obj();
1176   markWord mark = read_stable_mark(obj);
1177 
1178   if (LockingMode == LM_LEGACY && mark.has_locker()) {
1179     // stack-locked so header points into owner's stack.
1180     // owning_thread_from_monitor_owner() may also return null here:
1181     return Threads::owning_thread_from_stacklock(t_list, (address) mark.locker());
1182   }
1183 
1184   if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
1185     // fast-locked so get owner from the object.
1186     // owning_thread_from_object() may also return null here:
1187     return Threads::owning_thread_from_object(t_list, h_obj());
1188   }
1189 
1190   while (LockingMode == LM_LIGHTWEIGHT && mark.has_monitor()) {
1191     ObjectMonitor* monitor = read_monitor(Thread::current(), obj, mark);
1192     if (monitor != nullptr) {
1193       return Threads::owning_thread_from_monitor(t_list, monitor);
1194     }
1195     // Racing with inflation/deflation, retry
1196     mark = obj->mark_acquire();
1197 
1198     if (mark.is_fast_locked()) {
1199       // Some other thread fast_locked
1200       return Threads::owning_thread_from_object(t_list, h_obj());
1201     }
1202   }
1203 
1204   if (LockingMode != LM_LIGHTWEIGHT && mark.has_monitor()) {
1205     // Inflated monitor so header points to ObjectMonitor (tagged pointer).
1206     // The first stage of async deflation does not affect any field
1207     // used by this comparison so the ObjectMonitor* is usable here.
1208     ObjectMonitor* monitor = read_monitor(mark);
1209     assert(monitor != nullptr, "monitor should be non-null");
1210     // owning_thread_from_monitor() may also return null here:
1211     return Threads::owning_thread_from_monitor(t_list, monitor);
1212   }
1213 
1214   // Unlocked case, header in place
1215   // Cannot have assertion since this object may have been
1216   // locked by another thread when reaching here.
1217   // assert(mark.is_unlocked(), "sanity check");
1218 
1219   return nullptr;
1220 }
1221 
1222 // Visitors ...
1223 
1224 // Iterate over all ObjectMonitors.
1225 template <typename Function>
1226 void ObjectSynchronizer::monitors_iterate(Function function) {
1227   MonitorList::Iterator iter = _in_use_list.iterator();
1228   while (iter.has_next()) {
1229     ObjectMonitor* monitor = iter.next();
1230     function(monitor);
1231   }
1232 }
1233 
1234 // Iterate ObjectMonitors owned by any thread and where the owner `filter`
1235 // returns true.
1236 template <typename OwnerFilter>
1237 void ObjectSynchronizer::owned_monitors_iterate_filtered(MonitorClosure* closure, OwnerFilter filter) {
1238   monitors_iterate([&](ObjectMonitor* monitor) {
1239     // This function is only called at a safepoint or when the
1240     // target thread is suspended or when the target thread is
1241     // operating on itself. The current closures in use today are
1242     // only interested in an owned ObjectMonitor and ownership
1243     // cannot be dropped under the calling contexts so the
1244     // ObjectMonitor cannot be async deflated.
1245     if (monitor->has_owner() && filter(monitor)) {
1246       assert(!monitor->is_being_async_deflated(), "Owned monitors should not be deflating");
1247 
1248       closure->do_monitor(monitor);
1249     }
1250   });
1251 }
1252 
1253 // Iterate ObjectMonitors where the owner == thread; this does NOT include
1254 // ObjectMonitors where owner is set to a stack-lock address in thread.
1255 void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure, JavaThread* thread) {
1256   int64_t key = ObjectMonitor::owner_id_from(thread);
1257   auto thread_filter = [&](ObjectMonitor* monitor) { return monitor->owner() == key; };
1258   return owned_monitors_iterate_filtered(closure, thread_filter);
1259 }
1260 
1261 void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure, oop vthread) {
1262   int64_t key = ObjectMonitor::owner_id_from(vthread);
1263   auto thread_filter = [&](ObjectMonitor* monitor) { return monitor->owner() == key; };
1264   return owned_monitors_iterate_filtered(closure, thread_filter);
1265 }
1266 
1267 // Iterate ObjectMonitors owned by any thread.
1268 void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure) {
1269   auto all_filter = [&](ObjectMonitor* monitor) { return true; };
1270   return owned_monitors_iterate_filtered(closure, all_filter);
1271 }
1272 
1273 static bool monitors_used_above_threshold(MonitorList* list) {
1274   if (MonitorUsedDeflationThreshold == 0) {  // disabled case is easy
1275     return false;
1276   }
1277   size_t monitors_used = list->count();
1278   if (monitors_used == 0) {  // empty list is easy
1279     return false;
1280   }
1281   size_t old_ceiling = ObjectSynchronizer::in_use_list_ceiling();
1282   // Make sure that we use a ceiling value that is not lower than
1283   // previous, not lower than the recorded max used by the system, and
1284   // not lower than the current number of monitors in use (which can
1285   // race ahead of max). The result is guaranteed > 0.
1286   size_t ceiling = MAX3(old_ceiling, list->max(), monitors_used);
1287 
1288   // Check if our monitor usage is above the threshold:
1289   size_t monitor_usage = (monitors_used * 100LL) / ceiling;
1290   if (int(monitor_usage) > MonitorUsedDeflationThreshold) {
1291     // Deflate monitors if over the threshold percentage, unless no
1292     // progress on previous deflations.
1293     bool is_above_threshold = true;
1294 
1295     // Check if it's time to adjust the in_use_list_ceiling up, due
1296     // to too many async deflation attempts without any progress.
1297     if (NoAsyncDeflationProgressMax != 0 &&
1298         _no_progress_cnt >= NoAsyncDeflationProgressMax) {
1299       double remainder = (100.0 - MonitorUsedDeflationThreshold) / 100.0;
1300       size_t delta = (size_t)(ceiling * remainder) + 1;
1301       size_t new_ceiling = (ceiling > SIZE_MAX - delta)
1302         ? SIZE_MAX         // Overflow, let's clamp new_ceiling.
1303         : ceiling + delta;
1304 
1305       ObjectSynchronizer::set_in_use_list_ceiling(new_ceiling);
1306       log_info(monitorinflation)("Too many deflations without progress; "
1307                                  "bumping in_use_list_ceiling from %zu"
1308                                  " to %zu", old_ceiling, new_ceiling);
1309       _no_progress_cnt = 0;
1310       ceiling = new_ceiling;
1311 
1312       // Check if our monitor usage is still above the threshold:
1313       monitor_usage = (monitors_used * 100LL) / ceiling;
1314       is_above_threshold = int(monitor_usage) > MonitorUsedDeflationThreshold;
1315     }
1316     log_info(monitorinflation)("monitors_used=%zu, ceiling=%zu"
1317                                ", monitor_usage=%zu, threshold=%d",
1318                                monitors_used, ceiling, monitor_usage, MonitorUsedDeflationThreshold);
1319     return is_above_threshold;
1320   }
1321 
1322   return false;
1323 }
1324 
1325 size_t ObjectSynchronizer::in_use_list_count() {
1326   return _in_use_list.count();
1327 }
1328 
1329 size_t ObjectSynchronizer::in_use_list_max() {
1330   return _in_use_list.max();
1331 }
1332 
1333 size_t ObjectSynchronizer::in_use_list_ceiling() {
1334   return _in_use_list_ceiling;
1335 }
1336 
1337 void ObjectSynchronizer::dec_in_use_list_ceiling() {
1338   Atomic::sub(&_in_use_list_ceiling, AvgMonitorsPerThreadEstimate);
1339 }
1340 
1341 void ObjectSynchronizer::inc_in_use_list_ceiling() {
1342   Atomic::add(&_in_use_list_ceiling, AvgMonitorsPerThreadEstimate);
1343 }
1344 
1345 void ObjectSynchronizer::set_in_use_list_ceiling(size_t new_value) {
1346   _in_use_list_ceiling = new_value;
1347 }
1348 
1349 bool ObjectSynchronizer::is_async_deflation_needed() {
1350   if (is_async_deflation_requested()) {
1351     // Async deflation request.
1352     log_info(monitorinflation)("Async deflation needed: explicit request");
1353     return true;
1354   }
1355 
1356   jlong time_since_last = time_since_last_async_deflation_ms();
1357 
1358   if (AsyncDeflationInterval > 0 &&
1359       time_since_last > AsyncDeflationInterval &&
1360       monitors_used_above_threshold(&_in_use_list)) {
1361     // It's been longer than our specified deflate interval and there
1362     // are too many monitors in use. We don't deflate more frequently
1363     // than AsyncDeflationInterval (unless is_async_deflation_requested)
1364     // in order to not swamp the MonitorDeflationThread.
1365     log_info(monitorinflation)("Async deflation needed: monitors used are above the threshold");
1366     return true;
1367   }
1368 
1369   if (GuaranteedAsyncDeflationInterval > 0 &&
1370       time_since_last > GuaranteedAsyncDeflationInterval) {
1371     // It's been longer than our specified guaranteed deflate interval.
1372     // We need to clean up the used monitors even if the threshold is
1373     // not reached, to keep the memory utilization at bay when many threads
1374     // touched many monitors.
1375     log_info(monitorinflation)("Async deflation needed: guaranteed interval (%zd ms) "
1376                                "is greater than time since last deflation (" JLONG_FORMAT " ms)",
1377                                GuaranteedAsyncDeflationInterval, time_since_last);
1378 
1379     // If this deflation has no progress, then it should not affect the no-progress
1380     // tracking, otherwise threshold heuristics would think it was triggered, experienced
1381     // no progress, and needs to backoff more aggressively. In this "no progress" case,
1382     // the generic code would bump the no-progress counter, and we compensate for that
1383     // by telling it to skip the update.
1384     //
1385     // If this deflation has progress, then it should let non-progress tracking
1386     // know about this, otherwise the threshold heuristics would kick in, potentially
1387     // experience no-progress due to aggressive cleanup by this deflation, and think
1388     // it is still in no-progress stride. In this "progress" case, the generic code would
1389     // zero the counter, and we allow it to happen.
1390     _no_progress_skip_increment = true;
1391 
1392     return true;
1393   }
1394 
1395   return false;
1396 }
1397 
1398 void ObjectSynchronizer::request_deflate_idle_monitors() {
1399   MonitorLocker ml(MonitorDeflation_lock, Mutex::_no_safepoint_check_flag);
1400   set_is_async_deflation_requested(true);
1401   ml.notify_all();
1402 }
1403 
1404 bool ObjectSynchronizer::request_deflate_idle_monitors_from_wb() {
1405   JavaThread* current = JavaThread::current();
1406   bool ret_code = false;
1407 
1408   jlong last_time = last_async_deflation_time_ns();
1409 
1410   request_deflate_idle_monitors();
1411 
1412   const int N_CHECKS = 5;
1413   for (int i = 0; i < N_CHECKS; i++) {  // sleep for at most 5 seconds
1414     if (last_async_deflation_time_ns() > last_time) {
1415       log_info(monitorinflation)("Async Deflation happened after %d check(s).", i);
1416       ret_code = true;
1417       break;
1418     }
1419     {
1420       // JavaThread has to honor the blocking protocol.
1421       ThreadBlockInVM tbivm(current);
1422       os::naked_short_sleep(999);  // sleep for almost 1 second
1423     }
1424   }
1425   if (!ret_code) {
1426     log_info(monitorinflation)("Async Deflation DID NOT happen after %d checks.", N_CHECKS);
1427   }
1428 
1429   return ret_code;
1430 }
1431 
1432 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1433   return (os::javaTimeNanos() - last_async_deflation_time_ns()) / (NANOUNITS / MILLIUNITS);
1434 }
1435 
1436 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1437                                        const oop obj,
1438                                        ObjectSynchronizer::InflateCause cause) {
1439   assert(event != nullptr, "invariant");
1440   const Klass* monitor_klass = obj->klass();
1441   if (ObjectMonitor::is_jfr_excluded(monitor_klass)) {
1442     return;
1443   }
1444   event->set_monitorClass(monitor_klass);
1445   event->set_address((uintptr_t)(void*)obj);
1446   event->set_cause((u1)cause);
1447   event->commit();
1448 }
1449 
1450 // Fast path code shared by multiple functions
1451 void ObjectSynchronizer::inflate_helper(oop obj) {
1452   assert(LockingMode != LM_LIGHTWEIGHT, "only inflate through enter");
1453   markWord mark = obj->mark_acquire();
1454   if (mark.has_monitor()) {
1455     ObjectMonitor* monitor = read_monitor(mark);
1456     markWord dmw = monitor->header();
1457     assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1458     return;
1459   }
1460   (void)inflate(Thread::current(), obj, inflate_cause_vm_internal);
1461 }
1462 
1463 ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop obj, const InflateCause cause) {
1464   assert(current == Thread::current(), "must be");
1465   assert(LockingMode != LM_LIGHTWEIGHT, "only inflate through enter");
1466   return inflate_impl(current->is_Java_thread() ? JavaThread::cast(current) : nullptr, obj, cause);
1467 }
1468 
1469 ObjectMonitor* ObjectSynchronizer::inflate_for(JavaThread* thread, oop obj, const InflateCause cause) {
1470   assert(thread == Thread::current() || thread->is_obj_deopt_suspend(), "must be");
1471   assert(LockingMode != LM_LIGHTWEIGHT, "LM_LIGHTWEIGHT cannot use inflate_for");
1472   return inflate_impl(thread, obj, cause);
1473 }
1474 
1475 ObjectMonitor* ObjectSynchronizer::inflate_impl(JavaThread* locking_thread, oop object, const InflateCause cause) {
1476   // The JavaThread* locking_thread requires that the locking_thread == Thread::current() or
1477   // is suspended throughout the call by some other mechanism.
1478   // The thread might be nullptr when called from a non JavaThread. (As may still be
1479   // the case from FastHashCode). However it is only important for correctness that the
1480   // thread is set when called from ObjectSynchronizer::enter from the owning thread,
1481   // ObjectSynchronizer::enter_for from any thread, or ObjectSynchronizer::exit.
1482   assert(LockingMode != LM_LIGHTWEIGHT, "LM_LIGHTWEIGHT cannot use inflate_impl");
1483   EventJavaMonitorInflate event;
1484 
1485   for (;;) {
1486     const markWord mark = object->mark_acquire();
1487 
1488     // The mark can be in one of the following states:
1489     // *  inflated     - If the ObjectMonitor owner is anonymous and the
1490     //                   locking_thread owns the object lock, then we
1491     //                   make the locking_thread the ObjectMonitor owner.
1492     // *  stack-locked - Coerce it to inflated from stack-locked.
1493     // *  INFLATING    - Busy wait for conversion from stack-locked to
1494     //                   inflated.
1495     // *  unlocked     - Aggressively inflate the object.
1496 
1497     // CASE: inflated
1498     if (mark.has_monitor()) {
1499       ObjectMonitor* inf = mark.monitor();
1500       markWord dmw = inf->header();
1501       assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1502       if (inf->has_anonymous_owner() && locking_thread != nullptr) {
1503         assert(LockingMode == LM_LEGACY, "invariant");
1504         if (locking_thread->is_lock_owned((address)inf->stack_locker())) {
1505           inf->set_stack_locker(nullptr);
1506           inf->set_owner_from_anonymous(locking_thread);
1507         }
1508       }
1509       return inf;
1510     }
1511 
1512     // CASE: inflation in progress - inflating over a stack-lock.
1513     // Some other thread is converting from stack-locked to inflated.
1514     // Only that thread can complete inflation -- other threads must wait.
1515     // The INFLATING value is transient.
1516     // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1517     // We could always eliminate polling by parking the thread on some auxiliary list.
1518     if (mark == markWord::INFLATING()) {
1519       read_stable_mark(object);
1520       continue;
1521     }
1522 
1523     // CASE: stack-locked
1524     // Could be stack-locked either by current or by some other thread.
1525     //
1526     // Note that we allocate the ObjectMonitor speculatively, _before_ attempting
1527     // to install INFLATING into the mark word.  We originally installed INFLATING,
1528     // allocated the ObjectMonitor, and then finally STed the address of the
1529     // ObjectMonitor into the mark.  This was correct, but artificially lengthened
1530     // the interval in which INFLATING appeared in the mark, thus increasing
1531     // the odds of inflation contention. If we lose the race to set INFLATING,
1532     // then we just delete the ObjectMonitor and loop around again.
1533     //
1534     LogStreamHandle(Trace, monitorinflation) lsh;
1535     if (LockingMode == LM_LEGACY && mark.has_locker()) {
1536       ObjectMonitor* m = new ObjectMonitor(object);
1537       // Optimistically prepare the ObjectMonitor - anticipate successful CAS
1538       // We do this before the CAS in order to minimize the length of time
1539       // in which INFLATING appears in the mark.
1540 
1541       markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1542       if (cmp != mark) {
1543         delete m;
1544         continue;       // Interference -- just retry
1545       }
1546 
1547       // We've successfully installed INFLATING (0) into the mark-word.
1548       // This is the only case where 0 will appear in a mark-word.
1549       // Only the singular thread that successfully swings the mark-word
1550       // to 0 can perform (or more precisely, complete) inflation.
1551       //
1552       // Why do we CAS a 0 into the mark-word instead of just CASing the
1553       // mark-word from the stack-locked value directly to the new inflated state?
1554       // Consider what happens when a thread unlocks a stack-locked object.
1555       // It attempts to use CAS to swing the displaced header value from the
1556       // on-stack BasicLock back into the object header.  Recall also that the
1557       // header value (hash code, etc) can reside in (a) the object header, or
1558       // (b) a displaced header associated with the stack-lock, or (c) a displaced
1559       // header in an ObjectMonitor.  The inflate() routine must copy the header
1560       // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1561       // the while preserving the hashCode stability invariants.  If the owner
1562       // decides to release the lock while the value is 0, the unlock will fail
1563       // and control will eventually pass from slow_exit() to inflate.  The owner
1564       // will then spin, waiting for the 0 value to disappear.   Put another way,
1565       // the 0 causes the owner to stall if the owner happens to try to
1566       // drop the lock (restoring the header from the BasicLock to the object)
1567       // while inflation is in-progress.  This protocol avoids races that might
1568       // would otherwise permit hashCode values to change or "flicker" for an object.
1569       // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1570       // 0 serves as a "BUSY" inflate-in-progress indicator.
1571 
1572 
1573       // fetch the displaced mark from the owner's stack.
1574       // The owner can't die or unwind past the lock while our INFLATING
1575       // object is in the mark.  Furthermore the owner can't complete
1576       // an unlock on the object, either.
1577       markWord dmw = mark.displaced_mark_helper();
1578       // Catch if the object's header is not neutral (not locked and
1579       // not marked is what we care about here).
1580       assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1581 
1582       // Setup monitor fields to proper values -- prepare the monitor
1583       m->set_header(dmw);
1584 
1585       // Note that a thread can inflate an object
1586       // that it has stack-locked -- as might happen in wait() -- directly
1587       // with CAS.  That is, we can avoid the xchg-nullptr .... ST idiom.
1588       if (locking_thread != nullptr && locking_thread->is_lock_owned((address)mark.locker())) {
1589         m->set_owner(locking_thread);
1590       } else {
1591         // Use ANONYMOUS_OWNER to indicate that the owner is the BasicLock on the stack,
1592         // and set the stack locker field in the monitor.
1593         m->set_stack_locker(mark.locker());
1594         m->set_anonymous_owner();
1595       }
1596       // TODO-FIXME: assert BasicLock->dhw != 0.
1597 
1598       // Must preserve store ordering. The monitor state must
1599       // be stable at the time of publishing the monitor address.
1600       guarantee(object->mark() == markWord::INFLATING(), "invariant");
1601       // Release semantics so that above set_object() is seen first.
1602       object->release_set_mark(markWord::encode(m));
1603 
1604       // Once ObjectMonitor is configured and the object is associated
1605       // with the ObjectMonitor, it is safe to allow async deflation:
1606       _in_use_list.add(m);
1607 
1608       if (log_is_enabled(Trace, monitorinflation)) {
1609         ResourceMark rm;
1610         lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1611                      INTPTR_FORMAT ", type='%s'", p2i(object),
1612                      object->mark().value(), object->klass()->external_name());
1613       }
1614       if (event.should_commit()) {
1615         post_monitor_inflate_event(&event, object, cause);
1616       }
1617       return m;
1618     }
1619 
1620     // CASE: unlocked
1621     // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1622     // If we know we're inflating for entry it's better to inflate by swinging a
1623     // pre-locked ObjectMonitor pointer into the object header.   A successful
1624     // CAS inflates the object *and* confers ownership to the inflating thread.
1625     // In the current implementation we use a 2-step mechanism where we CAS()
1626     // to inflate and then CAS() again to try to swing _owner from null to current.
1627     // An inflateTry() method that we could call from enter() would be useful.
1628 
1629     assert(mark.is_unlocked(), "invariant: header=" INTPTR_FORMAT, mark.value());
1630     ObjectMonitor* m = new ObjectMonitor(object);
1631     // prepare m for installation - set monitor to initial state
1632     m->set_header(mark);
1633 
1634     if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1635       delete m;
1636       m = nullptr;
1637       continue;
1638       // interference - the markword changed - just retry.
1639       // The state-transitions are one-way, so there's no chance of
1640       // live-lock -- "Inflated" is an absorbing state.
1641     }
1642 
1643     // Once the ObjectMonitor is configured and object is associated
1644     // with the ObjectMonitor, it is safe to allow async deflation:
1645     _in_use_list.add(m);
1646 
1647     if (log_is_enabled(Trace, monitorinflation)) {
1648       ResourceMark rm;
1649       lsh.print_cr("inflate(unlocked): object=" INTPTR_FORMAT ", mark="
1650                    INTPTR_FORMAT ", type='%s'", p2i(object),
1651                    object->mark().value(), object->klass()->external_name());
1652     }
1653     if (event.should_commit()) {
1654       post_monitor_inflate_event(&event, object, cause);
1655     }
1656     return m;
1657   }
1658 }
1659 
1660 // Walk the in-use list and deflate (at most MonitorDeflationMax) idle
1661 // ObjectMonitors. Returns the number of deflated ObjectMonitors.
1662 //
1663 size_t ObjectSynchronizer::deflate_monitor_list(ObjectMonitorDeflationSafepointer* safepointer) {
1664   MonitorList::Iterator iter = _in_use_list.iterator();
1665   size_t deflated_count = 0;
1666   Thread* current = Thread::current();
1667 
1668   while (iter.has_next()) {
1669     if (deflated_count >= (size_t)MonitorDeflationMax) {
1670       break;
1671     }
1672     ObjectMonitor* mid = iter.next();
1673     if (mid->deflate_monitor(current)) {
1674       deflated_count++;
1675     }
1676 
1677     // Must check for a safepoint/handshake and honor it.
1678     safepointer->block_for_safepoint("deflation", "deflated_count", deflated_count);
1679   }
1680 
1681   return deflated_count;
1682 }
1683 
1684 class DeflationHandshakeClosure : public HandshakeClosure {
1685  public:
1686   DeflationHandshakeClosure() : HandshakeClosure("DeflationHandshakeClosure") {}
1687 
1688   void do_thread(Thread* thread) {
1689     log_trace(monitorinflation)("DeflationHandshakeClosure::do_thread: thread="
1690                                 INTPTR_FORMAT, p2i(thread));
1691     if (thread->is_Java_thread()) {
1692       // Clear OM cache
1693       JavaThread* jt = JavaThread::cast(thread);
1694       jt->om_clear_monitor_cache();
1695     }
1696   }
1697 };
1698 
1699 class VM_RendezvousGCThreads : public VM_Operation {
1700 public:
1701   bool evaluate_at_safepoint() const override { return false; }
1702   VMOp_Type type() const override { return VMOp_RendezvousGCThreads; }
1703   void doit() override {
1704     Universe::heap()->safepoint_synchronize_begin();
1705     Universe::heap()->safepoint_synchronize_end();
1706   };
1707 };
1708 
1709 static size_t delete_monitors(GrowableArray<ObjectMonitor*>* delete_list,
1710                               ObjectMonitorDeflationSafepointer* safepointer) {
1711   NativeHeapTrimmer::SuspendMark sm("monitor deletion");
1712   size_t deleted_count = 0;
1713   for (ObjectMonitor* monitor: *delete_list) {
1714     delete monitor;
1715     deleted_count++;
1716     // A JavaThread must check for a safepoint/handshake and honor it.
1717     safepointer->block_for_safepoint("deletion", "deleted_count", deleted_count);
1718   }
1719   return deleted_count;
1720 }
1721 
1722 class ObjectMonitorDeflationLogging: public StackObj {
1723   LogStreamHandle(Debug, monitorinflation) _debug;
1724   LogStreamHandle(Info, monitorinflation)  _info;
1725   LogStream*                               _stream;
1726   elapsedTimer                             _timer;
1727 
1728   size_t ceiling() const { return ObjectSynchronizer::in_use_list_ceiling(); }
1729   size_t count() const   { return ObjectSynchronizer::in_use_list_count(); }
1730   size_t max() const     { return ObjectSynchronizer::in_use_list_max(); }
1731 
1732 public:
1733   ObjectMonitorDeflationLogging()
1734     : _debug(), _info(), _stream(nullptr) {
1735     if (_debug.is_enabled()) {
1736       _stream = &_debug;
1737     } else if (_info.is_enabled()) {
1738       _stream = &_info;
1739     }
1740   }
1741 
1742   void begin() {
1743     if (_stream != nullptr) {
1744       _stream->print_cr("begin deflating: in_use_list stats: ceiling=%zu, count=%zu, max=%zu",
1745                         ceiling(), count(), max());
1746       _timer.start();
1747     }
1748   }
1749 
1750   void before_handshake(size_t unlinked_count) {
1751     if (_stream != nullptr) {
1752       _timer.stop();
1753       _stream->print_cr("before handshaking: unlinked_count=%zu"
1754                         ", in_use_list stats: ceiling=%zu, count="
1755                         "%zu, max=%zu",
1756                         unlinked_count, ceiling(), count(), max());
1757     }
1758   }
1759 
1760   void after_handshake() {
1761     if (_stream != nullptr) {
1762       _stream->print_cr("after handshaking: in_use_list stats: ceiling="
1763                         "%zu, count=%zu, max=%zu",
1764                         ceiling(), count(), max());
1765       _timer.start();
1766     }
1767   }
1768 
1769   void end(size_t deflated_count, size_t unlinked_count) {
1770     if (_stream != nullptr) {
1771       _timer.stop();
1772       if (deflated_count != 0 || unlinked_count != 0 || _debug.is_enabled()) {
1773         _stream->print_cr("deflated_count=%zu, {unlinked,deleted}_count=%zu monitors in %3.7f secs",
1774                           deflated_count, unlinked_count, _timer.seconds());
1775       }
1776       _stream->print_cr("end deflating: in_use_list stats: ceiling=%zu, count=%zu, max=%zu",
1777                         ceiling(), count(), max());
1778     }
1779   }
1780 
1781   void before_block_for_safepoint(const char* op_name, const char* cnt_name, size_t cnt) {
1782     if (_stream != nullptr) {
1783       _timer.stop();
1784       _stream->print_cr("pausing %s: %s=%zu, in_use_list stats: ceiling="
1785                         "%zu, count=%zu, max=%zu",
1786                         op_name, cnt_name, cnt, ceiling(), count(), max());
1787     }
1788   }
1789 
1790   void after_block_for_safepoint(const char* op_name) {
1791     if (_stream != nullptr) {
1792       _stream->print_cr("resuming %s: in_use_list stats: ceiling=%zu"
1793                         ", count=%zu, max=%zu", op_name,
1794                         ceiling(), count(), max());
1795       _timer.start();
1796     }
1797   }
1798 };
1799 
1800 void ObjectMonitorDeflationSafepointer::block_for_safepoint(const char* op_name, const char* count_name, size_t counter) {
1801   if (!SafepointMechanism::should_process(_current)) {
1802     return;
1803   }
1804 
1805   // A safepoint/handshake has started.
1806   _log->before_block_for_safepoint(op_name, count_name, counter);
1807 
1808   {
1809     // Honor block request.
1810     ThreadBlockInVM tbivm(_current);
1811   }
1812 
1813   _log->after_block_for_safepoint(op_name);
1814 }
1815 
1816 // This function is called by the MonitorDeflationThread to deflate
1817 // ObjectMonitors.
1818 size_t ObjectSynchronizer::deflate_idle_monitors() {
1819   JavaThread* current = JavaThread::current();
1820   assert(current->is_monitor_deflation_thread(), "The only monitor deflater");
1821 
1822   // The async deflation request has been processed.
1823   _last_async_deflation_time_ns = os::javaTimeNanos();
1824   set_is_async_deflation_requested(false);
1825 
1826   ObjectMonitorDeflationLogging log;
1827   ObjectMonitorDeflationSafepointer safepointer(current, &log);
1828 
1829   log.begin();
1830 
1831   // Deflate some idle ObjectMonitors.
1832   size_t deflated_count = deflate_monitor_list(&safepointer);
1833 
1834   // Unlink the deflated ObjectMonitors from the in-use list.
1835   size_t unlinked_count = 0;
1836   size_t deleted_count = 0;
1837   if (deflated_count > 0) {
1838     ResourceMark rm(current);
1839     GrowableArray<ObjectMonitor*> delete_list((int)deflated_count);
1840     unlinked_count = _in_use_list.unlink_deflated(deflated_count, &delete_list, &safepointer);
1841 
1842 #ifdef ASSERT
1843     if (UseObjectMonitorTable) {
1844       for (ObjectMonitor* monitor : delete_list) {
1845         assert(!LightweightSynchronizer::contains_monitor(current, monitor), "Should have been removed");
1846       }
1847     }
1848 #endif
1849 
1850     log.before_handshake(unlinked_count);
1851 
1852     // A JavaThread needs to handshake in order to safely free the
1853     // ObjectMonitors that were deflated in this cycle.
1854     DeflationHandshakeClosure dhc;
1855     Handshake::execute(&dhc);
1856     // Also, we sync and desync GC threads around the handshake, so that they can
1857     // safely read the mark-word and look-through to the object-monitor, without
1858     // being afraid that the object-monitor is going away.
1859     VM_RendezvousGCThreads sync_gc;
1860     VMThread::execute(&sync_gc);
1861 
1862     log.after_handshake();
1863 
1864     // After the handshake, safely free the ObjectMonitors that were
1865     // deflated and unlinked in this cycle.
1866 
1867     // Delete the unlinked ObjectMonitors.
1868     deleted_count = delete_monitors(&delete_list, &safepointer);
1869     assert(unlinked_count == deleted_count, "must be");
1870   }
1871 
1872   log.end(deflated_count, unlinked_count);
1873 
1874   GVars.stw_random = os::random();
1875 
1876   if (deflated_count != 0) {
1877     _no_progress_cnt = 0;
1878   } else if (_no_progress_skip_increment) {
1879     _no_progress_skip_increment = false;
1880   } else {
1881     _no_progress_cnt++;
1882   }
1883 
1884   return deflated_count;
1885 }
1886 
1887 // Monitor cleanup on JavaThread::exit
1888 
1889 // Iterate through monitor cache and attempt to release thread's monitors
1890 class ReleaseJavaMonitorsClosure: public MonitorClosure {
1891  private:
1892   JavaThread* _thread;
1893 
1894  public:
1895   ReleaseJavaMonitorsClosure(JavaThread* thread) : _thread(thread) {}
1896   void do_monitor(ObjectMonitor* mid) {
1897     intx rec = mid->complete_exit(_thread);
1898     _thread->dec_held_monitor_count(rec + 1);
1899   }
1900 };
1901 
1902 // Release all inflated monitors owned by current thread.  Lightweight monitors are
1903 // ignored.  This is meant to be called during JNI thread detach which assumes
1904 // all remaining monitors are heavyweight.  All exceptions are swallowed.
1905 // Scanning the extant monitor list can be time consuming.
1906 // A simple optimization is to add a per-thread flag that indicates a thread
1907 // called jni_monitorenter() during its lifetime.
1908 //
1909 // Instead of NoSafepointVerifier it might be cheaper to
1910 // use an idiom of the form:
1911 //   auto int tmp = SafepointSynchronize::_safepoint_counter ;
1912 //   <code that must not run at safepoint>
1913 //   guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
1914 // Since the tests are extremely cheap we could leave them enabled
1915 // for normal product builds.
1916 
1917 void ObjectSynchronizer::release_monitors_owned_by_thread(JavaThread* current) {
1918   assert(current == JavaThread::current(), "must be current Java thread");
1919   NoSafepointVerifier nsv;
1920   ReleaseJavaMonitorsClosure rjmc(current);
1921   ObjectSynchronizer::owned_monitors_iterate(&rjmc, current);
1922   assert(!current->has_pending_exception(), "Should not be possible");
1923   current->clear_pending_exception();
1924   assert(current->held_monitor_count() == 0, "Should not be possible");
1925   // All monitors (including entered via JNI) have been unlocked above, so we need to clear jni count.
1926   current->clear_jni_monitor_count();
1927 }
1928 
1929 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
1930   switch (cause) {
1931     case inflate_cause_vm_internal:    return "VM Internal";
1932     case inflate_cause_monitor_enter:  return "Monitor Enter";
1933     case inflate_cause_wait:           return "Monitor Wait";
1934     case inflate_cause_notify:         return "Monitor Notify";
1935     case inflate_cause_hash_code:      return "Monitor Hash Code";
1936     case inflate_cause_jni_enter:      return "JNI Monitor Enter";
1937     case inflate_cause_jni_exit:       return "JNI Monitor Exit";
1938     default:
1939       ShouldNotReachHere();
1940   }
1941   return "Unknown";
1942 }
1943 
1944 //------------------------------------------------------------------------------
1945 // Debugging code
1946 
1947 u_char* ObjectSynchronizer::get_gvars_addr() {
1948   return (u_char*)&GVars;
1949 }
1950 
1951 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
1952   return (u_char*)&GVars.hc_sequence;
1953 }
1954 
1955 size_t ObjectSynchronizer::get_gvars_size() {
1956   return sizeof(SharedGlobals);
1957 }
1958 
1959 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
1960   return (u_char*)&GVars.stw_random;
1961 }
1962 
1963 // Do the final audit and print of ObjectMonitor stats; must be done
1964 // by the VMThread at VM exit time.
1965 void ObjectSynchronizer::do_final_audit_and_print_stats() {
1966   assert(Thread::current()->is_VM_thread(), "sanity check");
1967 
1968   if (is_final_audit()) {  // Only do the audit once.
1969     return;
1970   }
1971   set_is_final_audit();
1972   log_info(monitorinflation)("Starting the final audit.");
1973 
1974   if (log_is_enabled(Info, monitorinflation)) {
1975     LogStreamHandle(Info, monitorinflation) ls;
1976     audit_and_print_stats(&ls, true /* on_exit */);
1977   }
1978 }
1979 
1980 // This function can be called by the MonitorDeflationThread or it can be called when
1981 // we are trying to exit the VM. The list walker functions can run in parallel with
1982 // the other list operations.
1983 // Calls to this function can be added in various places as a debugging
1984 // aid.
1985 //
1986 void ObjectSynchronizer::audit_and_print_stats(outputStream* ls, bool on_exit) {
1987   int error_cnt = 0;
1988 
1989   ls->print_cr("Checking in_use_list:");
1990   chk_in_use_list(ls, &error_cnt);
1991 
1992   if (error_cnt == 0) {
1993     ls->print_cr("No errors found in in_use_list checks.");
1994   } else {
1995     log_error(monitorinflation)("found in_use_list errors: error_cnt=%d", error_cnt);
1996   }
1997 
1998   // When exiting, only log the interesting entries at the Info level.
1999   // When called at intervals by the MonitorDeflationThread, log output
2000   // at the Trace level since there can be a lot of it.
2001   if (!on_exit && log_is_enabled(Trace, monitorinflation)) {
2002     LogStreamHandle(Trace, monitorinflation) ls_tr;
2003     log_in_use_monitor_details(&ls_tr, true /* log_all */);
2004   } else if (on_exit) {
2005     log_in_use_monitor_details(ls, false /* log_all */);
2006   }
2007 
2008   ls->flush();
2009 
2010   guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
2011 }
2012 
2013 // Check the in_use_list; log the results of the checks.
2014 void ObjectSynchronizer::chk_in_use_list(outputStream* out, int *error_cnt_p) {
2015   size_t l_in_use_count = _in_use_list.count();
2016   size_t l_in_use_max = _in_use_list.max();
2017   out->print_cr("count=%zu, max=%zu", l_in_use_count,
2018                 l_in_use_max);
2019 
2020   size_t ck_in_use_count = 0;
2021   MonitorList::Iterator iter = _in_use_list.iterator();
2022   while (iter.has_next()) {
2023     ObjectMonitor* mid = iter.next();
2024     chk_in_use_entry(mid, out, error_cnt_p);
2025     ck_in_use_count++;
2026   }
2027 
2028   if (l_in_use_count == ck_in_use_count) {
2029     out->print_cr("in_use_count=%zu equals ck_in_use_count=%zu",
2030                   l_in_use_count, ck_in_use_count);
2031   } else {
2032     out->print_cr("WARNING: in_use_count=%zu is not equal to "
2033                   "ck_in_use_count=%zu", l_in_use_count,
2034                   ck_in_use_count);
2035   }
2036 
2037   size_t ck_in_use_max = _in_use_list.max();
2038   if (l_in_use_max == ck_in_use_max) {
2039     out->print_cr("in_use_max=%zu equals ck_in_use_max=%zu",
2040                   l_in_use_max, ck_in_use_max);
2041   } else {
2042     out->print_cr("WARNING: in_use_max=%zu is not equal to "
2043                   "ck_in_use_max=%zu", l_in_use_max, ck_in_use_max);
2044   }
2045 }
2046 
2047 // Check an in-use monitor entry; log any errors.
2048 void ObjectSynchronizer::chk_in_use_entry(ObjectMonitor* n, outputStream* out,
2049                                           int* error_cnt_p) {
2050   if (n->owner_is_DEFLATER_MARKER()) {
2051     // This could happen when monitor deflation blocks for a safepoint.
2052     return;
2053   }
2054 
2055 
2056   if (n->metadata() == 0) {
2057     out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use monitor must "
2058                   "have non-null _metadata (header/hash) field.", p2i(n));
2059     *error_cnt_p = *error_cnt_p + 1;
2060   }
2061 
2062   const oop obj = n->object_peek();
2063   if (obj == nullptr) {
2064     return;
2065   }
2066 
2067   const markWord mark = obj->mark();
2068   if (!mark.has_monitor()) {
2069     out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use monitor's "
2070                   "object does not think it has a monitor: obj="
2071                   INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
2072                   p2i(obj), mark.value());
2073     *error_cnt_p = *error_cnt_p + 1;
2074     return;
2075   }
2076 
2077   ObjectMonitor* const obj_mon = read_monitor(Thread::current(), obj, mark);
2078   if (n != obj_mon) {
2079     out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use monitor's "
2080                   "object does not refer to the same monitor: obj="
2081                   INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
2082                   INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2083     *error_cnt_p = *error_cnt_p + 1;
2084   }
2085 }
2086 
2087 // Log details about ObjectMonitors on the in_use_list. The 'BHL'
2088 // flags indicate why the entry is in-use, 'object' and 'object type'
2089 // indicate the associated object and its type.
2090 void ObjectSynchronizer::log_in_use_monitor_details(outputStream* out, bool log_all) {
2091   if (_in_use_list.count() > 0) {
2092     stringStream ss;
2093     out->print_cr("In-use monitor info%s:", log_all ? "" : " (eliding idle monitors)");
2094     out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2095     out->print_cr("%18s  %s  %18s  %18s",
2096                   "monitor", "BHL", "object", "object type");
2097     out->print_cr("==================  ===  ==================  ==================");
2098 
2099     auto is_interesting = [&](ObjectMonitor* monitor) {
2100       return log_all || monitor->has_owner() || monitor->is_busy();
2101     };
2102 
2103     monitors_iterate([&](ObjectMonitor* monitor) {
2104       if (is_interesting(monitor)) {
2105         const oop obj = monitor->object_peek();
2106         const intptr_t hash = UseObjectMonitorTable ? monitor->hash() : monitor->header().hash();
2107         ResourceMark rm;
2108         out->print(INTPTR_FORMAT "  %d%d%d  " INTPTR_FORMAT "  %s", p2i(monitor),
2109                    monitor->is_busy(), hash != 0, monitor->has_owner(),
2110                    p2i(obj), obj == nullptr ? "" : obj->klass()->external_name());
2111         if (monitor->is_busy()) {
2112           out->print(" (%s)", monitor->is_busy_to_string(&ss));
2113           ss.reset();
2114         }
2115         out->cr();
2116       }
2117     });
2118   }
2119 
2120   out->flush();
2121 }