1 /* 2 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "classfile/vmSymbols.hpp" 26 #include "jfrfiles/jfrEventClasses.hpp" 27 #include "logging/log.hpp" 28 #include "memory/allStatic.hpp" 29 #include "memory/resourceArea.hpp" 30 #include "nmt/memTag.hpp" 31 #include "oops/oop.inline.hpp" 32 #include "runtime/atomic.hpp" 33 #include "runtime/basicLock.inline.hpp" 34 #include "runtime/globals_extension.hpp" 35 #include "runtime/interfaceSupport.inline.hpp" 36 #include "runtime/javaThread.inline.hpp" 37 #include "runtime/lightweightSynchronizer.hpp" 38 #include "runtime/lockStack.inline.hpp" 39 #include "runtime/mutexLocker.hpp" 40 #include "runtime/objectMonitor.inline.hpp" 41 #include "runtime/os.hpp" 42 #include "runtime/safepointMechanism.inline.hpp" 43 #include "runtime/safepointVerifiers.hpp" 44 #include "runtime/synchronizer.inline.hpp" 45 #include "runtime/timerTrace.hpp" 46 #include "runtime/trimNativeHeap.hpp" 47 #include "utilities/concurrentHashTable.inline.hpp" 48 #include "utilities/concurrentHashTableTasks.inline.hpp" 49 #include "utilities/globalDefinitions.hpp" 50 51 // ConcurrentHashTable storing links from objects to ObjectMonitors 52 class ObjectMonitorTable : AllStatic { 53 struct Config { 54 using Value = ObjectMonitor*; 55 static uintx get_hash(Value const& value, bool* is_dead) { 56 return (uintx)value->hash(); 57 } 58 static void* allocate_node(void* context, size_t size, Value const& value) { 59 ObjectMonitorTable::inc_items_count(); 60 return AllocateHeap(size, mtObjectMonitor); 61 }; 62 static void free_node(void* context, void* memory, Value const& value) { 63 ObjectMonitorTable::dec_items_count(); 64 FreeHeap(memory); 65 } 66 }; 67 using ConcurrentTable = ConcurrentHashTable<Config, mtObjectMonitor>; 68 69 static ConcurrentTable* _table; 70 static volatile size_t _items_count; 71 static size_t _table_size; 72 static volatile bool _resize; 73 74 class Lookup : public StackObj { 75 oop _obj; 76 77 public: 78 explicit Lookup(oop obj) : _obj(obj) {} 79 80 uintx get_hash() const { 81 uintx hash = _obj->mark().hash(); 82 assert(hash != 0, "should have a hash"); 83 return hash; 84 } 85 86 bool equals(ObjectMonitor** value) { 87 assert(*value != nullptr, "must be"); 88 return (*value)->object_refers_to(_obj); 89 } 90 91 bool is_dead(ObjectMonitor** value) { 92 assert(*value != nullptr, "must be"); 93 return false; 94 } 95 }; 96 97 class LookupMonitor : public StackObj { 98 ObjectMonitor* _monitor; 99 100 public: 101 explicit LookupMonitor(ObjectMonitor* monitor) : _monitor(monitor) {} 102 103 uintx get_hash() const { 104 return _monitor->hash(); 105 } 106 107 bool equals(ObjectMonitor** value) { 108 return (*value) == _monitor; 109 } 110 111 bool is_dead(ObjectMonitor** value) { 112 assert(*value != nullptr, "must be"); 113 return (*value)->object_is_dead(); 114 } 115 }; 116 117 static void inc_items_count() { 118 Atomic::inc(&_items_count, memory_order_relaxed); 119 } 120 121 static void dec_items_count() { 122 Atomic::dec(&_items_count, memory_order_relaxed); 123 } 124 125 static double get_load_factor() { 126 size_t count = Atomic::load(&_items_count); 127 return (double)count / (double)_table_size; 128 } 129 130 static size_t table_size(Thread* current = Thread::current()) { 131 return ((size_t)1) << _table->get_size_log2(current); 132 } 133 134 static size_t max_log_size() { 135 // TODO[OMTable]: Evaluate the max size. 136 // TODO[OMTable]: Need to fix init order to use Universe::heap()->max_capacity(); 137 // Using MaxHeapSize directly this early may be wrong, and there 138 // are definitely rounding errors (alignment). 139 const size_t max_capacity = MaxHeapSize; 140 const size_t min_object_size = CollectedHeap::min_dummy_object_size() * HeapWordSize; 141 const size_t max_objects = max_capacity / MAX2(MinObjAlignmentInBytes, checked_cast<int>(min_object_size)); 142 const size_t log_max_objects = log2i_graceful(max_objects); 143 144 return MAX2(MIN2<size_t>(SIZE_BIG_LOG2, log_max_objects), min_log_size()); 145 } 146 147 static size_t min_log_size() { 148 // ~= log(AvgMonitorsPerThreadEstimate default) 149 return 10; 150 } 151 152 template<typename V> 153 static size_t clamp_log_size(V log_size) { 154 return MAX2(MIN2(log_size, checked_cast<V>(max_log_size())), checked_cast<V>(min_log_size())); 155 } 156 157 static size_t initial_log_size() { 158 const size_t estimate = log2i(MAX2(os::processor_count(), 1)) + log2i(MAX2(AvgMonitorsPerThreadEstimate, size_t(1))); 159 return clamp_log_size(estimate); 160 } 161 162 static size_t grow_hint () { 163 return ConcurrentTable::DEFAULT_GROW_HINT; 164 } 165 166 public: 167 static void create() { 168 _table = new ConcurrentTable(initial_log_size(), max_log_size(), grow_hint()); 169 _items_count = 0; 170 _table_size = table_size(); 171 _resize = false; 172 } 173 174 static void verify_monitor_get_result(oop obj, ObjectMonitor* monitor) { 175 #ifdef ASSERT 176 if (SafepointSynchronize::is_at_safepoint()) { 177 bool has_monitor = obj->mark().has_monitor(); 178 assert(has_monitor == (monitor != nullptr), 179 "Inconsistency between markWord and ObjectMonitorTable has_monitor: %s monitor: " PTR_FORMAT, 180 BOOL_TO_STR(has_monitor), p2i(monitor)); 181 } 182 #endif 183 } 184 185 static ObjectMonitor* monitor_get(Thread* current, oop obj) { 186 ObjectMonitor* result = nullptr; 187 Lookup lookup_f(obj); 188 auto found_f = [&](ObjectMonitor** found) { 189 assert((*found)->object_peek() == obj, "must be"); 190 result = *found; 191 }; 192 _table->get(current, lookup_f, found_f); 193 verify_monitor_get_result(obj, result); 194 return result; 195 } 196 197 static void try_notify_grow() { 198 if (!_table->is_max_size_reached() && !Atomic::load(&_resize)) { 199 Atomic::store(&_resize, true); 200 if (Service_lock->try_lock()) { 201 Service_lock->notify(); 202 Service_lock->unlock(); 203 } 204 } 205 } 206 207 static bool should_shrink() { 208 // Not implemented; 209 return false; 210 } 211 212 static constexpr double GROW_LOAD_FACTOR = 0.75; 213 214 static bool should_grow() { 215 return get_load_factor() > GROW_LOAD_FACTOR && !_table->is_max_size_reached(); 216 } 217 218 static bool should_resize() { 219 return should_grow() || should_shrink() || Atomic::load(&_resize); 220 } 221 222 template<typename Task, typename... Args> 223 static bool run_task(JavaThread* current, Task& task, const char* task_name, Args&... args) { 224 if (task.prepare(current)) { 225 log_trace(monitortable)("Started to %s", task_name); 226 TraceTime timer(task_name, TRACETIME_LOG(Debug, monitortable, perf)); 227 while (task.do_task(current, args...)) { 228 task.pause(current); 229 { 230 ThreadBlockInVM tbivm(current); 231 } 232 task.cont(current); 233 } 234 task.done(current); 235 return true; 236 } 237 return false; 238 } 239 240 static bool grow(JavaThread* current) { 241 ConcurrentTable::GrowTask grow_task(_table); 242 if (run_task(current, grow_task, "Grow")) { 243 _table_size = table_size(current); 244 log_info(monitortable)("Grown to size: %zu", _table_size); 245 return true; 246 } 247 return false; 248 } 249 250 static bool clean(JavaThread* current) { 251 ConcurrentTable::BulkDeleteTask clean_task(_table); 252 auto is_dead = [&](ObjectMonitor** monitor) { 253 return (*monitor)->object_is_dead(); 254 }; 255 auto do_nothing = [&](ObjectMonitor** monitor) {}; 256 NativeHeapTrimmer::SuspendMark sm("ObjectMonitorTable"); 257 return run_task(current, clean_task, "Clean", is_dead, do_nothing); 258 } 259 260 static bool resize(JavaThread* current) { 261 LogTarget(Info, monitortable) lt; 262 bool success = false; 263 264 if (should_grow()) { 265 lt.print("Start growing with load factor %f", get_load_factor()); 266 success = grow(current); 267 } else { 268 if (!_table->is_max_size_reached() && Atomic::load(&_resize)) { 269 lt.print("WARNING: Getting resize hints with load factor %f", get_load_factor()); 270 } 271 lt.print("Start cleaning with load factor %f", get_load_factor()); 272 success = clean(current); 273 } 274 275 Atomic::store(&_resize, false); 276 277 return success; 278 } 279 280 static ObjectMonitor* monitor_put_get(Thread* current, ObjectMonitor* monitor, oop obj) { 281 // Enter the monitor into the concurrent hashtable. 282 ObjectMonitor* result = monitor; 283 Lookup lookup_f(obj); 284 auto found_f = [&](ObjectMonitor** found) { 285 assert((*found)->object_peek() == obj, "must be"); 286 result = *found; 287 }; 288 bool grow; 289 _table->insert_get(current, lookup_f, monitor, found_f, &grow); 290 verify_monitor_get_result(obj, result); 291 if (grow) { 292 try_notify_grow(); 293 } 294 return result; 295 } 296 297 static bool remove_monitor_entry(Thread* current, ObjectMonitor* monitor) { 298 LookupMonitor lookup_f(monitor); 299 return _table->remove(current, lookup_f); 300 } 301 302 static bool contains_monitor(Thread* current, ObjectMonitor* monitor) { 303 LookupMonitor lookup_f(monitor); 304 bool result = false; 305 auto found_f = [&](ObjectMonitor** found) { 306 result = true; 307 }; 308 _table->get(current, lookup_f, found_f); 309 return result; 310 } 311 312 static void print_on(outputStream* st) { 313 auto printer = [&] (ObjectMonitor** entry) { 314 ObjectMonitor* om = *entry; 315 oop obj = om->object_peek(); 316 st->print("monitor=" PTR_FORMAT ", ", p2i(om)); 317 st->print("object=" PTR_FORMAT, p2i(obj)); 318 assert(obj->mark().hash() == om->hash(), "hash must match"); 319 st->cr(); 320 return true; 321 }; 322 if (SafepointSynchronize::is_at_safepoint()) { 323 _table->do_safepoint_scan(printer); 324 } else { 325 _table->do_scan(Thread::current(), printer); 326 } 327 } 328 }; 329 330 ObjectMonitorTable::ConcurrentTable* ObjectMonitorTable::_table = nullptr; 331 volatile size_t ObjectMonitorTable::_items_count = 0; 332 size_t ObjectMonitorTable::_table_size = 0; 333 volatile bool ObjectMonitorTable::_resize = false; 334 335 ObjectMonitor* LightweightSynchronizer::get_or_insert_monitor_from_table(oop object, JavaThread* current, bool* inserted) { 336 assert(LockingMode == LM_LIGHTWEIGHT, "must be"); 337 338 ObjectMonitor* monitor = get_monitor_from_table(current, object); 339 if (monitor != nullptr) { 340 *inserted = false; 341 return monitor; 342 } 343 344 ObjectMonitor* alloced_monitor = new ObjectMonitor(object); 345 alloced_monitor->set_anonymous_owner(); 346 347 // Try insert monitor 348 monitor = add_monitor(current, alloced_monitor, object); 349 350 *inserted = alloced_monitor == monitor; 351 if (!*inserted) { 352 delete alloced_monitor; 353 } 354 355 return monitor; 356 } 357 358 static void log_inflate(Thread* current, oop object, ObjectSynchronizer::InflateCause cause) { 359 if (log_is_enabled(Trace, monitorinflation)) { 360 ResourceMark rm(current); 361 log_trace(monitorinflation)("inflate: object=" INTPTR_FORMAT ", mark=" 362 INTPTR_FORMAT ", type='%s' cause=%s", p2i(object), 363 object->mark().value(), object->klass()->external_name(), 364 ObjectSynchronizer::inflate_cause_name(cause)); 365 } 366 } 367 368 static void post_monitor_inflate_event(EventJavaMonitorInflate* event, 369 const oop obj, 370 ObjectSynchronizer::InflateCause cause) { 371 assert(event != nullptr, "invariant"); 372 const Klass* monitor_klass = obj->klass(); 373 if (ObjectMonitor::is_jfr_excluded(monitor_klass)) { 374 return; 375 } 376 event->set_monitorClass(monitor_klass); 377 event->set_address((uintptr_t)(void*)obj); 378 event->set_cause((u1)cause); 379 event->commit(); 380 } 381 382 ObjectMonitor* LightweightSynchronizer::get_or_insert_monitor(oop object, JavaThread* current, ObjectSynchronizer::InflateCause cause) { 383 assert(UseObjectMonitorTable, "must be"); 384 385 EventJavaMonitorInflate event; 386 387 bool inserted; 388 ObjectMonitor* monitor = get_or_insert_monitor_from_table(object, current, &inserted); 389 390 if (inserted) { 391 log_inflate(current, object, cause); 392 if (event.should_commit()) { 393 post_monitor_inflate_event(&event, object, cause); 394 } 395 396 // The monitor has an anonymous owner so it is safe from async deflation. 397 ObjectSynchronizer::_in_use_list.add(monitor); 398 } 399 400 return monitor; 401 } 402 403 // Add the hashcode to the monitor to match the object and put it in the hashtable. 404 ObjectMonitor* LightweightSynchronizer::add_monitor(JavaThread* current, ObjectMonitor* monitor, oop obj) { 405 assert(UseObjectMonitorTable, "must be"); 406 assert(obj == monitor->object(), "must be"); 407 408 intptr_t hash = obj->mark().hash(); 409 assert(hash != 0, "must be set when claiming the object monitor"); 410 monitor->set_hash(hash); 411 412 return ObjectMonitorTable::monitor_put_get(current, monitor, obj); 413 } 414 415 bool LightweightSynchronizer::remove_monitor(Thread* current, ObjectMonitor* monitor, oop obj) { 416 assert(UseObjectMonitorTable, "must be"); 417 assert(monitor->object_peek() == obj, "must be, cleared objects are removed by is_dead"); 418 419 return ObjectMonitorTable::remove_monitor_entry(current, monitor); 420 } 421 422 void LightweightSynchronizer::deflate_mark_word(oop obj) { 423 assert(UseObjectMonitorTable, "must be"); 424 425 markWord mark = obj->mark_acquire(); 426 assert(!mark.has_no_hash(), "obj with inflated monitor must have had a hash"); 427 428 while (mark.has_monitor()) { 429 const markWord new_mark = mark.clear_lock_bits().set_unlocked(); 430 mark = obj->cas_set_mark(new_mark, mark); 431 } 432 } 433 434 void LightweightSynchronizer::initialize() { 435 if (!UseObjectMonitorTable) { 436 return; 437 } 438 ObjectMonitorTable::create(); 439 } 440 441 bool LightweightSynchronizer::needs_resize() { 442 if (!UseObjectMonitorTable) { 443 return false; 444 } 445 return ObjectMonitorTable::should_resize(); 446 } 447 448 bool LightweightSynchronizer::resize_table(JavaThread* current) { 449 if (!UseObjectMonitorTable) { 450 return true; 451 } 452 return ObjectMonitorTable::resize(current); 453 } 454 455 class LightweightSynchronizer::LockStackInflateContendedLocks : private OopClosure { 456 private: 457 oop _contended_oops[LockStack::CAPACITY]; 458 int _length; 459 460 void do_oop(oop* o) final { 461 oop obj = *o; 462 if (obj->mark_acquire().has_monitor()) { 463 if (_length > 0 && _contended_oops[_length - 1] == obj) { 464 // Recursive 465 return; 466 } 467 _contended_oops[_length++] = obj; 468 } 469 } 470 471 void do_oop(narrowOop* o) final { 472 ShouldNotReachHere(); 473 } 474 475 public: 476 LockStackInflateContendedLocks() : 477 _contended_oops(), 478 _length(0) {}; 479 480 void inflate(JavaThread* current) { 481 assert(current == JavaThread::current(), "must be"); 482 current->lock_stack().oops_do(this); 483 for (int i = 0; i < _length; i++) { 484 LightweightSynchronizer:: 485 inflate_fast_locked_object(_contended_oops[i], ObjectSynchronizer::inflate_cause_vm_internal, current, current); 486 } 487 } 488 }; 489 490 void LightweightSynchronizer::ensure_lock_stack_space(JavaThread* current) { 491 assert(current == JavaThread::current(), "must be"); 492 LockStack& lock_stack = current->lock_stack(); 493 494 // Make room on lock_stack 495 if (lock_stack.is_full()) { 496 // Inflate contended objects 497 LockStackInflateContendedLocks().inflate(current); 498 if (lock_stack.is_full()) { 499 // Inflate the oldest object 500 inflate_fast_locked_object(lock_stack.bottom(), ObjectSynchronizer::inflate_cause_vm_internal, current, current); 501 } 502 } 503 } 504 505 class LightweightSynchronizer::CacheSetter : StackObj { 506 JavaThread* const _thread; 507 BasicLock* const _lock; 508 ObjectMonitor* _monitor; 509 510 NONCOPYABLE(CacheSetter); 511 512 public: 513 CacheSetter(JavaThread* thread, BasicLock* lock) : 514 _thread(thread), 515 _lock(lock), 516 _monitor(nullptr) {} 517 518 ~CacheSetter() { 519 // Only use the cache if using the table. 520 if (UseObjectMonitorTable) { 521 if (_monitor != nullptr) { 522 // If the monitor is already in the BasicLock cache then it is most 523 // likely in the thread cache, do not set it again to avoid reordering. 524 if (_monitor != _lock->object_monitor_cache()) { 525 _thread->om_set_monitor_cache(_monitor); 526 _lock->set_object_monitor_cache(_monitor); 527 } 528 } else { 529 _lock->clear_object_monitor_cache(); 530 } 531 } 532 } 533 534 void set_monitor(ObjectMonitor* monitor) { 535 assert(_monitor == nullptr, "only set once"); 536 _monitor = monitor; 537 } 538 539 }; 540 541 // Reads first from the BasicLock cache then from the OMCache in the current thread. 542 // C2 fast-path may have put the monitor in the cache in the BasicLock. 543 inline static ObjectMonitor* read_caches(JavaThread* current, BasicLock* lock, oop object) { 544 ObjectMonitor* monitor = lock->object_monitor_cache(); 545 if (monitor == nullptr) { 546 monitor = current->om_get_from_monitor_cache(object); 547 } 548 return monitor; 549 } 550 551 class LightweightSynchronizer::VerifyThreadState { 552 bool _no_safepoint; 553 554 public: 555 VerifyThreadState(JavaThread* locking_thread, JavaThread* current) : _no_safepoint(locking_thread != current) { 556 assert(current == Thread::current(), "must be"); 557 assert(locking_thread == current || locking_thread->is_obj_deopt_suspend(), "locking_thread may not run concurrently"); 558 if (_no_safepoint) { 559 DEBUG_ONLY(JavaThread::current()->inc_no_safepoint_count();) 560 } 561 } 562 ~VerifyThreadState() { 563 if (_no_safepoint){ 564 DEBUG_ONLY(JavaThread::current()->dec_no_safepoint_count();) 565 } 566 } 567 }; 568 569 inline bool LightweightSynchronizer::fast_lock_try_enter(oop obj, LockStack& lock_stack, JavaThread* current) { 570 markWord mark = obj->mark(); 571 while (mark.is_unlocked()) { 572 ensure_lock_stack_space(current); 573 assert(!lock_stack.is_full(), "must have made room on the lock stack"); 574 assert(!lock_stack.contains(obj), "thread must not already hold the lock"); 575 // Try to swing into 'fast-locked' state. 576 markWord locked_mark = mark.set_fast_locked(); 577 markWord old_mark = mark; 578 mark = obj->cas_set_mark(locked_mark, old_mark); 579 if (old_mark == mark) { 580 // Successfully fast-locked, push object to lock-stack and return. 581 lock_stack.push(obj); 582 return true; 583 } 584 } 585 return false; 586 } 587 588 bool LightweightSynchronizer::fast_lock_spin_enter(oop obj, LockStack& lock_stack, JavaThread* current, bool observed_deflation) { 589 assert(UseObjectMonitorTable, "must be"); 590 // Will spin with exponential backoff with an accumulative O(2^spin_limit) spins. 591 const int log_spin_limit = os::is_MP() ? LightweightFastLockingSpins : 1; 592 const int log_min_safepoint_check_interval = 10; 593 594 markWord mark = obj->mark(); 595 const auto should_spin = [&]() { 596 if (!mark.has_monitor()) { 597 // Spin while not inflated. 598 return true; 599 } else if (observed_deflation) { 600 // Spin while monitor is being deflated. 601 ObjectMonitor* monitor = ObjectSynchronizer::read_monitor(current, obj, mark); 602 return monitor == nullptr || monitor->is_being_async_deflated(); 603 } 604 // Else stop spinning. 605 return false; 606 }; 607 // Always attempt to lock once even when safepoint synchronizing. 608 bool should_process = false; 609 for (int i = 0; should_spin() && !should_process && i < log_spin_limit; i++) { 610 // Spin with exponential backoff. 611 const int total_spin_count = 1 << i; 612 const int inner_spin_count = MIN2(1 << log_min_safepoint_check_interval, total_spin_count); 613 const int outer_spin_count = total_spin_count / inner_spin_count; 614 for (int outer = 0; outer < outer_spin_count; outer++) { 615 should_process = SafepointMechanism::should_process(current); 616 if (should_process) { 617 // Stop spinning for safepoint. 618 break; 619 } 620 for (int inner = 1; inner < inner_spin_count; inner++) { 621 SpinPause(); 622 } 623 } 624 625 if (fast_lock_try_enter(obj, lock_stack, current)) return true; 626 } 627 return false; 628 } 629 630 void LightweightSynchronizer::enter_for(Handle obj, BasicLock* lock, JavaThread* locking_thread) { 631 assert(LockingMode == LM_LIGHTWEIGHT, "must be"); 632 assert(!UseObjectMonitorTable || lock->object_monitor_cache() == nullptr, "must be cleared"); 633 JavaThread* current = JavaThread::current(); 634 VerifyThreadState vts(locking_thread, current); 635 636 if (obj->klass()->is_value_based()) { 637 ObjectSynchronizer::handle_sync_on_value_based_class(obj, locking_thread); 638 } 639 640 LockStack& lock_stack = locking_thread->lock_stack(); 641 642 ObjectMonitor* monitor = nullptr; 643 if (lock_stack.contains(obj())) { 644 monitor = inflate_fast_locked_object(obj(), ObjectSynchronizer::inflate_cause_monitor_enter, locking_thread, current); 645 bool entered = monitor->enter_for(locking_thread); 646 assert(entered, "recursive ObjectMonitor::enter_for must succeed"); 647 } else { 648 do { 649 // It is assumed that enter_for must enter on an object without contention. 650 monitor = inflate_and_enter(obj(), lock, ObjectSynchronizer::inflate_cause_monitor_enter, locking_thread, current); 651 // But there may still be a race with deflation. 652 } while (monitor == nullptr); 653 } 654 655 assert(monitor != nullptr, "LightweightSynchronizer::enter_for must succeed"); 656 assert(!UseObjectMonitorTable || lock->object_monitor_cache() == nullptr, "unused. already cleared"); 657 } 658 659 void LightweightSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current) { 660 assert(LockingMode == LM_LIGHTWEIGHT, "must be"); 661 assert(current == JavaThread::current(), "must be"); 662 663 if (obj->klass()->is_value_based()) { 664 ObjectSynchronizer::handle_sync_on_value_based_class(obj, current); 665 } 666 667 CacheSetter cache_setter(current, lock); 668 669 // Used when deflation is observed. Progress here requires progress 670 // from the deflator. After observing that the deflator is not 671 // making progress (after two yields), switch to sleeping. 672 SpinYield spin_yield(0, 2); 673 bool observed_deflation = false; 674 675 LockStack& lock_stack = current->lock_stack(); 676 677 if (!lock_stack.is_full() && lock_stack.try_recursive_enter(obj())) { 678 // Recursively fast locked 679 return; 680 } 681 682 if (lock_stack.contains(obj())) { 683 ObjectMonitor* monitor = inflate_fast_locked_object(obj(), ObjectSynchronizer::inflate_cause_monitor_enter, current, current); 684 bool entered = monitor->enter(current); 685 assert(entered, "recursive ObjectMonitor::enter must succeed"); 686 cache_setter.set_monitor(monitor); 687 return; 688 } 689 690 while (true) { 691 // Fast-locking does not use the 'lock' argument. 692 // Fast-lock spinning to avoid inflating for short critical sections. 693 // The goal is to only inflate when the extra cost of using ObjectMonitors 694 // is worth it. 695 // If deflation has been observed we also spin while deflation is ongoing. 696 if (fast_lock_try_enter(obj(), lock_stack, current)) { 697 return; 698 } else if (UseObjectMonitorTable && fast_lock_spin_enter(obj(), lock_stack, current, observed_deflation)) { 699 return; 700 } 701 702 if (observed_deflation) { 703 spin_yield.wait(); 704 } 705 706 ObjectMonitor* monitor = inflate_and_enter(obj(), lock, ObjectSynchronizer::inflate_cause_monitor_enter, current, current); 707 if (monitor != nullptr) { 708 cache_setter.set_monitor(monitor); 709 return; 710 } 711 712 // If inflate_and_enter returns nullptr it is because a deflated monitor 713 // was encountered. Fallback to fast locking. The deflater is responsible 714 // for clearing out the monitor and transitioning the markWord back to 715 // fast locking. 716 observed_deflation = true; 717 } 718 } 719 720 void LightweightSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current) { 721 assert(LockingMode == LM_LIGHTWEIGHT, "must be"); 722 assert(current == Thread::current(), "must be"); 723 724 markWord mark = object->mark(); 725 assert(!mark.is_unlocked(), "must be"); 726 727 LockStack& lock_stack = current->lock_stack(); 728 if (mark.is_fast_locked()) { 729 if (lock_stack.try_recursive_exit(object)) { 730 // This is a recursive exit which succeeded 731 return; 732 } 733 if (lock_stack.is_recursive(object)) { 734 // Must inflate recursive locks if try_recursive_exit fails 735 // This happens for un-structured unlocks, could potentially 736 // fix try_recursive_exit to handle these. 737 inflate_fast_locked_object(object, ObjectSynchronizer::inflate_cause_vm_internal, current, current); 738 } 739 } 740 741 while (mark.is_fast_locked()) { 742 markWord unlocked_mark = mark.set_unlocked(); 743 markWord old_mark = mark; 744 mark = object->cas_set_mark(unlocked_mark, old_mark); 745 if (old_mark == mark) { 746 // CAS successful, remove from lock_stack 747 size_t recursion = lock_stack.remove(object) - 1; 748 assert(recursion == 0, "Should not have unlocked here"); 749 return; 750 } 751 } 752 753 assert(mark.has_monitor(), "must be"); 754 // The monitor exists 755 ObjectMonitor* monitor; 756 if (UseObjectMonitorTable) { 757 monitor = read_caches(current, lock, object); 758 if (monitor == nullptr) { 759 monitor = get_monitor_from_table(current, object); 760 } 761 } else { 762 monitor = ObjectSynchronizer::read_monitor(mark); 763 } 764 if (monitor->has_anonymous_owner()) { 765 assert(current->lock_stack().contains(object), "current must have object on its lock stack"); 766 monitor->set_owner_from_anonymous(current); 767 monitor->set_recursions(current->lock_stack().remove(object) - 1); 768 } 769 770 monitor->exit(current); 771 } 772 773 // LightweightSynchronizer::inflate_locked_or_imse is used to to get an inflated 774 // ObjectMonitor* with LM_LIGHTWEIGHT. It is used from contexts which require 775 // an inflated ObjectMonitor* for a monitor, and expects to throw a 776 // java.lang.IllegalMonitorStateException if it is not held by the current 777 // thread. Such as notify/wait and jni_exit. LM_LIGHTWEIGHT keeps it invariant 778 // that it only inflates if it is already locked by the current thread or the 779 // current thread is in the process of entering. To maintain this invariant we 780 // need to throw a java.lang.IllegalMonitorStateException before inflating if 781 // the current thread is not the owner. 782 // LightweightSynchronizer::inflate_locked_or_imse facilitates this. 783 ObjectMonitor* LightweightSynchronizer::inflate_locked_or_imse(oop obj, ObjectSynchronizer::InflateCause cause, TRAPS) { 784 assert(LockingMode == LM_LIGHTWEIGHT, "must be"); 785 JavaThread* current = THREAD; 786 787 for (;;) { 788 markWord mark = obj->mark_acquire(); 789 if (mark.is_unlocked()) { 790 // No lock, IMSE. 791 THROW_MSG_(vmSymbols::java_lang_IllegalMonitorStateException(), 792 "current thread is not owner", nullptr); 793 } 794 795 if (mark.is_fast_locked()) { 796 if (!current->lock_stack().contains(obj)) { 797 // Fast locked by other thread, IMSE. 798 THROW_MSG_(vmSymbols::java_lang_IllegalMonitorStateException(), 799 "current thread is not owner", nullptr); 800 } else { 801 // Current thread owns the lock, must inflate 802 return inflate_fast_locked_object(obj, cause, current, current); 803 } 804 } 805 806 assert(mark.has_monitor(), "must be"); 807 ObjectMonitor* monitor = ObjectSynchronizer::read_monitor(current, obj, mark); 808 if (monitor != nullptr) { 809 if (monitor->has_anonymous_owner()) { 810 LockStack& lock_stack = current->lock_stack(); 811 if (lock_stack.contains(obj)) { 812 // Current thread owns the lock but someone else inflated it. 813 // Fix owner and pop lock stack. 814 monitor->set_owner_from_anonymous(current); 815 monitor->set_recursions(lock_stack.remove(obj) - 1); 816 } else { 817 // Fast locked (and inflated) by other thread, or deflation in progress, IMSE. 818 THROW_MSG_(vmSymbols::java_lang_IllegalMonitorStateException(), 819 "current thread is not owner", nullptr); 820 } 821 } 822 return monitor; 823 } 824 } 825 } 826 827 ObjectMonitor* LightweightSynchronizer::inflate_into_object_header(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, Thread* current) { 828 829 // The JavaThread* locking_thread parameter is only used by LM_LIGHTWEIGHT and requires 830 // that the locking_thread == Thread::current() or is suspended throughout the call by 831 // some other mechanism. 832 // Even with LM_LIGHTWEIGHT the thread might be nullptr when called from a non 833 // JavaThread. (As may still be the case from FastHashCode). However it is only 834 // important for the correctness of the LM_LIGHTWEIGHT algorithm that the thread 835 // is set when called from ObjectSynchronizer::enter from the owning thread, 836 // ObjectSynchronizer::enter_for from any thread, or ObjectSynchronizer::exit. 837 EventJavaMonitorInflate event; 838 839 for (;;) { 840 const markWord mark = object->mark_acquire(); 841 842 // The mark can be in one of the following states: 843 // * inflated - Just return if using stack-locking. 844 // If using fast-locking and the ObjectMonitor owner 845 // is anonymous and the locking_thread owns the 846 // object lock, then we make the locking_thread 847 // the ObjectMonitor owner and remove the lock from 848 // the locking_thread's lock stack. 849 // * fast-locked - Coerce it to inflated from fast-locked. 850 // * unlocked - Aggressively inflate the object. 851 852 // CASE: inflated 853 if (mark.has_monitor()) { 854 ObjectMonitor* inf = mark.monitor(); 855 markWord dmw = inf->header(); 856 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); 857 if (inf->has_anonymous_owner() && 858 locking_thread != nullptr && locking_thread->lock_stack().contains(object)) { 859 inf->set_owner_from_anonymous(locking_thread); 860 size_t removed = locking_thread->lock_stack().remove(object); 861 inf->set_recursions(removed - 1); 862 } 863 return inf; 864 } 865 866 // CASE: fast-locked 867 // Could be fast-locked either by the locking_thread or by some other thread. 868 // 869 // Note that we allocate the ObjectMonitor speculatively, _before_ 870 // attempting to set the object's mark to the new ObjectMonitor. If 871 // the locking_thread owns the monitor, then we set the ObjectMonitor's 872 // owner to the locking_thread. Otherwise, we set the ObjectMonitor's owner 873 // to anonymous. If we lose the race to set the object's mark to the 874 // new ObjectMonitor, then we just delete it and loop around again. 875 // 876 if (mark.is_fast_locked()) { 877 ObjectMonitor* monitor = new ObjectMonitor(object); 878 monitor->set_header(mark.set_unlocked()); 879 bool own = locking_thread != nullptr && locking_thread->lock_stack().contains(object); 880 if (own) { 881 // Owned by locking_thread. 882 monitor->set_owner(locking_thread); 883 } else { 884 // Owned by somebody else. 885 monitor->set_anonymous_owner(); 886 } 887 markWord monitor_mark = markWord::encode(monitor); 888 markWord old_mark = object->cas_set_mark(monitor_mark, mark); 889 if (old_mark == mark) { 890 // Success! Return inflated monitor. 891 if (own) { 892 size_t removed = locking_thread->lock_stack().remove(object); 893 monitor->set_recursions(removed - 1); 894 } 895 // Once the ObjectMonitor is configured and object is associated 896 // with the ObjectMonitor, it is safe to allow async deflation: 897 ObjectSynchronizer::_in_use_list.add(monitor); 898 899 log_inflate(current, object, cause); 900 if (event.should_commit()) { 901 post_monitor_inflate_event(&event, object, cause); 902 } 903 return monitor; 904 } else { 905 delete monitor; 906 continue; // Interference -- just retry 907 } 908 } 909 910 // CASE: unlocked 911 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner. 912 // If we know we're inflating for entry it's better to inflate by swinging a 913 // pre-locked ObjectMonitor pointer into the object header. A successful 914 // CAS inflates the object *and* confers ownership to the inflating thread. 915 // In the current implementation we use a 2-step mechanism where we CAS() 916 // to inflate and then CAS() again to try to swing _owner from null to current. 917 // An inflateTry() method that we could call from enter() would be useful. 918 919 assert(mark.is_unlocked(), "invariant: header=" INTPTR_FORMAT, mark.value()); 920 ObjectMonitor* m = new ObjectMonitor(object); 921 // prepare m for installation - set monitor to initial state 922 m->set_header(mark); 923 924 if (object->cas_set_mark(markWord::encode(m), mark) != mark) { 925 delete m; 926 m = nullptr; 927 continue; 928 // interference - the markword changed - just retry. 929 // The state-transitions are one-way, so there's no chance of 930 // live-lock -- "Inflated" is an absorbing state. 931 } 932 933 // Once the ObjectMonitor is configured and object is associated 934 // with the ObjectMonitor, it is safe to allow async deflation: 935 ObjectSynchronizer::_in_use_list.add(m); 936 937 log_inflate(current, object, cause); 938 if (event.should_commit()) { 939 post_monitor_inflate_event(&event, object, cause); 940 } 941 return m; 942 } 943 } 944 945 ObjectMonitor* LightweightSynchronizer::inflate_fast_locked_object(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, JavaThread* current) { 946 assert(LockingMode == LM_LIGHTWEIGHT, "only used for lightweight"); 947 VerifyThreadState vts(locking_thread, current); 948 assert(locking_thread->lock_stack().contains(object), "locking_thread must have object on its lock stack"); 949 950 ObjectMonitor* monitor; 951 952 if (!UseObjectMonitorTable) { 953 return inflate_into_object_header(object, cause, locking_thread, current); 954 } 955 956 // Inflating requires a hash code 957 ObjectSynchronizer::FastHashCode(current, object); 958 959 markWord mark = object->mark_acquire(); 960 assert(!mark.is_unlocked(), "Cannot be unlocked"); 961 962 for (;;) { 963 // Fetch the monitor from the table 964 monitor = get_or_insert_monitor(object, current, cause); 965 966 // ObjectMonitors are always inserted as anonymously owned, this thread is 967 // the current holder of the monitor. So unless the entry is stale and 968 // contains a deflating monitor it must be anonymously owned. 969 if (monitor->has_anonymous_owner()) { 970 // The monitor must be anonymously owned if it was added 971 assert(monitor == get_monitor_from_table(current, object), "The monitor must be found"); 972 // New fresh monitor 973 break; 974 } 975 976 // If the monitor was not anonymously owned then we got a deflating monitor 977 // from the table. We need to let the deflator make progress and remove this 978 // entry before we are allowed to add a new one. 979 os::naked_yield(); 980 assert(monitor->is_being_async_deflated(), "Should be the reason"); 981 } 982 983 // Set the mark word; loop to handle concurrent updates to other parts of the mark word 984 while (mark.is_fast_locked()) { 985 mark = object->cas_set_mark(mark.set_has_monitor(), mark); 986 } 987 988 // Indicate that the monitor now has a known owner 989 monitor->set_owner_from_anonymous(locking_thread); 990 991 // Remove the entry from the thread's lock stack 992 monitor->set_recursions(locking_thread->lock_stack().remove(object) - 1); 993 994 if (locking_thread == current) { 995 // Only change the thread local state of the current thread. 996 locking_thread->om_set_monitor_cache(monitor); 997 } 998 999 return monitor; 1000 } 1001 1002 ObjectMonitor* LightweightSynchronizer::inflate_and_enter(oop object, BasicLock* lock, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, JavaThread* current) { 1003 assert(LockingMode == LM_LIGHTWEIGHT, "only used for lightweight"); 1004 VerifyThreadState vts(locking_thread, current); 1005 1006 // Note: In some paths (deoptimization) the 'current' thread inflates and 1007 // enters the lock on behalf of the 'locking_thread' thread. 1008 1009 ObjectMonitor* monitor = nullptr; 1010 1011 if (!UseObjectMonitorTable) { 1012 // Do the old inflate and enter. 1013 monitor = inflate_into_object_header(object, cause, locking_thread, current); 1014 1015 bool entered; 1016 if (locking_thread == current) { 1017 entered = monitor->enter(locking_thread); 1018 } else { 1019 entered = monitor->enter_for(locking_thread); 1020 } 1021 1022 // enter returns false for deflation found. 1023 return entered ? monitor : nullptr; 1024 } 1025 1026 NoSafepointVerifier nsv; 1027 1028 // Try to get the monitor from the thread-local cache. 1029 // There's no need to use the cache if we are locking 1030 // on behalf of another thread. 1031 if (current == locking_thread) { 1032 monitor = read_caches(current, lock, object); 1033 } 1034 1035 // Get or create the monitor 1036 if (monitor == nullptr) { 1037 // Lightweight monitors require that hash codes are installed first 1038 ObjectSynchronizer::FastHashCode(locking_thread, object); 1039 monitor = get_or_insert_monitor(object, current, cause); 1040 } 1041 1042 if (monitor->try_enter(locking_thread)) { 1043 return monitor; 1044 } 1045 1046 // Holds is_being_async_deflated() stable throughout this function. 1047 ObjectMonitorContentionMark contention_mark(monitor); 1048 1049 /// First handle the case where the monitor from the table is deflated 1050 if (monitor->is_being_async_deflated()) { 1051 // The MonitorDeflation thread is deflating the monitor. The locking thread 1052 // must spin until further progress has been made. 1053 1054 // Clear the BasicLock cache as it may contain this monitor. 1055 lock->clear_object_monitor_cache(); 1056 1057 const markWord mark = object->mark_acquire(); 1058 1059 if (mark.has_monitor()) { 1060 // Waiting on the deflation thread to remove the deflated monitor from the table. 1061 os::naked_yield(); 1062 1063 } else if (mark.is_fast_locked()) { 1064 // Some other thread managed to fast-lock the lock, or this is a 1065 // recursive lock from the same thread; yield for the deflation 1066 // thread to remove the deflated monitor from the table. 1067 os::naked_yield(); 1068 1069 } else { 1070 assert(mark.is_unlocked(), "Implied"); 1071 // Retry immediately 1072 } 1073 1074 // Retry 1075 return nullptr; 1076 } 1077 1078 for (;;) { 1079 const markWord mark = object->mark_acquire(); 1080 // The mark can be in one of the following states: 1081 // * inflated - If the ObjectMonitor owner is anonymous 1082 // and the locking_thread owns the object 1083 // lock, then we make the locking_thread 1084 // the ObjectMonitor owner and remove the 1085 // lock from the locking_thread's lock stack. 1086 // * fast-locked - Coerce it to inflated from fast-locked. 1087 // * neutral - Inflate the object. Successful CAS is locked 1088 1089 // CASE: inflated 1090 if (mark.has_monitor()) { 1091 LockStack& lock_stack = locking_thread->lock_stack(); 1092 if (monitor->has_anonymous_owner() && lock_stack.contains(object)) { 1093 // The lock is fast-locked by the locking thread, 1094 // convert it to a held monitor with a known owner. 1095 monitor->set_owner_from_anonymous(locking_thread); 1096 monitor->set_recursions(lock_stack.remove(object) - 1); 1097 } 1098 1099 break; // Success 1100 } 1101 1102 // CASE: fast-locked 1103 // Could be fast-locked either by locking_thread or by some other thread. 1104 // 1105 if (mark.is_fast_locked()) { 1106 markWord old_mark = object->cas_set_mark(mark.set_has_monitor(), mark); 1107 if (old_mark != mark) { 1108 // CAS failed 1109 continue; 1110 } 1111 1112 // Success! Return inflated monitor. 1113 LockStack& lock_stack = locking_thread->lock_stack(); 1114 if (lock_stack.contains(object)) { 1115 // The lock is fast-locked by the locking thread, 1116 // convert it to a held monitor with a known owner. 1117 monitor->set_owner_from_anonymous(locking_thread); 1118 monitor->set_recursions(lock_stack.remove(object) - 1); 1119 } 1120 1121 break; // Success 1122 } 1123 1124 // CASE: neutral (unlocked) 1125 1126 // Catch if the object's header is not neutral (not locked and 1127 // not marked is what we care about here). 1128 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); 1129 markWord old_mark = object->cas_set_mark(mark.set_has_monitor(), mark); 1130 if (old_mark != mark) { 1131 // CAS failed 1132 continue; 1133 } 1134 1135 // Transitioned from unlocked to monitor means locking_thread owns the lock. 1136 monitor->set_owner_from_anonymous(locking_thread); 1137 1138 return monitor; 1139 } 1140 1141 if (current == locking_thread) { 1142 // One round of spinning 1143 if (monitor->spin_enter(locking_thread)) { 1144 return monitor; 1145 } 1146 1147 // Monitor is contended, take the time before entering to fix the lock stack. 1148 LockStackInflateContendedLocks().inflate(current); 1149 } 1150 1151 // enter can block for safepoints; clear the unhandled object oop 1152 PauseNoSafepointVerifier pnsv(&nsv); 1153 object = nullptr; 1154 1155 if (current == locking_thread) { 1156 monitor->enter_with_contention_mark(locking_thread, contention_mark); 1157 } else { 1158 monitor->enter_for_with_contention_mark(locking_thread, contention_mark); 1159 } 1160 1161 return monitor; 1162 } 1163 1164 void LightweightSynchronizer::deflate_monitor(Thread* current, oop obj, ObjectMonitor* monitor) { 1165 if (obj != nullptr) { 1166 deflate_mark_word(obj); 1167 } 1168 bool removed = remove_monitor(current, monitor, obj); 1169 if (obj != nullptr) { 1170 assert(removed, "Should have removed the entry if obj was alive"); 1171 } 1172 } 1173 1174 ObjectMonitor* LightweightSynchronizer::get_monitor_from_table(Thread* current, oop obj) { 1175 assert(UseObjectMonitorTable, "must be"); 1176 return ObjectMonitorTable::monitor_get(current, obj); 1177 } 1178 1179 bool LightweightSynchronizer::contains_monitor(Thread* current, ObjectMonitor* monitor) { 1180 assert(UseObjectMonitorTable, "must be"); 1181 return ObjectMonitorTable::contains_monitor(current, monitor); 1182 } 1183 1184 bool LightweightSynchronizer::quick_enter(oop obj, BasicLock* lock, JavaThread* current) { 1185 assert(current->thread_state() == _thread_in_Java, "must be"); 1186 assert(obj != nullptr, "must be"); 1187 NoSafepointVerifier nsv; 1188 1189 LockStack& lock_stack = current->lock_stack(); 1190 if (lock_stack.is_full()) { 1191 // Always go into runtime if the lock stack is full. 1192 return false; 1193 } 1194 1195 const markWord mark = obj->mark(); 1196 1197 #ifndef _LP64 1198 // Only for 32bit which has limited support for fast locking outside the runtime. 1199 if (lock_stack.try_recursive_enter(obj)) { 1200 // Recursive lock successful. 1201 return true; 1202 } 1203 1204 if (mark.is_unlocked()) { 1205 markWord locked_mark = mark.set_fast_locked(); 1206 if (obj->cas_set_mark(locked_mark, mark) == mark) { 1207 // Successfully fast-locked, push object to lock-stack and return. 1208 lock_stack.push(obj); 1209 return true; 1210 } 1211 } 1212 #endif 1213 1214 if (mark.has_monitor()) { 1215 ObjectMonitor* monitor; 1216 if (UseObjectMonitorTable) { 1217 monitor = read_caches(current, lock, obj); 1218 } else { 1219 monitor = ObjectSynchronizer::read_monitor(mark); 1220 } 1221 1222 if (monitor == nullptr) { 1223 // Take the slow-path on a cache miss. 1224 return false; 1225 } 1226 1227 if (UseObjectMonitorTable) { 1228 // Set the monitor regardless of success. 1229 // Either we successfully lock on the monitor, or we retry with the 1230 // monitor in the slow path. If the monitor gets deflated, it will be 1231 // cleared, either by the CacheSetter if we fast lock in enter or in 1232 // inflate_and_enter when we see that the monitor is deflated. 1233 lock->set_object_monitor_cache(monitor); 1234 } 1235 1236 if (monitor->spin_enter(current)) { 1237 return true; 1238 } 1239 } 1240 1241 // Slow-path. 1242 return false; 1243 }