2322
2323 and_(R0/*==0?*/, current_header, temp);
2324 // If condition is true we are cont and hence we can store 0 as the
2325 // displaced header in the box, which indicates that it is a recursive lock.
2326 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2327
2328 if (flag != CCR0) {
2329 mcrf(flag, CCR0);
2330 }
2331 beq(CCR0, success);
2332 b(failure);
2333 }
2334
2335 // Handle existing monitor.
2336 bind(object_has_monitor);
2337 // The object's monitor m is unlocked iff m->owner is null,
2338 // otherwise m->owner may contain a thread or a stack address.
2339
2340 // Try to CAS m->owner from null to current thread.
2341 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2342 cmpxchgd(/*flag=*/flag,
2343 /*current_value=*/current_header,
2344 /*compare_value=*/(intptr_t)0,
2345 /*exchange_value=*/R16_thread,
2346 /*where=*/temp,
2347 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2348 MacroAssembler::cmpxchgx_hint_acquire_lock());
2349
2350 // Store a non-null value into the box.
2351 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2352 beq(flag, success);
2353
2354 // Check for recursive locking.
2355 cmpd(flag, current_header, R16_thread);
2356 bne(flag, failure);
2357
2358 // Current thread already owns the lock. Just increment recursions.
2359 Register recursions = displaced_header;
2360 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2361 addi(recursions, recursions, 1);
2362 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2363
2364 // flag == EQ indicates success, increment held monitor count
2365 // flag == NE indicates failure
2366 bind(success);
2367 inc_held_monitor_count(temp);
2368 bind(failure);
2369 }
2370
2371 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2372 Register temp, Register displaced_header, Register current_header) {
2373 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2374 assert_different_registers(oop, box, temp, displaced_header, current_header);
2375 Label success, failure, object_has_monitor, notRecursive;
2402 /*current_value=*/current_header,
2403 /*compare_value=*/box,
2404 /*exchange_value=*/displaced_header,
2405 /*where=*/oop,
2406 MacroAssembler::MemBarRel,
2407 MacroAssembler::cmpxchgx_hint_release_lock(),
2408 noreg,
2409 &failure);
2410 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2411 b(success);
2412 }
2413
2414 // Handle existing monitor.
2415 bind(object_has_monitor);
2416 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2417 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2418 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2419
2420 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2421 // This is handled like owner thread mismatches: We take the slow path.
2422 cmpd(flag, temp, R16_thread);
2423 bne(flag, failure);
2424
2425 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2426
2427 addic_(displaced_header, displaced_header, -1);
2428 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2429 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2430 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2431 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2432 }
2433 b(success);
2434
2435 bind(notRecursive);
2436 ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2437 ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2438 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2439 cmpdi(flag, temp, 0);
2440 bne(flag, failure);
2441 release();
2442 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2506
2507 bind(push);
2508 // After successful lock, push object on lock-stack.
2509 stdx(obj, R16_thread, top);
2510 addi(top, top, oopSize);
2511 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2512 b(locked);
2513 }
2514
2515 { // Handle inflated monitor.
2516 bind(inflated);
2517
2518 // mark contains the tagged ObjectMonitor*.
2519 const Register tagged_monitor = mark;
2520 const uintptr_t monitor_tag = markWord::monitor_value;
2521 const Register owner_addr = tmp2;
2522
2523 // Compute owner address.
2524 addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2525
2526 // CAS owner (null => current thread).
2527 cmpxchgd(/*flag=*/flag,
2528 /*current_value=*/t,
2529 /*compare_value=*/(intptr_t)0,
2530 /*exchange_value=*/R16_thread,
2531 /*where=*/owner_addr,
2532 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2533 MacroAssembler::cmpxchgx_hint_acquire_lock());
2534 beq(flag, locked);
2535
2536 // Check if recursive.
2537 cmpd(flag, t, R16_thread);
2538 bne(flag, slow_path);
2539
2540 // Recursive.
2541 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2542 addi(tmp1, tmp1, 1);
2543 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2544 }
2545
2546 bind(locked);
2547 inc_held_monitor_count(tmp1);
2548
2549 #ifdef ASSERT
2550 // Check that locked label is reached with flag == EQ.
2551 Label flag_correct;
2552 beq(flag, flag_correct);
2553 stop("Fast Lock Flag != EQ");
2554 #endif
2555 bind(slow_path);
2556 #ifdef ASSERT
2557 // Check that slow_path label is reached with flag == NE.
2672 // Recursive unlock.
2673 std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2674 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2675 b(unlocked);
2676
2677 bind(not_recursive);
2678
2679 Label release_;
2680 const Register t2 = tmp2;
2681
2682 // Check if the entry lists are empty.
2683 ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2684 ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2685 orr(t, t, t2);
2686 cmpdi(flag, t, 0);
2687 beq(flag, release_);
2688
2689 // The owner may be anonymous and we removed the last obj entry in
2690 // the lock-stack. This loses the information about the owner.
2691 // Write the thread to the owner field so the runtime knows the owner.
2692 std(R16_thread, in_bytes(ObjectMonitor::owner_offset()), monitor);
2693 b(slow_path);
2694
2695 bind(release_);
2696 // Set owner to null.
2697 release();
2698 // t contains 0
2699 std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2700 }
2701
2702 bind(unlocked);
2703 dec_held_monitor_count(t);
2704
2705 #ifdef ASSERT
2706 // Check that unlocked label is reached with flag == EQ.
2707 Label flag_correct;
2708 beq(flag, flag_correct);
2709 stop("Fast Lock Flag != EQ");
2710 #endif
2711 bind(slow_path);
2712 #ifdef ASSERT
|
2322
2323 and_(R0/*==0?*/, current_header, temp);
2324 // If condition is true we are cont and hence we can store 0 as the
2325 // displaced header in the box, which indicates that it is a recursive lock.
2326 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2327
2328 if (flag != CCR0) {
2329 mcrf(flag, CCR0);
2330 }
2331 beq(CCR0, success);
2332 b(failure);
2333 }
2334
2335 // Handle existing monitor.
2336 bind(object_has_monitor);
2337 // The object's monitor m is unlocked iff m->owner is null,
2338 // otherwise m->owner may contain a thread or a stack address.
2339
2340 // Try to CAS m->owner from null to current thread.
2341 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2342 Register thread_id = displaced_header;
2343 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2344 cmpxchgd(/*flag=*/flag,
2345 /*current_value=*/current_header,
2346 /*compare_value=*/(intptr_t)0,
2347 /*exchange_value=*/thread_id,
2348 /*where=*/temp,
2349 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2350 MacroAssembler::cmpxchgx_hint_acquire_lock());
2351
2352 // Store a non-null value into the box.
2353 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2354 beq(flag, success);
2355
2356 // Check for recursive locking.
2357 cmpd(flag, current_header, thread_id);
2358 bne(flag, failure);
2359
2360 // Current thread already owns the lock. Just increment recursions.
2361 Register recursions = displaced_header;
2362 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2363 addi(recursions, recursions, 1);
2364 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2365
2366 // flag == EQ indicates success, increment held monitor count
2367 // flag == NE indicates failure
2368 bind(success);
2369 inc_held_monitor_count(temp);
2370 bind(failure);
2371 }
2372
2373 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2374 Register temp, Register displaced_header, Register current_header) {
2375 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2376 assert_different_registers(oop, box, temp, displaced_header, current_header);
2377 Label success, failure, object_has_monitor, notRecursive;
2404 /*current_value=*/current_header,
2405 /*compare_value=*/box,
2406 /*exchange_value=*/displaced_header,
2407 /*where=*/oop,
2408 MacroAssembler::MemBarRel,
2409 MacroAssembler::cmpxchgx_hint_release_lock(),
2410 noreg,
2411 &failure);
2412 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2413 b(success);
2414 }
2415
2416 // Handle existing monitor.
2417 bind(object_has_monitor);
2418 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2419 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2420 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2421
2422 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2423 // This is handled like owner thread mismatches: We take the slow path.
2424 Register thread_id = displaced_header;
2425 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2426 cmpd(flag, temp, thread_id);
2427 bne(flag, failure);
2428
2429 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2430
2431 addic_(displaced_header, displaced_header, -1);
2432 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2433 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2434 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2435 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2436 }
2437 b(success);
2438
2439 bind(notRecursive);
2440 ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2441 ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2442 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2443 cmpdi(flag, temp, 0);
2444 bne(flag, failure);
2445 release();
2446 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2510
2511 bind(push);
2512 // After successful lock, push object on lock-stack.
2513 stdx(obj, R16_thread, top);
2514 addi(top, top, oopSize);
2515 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2516 b(locked);
2517 }
2518
2519 { // Handle inflated monitor.
2520 bind(inflated);
2521
2522 // mark contains the tagged ObjectMonitor*.
2523 const Register tagged_monitor = mark;
2524 const uintptr_t monitor_tag = markWord::monitor_value;
2525 const Register owner_addr = tmp2;
2526
2527 // Compute owner address.
2528 addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2529
2530 // CAS owner (null => current thread id).
2531 Register thread_id = tmp1;
2532 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2533 cmpxchgd(/*flag=*/flag,
2534 /*current_value=*/t,
2535 /*compare_value=*/(intptr_t)0,
2536 /*exchange_value=*/thread_id,
2537 /*where=*/owner_addr,
2538 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2539 MacroAssembler::cmpxchgx_hint_acquire_lock());
2540 beq(flag, locked);
2541
2542 // Check if recursive.
2543 cmpd(flag, t, thread_id);
2544 bne(flag, slow_path);
2545
2546 // Recursive.
2547 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2548 addi(tmp1, tmp1, 1);
2549 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2550 }
2551
2552 bind(locked);
2553 inc_held_monitor_count(tmp1);
2554
2555 #ifdef ASSERT
2556 // Check that locked label is reached with flag == EQ.
2557 Label flag_correct;
2558 beq(flag, flag_correct);
2559 stop("Fast Lock Flag != EQ");
2560 #endif
2561 bind(slow_path);
2562 #ifdef ASSERT
2563 // Check that slow_path label is reached with flag == NE.
2678 // Recursive unlock.
2679 std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2680 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2681 b(unlocked);
2682
2683 bind(not_recursive);
2684
2685 Label release_;
2686 const Register t2 = tmp2;
2687
2688 // Check if the entry lists are empty.
2689 ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2690 ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2691 orr(t, t, t2);
2692 cmpdi(flag, t, 0);
2693 beq(flag, release_);
2694
2695 // The owner may be anonymous and we removed the last obj entry in
2696 // the lock-stack. This loses the information about the owner.
2697 // Write the thread to the owner field so the runtime knows the owner.
2698 Register thread_id = tmp2;
2699 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2700 std(thread_id, in_bytes(ObjectMonitor::owner_offset()), monitor);
2701 b(slow_path);
2702
2703 bind(release_);
2704 // Set owner to null.
2705 release();
2706 // t contains 0
2707 std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2708 }
2709
2710 bind(unlocked);
2711 dec_held_monitor_count(t);
2712
2713 #ifdef ASSERT
2714 // Check that unlocked label is reached with flag == EQ.
2715 Label flag_correct;
2716 beq(flag, flag_correct);
2717 stop("Fast Lock Flag != EQ");
2718 #endif
2719 bind(slow_path);
2720 #ifdef ASSERT
|