306
307 // Try to lock. Transition lock-bits 0b01 => 0b00
308 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
309 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
310 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
311 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
312 bne(tmp1_mark, tmp3_t, slow_path);
313
314 bind(push);
315 // After successful lock, push object on lock-stack.
316 add(tmp3_t, xthread, tmp2_top);
317 sd(obj, Address(tmp3_t));
318 addw(tmp2_top, tmp2_top, oopSize);
319 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
320 j(locked);
321 }
322
323 { // Handle inflated monitor.
324 bind(inflated);
325
326 // mark contains the tagged ObjectMonitor*.
327 const Register tmp1_tagged_monitor = tmp1_mark;
328 const uintptr_t monitor_tag = markWord::monitor_value;
329 const Register tmp2_owner_addr = tmp2;
330 const Register tmp3_owner = tmp3;
331
332 // Compute owner address.
333 la(tmp2_owner_addr, Address(tmp1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
334
335 // CAS owner (null => current thread).
336 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ xthread, Assembler::int64,
337 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
338 beqz(tmp3_owner, locked);
339
340 // Check if recursive.
341 bne(tmp3_owner, xthread, slow_path);
342
343 // Recursive.
344 increment(Address(tmp1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1, tmp2, tmp3);
345 }
346
347 bind(locked);
348 mv(flag, zr);
349 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
350
351 #ifdef ASSERT
352 // Check that locked label is reached with flag == 0.
353 Label flag_correct;
354 beqz(flag, flag_correct);
355 stop("Fast Lock Flag != 0");
356 #endif
357
358 bind(slow_path);
359 #ifdef ASSERT
360 // Check that slow_path label is reached with flag != 0.
361 bnez(flag, flag_correct);
362 stop("Fast Lock Flag == 0");
363 bind(flag_correct);
364 #endif
436 #ifdef ASSERT
437 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
438 bnez(tmp3_t, inflated);
439 stop("Fast Unlock not monitor");
440 #endif
441
442 bind(inflated);
443
444 #ifdef ASSERT
445 Label check_done;
446 subw(tmp2_top, tmp2_top, oopSize);
447 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
448 blt(tmp2_top, tmp3_t, check_done);
449 add(tmp3_t, xthread, tmp2_top);
450 ld(tmp3_t, Address(tmp3_t));
451 bne(obj, tmp3_t, inflated);
452 stop("Fast Unlock lock on stack");
453 bind(check_done);
454 #endif
455
456 // mark contains the tagged ObjectMonitor*.
457 const Register tmp1_monitor = tmp1_mark;
458 const uintptr_t monitor_tag = markWord::monitor_value;
459
460 // Untag the monitor.
461 sub(tmp1_monitor, tmp1_mark, monitor_tag);
462
463 const Register tmp2_recursions = tmp2;
464 Label not_recursive;
465
466 // Check if recursive.
467 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
468 beqz(tmp2_recursions, not_recursive);
469
470 // Recursive unlock.
471 addi(tmp2_recursions, tmp2_recursions, -1);
472 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
473 j(unlocked);
474
475 bind(not_recursive);
476
477 Label release;
478 const Register tmp2_owner_addr = tmp2;
479
480 // Compute owner address.
481 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
482
483 // Check if the entry lists are empty.
484 ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
485 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
486 orr(t0, t0, tmp3_t);
487 beqz(t0, release);
488
489 // The owner may be anonymous and we removed the last obj entry in
490 // the lock-stack. This loses the information about the owner.
491 // Write the thread to the owner field so the runtime knows the owner.
492 sd(xthread, Address(tmp2_owner_addr));
493 j(slow_path);
494
495 bind(release);
496 // Set owner to null.
497 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
498 sd(zr, Address(tmp2_owner_addr));
499 }
500
501 bind(unlocked);
502 mv(flag, zr);
503 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
504
505 #ifdef ASSERT
506 // Check that unlocked label is reached with flag == 0.
507 Label flag_correct;
508 beqz(flag, flag_correct);
509 stop("Fast Lock Flag != 0");
510 #endif
511
512 bind(slow_path);
513 #ifdef ASSERT
514 // Check that slow_path label is reached with flag != 0.
515 bnez(flag, flag_correct);
516 stop("Fast Lock Flag == 0");
517 bind(flag_correct);
518 #endif
|
306
307 // Try to lock. Transition lock-bits 0b01 => 0b00
308 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
309 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
310 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
311 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
312 bne(tmp1_mark, tmp3_t, slow_path);
313
314 bind(push);
315 // After successful lock, push object on lock-stack.
316 add(tmp3_t, xthread, tmp2_top);
317 sd(obj, Address(tmp3_t));
318 addw(tmp2_top, tmp2_top, oopSize);
319 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
320 j(locked);
321 }
322
323 { // Handle inflated monitor.
324 bind(inflated);
325
326 if (!UseObjectMonitorTable) {
327 // mark contains the tagged ObjectMonitor*.
328 const Register tmp1_tagged_monitor = tmp1_mark;
329 const uintptr_t monitor_tag = markWord::monitor_value;
330 const Register tmp2_owner_addr = tmp2;
331 const Register tmp3_owner = tmp3;
332
333 // Compute owner address.
334 la(tmp2_owner_addr, Address(tmp1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
335
336 // CAS owner (null => current thread).
337 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ xthread, Assembler::int64,
338 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
339 beqz(tmp3_owner, locked);
340
341 // Check if recursive.
342 bne(tmp3_owner, xthread, slow_path);
343
344 // Recursive.
345 increment(Address(tmp1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1, tmp2, tmp3);
346 } else {
347 // OMCache lookup not supported yet. Take the slowpath.
348 j(slow_path);
349 }
350 }
351
352 bind(locked);
353 mv(flag, zr);
354 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
355
356 #ifdef ASSERT
357 // Check that locked label is reached with flag == 0.
358 Label flag_correct;
359 beqz(flag, flag_correct);
360 stop("Fast Lock Flag != 0");
361 #endif
362
363 bind(slow_path);
364 #ifdef ASSERT
365 // Check that slow_path label is reached with flag != 0.
366 bnez(flag, flag_correct);
367 stop("Fast Lock Flag == 0");
368 bind(flag_correct);
369 #endif
441 #ifdef ASSERT
442 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
443 bnez(tmp3_t, inflated);
444 stop("Fast Unlock not monitor");
445 #endif
446
447 bind(inflated);
448
449 #ifdef ASSERT
450 Label check_done;
451 subw(tmp2_top, tmp2_top, oopSize);
452 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
453 blt(tmp2_top, tmp3_t, check_done);
454 add(tmp3_t, xthread, tmp2_top);
455 ld(tmp3_t, Address(tmp3_t));
456 bne(obj, tmp3_t, inflated);
457 stop("Fast Unlock lock on stack");
458 bind(check_done);
459 #endif
460
461 if (!UseObjectMonitorTable) {
462 // mark contains the tagged ObjectMonitor*.
463 const Register tmp1_monitor = tmp1_mark;
464 const uintptr_t monitor_tag = markWord::monitor_value;
465
466 // Untag the monitor.
467 sub(tmp1_monitor, tmp1_mark, monitor_tag);
468
469 const Register tmp2_recursions = tmp2;
470 Label not_recursive;
471
472 // Check if recursive.
473 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
474 beqz(tmp2_recursions, not_recursive);
475
476 // Recursive unlock.
477 addi(tmp2_recursions, tmp2_recursions, -1);
478 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
479 j(unlocked);
480
481 bind(not_recursive);
482
483 Label release;
484 const Register tmp2_owner_addr = tmp2;
485
486 // Compute owner address.
487 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
488
489 // Check if the entry lists are empty.
490 ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
491 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
492 orr(t0, t0, tmp3_t);
493 beqz(t0, release);
494
495 // The owner may be anonymous and we removed the last obj entry in
496 // the lock-stack. This loses the information about the owner.
497 // Write the thread to the owner field so the runtime knows the owner.
498 sd(xthread, Address(tmp2_owner_addr));
499 j(slow_path);
500
501 bind(release);
502 // Set owner to null.
503 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
504 sd(zr, Address(tmp2_owner_addr));
505 } else {
506 // OMCache lookup not supported yet. Take the slowpath.
507 j(slow_path);
508 }
509 }
510
511 bind(unlocked);
512 mv(flag, zr);
513 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
514
515 #ifdef ASSERT
516 // Check that unlocked label is reached with flag == 0.
517 Label flag_correct;
518 beqz(flag, flag_correct);
519 stop("Fast Lock Flag != 0");
520 #endif
521
522 bind(slow_path);
523 #ifdef ASSERT
524 // Check that slow_path label is reached with flag != 0.
525 bnez(flag, flag_correct);
526 stop("Fast Lock Flag == 0");
527 bind(flag_correct);
528 #endif
|