src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

*** 63,10 ***
--- 63,11 ---
  int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
    assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
    int null_check_offset = -1;
+   Label count_locking, done;
  
    verify_oop(obj);
  
    // save object being locked into the BasicObjectLock
    str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));

*** 80,23 ***
      br(Assembler::NE, slow_case);
    }
  
    if (LockingMode == LM_LIGHTWEIGHT) {
      lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
    } else if (LockingMode == LM_LEGACY) {
-     Label done;
      // Load object header
      ldr(hdr, Address(obj, hdr_offset));
      // and mark it as unlocked
      orr(hdr, hdr, markWord::unlocked_value);
      // save unlocked object header into the displaced header location on the stack
      str(hdr, Address(disp_hdr, 0));
      // test if object header is still the same (i.e. unlocked), and if so, store the
      // displaced header address in the object header - if it is not the same, get the
      // object header instead
      lea(rscratch2, Address(obj, hdr_offset));
!     cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/nullptr);
      // if the object header was the same, we're done
      // if the object header was not the same, it is now in the hdr register
      // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
      //
      // 1) (hdr & aligned_mask) == 0
--- 81,24 ---
      br(Assembler::NE, slow_case);
    }
  
    if (LockingMode == LM_LIGHTWEIGHT) {
      lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
+     b(done);
    } else if (LockingMode == LM_LEGACY) {
      // Load object header
      ldr(hdr, Address(obj, hdr_offset));
      // and mark it as unlocked
      orr(hdr, hdr, markWord::unlocked_value);
      // save unlocked object header into the displaced header location on the stack
      str(hdr, Address(disp_hdr, 0));
      // test if object header is still the same (i.e. unlocked), and if so, store the
      // displaced header address in the object header - if it is not the same, get the
      // object header instead
      lea(rscratch2, Address(obj, hdr_offset));
!     // if the object header was the same, we're done
+     cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, count_locking, /*fallthough*/nullptr);
      // if the object header was the same, we're done
      // if the object header was not the same, it is now in the hdr register
      // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
      //
      // 1) (hdr & aligned_mask) == 0

*** 116,22 ***
      // location (null in the displaced hdr location indicates recursive locking)
      str(hdr, Address(disp_hdr, 0));
      // otherwise we don't care about the result and handle locking via runtime call
      cbnz(hdr, slow_case);
      // done
!     bind(done);
    }
!   increment(Address(rthread, JavaThread::held_monitor_count_offset()));
    return null_check_offset;
  }
  
  
  void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
    assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
!   Label done;
  
    if (LockingMode != LM_LIGHTWEIGHT) {
      // load displaced header
      ldr(hdr, Address(disp_hdr, 0));
      // if the loaded hdr is null we had recursive locking
--- 118,24 ---
      // location (null in the displaced hdr location indicates recursive locking)
      str(hdr, Address(disp_hdr, 0));
      // otherwise we don't care about the result and handle locking via runtime call
      cbnz(hdr, slow_case);
      // done
!     b(done);
    }
!   bind(count_locking);
+   inc_held_monitor_count();
+   bind(done);
    return null_check_offset;
  }
  
  
  void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
    assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
!   Label count_locking, done;
  
    if (LockingMode != LM_LIGHTWEIGHT) {
      // load displaced header
      ldr(hdr, Address(disp_hdr, 0));
      // if the loaded hdr is null we had recursive locking

*** 143,26 ***
    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
    verify_oop(obj);
  
    if (LockingMode == LM_LIGHTWEIGHT) {
      lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
    } else if (LockingMode == LM_LEGACY) {
      // test if object header is pointing to the displaced header, and if so, restore
      // the displaced header in the object - if the object header is not pointing to
      // the displaced header, get the object header instead
      // if the object header was not pointing to the displaced header,
      // we do unlocking via runtime call
      if (hdr_offset) {
        lea(rscratch1, Address(obj, hdr_offset));
!       cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
      } else {
!       cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
      }
      // done
!     bind(done);
    }
!   decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
  }
  
  
  // Defines obj, preserves var_size_in_bytes
  void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
--- 147,28 ---
    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
    verify_oop(obj);
  
    if (LockingMode == LM_LIGHTWEIGHT) {
      lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
+     b(done);
    } else if (LockingMode == LM_LEGACY) {
      // test if object header is pointing to the displaced header, and if so, restore
      // the displaced header in the object - if the object header is not pointing to
      // the displaced header, get the object header instead
      // if the object header was not pointing to the displaced header,
      // we do unlocking via runtime call
      if (hdr_offset) {
        lea(rscratch1, Address(obj, hdr_offset));
!       cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, count_locking, &slow_case);
      } else {
!       cmpxchgptr(disp_hdr, hdr, obj, rscratch2, count_locking, &slow_case);
      }
      // done
!     bind(count_locking);
+     dec_held_monitor_count();
    }
!   bind(done);
  }
  
  
  // Defines obj, preserves var_size_in_bytes
  void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {