< prev index next >

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

Print this page
*** 59,14 ***
      cset(result, NE);  // Not equal or unordered
      cneg(result, result, LO);  // Less than
    }
  }
  
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
!   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
    Label done;
    int null_check_offset = -1;
  
    verify_oop(obj);
  
--- 59,14 ---
      cset(result, NE);  // Not equal or unordered
      cneg(result, result, LO);  // Less than
    }
  }
  
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
!   assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
    Label done;
    int null_check_offset = -1;
  
    verify_oop(obj);
  

*** 81,92 ***
      tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
      br(Assembler::NE, slow_case);
    }
  
    if (UseBiasedLocking) {
!     assert(scratch != noreg, "should have scratch register at this point");
!     biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
    }
  
    // Load object header
    ldr(hdr, Address(obj, hdr_offset));
!   // and mark it as unlocked
!   orr(hdr, hdr, markWord::unlocked_value);
!   // save unlocked object header into the displaced header location on the stack
!   str(hdr, Address(disp_hdr, 0));
!   // test if object header is still the same (i.e. unlocked), and if so, store the
!   // displaced header address in the object header - if it is not the same, get the
!   // object header instead
!   lea(rscratch2, Address(obj, hdr_offset));
!   cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
!   // if the object header was the same, we're done
!   // if the object header was not the same, it is now in the hdr register
!   // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
!   //
!   // 1) (hdr & aligned_mask) == 0
!   // 2) sp <= hdr
!   // 3) hdr <= sp + page_size
!   //
!   // these 3 tests can be done by evaluating the following expression:
!   //
!   // (hdr - sp) & (aligned_mask - page_size)
!   //
!   // assuming both the stack pointer and page_size have their least
!   // significant 2 bits cleared and page_size is a power of 2
!   mov(rscratch1, sp);
!   sub(hdr, hdr, rscratch1);
!   ands(hdr, hdr, aligned_mask - os::vm_page_size());
!   // for recursive locking, the result is zero => save it in the displaced header
!   // location (NULL in the displaced hdr location indicates recursive locking)
!   str(hdr, Address(disp_hdr, 0));
!   // otherwise we don't care about the result and handle locking via runtime call
!   cbnz(hdr, slow_case);
!   // done
!   bind(done);
    if (PrintBiasedLockingStatistics) {
      lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
      addmw(Address(rscratch2, 0), 1, rscratch1);
    }
    return null_check_offset;
  }
  
  
! void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
!   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
    Label done;
  
    if (UseBiasedLocking) {
      // load object
      ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
      biased_locking_exit(obj, hdr, done);
    }
  
!   // load displaced header
!   ldr(hdr, Address(disp_hdr, 0));
!   // if the loaded hdr is NULL we had recursive locking
!   // if we had recursive locking, we are done
!   cbz(hdr, done);
    if (!UseBiasedLocking) {
      // load object
      ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
    }
    verify_oop(obj);
!   // test if object header is pointing to the displaced header, and if so, restore
!   // the displaced header in the object - if the object header is not pointing to
!   // the displaced header, get the object header instead
!   // if the object header was not pointing to the displaced header,
!   // we do unlocking via runtime call
!   if (hdr_offset) {
!     lea(rscratch1, Address(obj, hdr_offset));
!     cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
    } else {
!     cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
    }
-   // done
-   bind(done);
  }
  
  
  // Defines obj, preserves var_size_in_bytes
  void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
--- 81,109 ---
      tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
      br(Assembler::NE, slow_case);
    }
  
    if (UseBiasedLocking) {
!     assert(temp != noreg, "should have temp register at this point");
!     biased_locking_enter(disp_hdr, obj, hdr, temp, false, done, &slow_case);
    }
  
    // Load object header
    ldr(hdr, Address(obj, hdr_offset));
!   if (LockingMode == LM_LIGHTWEIGHT) {
!     lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
!   } else {
!     // and mark it as unlocked
!     orr(hdr, hdr, markWord::unlocked_value);
!     // save unlocked object header into the displaced header location on the stack
!     str(hdr, Address(disp_hdr, 0));
!     // test if object header is still the same (i.e. unlocked), and if so, store the
!     // displaced header address in the object header - if it is not the same, get the
!     // object header instead
!     lea(rscratch2, Address(obj, hdr_offset));
!     cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
!     // if the object header was the same, we're done
!     // if the object header was not the same, it is now in the hdr register
!     // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
!     //
!     // 1) (hdr & aligned_mask) == 0
!     // 2) sp <= hdr
!     // 3) hdr <= sp + page_size
!     //
!     // these 3 tests can be done by evaluating the following expression:
!     //
!     // (hdr - sp) & (aligned_mask - page_size)
!     //
!     // assuming both the stack pointer and page_size have their least
!     // significant 2 bits cleared and page_size is a power of 2
!     mov(rscratch1, sp);
!     sub(hdr, hdr, rscratch1);
!     ands(hdr, hdr, aligned_mask - os::vm_page_size());
!     // for recursive locking, the result is zero => save it in the displaced header
!     // location (NULL in the displaced hdr location indicates recursive locking)
!     str(hdr, Address(disp_hdr, 0));
!     // otherwise we don't care about the result and handle locking via runtime call
+     cbnz(hdr, slow_case);
+     // done
+     bind(done);
+   }
    if (PrintBiasedLockingStatistics) {
      lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
      addmw(Address(rscratch2, 0), 1, rscratch1);
    }
    return null_check_offset;
  }
  
  
! void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
!   assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
    Label done;
  
    if (UseBiasedLocking) {
      // load object
      ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
      biased_locking_exit(obj, hdr, done);
    }
  
!   if (LockingMode != LM_LIGHTWEIGHT) {
!     // load displaced header
!     ldr(hdr, Address(disp_hdr, 0));
!     // if the loaded hdr is null we had recursive locking
!     // if we had recursive locking, we are done
+     cbz(hdr, done);
+   }
+ 
    if (!UseBiasedLocking) {
      // load object
      ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
    }
    verify_oop(obj);
! 
!   if (LockingMode == LM_LIGHTWEIGHT) {
!     ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
!     // We cannot use tbnz here, the target might be too far away and cannot
!     // be encoded.
!     tst(hdr, markWord::monitor_value);
!     br(Assembler::NE, slow_case);
!     lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
    } else {
!     // test if object header is pointing to the displaced header, and if so, restore
+     // the displaced header in the object - if the object header is not pointing to
+     // the displaced header, get the object header instead
+     // if the object header was not pointing to the displaced header,
+     // we do unlocking via runtime call
+     if (hdr_offset) {
+       lea(rscratch1, Address(obj, hdr_offset));
+       cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+     } else {
+       cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+     }
+     // done
+     bind(done);
    }
  }
  
  
  // Defines obj, preserves var_size_in_bytes
  void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {

*** 177,29 ***
    }
  }
  
  void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
    assert_different_registers(obj, klass, len);
!   if (UseBiasedLocking && !len->is_valid()) {
      assert_different_registers(obj, klass, len, t1, t2);
      ldr(t1, Address(klass, Klass::prototype_header_offset()));
    } else {
      // This assumes that all prototype bits fit in an int32_t
      mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
    }
    str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
  
!   if (UseCompressedClassPointers) { // Take care not to kill klass
!     encode_klass_not_null(t1, klass);
!     strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
!   } else {
!     str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
    }
  
    if (len->is_valid()) {
      strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
!   } else if (UseCompressedClassPointers) {
      store_klass_gap(obj, zr);
    }
  }
  
  // preserves obj, destroys len_in_bytes
--- 194,31 ---
    }
  }
  
  void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
    assert_different_registers(obj, klass, len);
!   if (UseCompactObjectHeaders || (UseBiasedLocking && !len->is_valid())) {
      assert_different_registers(obj, klass, len, t1, t2);
      ldr(t1, Address(klass, Klass::prototype_header_offset()));
    } else {
      // This assumes that all prototype bits fit in an int32_t
      mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
    }
    str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
  
!   if (!UseCompactObjectHeaders) {
!     if (UseCompressedClassPointers) { // Take care not to kill klass
!       encode_klass_not_null(t1, klass);
!       strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
!     } else {
+       str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+     }
    }
  
    if (len->is_valid()) {
      strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
!   } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
      store_klass_gap(obj, zr);
    }
  }
  
  // preserves obj, destroys len_in_bytes

*** 214,10 ***
--- 233,16 ---
  
    // len_in_bytes is positive and ptr sized
    subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
    br(Assembler::EQ, done);
  
+   // Zero first 4 bytes, if start offset is not word aligned.
+   if (!is_aligned(hdr_size_in_bytes, BytesPerWord)) {
+     strw(zr, Address(obj, hdr_size_in_bytes));
+     hdr_size_in_bytes += BytesPerInt;
+   }
+ 
    // zero_words() takes ptr in r10 and count in words in r11
    mov(rscratch1, len_in_bytes);
    lea(t1, Address(obj, hdr_size_in_bytes));
    lsr(t2, rscratch1, LogBytesPerWord);
    zero_words(t1, t2);

*** 263,11 ***
      far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
    }
  
    verify_oop(obj);
  }
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
    assert_different_registers(obj, len, t1, t2, klass);
  
    // determine alignment mask
    assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
  
--- 288,11 ---
      far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
    }
  
    verify_oop(obj);
  }
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
    assert_different_registers(obj, len, t1, t2, klass);
  
    // determine alignment mask
    assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
  

*** 276,20 ***
    cmp(len, rscratch1);
    br(Assembler::HS, slow_case);
  
    const Register arr_size = t2; // okay to be the same
    // align object end
!   mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
    add(arr_size, arr_size, len, ext::uxtw, f);
    andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
  
    try_allocate(obj, arr_size, 0, t1, t2, slow_case);
  
    initialize_header(obj, klass, len, t1, t2);
  
    // clear rest of allocated space
!   initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
  
    membar(StoreStore);
  
    if (CURRENT_ENV->dtrace_alloc_probes()) {
      assert(obj == r0, "must be");
--- 301,20 ---
    cmp(len, rscratch1);
    br(Assembler::HS, slow_case);
  
    const Register arr_size = t2; // okay to be the same
    // align object end
!   mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
    add(arr_size, arr_size, len, ext::uxtw, f);
    andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
  
    try_allocate(obj, arr_size, 0, t1, t2, slow_case);
  
    initialize_header(obj, klass, len, t1, t2);
  
    // clear rest of allocated space
!   initialize_body(obj, arr_size, base_offset_in_bytes, t1, t2);
  
    membar(StoreStore);
  
    if (CURRENT_ENV->dtrace_alloc_probes()) {
      assert(obj == r0, "must be");

*** 302,12 ***
  
  void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
    verify_oop(receiver);
    // explicit NULL check not needed since load from [klass_offset] causes a trap
    // check against inline cache
!   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
! 
    cmp_klass(receiver, iCache, rscratch1);
  }
  
  
  void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
--- 327,15 ---
  
  void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
    verify_oop(receiver);
    // explicit NULL check not needed since load from [klass_offset] causes a trap
    // check against inline cache
!   if (UseCompactObjectHeaders) {
!     assert(!MacroAssembler::needs_explicit_null_check(oopDesc::mark_offset_in_bytes()), "must add explicit null check");
+   } else {
+     assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
+   }
    cmp_klass(receiver, iCache, rscratch1);
  }
  
  
  void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
< prev index next >