< prev index next >

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

Print this page
*** 81,41 ***
      br(Assembler::NE, slow_case);
    }
  
    // Load object header
    ldr(hdr, Address(obj, hdr_offset));
!   // and mark it as unlocked
!   orr(hdr, hdr, markWord::unlocked_value);
!   // save unlocked object header into the displaced header location on the stack
!   str(hdr, Address(disp_hdr, 0));
!   // test if object header is still the same (i.e. unlocked), and if so, store the
!   // displaced header address in the object header - if it is not the same, get the
!   // object header instead
!   lea(rscratch2, Address(obj, hdr_offset));
!   cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
!   // if the object header was the same, we're done
!   // if the object header was not the same, it is now in the hdr register
!   // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
!   //
!   // 1) (hdr & aligned_mask) == 0
!   // 2) sp <= hdr
!   // 3) hdr <= sp + page_size
!   //
!   // these 3 tests can be done by evaluating the following expression:
!   //
!   // (hdr - sp) & (aligned_mask - page_size)
!   //
!   // assuming both the stack pointer and page_size have their least
!   // significant 2 bits cleared and page_size is a power of 2
!   mov(rscratch1, sp);
!   sub(hdr, hdr, rscratch1);
!   ands(hdr, hdr, aligned_mask - os::vm_page_size());
!   // for recursive locking, the result is zero => save it in the displaced header
!   // location (NULL in the displaced hdr location indicates recursive locking)
!   str(hdr, Address(disp_hdr, 0));
!   // otherwise we don't care about the result and handle locking via runtime call
!   cbnz(hdr, slow_case);
    // done
    bind(done);
    increment(Address(rthread, JavaThread::held_monitor_count_offset()));
    return null_check_offset;
  }
--- 81,45 ---
      br(Assembler::NE, slow_case);
    }
  
    // Load object header
    ldr(hdr, Address(obj, hdr_offset));
!   if (UseFastLocking) {
!     fast_lock(obj, hdr, rscratch1, rscratch2, slow_case, false);
!   } else {
!     // and mark it as unlocked
!     orr(hdr, hdr, markWord::unlocked_value);
!     // save unlocked object header into the displaced header location on the stack
!     str(hdr, Address(disp_hdr, 0));
!     // test if object header is still the same (i.e. unlocked), and if so, store the
!     // displaced header address in the object header - if it is not the same, get the
!     // object header instead
!     lea(rscratch2, Address(obj, hdr_offset));
!     cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
!     // if the object header was the same, we're done
!     // if the object header was not the same, it is now in the hdr register
!     // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
!     //
!     // 1) (hdr & aligned_mask) == 0
!     // 2) sp <= hdr
!     // 3) hdr <= sp + page_size
!     //
!     // these 3 tests can be done by evaluating the following expression:
!     //
!     // (hdr - sp) & (aligned_mask - page_size)
!     //
!     // assuming both the stack pointer and page_size have their least
!     // significant 2 bits cleared and page_size is a power of 2
!     mov(rscratch1, sp);
!     sub(hdr, hdr, rscratch1);
!     ands(hdr, hdr, aligned_mask - os::vm_page_size());
!     // for recursive locking, the result is zero => save it in the displaced header
!     // location (NULL in the displaced hdr location indicates recursive locking)
+     str(hdr, Address(disp_hdr, 0));
+     // otherwise we don't care about the result and handle locking via runtime call
+     cbnz(hdr, slow_case);
+   }
    // done
    bind(done);
    increment(Address(rthread, JavaThread::held_monitor_count_offset()));
    return null_check_offset;
  }

*** 125,31 ***
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
    assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
    Label done;
  
!   // load displaced header
!   ldr(hdr, Address(disp_hdr, 0));
!   // if the loaded hdr is NULL we had recursive locking
!   // if we had recursive locking, we are done
!   cbz(hdr, done);
    // load object
    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
    verify_oop(obj);
!   // test if object header is pointing to the displaced header, and if so, restore
!   // the displaced header in the object - if the object header is not pointing to
!   // the displaced header, get the object header instead
!   // if the object header was not pointing to the displaced header,
-   // we do unlocking via runtime call
-   if (hdr_offset) {
-     lea(rscratch1, Address(obj, hdr_offset));
-     cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
    } else {
!     cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
    }
-   // done
-   bind(done);
    decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
  }
  
  
  // Defines obj, preserves var_size_in_bytes
--- 129,40 ---
    const int aligned_mask = BytesPerWord -1;
    const int hdr_offset = oopDesc::mark_offset_in_bytes();
    assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
    Label done;
  
!   if (!UseFastLocking) {
!     // load displaced header
!     ldr(hdr, Address(disp_hdr, 0));
!     // if the loaded hdr is NULL we had recursive locking
!     // if we had recursive locking, we are done
+     cbz(hdr, done);
+   }
+ 
    // load object
    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
    verify_oop(obj);
! 
!   if (UseFastLocking) {
!     ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
!     fast_unlock(obj, hdr, rscratch1, rscratch2, slow_case);
    } else {
!     // test if object header is pointing to the displaced header, and if so, restore
+     // the displaced header in the object - if the object header is not pointing to
+     // the displaced header, get the object header instead
+     // if the object header was not pointing to the displaced header,
+     // we do unlocking via runtime call
+     if (hdr_offset) {
+       lea(rscratch1, Address(obj, hdr_offset));
+       cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+     } else {
+       cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+     }
+     // done
+     bind(done);
    }
    decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
  }
  
  
  // Defines obj, preserves var_size_in_bytes

*** 161,25 ***
    }
  }
  
  void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
    assert_different_registers(obj, klass, len);
!   // This assumes that all prototype bits fit in an int32_t
-   mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
    str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
  
-   if (UseCompressedClassPointers) { // Take care not to kill klass
-     encode_klass_not_null(t1, klass);
-     strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
-   } else {
-     str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
-   }
- 
    if (len->is_valid()) {
      strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
-   } else if (UseCompressedClassPointers) {
-     store_klass_gap(obj, zr);
    }
  }
  
  // preserves obj, destroys len_in_bytes
  //
--- 174,15 ---
    }
  }
  
  void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
    assert_different_registers(obj, klass, len);
!   ldr(t1, Address(klass, Klass::prototype_header_offset()));
    str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
  
    if (len->is_valid()) {
      strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
    }
  }
  
  // preserves obj, destroys len_in_bytes
  //

*** 193,10 ***
--- 196,16 ---
  
    // len_in_bytes is positive and ptr sized
    subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
    br(Assembler::EQ, done);
  
+   // Zero first 4 bytes, if start offset is not word aligned.
+   if (!is_aligned(hdr_size_in_bytes, BytesPerWord)) {
+     strw(zr, Address(obj, hdr_size_in_bytes));
+     hdr_size_in_bytes += BytesPerInt;
+   }
+ 
    // zero_words() takes ptr in r10 and count in words in r11
    mov(rscratch1, len_in_bytes);
    lea(t1, Address(obj, hdr_size_in_bytes));
    lsr(t2, rscratch1, LogBytesPerWord);
    address tpc = zero_words(t1, t2);

*** 252,11 ***
      far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
    }
  
    verify_oop(obj);
  }
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
    assert_different_registers(obj, len, t1, t2, klass);
  
    // determine alignment mask
    assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
  
--- 261,11 ---
      far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
    }
  
    verify_oop(obj);
  }
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
    assert_different_registers(obj, len, t1, t2, klass);
  
    // determine alignment mask
    assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
  

*** 265,20 ***
    cmp(len, rscratch1);
    br(Assembler::HS, slow_case);
  
    const Register arr_size = t2; // okay to be the same
    // align object end
!   mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
    add(arr_size, arr_size, len, ext::uxtw, f);
    andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
  
    try_allocate(obj, arr_size, 0, t1, t2, slow_case);
  
    initialize_header(obj, klass, len, t1, t2);
  
    // clear rest of allocated space
!   initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
    if (Compilation::current()->bailed_out()) {
      return;
    }
  
    membar(StoreStore);
--- 274,20 ---
    cmp(len, rscratch1);
    br(Assembler::HS, slow_case);
  
    const Register arr_size = t2; // okay to be the same
    // align object end
!   mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
    add(arr_size, arr_size, len, ext::uxtw, f);
    andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
  
    try_allocate(obj, arr_size, 0, t1, t2, slow_case);
  
    initialize_header(obj, klass, len, t1, t2);
  
    // clear rest of allocated space
!   initialize_body(obj, arr_size, base_offset_in_bytes, t1, t2);
    if (Compilation::current()->bailed_out()) {
      return;
    }
  
    membar(StoreStore);

*** 294,23 ***
  
  void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
    verify_oop(receiver);
    // explicit NULL check not needed since load from [klass_offset] causes a trap
    // check against inline cache
!   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
  
    cmp_klass(receiver, iCache, rscratch1);
  }
  
  
! void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
    assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
    // Make sure there is enough stack space for this method's activation.
    // Note that we do this before creating a frame.
    generate_stack_overflow_check(bang_size_in_bytes);
    MacroAssembler::build_frame(framesize);
  
    // Insert nmethod entry barrier into frame.
    BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */);
  }
  
--- 303,35 ---
  
  void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
    verify_oop(receiver);
    // explicit NULL check not needed since load from [klass_offset] causes a trap
    // check against inline cache
!   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::mark_offset_in_bytes()), "must add explicit null check");
  
    cmp_klass(receiver, iCache, rscratch1);
  }
  
  
! void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes, int max_monitors) {
    assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
    // Make sure there is enough stack space for this method's activation.
    // Note that we do this before creating a frame.
    generate_stack_overflow_check(bang_size_in_bytes);
    MacroAssembler::build_frame(framesize);
  
+   if (UseFastLocking && max_monitors > 0) {
+     Label ok;
+     ldr(r9, Address(rthread, JavaThread::lock_stack_current_offset()));
+     ldr(r10, Address(rthread, JavaThread::lock_stack_limit_offset()));
+     add(r9, r9, max_monitors * oopSize);
+     cmp(r9, r10);
+     br(Assembler::LT, ok);
+     assert(StubRoutines::aarch64::check_lock_stack() != NULL, "need runtime call stub");
+     far_call(StubRoutines::aarch64::check_lock_stack());
+     bind(ok);
+   }
+ 
    // Insert nmethod entry barrier into frame.
    BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */);
  }
  
< prev index next >