< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

        

*** 44,53 **** --- 44,54 ---- #include "runtime/biasedLocking.hpp" #include "runtime/icache.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/jniHandles.inline.hpp" #include "runtime/sharedRuntime.hpp" + #include "runtime/signature_cc.hpp" #include "runtime/thread.hpp" #ifdef COMPILER1 #include "c1/c1_LIRAssembler.hpp" #endif #ifdef COMPILER2
*** 1307,1317 **** bind(L_fallthrough); } void MacroAssembler::verify_oop(Register reg, const char* s) { ! if (!VerifyOops) return; // Pass register number to verify_oop_subroutine const char* b = NULL; { ResourceMark rm; --- 1308,1322 ---- bind(L_fallthrough); } void MacroAssembler::verify_oop(Register reg, const char* s) { ! if (!VerifyOops || VerifyAdapterSharing) { ! // Below address of the code string confuses VerifyAdapterSharing ! // because it may differ between otherwise equivalent adapters. ! return; ! } // Pass register number to verify_oop_subroutine const char* b = NULL; { ResourceMark rm;
*** 1337,1347 **** BLOCK_COMMENT("} verify_oop"); } void MacroAssembler::verify_oop_addr(Address addr, const char* s) { ! if (!VerifyOops) return; const char* b = NULL; { ResourceMark rm; stringStream ss; --- 1342,1356 ---- BLOCK_COMMENT("} verify_oop"); } void MacroAssembler::verify_oop_addr(Address addr, const char* s) { ! if (!VerifyOops || VerifyAdapterSharing) { ! // Below address of the code string confuses VerifyAdapterSharing ! // because it may differ between otherwise equivalent adapters. ! return; ! } const char* b = NULL; { ResourceMark rm; stringStream ss;
*** 1440,1449 **** --- 1449,1462 ---- pass_arg1(this, arg_1); pass_arg2(this, arg_2); call_VM_leaf_base(entry_point, 3); } + void MacroAssembler::super_call_VM_leaf(address entry_point) { + MacroAssembler::call_VM_leaf_base(entry_point, 1); + } + void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { pass_arg0(this, arg_0); MacroAssembler::call_VM_leaf_base(entry_point, 1); }
*** 1489,1498 **** --- 1502,1544 ---- // nothing to do, (later) access of M[reg + offset] // will provoke OS NULL exception if reg = NULL } } + void MacroAssembler::test_klass_is_value(Register klass, Register temp_reg, Label& is_value) { + ldrw(temp_reg, Address(klass, Klass::access_flags_offset())); + andr(temp_reg, temp_reg, JVM_ACC_VALUE); + cbnz(temp_reg, is_value); + } + + void MacroAssembler::test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable) { + (void) temp_reg; // keep signature uniform with x86 + tbnz(flags, ConstantPoolCacheEntry::is_flattenable_field_shift, is_flattenable); + } + + void MacroAssembler::test_field_is_not_flattenable(Register flags, Register temp_reg, Label& not_flattenable) { + (void) temp_reg; // keep signature uniform with x86 + tbz(flags, ConstantPoolCacheEntry::is_flattenable_field_shift, not_flattenable); + } + + void MacroAssembler::test_field_is_flattened(Register flags, Register temp_reg, Label& is_flattened) { + (void) temp_reg; // keep signature uniform with x86 + tbnz(flags, ConstantPoolCacheEntry::is_flattened_field_shift, is_flattened); + } + + void MacroAssembler::test_flattened_array_oop(Register oop, Register temp_reg, Label& is_flattened_array) { + load_storage_props(temp_reg, oop); + andr(temp_reg, temp_reg, ArrayStorageProperties::flattened_value); + cbnz(temp_reg, is_flattened_array); + } + + void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) { + load_storage_props(temp_reg, oop); + andr(temp_reg, temp_reg, ArrayStorageProperties::null_free_value); + cbnz(temp_reg, is_null_free_array); + } + // MacroAssembler protected routines needed to implement // public methods void MacroAssembler::mov(Register r, Address dest) { code_section()->relocate(pc(), dest.rspec());
*** 3681,3699 **** void MacroAssembler::cmpoop(Register obj1, Register obj2) { BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->obj_equals(this, obj1, obj2); } ! void MacroAssembler::load_klass(Register dst, Register src) { if (UseCompressedClassPointers) { ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); - decode_klass_not_null(dst); } else { ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); } } // ((OopHandle)result).resolve(); void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { // OopHandle::resolve is an indirection. access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg); } --- 3727,3754 ---- void MacroAssembler::cmpoop(Register obj1, Register obj2) { BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->obj_equals(this, obj1, obj2); } ! void MacroAssembler::load_metadata(Register dst, Register src) { if (UseCompressedClassPointers) { ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); } else { ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); } } + void MacroAssembler::load_klass(Register dst, Register src) { + load_metadata(dst, src); + if (UseCompressedClassPointers) { + andr(dst, dst, oopDesc::compressed_klass_mask()); + decode_klass_not_null(dst); + } else { + ubfm(dst, dst, 0, 63 - oopDesc::storage_props_nof_bits); + } + } + // ((OopHandle)result).resolve(); void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { // OopHandle::resolve is an indirection. access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg); }
*** 3705,3714 **** --- 3760,3778 ---- ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes())); ldr(dst, Address(dst, mirror_offset)); resolve_oop_handle(dst, tmp); } + void MacroAssembler::load_storage_props(Register dst, Register src) { + load_metadata(dst, src); + if (UseCompressedClassPointers) { + asrw(dst, dst, oopDesc::narrow_storage_props_shift); + } else { + asr(dst, dst, oopDesc::wide_storage_props_shift); + } + } + void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) { if (UseCompressedClassPointers) { ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes())); if (CompressedKlassPointers::base() == NULL) { cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
*** 4022,4039 **** } } void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src, ! Register tmp1, Register thread_tmp) { BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); decorators = AccessInternal::decorator_fixup(decorators); bool as_raw = (decorators & AS_RAW) != 0; if (as_raw) { ! bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp); } else { ! bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp); } } void MacroAssembler::resolve(DecoratorSet decorators, Register obj) { // Use stronger ACCESS_WRITE|ACCESS_READ by default. --- 4086,4104 ---- } } void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src, ! Register tmp1, Register thread_tmp, Register tmp3) { ! BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); decorators = AccessInternal::decorator_fixup(decorators); bool as_raw = (decorators & AS_RAW) != 0; if (as_raw) { ! bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp, tmp3); } else { ! bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp, tmp3); } } void MacroAssembler::resolve(DecoratorSet decorators, Register obj) { // Use stronger ACCESS_WRITE|ACCESS_READ by default.
*** 4053,4069 **** Register thread_tmp, DecoratorSet decorators) { access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); } void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1, ! Register thread_tmp, DecoratorSet decorators) { ! access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); } // Used for storing NULLs. void MacroAssembler::store_heap_oop_null(Address dst) { ! access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg); } Address MacroAssembler::allocate_metadata_address(Metadata* obj) { assert(oop_recorder() != NULL, "this assembler needs a Recorder"); int index = oop_recorder()->allocate_metadata_index(obj); --- 4118,4134 ---- Register thread_tmp, DecoratorSet decorators) { access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); } void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1, ! Register thread_tmp, Register tmp3, DecoratorSet decorators) { ! access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp, tmp3); } // Used for storing NULLs. void MacroAssembler::store_heap_oop_null(Address dst) { ! access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); } Address MacroAssembler::allocate_metadata_address(Metadata* obj) { assert(oop_recorder() != NULL, "this assembler needs a Recorder"); int index = oop_recorder()->allocate_metadata_index(obj);
*** 5863,5867 **** --- 5928,6326 ---- mov(dst, c_rarg0); } pop(saved_regs, sp); } + + // C2 compiled method's prolog code + // Moved here from aarch64.ad to support Valhalla code belows + void MacroAssembler::verified_entry(Compile* C, int sp_inc) { + + // n.b. frame size includes space for return pc and rfp + const long framesize = C->frame_size_in_bytes(); + assert(framesize % (2 * wordSize) == 0, "must preserve 2 * wordSize alignment"); + + // insert a nop at the start of the prolog so we can patch in a + // branch if we need to invalidate the method later + nop(); + + int bangsize = C->bang_size_in_bytes(); + if (C->need_stack_bang(bangsize) && UseStackBanging) + generate_stack_overflow_check(bangsize); + + build_frame(framesize); + + if (NotifySimulator) { + notify(Assembler::method_entry); + } + + if (VerifyStackAtCalls) { + Unimplemented(); + } + } + + int MacroAssembler::store_value_type_fields_to_buf(ciValueKlass* vk, bool from_interpreter) { + // A value type might be returned. If fields are in registers we + // need to allocate a value type instance and initialize it with + // the value of the fields. + Label skip; + // We only need a new buffered value if a new one is not returned + cmp(r0, (u1) 1); + br(Assembler::EQ, skip); + int call_offset = -1; + + Label slow_case; + + // Try to allocate a new buffered value (from the heap) + if (UseTLAB) { + + if (vk != NULL) { + // Called from C1, where the return type is statically known. + mov(r1, (intptr_t)vk->get_ValueKlass()); + jint lh = vk->layout_helper(); + assert(lh != Klass::_lh_neutral_value, "inline class in return type must have been resolved"); + mov(r14, lh); + } else { + // Call from interpreter. R0 contains ((the ValueKlass* of the return type) | 0x01) + andr(r1, r0, -2); + // get obj size + ldrw(r14, Address(rscratch1 /*klass*/, Klass::layout_helper_offset())); + } + + ldr(r13, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); + + // check whether we have space in TLAB, + // rscratch1 contains pointer to just allocated obj + lea(r14, Address(r13, r14)); + ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_end_offset()))); + + cmp(r14, rscratch1); + br(Assembler::GT, slow_case); + + // OK we have room in TLAB, + // Set new TLAB top + str(r14, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); + + // Set new class always locked + mov(rscratch1, (uint64_t) markOopDesc::always_locked_prototype()); + str(rscratch1, Address(r13, oopDesc::mark_offset_in_bytes())); + + store_klass_gap(r13, zr); // zero klass gap for compressed oops + if (vk == NULL) { + // store_klass corrupts rbx, so save it in rax for later use (interpreter case only). + mov(r0, r1); + } + + store_klass(r13, r1); // klass + + if (vk != NULL) { + // FIXME -- do the packing in-line to avoid the runtime call + mov(r0, r13); + far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint. + } else { + + // We have our new buffered value, initialize its fields with a + // value class specific handler + ldr(r1, Address(r0, InstanceKlass::adr_valueklass_fixed_block_offset())); + ldr(r1, Address(r1, ValueKlass::pack_handler_offset())); + + // Mov new class to r0 and call pack_handler + mov(r0, r13); + blr(r1); + } + b(skip); + } + + bind(slow_case); + // We failed to allocate a new value, fall back to a runtime + // call. Some oop field may be live in some registers but we can't + // tell. That runtime call will take care of preserving them + // across a GC if there's one. + + + if (from_interpreter) { + super_call_VM_leaf(StubRoutines::store_value_type_fields_to_buf()); + } else { + ldr(rscratch1, RuntimeAddress(StubRoutines::store_value_type_fields_to_buf())); + blr(rscratch1); + call_offset = offset(); + } + + bind(skip); + return call_offset; + } + + // Move a value between registers/stack slots and update the reg_state + bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[], int ret_off, int extra_stack_offset) { + if (reg_state[to->value()] == reg_written) { + return true; // Already written + } + + if (from != to && bt != T_VOID) { + if (reg_state[to->value()] == reg_readonly) { + return false; // Not yet writable + } + if (from->is_reg()) { + if (to->is_reg()) { + mov(to->as_Register(), from->as_Register()); + } else { + int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset; + Address to_addr = Address(sp, st_off); + if (from->is_FloatRegister()) { + if (bt == T_DOUBLE) { + strd(from->as_FloatRegister(), to_addr); + } else { + assert(bt == T_FLOAT, "must be float"); + strs(from->as_FloatRegister(), to_addr); + } + } else { + str(from->as_Register(), to_addr); + } + } + } else { + Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset); + if (to->is_reg()) { + if (to->is_FloatRegister()) { + if (bt == T_DOUBLE) { + ldrd(to->as_FloatRegister(), from_addr); + } else { + assert(bt == T_FLOAT, "must be float"); + ldrs(to->as_FloatRegister(), from_addr); + } + } else { + ldr(to->as_Register(), from_addr); + } + } else { + int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset; + ldr(rscratch1, from_addr); + str(rscratch1, Address(sp, st_off)); + } + } + } + + // Update register states + reg_state[from->value()] = reg_writable; + reg_state[to->value()] = reg_written; + return true; + } + + // Read all fields from a value type oop and store the values in registers/stack slots + bool MacroAssembler::unpack_value_helper(const GrowableArray<SigEntry>* sig, int& sig_index, VMReg from, VMRegPair* regs_to, + int& to_index, RegState reg_state[], int ret_off, int extra_stack_offset) { + Register fromReg = from->is_reg() ? from->as_Register() : noreg; + assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter"); + + + int vt = 1; + bool done = true; + bool mark_done = true; + do { + sig_index--; + BasicType bt = sig->at(sig_index)._bt; + if (bt == T_VALUETYPE) { + vt--; + } else if (bt == T_VOID && + sig->at(sig_index-1)._bt != T_LONG && + sig->at(sig_index-1)._bt != T_DOUBLE) { + vt++; + } else if (SigEntry::is_reserved_entry(sig, sig_index)) { + to_index--; // Ignore this + } else { + assert(to_index >= 0, "invalid to_index"); + VMRegPair pair_to = regs_to[to_index--]; + VMReg to = pair_to.first(); + + if (bt == T_VOID) continue; + + int idx = (int) to->value(); + if (reg_state[idx] == reg_readonly) { + if (idx != from->value()) { + mark_done = false; + } + done = false; + continue; + } else if (reg_state[idx] == reg_written) { + continue; + } else { + assert(reg_state[idx] == reg_writable, "must be writable"); + reg_state[idx] = reg_written; + } + + if (fromReg == noreg) { + int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset; + ldr(rscratch2, Address(sp, st_off)); + fromReg = rscratch2; + } + + int off = sig->at(sig_index)._offset; + assert(off > 0, "offset in object should be positive"); + bool is_oop = (bt == T_OBJECT || bt == T_ARRAY); + + Address fromAddr = Address(fromReg, off); + bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN); + + if (!to->is_FloatRegister()) { + + Register dst = to->is_stack() ? rscratch1 : to->as_Register(); + + if (is_oop) { + load_heap_oop(dst, fromAddr); + } else { + load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed); + } + if (to->is_stack()) { + int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset; + str(dst, Address(sp, st_off)); + } + } else { + if (bt == T_DOUBLE) { + ldrd(to->as_FloatRegister(), fromAddr); + } else { + assert(bt == T_FLOAT, "must be float"); + ldrs(to->as_FloatRegister(), fromAddr); + } + } + + } + + } while (vt != 0); + + if (mark_done && reg_state[from->value()] != reg_written) { + // This is okay because no one else will write to that slot + reg_state[from->value()] = reg_writable; + } + return done; + } + + // Pack fields back into a value type oop + bool MacroAssembler::pack_value_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index, + VMReg to, VMRegPair* regs_from, int regs_from_count, int& from_index, RegState reg_state[], + int ret_off, int extra_stack_offset) { + assert(sig->at(sig_index)._bt == T_VALUETYPE, "should be at end delimiter"); + assert(to->is_valid(), "must be"); + + if (reg_state[to->value()] == reg_written) { + skip_unpacked_fields(sig, sig_index, regs_from, regs_from_count, from_index); + return true; // Already written + } + + Register val_array = r0; + Register val_obj_tmp = r11; + Register from_reg_tmp = r10; + Register tmp1 = r14; + Register tmp2 = r13; + Register tmp3 = r1; + Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register(); + + if (reg_state[to->value()] == reg_readonly) { + if (!is_reg_in_unpacked_fields(sig, sig_index, to, regs_from, regs_from_count, from_index)) { + skip_unpacked_fields(sig, sig_index, regs_from, regs_from_count, from_index); + return false; // Not yet writable + } + val_obj = val_obj_tmp; + } + + int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_VALUETYPE); + load_heap_oop(val_obj, Address(val_array, index)); + + ScalarizedValueArgsStream stream(sig, sig_index, regs_from, regs_from_count, from_index); + VMRegPair from_pair; + BasicType bt; + + while (stream.next(from_pair, bt)) { + int off = sig->at(stream.sig_cc_index())._offset; + assert(off > 0, "offset in object should be positive"); + bool is_oop = (bt == T_OBJECT || bt == T_ARRAY); + size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize; + + VMReg from_r1 = from_pair.first(); + VMReg from_r2 = from_pair.second(); + + // Pack the scalarized field into the value object. + Address dst(val_obj, off); + + if (!from_r1->is_FloatRegister()) { + Register from_reg; + if (from_r1->is_stack()) { + from_reg = from_reg_tmp; + int ld_off = from_r1->reg2stack() * VMRegImpl::stack_slot_size + extra_stack_offset; + load_sized_value(from_reg, Address(sp, ld_off), size_in_bytes, /* is_signed */ false); + } else { + from_reg = from_r1->as_Register(); + } + + if (is_oop) { + DecoratorSet decorators = IN_HEAP | ACCESS_WRITE; + store_heap_oop(dst, from_reg, tmp1, tmp2, tmp3, decorators); + } else { + store_sized_value(dst, from_reg, size_in_bytes); + } + } else { + if (from_r2->is_valid()) { + strd(from_r1->as_FloatRegister(), dst); + } else { + strs(from_r1->as_FloatRegister(), dst); + } + } + + reg_state[from_r1->value()] = reg_writable; + } + sig_index = stream.sig_cc_index(); + from_index = stream.regs_cc_index(); + + assert(reg_state[to->value()] == reg_writable, "must have already been read"); + bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state, ret_off, extra_stack_offset); + assert(success, "to register must be writeable"); + + return true; + } + + // Unpack all value type arguments passed as oops + void MacroAssembler::unpack_value_args(Compile* C, bool receiver_only) { + int sp_inc = unpack_value_args_common(C, receiver_only); + // Emit code for verified entry and save increment for stack repair on return + verified_entry(C, sp_inc); + } + + int MacroAssembler::shuffle_value_args(bool is_packing, bool receiver_only, int extra_stack_offset, + BasicType* sig_bt, const GrowableArray<SigEntry>* sig_cc, + int args_passed, int args_on_stack, VMRegPair* regs, // from + int args_passed_to, int args_on_stack_to, VMRegPair* regs_to) { // to + // Check if we need to extend the stack for packing/unpacking + int sp_inc = (args_on_stack_to - args_on_stack) * VMRegImpl::stack_slot_size; + if (sp_inc > 0) { + sp_inc = align_up(sp_inc, StackAlignmentInBytes); + if (!is_packing) { + // Save the return address, adjust the stack (make sure it is properly + // 16-byte aligned) and copy the return address to the new top of the stack. + // (Note: C1 does this in C1_MacroAssembler::scalarized_entry). + // FIXME: We need not to preserve return address on aarch64 + pop(rscratch1); + sub(sp, sp, sp_inc); + push(rscratch1); + } + } else { + // The scalarized calling convention needs less stack space than the unscalarized one. + // No need to extend the stack, the caller will take care of these adjustments. + sp_inc = 0; + } + + int ret_off; // make sure we don't overwrite the return address + if (is_packing) { + // For C1 code, the VVEP doesn't have reserved slots, so we store the returned address at + // rsp[0] during shuffling. + ret_off = 0; + } else { + // C2 code ensures that sp_inc is a reserved slot. + ret_off = sp_inc; + } + + return shuffle_value_args_common(is_packing, receiver_only, extra_stack_offset, + sig_bt, sig_cc, + args_passed, args_on_stack, regs, + args_passed_to, args_on_stack_to, regs_to, + sp_inc, ret_off); + } + + VMReg MacroAssembler::spill_reg_for(VMReg reg) { + return (reg->is_FloatRegister()) ? v0->as_VMReg() : r14->as_VMReg(); + }
< prev index next >