< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page




  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "compiler/disassembler.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "memory/universe.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/accessDecorators.hpp"
  42 #include "oops/compressedOops.inline.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "runtime/biasedLocking.hpp"
  45 #include "runtime/icache.hpp"
  46 #include "runtime/interfaceSupport.inline.hpp"
  47 #include "runtime/jniHandles.inline.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "runtime/thread.hpp"
  50 #ifdef COMPILER1
  51 #include "c1/c1_LIRAssembler.hpp"
  52 #endif
  53 #ifdef COMPILER2
  54 #include "oops/oop.hpp"
  55 #include "opto/compile.hpp"
  56 #include "opto/intrinsicnode.hpp"
  57 #include "opto/node.hpp"
  58 #endif
  59 


 796 
 797   // Now, create the trampoline stub's code:
 798   // - load the call
 799   // - call
 800   Label target;
 801   ldr(rscratch1, target);
 802   br(rscratch1);
 803   bind(target);
 804   assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
 805          "should be");
 806   emit_int64((int64_t)dest);
 807 
 808   const address stub_start_addr = addr_at(stub_start_offset);
 809 
 810   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
 811 
 812   end_a_stub();
 813   return stub_start_addr;
 814 }
 815 
 816 void MacroAssembler::emit_static_call_stub() {
 817   // CompiledDirectStaticCall::set_to_interpreted knows the
 818   // exact layout of this stub.
 819 
 820   isb();
 821   mov_metadata(rmethod, (Metadata*)NULL);
 822 
 823   // Jump to the entry point of the i2c stub.
 824   movptr(rscratch1, 0);
 825   br(rscratch1);
 826 }
 827 
 828 void MacroAssembler::c2bool(Register x) {
 829   // implements x == 0 ? 0 : 1
 830   // note: must only look at least-significant byte of x
 831   //       since C-style booleans are stored in one byte
 832   //       only! (was bug)
 833   tst(x, 0xff);
 834   cset(x, Assembler::NE);
 835 }
 836 
 837 address MacroAssembler::ic_call(address entry, jint method_index) {
 838   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 839   // address const_ptr = long_constant((jlong)Universe::non_oop_word());
 840   // unsigned long offset;
 841   // ldr_constant(rscratch2, const_ptr);
 842   movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
 843   return trampoline_call(Address(entry, rh));
 844 }
 845 
 846 // Implementation of call_VM versions
 847 


2115     words_pushed += 2;
2116   }
2117   if (count) {
2118     ldp(as_Register(regs[0]), as_Register(regs[1]),
2119        Address(post(stack, count * wordSize)));
2120     words_pushed += 2;
2121   }
2122 
2123   assert(words_pushed == count, "oops, pushed != count");
2124 
2125   return count;
2126 }
2127 #ifdef ASSERT
2128 void MacroAssembler::verify_heapbase(const char* msg) {
2129 #if 0
2130   assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
2131   assert (Universe::heap() != NULL, "java heap should be initialized");
2132   if (CheckCompressedOops) {
2133     Label ok;
2134     push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
2135     cmpptr(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
2136     br(Assembler::EQ, ok);
2137     stop(msg);
2138     bind(ok);
2139     pop(1 << rscratch1->encoding(), sp);
2140   }
2141 #endif
2142 }
2143 #endif
2144 
2145 void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) {
2146   Label done, not_weak;
2147   cbz(value, done);           // Use NULL as-is.
2148 
2149   STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2150   tbz(r0, 0, not_weak);    // Test for jweak tag.
2151 
2152   // Resolve jweak.
2153   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
2154                  Address(value, -JNIHandles::weak_tag_value), tmp, thread);
2155   verify_oop(value);


2248 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) {
2249   if (decrement.is_register()) {
2250     sub(Rd, Rn, decrement.as_register());
2251   } else {
2252     sub(Rd, Rn, decrement.as_constant());
2253   }
2254 }
2255 
2256 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) {
2257   if (decrement.is_register()) {
2258     subw(Rd, Rn, decrement.as_register());
2259   } else {
2260     subw(Rd, Rn, decrement.as_constant());
2261   }
2262 }
2263 
2264 void MacroAssembler::reinit_heapbase()
2265 {
2266   if (UseCompressedOops) {
2267     if (Universe::is_fully_initialized()) {
2268       mov(rheapbase, CompressedOops::ptrs_base());
2269     } else {
2270       lea(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
2271       ldr(rheapbase, Address(rheapbase));
2272     }
2273   }
2274 }
2275 
2276 // this simulates the behaviour of the x86 cmpxchg instruction using a
2277 // load linked/store conditional pair. we use the acquire/release
2278 // versions of these instructions so that we flush pending writes as
2279 // per Java semantics.
2280 
2281 // n.b the x86 version assumes the old value to be compared against is
2282 // in rax and updates rax with the value located in memory if the
2283 // cmpxchg fails. we supply a register for the old value explicitly
2284 
2285 // the aarch64 load linked/store conditional instructions do not
2286 // accept an offset. so, unlike x86, we must provide a plain register
2287 // to identify the memory word to be compared/exchanged rather than a
2288 // register+offset Address.
2289 
2290 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,


3693 }
3694 
3695 // ((OopHandle)result).resolve();
3696 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3697   // OopHandle::resolve is an indirection.
3698   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3699 }
3700 
3701 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3702   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3703   ldr(dst, Address(rmethod, Method::const_offset()));
3704   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3705   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3706   ldr(dst, Address(dst, mirror_offset));
3707   resolve_oop_handle(dst, tmp);
3708 }
3709 
3710 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3711   if (UseCompressedClassPointers) {
3712     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3713     if (CompressedKlassPointers::base() == NULL) {
3714       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3715       return;
3716     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3717                && CompressedKlassPointers::shift() == 0) {
3718       // Only the bottom 32 bits matter
3719       cmpw(trial_klass, tmp);
3720       return;
3721     }
3722     decode_klass_not_null(tmp);
3723   } else {
3724     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3725   }
3726   cmp(trial_klass, tmp);
3727 }
3728 
3729 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3730   load_klass(dst, src);
3731   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3732 }
3733 
3734 void MacroAssembler::store_klass(Register dst, Register src) {
3735   // FIXME: Should this be a store release?  concurrent gcs assumes
3736   // klass length is valid if klass field is not null.
3737   if (UseCompressedClassPointers) {
3738     encode_klass_not_null(src);
3739     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3740   } else {
3741     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3742   }
3743 }
3744 
3745 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3746   if (UseCompressedClassPointers) {
3747     // Store to klass gap in destination
3748     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3749   }
3750 }
3751 
3752 // Algorithm must match CompressedOops::encode.
3753 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3754 #ifdef ASSERT
3755   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3756 #endif
3757   verify_oop(s, "broken oop in encode_heap_oop");
3758   if (CompressedOops::base() == NULL) {
3759     if (CompressedOops::shift() != 0) {
3760       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3761       lsr(d, s, LogMinObjAlignmentInBytes);
3762     } else {
3763       mov(d, s);
3764     }
3765   } else {
3766     subs(d, s, rheapbase);
3767     csel(d, d, zr, Assembler::HS);
3768     lsr(d, d, LogMinObjAlignmentInBytes);
3769 
3770     /*  Old algorithm: is this any worse?
3771     Label nonnull;
3772     cbnz(r, nonnull);
3773     sub(r, r, rheapbase);
3774     bind(nonnull);
3775     lsr(r, r, LogMinObjAlignmentInBytes);
3776     */
3777   }
3778 }
3779 
3780 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3781 #ifdef ASSERT
3782   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
3783   if (CheckCompressedOops) {
3784     Label ok;
3785     cbnz(r, ok);
3786     stop("null oop passed to encode_heap_oop_not_null");
3787     bind(ok);
3788   }
3789 #endif
3790   verify_oop(r, "broken oop in encode_heap_oop_not_null");
3791   if (CompressedOops::base() != NULL) {
3792     sub(r, r, rheapbase);
3793   }
3794   if (CompressedOops::shift() != 0) {
3795     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3796     lsr(r, r, LogMinObjAlignmentInBytes);
3797   }
3798 }
3799 
3800 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3801 #ifdef ASSERT
3802   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
3803   if (CheckCompressedOops) {
3804     Label ok;
3805     cbnz(src, ok);
3806     stop("null oop passed to encode_heap_oop_not_null2");
3807     bind(ok);
3808   }
3809 #endif
3810   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
3811 
3812   Register data = src;
3813   if (CompressedOops::base() != NULL) {
3814     sub(dst, src, rheapbase);
3815     data = dst;
3816   }
3817   if (CompressedOops::shift() != 0) {
3818     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3819     lsr(dst, data, LogMinObjAlignmentInBytes);
3820     data = dst;
3821   }
3822   if (data == src)
3823     mov(dst, src);
3824 }
3825 
3826 void  MacroAssembler::decode_heap_oop(Register d, Register s) {
3827 #ifdef ASSERT
3828   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
3829 #endif
3830   if (CompressedOops::base() == NULL) {
3831     if (CompressedOops::shift() != 0 || d != s) {
3832       lsl(d, s, CompressedOops::shift());
3833     }
3834   } else {
3835     Label done;
3836     if (d != s)
3837       mov(d, s);
3838     cbz(s, done);
3839     add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes);
3840     bind(done);
3841   }
3842   verify_oop(d, "broken oop in decode_heap_oop");
3843 }
3844 
3845 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
3846   assert (UseCompressedOops, "should only be used for compressed headers");
3847   assert (Universe::heap() != NULL, "java heap should be initialized");
3848   // Cannot assert, unverified entry point counts instructions (see .ad file)
3849   // vtableStubs also counts instructions in pd_code_size_limit.
3850   // Also do not verify_oop as this is called by verify_oop.
3851   if (CompressedOops::shift() != 0) {
3852     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3853     if (CompressedOops::base() != NULL) {
3854       add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3855     } else {
3856       add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3857     }
3858   } else {
3859     assert (CompressedOops::base() == NULL, "sanity");
3860   }
3861 }
3862 
3863 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3864   assert (UseCompressedOops, "should only be used for compressed headers");
3865   assert (Universe::heap() != NULL, "java heap should be initialized");
3866   // Cannot assert, unverified entry point counts instructions (see .ad file)
3867   // vtableStubs also counts instructions in pd_code_size_limit.
3868   // Also do not verify_oop as this is called by verify_oop.
3869   if (CompressedOops::shift() != 0) {
3870     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3871     if (CompressedOops::base() != NULL) {
3872       add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3873     } else {
3874       add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3875     }
3876   } else {
3877     assert (CompressedOops::base() == NULL, "sanity");
3878     if (dst != src) {
3879       mov(dst, src);
3880     }
3881   }
3882 }
3883 
3884 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3885   if (CompressedKlassPointers::base() == NULL) {
3886     if (CompressedKlassPointers::shift() != 0) {
3887       assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3888       lsr(dst, src, LogKlassAlignmentInBytes);
3889     } else {
3890       if (dst != src) mov(dst, src);
3891     }
3892     return;
3893   }
3894 
3895   if (use_XOR_for_compressed_class_base) {
3896     if (CompressedKlassPointers::shift() != 0) {
3897       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3898       lsr(dst, dst, LogKlassAlignmentInBytes);
3899     } else {
3900       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3901     }
3902     return;
3903   }
3904 
3905   if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3906       && CompressedKlassPointers::shift() == 0) {
3907     movw(dst, src);
3908     return;
3909   }
3910 
3911 #ifdef ASSERT
3912   verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
3913 #endif
3914 
3915   Register rbase = dst;
3916   if (dst == src) rbase = rheapbase;
3917   mov(rbase, (uint64_t)CompressedKlassPointers::base());
3918   sub(dst, src, rbase);
3919   if (CompressedKlassPointers::shift() != 0) {
3920     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3921     lsr(dst, dst, LogKlassAlignmentInBytes);
3922   }
3923   if (dst == src) reinit_heapbase();
3924 }
3925 
3926 void MacroAssembler::encode_klass_not_null(Register r) {
3927   encode_klass_not_null(r, r);
3928 }
3929 
3930 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3931   Register rbase = dst;
3932   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3933 
3934   if (CompressedKlassPointers::base() == NULL) {
3935     if (CompressedKlassPointers::shift() != 0) {
3936       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3937       lsl(dst, src, LogKlassAlignmentInBytes);
3938     } else {
3939       if (dst != src) mov(dst, src);
3940     }
3941     return;
3942   }
3943 
3944   if (use_XOR_for_compressed_class_base) {
3945     if (CompressedKlassPointers::shift() != 0) {
3946       lsl(dst, src, LogKlassAlignmentInBytes);
3947       eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
3948     } else {
3949       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3950     }
3951     return;
3952   }
3953 
3954   if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3955       && CompressedKlassPointers::shift() == 0) {
3956     if (dst != src)
3957       movw(dst, src);
3958     movk(dst, (uint64_t)CompressedKlassPointers::base() >> 32, 32);
3959     return;
3960   }
3961 
3962   // Cannot assert, unverified entry point counts instructions (see .ad file)
3963   // vtableStubs also counts instructions in pd_code_size_limit.
3964   // Also do not verify_oop as this is called by verify_oop.
3965   if (dst == src) rbase = rheapbase;
3966   mov(rbase, (uint64_t)CompressedKlassPointers::base());
3967   if (CompressedKlassPointers::shift() != 0) {
3968     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3969     add(dst, rbase, src, Assembler::LSL, LogKlassAlignmentInBytes);
3970   } else {
3971     add(dst, rbase, src);
3972   }
3973   if (dst == src) reinit_heapbase();
3974 }
3975 
3976 void  MacroAssembler::decode_klass_not_null(Register r) {
3977   decode_klass_not_null(r, r);
3978 }
3979 
3980 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3981 #ifdef ASSERT
3982   {
3983     ThreadInVMfromUnknown tiv;
3984     assert (UseCompressedOops, "should only be used for compressed oops");
3985     assert (Universe::heap() != NULL, "java heap should be initialized");
3986     assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3987     assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3988   }
3989 #endif
3990   int oop_index = oop_recorder()->find_index(obj);
3991   InstructionMark im(this);
3992   RelocationHolder rspec = oop_Relocation::spec(oop_index);
3993   code_section()->relocate(inst_mark(), rspec);
3994   movz(dst, 0xDEAD, 16);
3995   movk(dst, 0xBEEF);
3996 }
3997 
3998 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3999   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4000   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4001   int index = oop_recorder()->find_index(k);
4002   assert(! Universe::heap()->is_in_reserved(k), "should not be an oop");
4003 
4004   InstructionMark im(this);
4005   RelocationHolder rspec = metadata_Relocation::spec(index);
4006   code_section()->relocate(inst_mark(), rspec);
4007   narrowKlass nk = CompressedKlassPointers::encode(k);
4008   movz(dst, (nk >> 16), 16);
4009   movk(dst, nk & 0xffff);
4010 }
4011 
4012 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
4013                                     Register dst, Address src,
4014                                     Register tmp1, Register thread_tmp) {
4015   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4016   decorators = AccessInternal::decorator_fixup(decorators);
4017   bool as_raw = (decorators & AS_RAW) != 0;
4018   if (as_raw) {
4019     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4020   } else {
4021     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4022   }
4023 }
4024 
4025 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4026                                      Address dst, Register src,
4027                                      Register tmp1, Register thread_tmp) {




  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "compiler/disassembler.hpp"
  38 #include "memory/resourceArea.hpp"

  39 #include "nativeInst_aarch64.hpp"
  40 #include "oops/accessDecorators.hpp"
  41 #include "oops/compressedOops.inline.hpp"
  42 #include "oops/klass.inline.hpp"
  43 #include "runtime/biasedLocking.hpp"
  44 #include "runtime/icache.hpp"
  45 #include "runtime/interfaceSupport.inline.hpp"
  46 #include "runtime/jniHandles.inline.hpp"
  47 #include "runtime/sharedRuntime.hpp"
  48 #include "runtime/thread.hpp"
  49 #ifdef COMPILER1
  50 #include "c1/c1_LIRAssembler.hpp"
  51 #endif
  52 #ifdef COMPILER2
  53 #include "oops/oop.hpp"
  54 #include "opto/compile.hpp"
  55 #include "opto/intrinsicnode.hpp"
  56 #include "opto/node.hpp"
  57 #endif
  58 


 795 
 796   // Now, create the trampoline stub's code:
 797   // - load the call
 798   // - call
 799   Label target;
 800   ldr(rscratch1, target);
 801   br(rscratch1);
 802   bind(target);
 803   assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
 804          "should be");
 805   emit_int64((int64_t)dest);
 806 
 807   const address stub_start_addr = addr_at(stub_start_offset);
 808 
 809   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
 810 
 811   end_a_stub();
 812   return stub_start_addr;
 813 }
 814 












 815 void MacroAssembler::c2bool(Register x) {
 816   // implements x == 0 ? 0 : 1
 817   // note: must only look at least-significant byte of x
 818   //       since C-style booleans are stored in one byte
 819   //       only! (was bug)
 820   tst(x, 0xff);
 821   cset(x, Assembler::NE);
 822 }
 823 
 824 address MacroAssembler::ic_call(address entry, jint method_index) {
 825   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 826   // address const_ptr = long_constant((jlong)Universe::non_oop_word());
 827   // unsigned long offset;
 828   // ldr_constant(rscratch2, const_ptr);
 829   movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
 830   return trampoline_call(Address(entry, rh));
 831 }
 832 
 833 // Implementation of call_VM versions
 834 


2102     words_pushed += 2;
2103   }
2104   if (count) {
2105     ldp(as_Register(regs[0]), as_Register(regs[1]),
2106        Address(post(stack, count * wordSize)));
2107     words_pushed += 2;
2108   }
2109 
2110   assert(words_pushed == count, "oops, pushed != count");
2111 
2112   return count;
2113 }
2114 #ifdef ASSERT
2115 void MacroAssembler::verify_heapbase(const char* msg) {
2116 #if 0
2117   assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
2118   assert (Universe::heap() != NULL, "java heap should be initialized");
2119   if (CheckCompressedOops) {
2120     Label ok;
2121     push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
2122     cmpptr(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
2123     br(Assembler::EQ, ok);
2124     stop(msg);
2125     bind(ok);
2126     pop(1 << rscratch1->encoding(), sp);
2127   }
2128 #endif
2129 }
2130 #endif
2131 
2132 void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) {
2133   Label done, not_weak;
2134   cbz(value, done);           // Use NULL as-is.
2135 
2136   STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2137   tbz(r0, 0, not_weak);    // Test for jweak tag.
2138 
2139   // Resolve jweak.
2140   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
2141                  Address(value, -JNIHandles::weak_tag_value), tmp, thread);
2142   verify_oop(value);


2235 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) {
2236   if (decrement.is_register()) {
2237     sub(Rd, Rn, decrement.as_register());
2238   } else {
2239     sub(Rd, Rn, decrement.as_constant());
2240   }
2241 }
2242 
2243 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) {
2244   if (decrement.is_register()) {
2245     subw(Rd, Rn, decrement.as_register());
2246   } else {
2247     subw(Rd, Rn, decrement.as_constant());
2248   }
2249 }
2250 
2251 void MacroAssembler::reinit_heapbase()
2252 {
2253   if (UseCompressedOops) {
2254     if (Universe::is_fully_initialized()) {
2255       mov(rheapbase, Universe::narrow_ptrs_base());
2256     } else {
2257       lea(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
2258       ldr(rheapbase, Address(rheapbase));
2259     }
2260   }
2261 }
2262 
2263 // this simulates the behaviour of the x86 cmpxchg instruction using a
2264 // load linked/store conditional pair. we use the acquire/release
2265 // versions of these instructions so that we flush pending writes as
2266 // per Java semantics.
2267 
2268 // n.b the x86 version assumes the old value to be compared against is
2269 // in rax and updates rax with the value located in memory if the
2270 // cmpxchg fails. we supply a register for the old value explicitly
2271 
2272 // the aarch64 load linked/store conditional instructions do not
2273 // accept an offset. so, unlike x86, we must provide a plain register
2274 // to identify the memory word to be compared/exchanged rather than a
2275 // register+offset Address.
2276 
2277 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,


3680 }
3681 
3682 // ((OopHandle)result).resolve();
3683 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3684   // OopHandle::resolve is an indirection.
3685   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3686 }
3687 
3688 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3689   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3690   ldr(dst, Address(rmethod, Method::const_offset()));
3691   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3692   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3693   ldr(dst, Address(dst, mirror_offset));
3694   resolve_oop_handle(dst, tmp);
3695 }
3696 
3697 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3698   if (UseCompressedClassPointers) {
3699     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3700     if (Universe::narrow_klass_base() == NULL) {
3701       cmp(trial_klass, tmp, LSL, Universe::narrow_klass_shift());
3702       return;
3703     } else if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3704                && Universe::narrow_klass_shift() == 0) {
3705       // Only the bottom 32 bits matter
3706       cmpw(trial_klass, tmp);
3707       return;
3708     }
3709     decode_klass_not_null(tmp);
3710   } else {
3711     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3712   }
3713   cmp(trial_klass, tmp);
3714 }
3715 
3716 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3717   load_klass(dst, src);
3718   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3719 }
3720 
3721 void MacroAssembler::store_klass(Register dst, Register src) {
3722   // FIXME: Should this be a store release?  concurrent gcs assumes
3723   // klass length is valid if klass field is not null.
3724   if (UseCompressedClassPointers) {
3725     encode_klass_not_null(src);
3726     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3727   } else {
3728     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3729   }
3730 }
3731 
3732 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3733   if (UseCompressedClassPointers) {
3734     // Store to klass gap in destination
3735     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3736   }
3737 }
3738 
3739 // Algorithm must match CompressedOops::encode.
3740 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3741 #ifdef ASSERT
3742   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3743 #endif
3744   verify_oop(s, "broken oop in encode_heap_oop");
3745   if (Universe::narrow_oop_base() == NULL) {
3746     if (Universe::narrow_oop_shift() != 0) {
3747       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3748       lsr(d, s, LogMinObjAlignmentInBytes);
3749     } else {
3750       mov(d, s);
3751     }
3752   } else {
3753     subs(d, s, rheapbase);
3754     csel(d, d, zr, Assembler::HS);
3755     lsr(d, d, LogMinObjAlignmentInBytes);
3756 
3757     /*  Old algorithm: is this any worse?
3758     Label nonnull;
3759     cbnz(r, nonnull);
3760     sub(r, r, rheapbase);
3761     bind(nonnull);
3762     lsr(r, r, LogMinObjAlignmentInBytes);
3763     */
3764   }
3765 }
3766 
3767 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3768 #ifdef ASSERT
3769   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
3770   if (CheckCompressedOops) {
3771     Label ok;
3772     cbnz(r, ok);
3773     stop("null oop passed to encode_heap_oop_not_null");
3774     bind(ok);
3775   }
3776 #endif
3777   verify_oop(r, "broken oop in encode_heap_oop_not_null");
3778   if (Universe::narrow_oop_base() != NULL) {
3779     sub(r, r, rheapbase);
3780   }
3781   if (Universe::narrow_oop_shift() != 0) {
3782     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3783     lsr(r, r, LogMinObjAlignmentInBytes);
3784   }
3785 }
3786 
3787 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3788 #ifdef ASSERT
3789   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
3790   if (CheckCompressedOops) {
3791     Label ok;
3792     cbnz(src, ok);
3793     stop("null oop passed to encode_heap_oop_not_null2");
3794     bind(ok);
3795   }
3796 #endif
3797   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
3798 
3799   Register data = src;
3800   if (Universe::narrow_oop_base() != NULL) {
3801     sub(dst, src, rheapbase);
3802     data = dst;
3803   }
3804   if (Universe::narrow_oop_shift() != 0) {
3805     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3806     lsr(dst, data, LogMinObjAlignmentInBytes);
3807     data = dst;
3808   }
3809   if (data == src)
3810     mov(dst, src);
3811 }
3812 
3813 void  MacroAssembler::decode_heap_oop(Register d, Register s) {
3814 #ifdef ASSERT
3815   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
3816 #endif
3817   if (Universe::narrow_oop_base() == NULL) {
3818     if (Universe::narrow_oop_shift() != 0 || d != s) {
3819       lsl(d, s, Universe::narrow_oop_shift());
3820     }
3821   } else {
3822     Label done;
3823     if (d != s)
3824       mov(d, s);
3825     cbz(s, done);
3826     add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes);
3827     bind(done);
3828   }
3829   verify_oop(d, "broken oop in decode_heap_oop");
3830 }
3831 
3832 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
3833   assert (UseCompressedOops, "should only be used for compressed headers");
3834   assert (Universe::heap() != NULL, "java heap should be initialized");
3835   // Cannot assert, unverified entry point counts instructions (see .ad file)
3836   // vtableStubs also counts instructions in pd_code_size_limit.
3837   // Also do not verify_oop as this is called by verify_oop.
3838   if (Universe::narrow_oop_shift() != 0) {
3839     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3840     if (Universe::narrow_oop_base() != NULL) {
3841       add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3842     } else {
3843       add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3844     }
3845   } else {
3846     assert (Universe::narrow_oop_base() == NULL, "sanity");
3847   }
3848 }
3849 
3850 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3851   assert (UseCompressedOops, "should only be used for compressed headers");
3852   assert (Universe::heap() != NULL, "java heap should be initialized");
3853   // Cannot assert, unverified entry point counts instructions (see .ad file)
3854   // vtableStubs also counts instructions in pd_code_size_limit.
3855   // Also do not verify_oop as this is called by verify_oop.
3856   if (Universe::narrow_oop_shift() != 0) {
3857     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3858     if (Universe::narrow_oop_base() != NULL) {
3859       add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3860     } else {
3861       add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3862     }
3863   } else {
3864     assert (Universe::narrow_oop_base() == NULL, "sanity");
3865     if (dst != src) {
3866       mov(dst, src);
3867     }
3868   }
3869 }
3870 
3871 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3872   if (Universe::narrow_klass_base() == NULL) {
3873     if (Universe::narrow_klass_shift() != 0) {
3874       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3875       lsr(dst, src, LogKlassAlignmentInBytes);
3876     } else {
3877       if (dst != src) mov(dst, src);
3878     }
3879     return;
3880   }
3881 
3882   if (use_XOR_for_compressed_class_base) {
3883     if (Universe::narrow_klass_shift() != 0) {
3884       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3885       lsr(dst, dst, LogKlassAlignmentInBytes);
3886     } else {
3887       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3888     }
3889     return;
3890   }
3891 
3892   if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3893       && Universe::narrow_klass_shift() == 0) {
3894     movw(dst, src);
3895     return;
3896   }
3897 
3898 #ifdef ASSERT
3899   verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
3900 #endif
3901 
3902   Register rbase = dst;
3903   if (dst == src) rbase = rheapbase;
3904   mov(rbase, (uint64_t)Universe::narrow_klass_base());
3905   sub(dst, src, rbase);
3906   if (Universe::narrow_klass_shift() != 0) {
3907     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3908     lsr(dst, dst, LogKlassAlignmentInBytes);
3909   }
3910   if (dst == src) reinit_heapbase();
3911 }
3912 
3913 void MacroAssembler::encode_klass_not_null(Register r) {
3914   encode_klass_not_null(r, r);
3915 }
3916 
3917 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3918   Register rbase = dst;
3919   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3920 
3921   if (Universe::narrow_klass_base() == NULL) {
3922     if (Universe::narrow_klass_shift() != 0) {
3923       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3924       lsl(dst, src, LogKlassAlignmentInBytes);
3925     } else {
3926       if (dst != src) mov(dst, src);
3927     }
3928     return;
3929   }
3930 
3931   if (use_XOR_for_compressed_class_base) {
3932     if (Universe::narrow_klass_shift() != 0) {
3933       lsl(dst, src, LogKlassAlignmentInBytes);
3934       eor(dst, dst, (uint64_t)Universe::narrow_klass_base());
3935     } else {
3936       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3937     }
3938     return;
3939   }
3940 
3941   if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3942       && Universe::narrow_klass_shift() == 0) {
3943     if (dst != src)
3944       movw(dst, src);
3945     movk(dst, (uint64_t)Universe::narrow_klass_base() >> 32, 32);
3946     return;
3947   }
3948 
3949   // Cannot assert, unverified entry point counts instructions (see .ad file)
3950   // vtableStubs also counts instructions in pd_code_size_limit.
3951   // Also do not verify_oop as this is called by verify_oop.
3952   if (dst == src) rbase = rheapbase;
3953   mov(rbase, (uint64_t)Universe::narrow_klass_base());
3954   if (Universe::narrow_klass_shift() != 0) {
3955     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3956     add(dst, rbase, src, Assembler::LSL, LogKlassAlignmentInBytes);
3957   } else {
3958     add(dst, rbase, src);
3959   }
3960   if (dst == src) reinit_heapbase();
3961 }
3962 
3963 void  MacroAssembler::decode_klass_not_null(Register r) {
3964   decode_klass_not_null(r, r);
3965 }
3966 
3967 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3968 #ifdef ASSERT
3969   {
3970     ThreadInVMfromUnknown tiv;
3971     assert (UseCompressedOops, "should only be used for compressed oops");
3972     assert (Universe::heap() != NULL, "java heap should be initialized");
3973     assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3974     assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3975   }
3976 #endif
3977   int oop_index = oop_recorder()->find_index(obj);
3978   InstructionMark im(this);
3979   RelocationHolder rspec = oop_Relocation::spec(oop_index);
3980   code_section()->relocate(inst_mark(), rspec);
3981   movz(dst, 0xDEAD, 16);
3982   movk(dst, 0xBEEF);
3983 }
3984 
3985 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3986   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3987   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3988   int index = oop_recorder()->find_index(k);
3989   assert(! Universe::heap()->is_in_reserved(k), "should not be an oop");
3990 
3991   InstructionMark im(this);
3992   RelocationHolder rspec = metadata_Relocation::spec(index);
3993   code_section()->relocate(inst_mark(), rspec);
3994   narrowKlass nk = Klass::encode_klass(k);
3995   movz(dst, (nk >> 16), 16);
3996   movk(dst, nk & 0xffff);
3997 }
3998 
3999 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
4000                                     Register dst, Address src,
4001                                     Register tmp1, Register thread_tmp) {
4002   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4003   decorators = AccessInternal::decorator_fixup(decorators);
4004   bool as_raw = (decorators & AS_RAW) != 0;
4005   if (as_raw) {
4006     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4007   } else {
4008     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4009   }
4010 }
4011 
4012 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4013                                      Address dst, Register src,
4014                                      Register tmp1, Register thread_tmp) {


< prev index next >