985
986 void MacroAssembler::c2bool(Register x) {
987 // implements x == 0 ? 0 : 1
988 // note: must only look at least-significant byte of x
989 // since C-style booleans are stored in one byte
990 // only! (was bug)
991 tst(x, 0xff);
992 cset(x, Assembler::NE);
993 }
994
995 address MacroAssembler::ic_call(address entry, jint method_index) {
996 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
997 // address const_ptr = long_constant((jlong)Universe::non_oop_word());
998 // uintptr_t offset;
999 // ldr_constant(rscratch2, const_ptr);
1000 movptr(rscratch2, (intptr_t)Universe::non_oop_word());
1001 return trampoline_call(Address(entry, rh));
1002 }
1003
1004 int MacroAssembler::ic_check_size() {
1005 if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) {
1006 return NativeInstruction::instruction_size * 7;
1007 } else {
1008 return NativeInstruction::instruction_size * 5;
1009 }
1010 }
1011
1012 int MacroAssembler::ic_check(int end_alignment) {
1013 Register receiver = j_rarg0;
1014 Register data = rscratch2;
1015 Register tmp1 = rscratch1;
1016 Register tmp2 = r10;
1017
1018 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
1019 // before the inline cache check, so we don't have to execute any nop instructions when dispatching
1020 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
1021 // before the inline cache check here, and not after
1022 align(end_alignment, offset() + ic_check_size());
1023
1024 int uep_offset = offset();
1025
1026 if (UseCompressedClassPointers) {
1027 ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
1028 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
1029 cmpw(tmp1, tmp2);
1030 } else {
1031 ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
1032 ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
1033 cmp(tmp1, tmp2);
1034 }
1035
1036 Label dont;
1037 br(Assembler::EQ, dont);
1038 far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1039 bind(dont);
1040 assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
1041
1042 return uep_offset;
1043 }
1044
1045 // Implementation of call_VM versions
1046
4821 adrp(rscratch1, src2, offset);
4822 ldr(rscratch1, Address(rscratch1, offset));
4823 cmp(src1, rscratch1);
4824 }
4825
4826 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4827 cmp(obj1, obj2);
4828 }
4829
4830 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4831 load_method_holder(rresult, rmethod);
4832 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4833 }
4834
4835 void MacroAssembler::load_method_holder(Register holder, Register method) {
4836 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
4837 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4838 ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
4839 }
4840
4841 void MacroAssembler::load_klass(Register dst, Register src) {
4842 if (UseCompressedClassPointers) {
4843 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4844 decode_klass_not_null(dst);
4845 } else {
4846 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4847 }
4848 }
4849
4850 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) {
4851 if (RestoreMXCSROnJNICalls) {
4852 Label OK;
4853 get_fpcr(tmp1);
4854 mov(tmp2, tmp1);
4855 // Set FPCR to the state we need. We do want Round to Nearest. We
4856 // don't want non-IEEE rounding modes or floating-point traps.
4857 bfi(tmp1, zr, 22, 4); // Clear DN, FZ, and Rmode
4858 bfi(tmp1, zr, 8, 5); // Clear exception-control bits (8-12)
4859 bfi(tmp1, zr, 0, 2); // Clear AH:FIZ
4860 eor(tmp2, tmp1, tmp2);
4861 cbz(tmp2, OK); // Only reset FPCR if it's wrong
4862 set_fpcr(tmp1);
4878 // A null weak handle resolves to null.
4879 cbz(result, resolved);
4880
4881 // Only 64 bit platforms support GCs that require a tmp register
4882 // WeakHandle::resolve is an indirection like jweak.
4883 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4884 result, Address(result), tmp1, tmp2);
4885 bind(resolved);
4886 }
4887
4888 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) {
4889 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4890 ldr(dst, Address(rmethod, Method::const_offset()));
4891 ldr(dst, Address(dst, ConstMethod::constants_offset()));
4892 ldr(dst, Address(dst, ConstantPool::pool_holder_offset()));
4893 ldr(dst, Address(dst, mirror_offset));
4894 resolve_oop_handle(dst, tmp1, tmp2);
4895 }
4896
4897 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4898 if (UseCompressedClassPointers) {
4899 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4900 if (CompressedKlassPointers::base() == nullptr) {
4901 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4902 return;
4903 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4904 && CompressedKlassPointers::shift() == 0) {
4905 // Only the bottom 32 bits matter
4906 cmpw(trial_klass, tmp);
4907 return;
4908 }
4909 decode_klass_not_null(tmp);
4910 } else {
4911 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4912 }
4913 cmp(trial_klass, tmp);
4914 }
4915
4916 void MacroAssembler::store_klass(Register dst, Register src) {
4917 // FIXME: Should this be a store release? concurrent gcs assumes
4918 // klass length is valid if klass field is not null.
4919 if (UseCompressedClassPointers) {
4920 encode_klass_not_null(src);
4921 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4922 } else {
4923 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4924 }
4925 }
4926
4927 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4928 if (UseCompressedClassPointers) {
4929 // Store to klass gap in destination
4930 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4931 }
4932 }
4933
4934 // Algorithm must match CompressedOops::encode.
4935 void MacroAssembler::encode_heap_oop(Register d, Register s) {
4936 #ifdef ASSERT
4937 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
4938 #endif
4939 verify_oop_msg(s, "broken oop in encode_heap_oop");
4940 if (CompressedOops::base() == nullptr) {
4941 if (CompressedOops::shift() != 0) {
4942 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4943 lsr(d, s, LogMinObjAlignmentInBytes);
4944 } else {
4945 mov(d, s);
4946 }
4947 } else {
5056 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
5057 }
5058 } else {
5059 assert (CompressedOops::base() == nullptr, "sanity");
5060 if (dst != src) {
5061 mov(dst, src);
5062 }
5063 }
5064 }
5065
5066 MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone);
5067
5068 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
5069 assert(UseCompressedClassPointers, "not using compressed class pointers");
5070 assert(Metaspace::initialized(), "metaspace not initialized yet");
5071
5072 if (_klass_decode_mode != KlassDecodeNone) {
5073 return _klass_decode_mode;
5074 }
5075
5076 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift()
5077 || 0 == CompressedKlassPointers::shift(), "decode alg wrong");
5078
5079 if (CompressedKlassPointers::base() == nullptr) {
5080 return (_klass_decode_mode = KlassDecodeZero);
5081 }
5082
5083 if (operand_valid_for_logical_immediate(
5084 /*is32*/false, (uint64_t)CompressedKlassPointers::base())) {
5085 const uint64_t range_mask =
5086 (1ULL << log2i(CompressedKlassPointers::range())) - 1;
5087 if (((uint64_t)CompressedKlassPointers::base() & range_mask) == 0) {
5088 return (_klass_decode_mode = KlassDecodeXor);
5089 }
5090 }
5091
5092 const uint64_t shifted_base =
5093 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5094 guarantee((shifted_base & 0xffff0000ffffffff) == 0,
5095 "compressed class base bad alignment");
5096
5097 return (_klass_decode_mode = KlassDecodeMovk);
5098 }
5099
5100 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
5101 switch (klass_decode_mode()) {
5102 case KlassDecodeZero:
5103 if (CompressedKlassPointers::shift() != 0) {
5104 lsr(dst, src, LogKlassAlignmentInBytes);
5105 } else {
5106 if (dst != src) mov(dst, src);
5107 }
5108 break;
5109
5110 case KlassDecodeXor:
5111 if (CompressedKlassPointers::shift() != 0) {
5112 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5113 lsr(dst, dst, LogKlassAlignmentInBytes);
5114 } else {
5115 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5116 }
5117 break;
5118
5119 case KlassDecodeMovk:
5120 if (CompressedKlassPointers::shift() != 0) {
5121 ubfx(dst, src, LogKlassAlignmentInBytes, 32);
5122 } else {
5123 movw(dst, src);
5124 }
5125 break;
5126
5127 case KlassDecodeNone:
5128 ShouldNotReachHere();
5129 break;
5130 }
5131 }
5132
5133 void MacroAssembler::encode_klass_not_null(Register r) {
5134 encode_klass_not_null(r, r);
5135 }
5136
5137 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
5138 assert (UseCompressedClassPointers, "should only be used for compressed headers");
5139
5140 switch (klass_decode_mode()) {
5141 case KlassDecodeZero:
5142 if (CompressedKlassPointers::shift() != 0) {
5143 lsl(dst, src, LogKlassAlignmentInBytes);
5144 } else {
5145 if (dst != src) mov(dst, src);
5146 }
5147 break;
5148
5149 case KlassDecodeXor:
5150 if (CompressedKlassPointers::shift() != 0) {
5151 lsl(dst, src, LogKlassAlignmentInBytes);
5152 eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
5153 } else {
5154 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5155 }
5156 break;
5157
5158 case KlassDecodeMovk: {
5159 const uint64_t shifted_base =
5160 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5161
5162 if (dst != src) movw(dst, src);
5163 movk(dst, shifted_base >> 32, 32);
5164
5165 if (CompressedKlassPointers::shift() != 0) {
5166 lsl(dst, dst, LogKlassAlignmentInBytes);
5167 }
5168
5169 break;
5170 }
5171
5172 case KlassDecodeNone:
5173 ShouldNotReachHere();
5174 break;
5175 }
5176 }
5177
5178 void MacroAssembler::decode_klass_not_null(Register r) {
5179 decode_klass_not_null(r, r);
5180 }
5181
5182 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5183 #ifdef ASSERT
5184 {
5185 ThreadInVMfromUnknown tiv;
5186 assert (UseCompressedOops, "should only be used for compressed oops");
6733 if (src.first()->is_stack()) {
6734 if (dst.first()->is_stack()) {
6735 ldr(tmp, Address(rfp, reg2offset_in(src.first())));
6736 str(tmp, Address(sp, reg2offset_out(dst.first())));
6737 } else {
6738 ldrd(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first())));
6739 }
6740 } else if (src.first() != dst.first()) {
6741 if (src.is_single_phys_reg() && dst.is_single_phys_reg())
6742 fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
6743 else
6744 strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
6745 }
6746 }
6747
6748 // Implements lightweight-locking.
6749 //
6750 // - obj: the object to be locked
6751 // - t1, t2, t3: temporary registers, will be destroyed
6752 // - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
6753 void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
6754 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
6755 assert_different_registers(obj, t1, t2, t3, rscratch1);
6756
6757 Label push;
6758 const Register top = t1;
6759 const Register mark = t2;
6760 const Register t = t3;
6761
6762 // Preload the markWord. It is important that this is the first
6763 // instruction emitted as it is part of C1's null check semantics.
6764 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
6765
6766 // Check if the lock-stack is full.
6767 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
6768 cmpw(top, (unsigned)LockStack::end_offset());
6769 br(Assembler::GE, slow);
6770
6771 // Check for recursion.
6772 subw(t, top, oopSize);
6773 ldr(t, Address(rthread, t));
6774 cmp(obj, t);
6775 br(Assembler::EQ, push);
6776
6777 // Check header for monitor (0b10).
6778 tst(mark, markWord::monitor_value);
6779 br(Assembler::NE, slow);
6780
6781 // Try to lock. Transition lock bits 0b01 => 0b00
6782 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
6783 orr(mark, mark, markWord::unlocked_value);
6784 eor(t, mark, markWord::unlocked_value);
6785 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
|
985
986 void MacroAssembler::c2bool(Register x) {
987 // implements x == 0 ? 0 : 1
988 // note: must only look at least-significant byte of x
989 // since C-style booleans are stored in one byte
990 // only! (was bug)
991 tst(x, 0xff);
992 cset(x, Assembler::NE);
993 }
994
995 address MacroAssembler::ic_call(address entry, jint method_index) {
996 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
997 // address const_ptr = long_constant((jlong)Universe::non_oop_word());
998 // uintptr_t offset;
999 // ldr_constant(rscratch2, const_ptr);
1000 movptr(rscratch2, (intptr_t)Universe::non_oop_word());
1001 return trampoline_call(Address(entry, rh));
1002 }
1003
1004 int MacroAssembler::ic_check_size() {
1005 int extra_instructions = UseCompactObjectHeaders ? 1 : 0;
1006 if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) {
1007 return NativeInstruction::instruction_size * (7 + extra_instructions);
1008 } else {
1009 return NativeInstruction::instruction_size * (5 + extra_instructions);
1010 }
1011 }
1012
1013 int MacroAssembler::ic_check(int end_alignment) {
1014 Register receiver = j_rarg0;
1015 Register data = rscratch2;
1016 Register tmp1 = rscratch1;
1017 Register tmp2 = r10;
1018
1019 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
1020 // before the inline cache check, so we don't have to execute any nop instructions when dispatching
1021 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
1022 // before the inline cache check here, and not after
1023 align(end_alignment, offset() + ic_check_size());
1024
1025 int uep_offset = offset();
1026
1027 if (UseCompactObjectHeaders) {
1028 load_nklass_compact(tmp1, receiver);
1029 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
1030 cmpw(tmp1, tmp2);
1031 } else if (UseCompressedClassPointers) {
1032 ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
1033 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
1034 cmpw(tmp1, tmp2);
1035 } else {
1036 ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
1037 ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
1038 cmp(tmp1, tmp2);
1039 }
1040
1041 Label dont;
1042 br(Assembler::EQ, dont);
1043 far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1044 bind(dont);
1045 assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
1046
1047 return uep_offset;
1048 }
1049
1050 // Implementation of call_VM versions
1051
4826 adrp(rscratch1, src2, offset);
4827 ldr(rscratch1, Address(rscratch1, offset));
4828 cmp(src1, rscratch1);
4829 }
4830
4831 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4832 cmp(obj1, obj2);
4833 }
4834
4835 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4836 load_method_holder(rresult, rmethod);
4837 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4838 }
4839
4840 void MacroAssembler::load_method_holder(Register holder, Register method) {
4841 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
4842 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4843 ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
4844 }
4845
4846 // Loads the obj's Klass* into dst.
4847 // Preserves all registers (incl src, rscratch1 and rscratch2).
4848 // Input:
4849 // src - the oop we want to load the klass from.
4850 // dst - output nklass.
4851 void MacroAssembler::load_nklass_compact(Register dst, Register src) {
4852 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
4853 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4854 lsr(dst, dst, markWord::klass_shift);
4855 }
4856
4857 void MacroAssembler::load_klass(Register dst, Register src) {
4858 if (UseCompactObjectHeaders) {
4859 load_nklass_compact(dst, src);
4860 decode_klass_not_null(dst);
4861 } else if (UseCompressedClassPointers) {
4862 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4863 decode_klass_not_null(dst);
4864 } else {
4865 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4866 }
4867 }
4868
4869 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) {
4870 if (RestoreMXCSROnJNICalls) {
4871 Label OK;
4872 get_fpcr(tmp1);
4873 mov(tmp2, tmp1);
4874 // Set FPCR to the state we need. We do want Round to Nearest. We
4875 // don't want non-IEEE rounding modes or floating-point traps.
4876 bfi(tmp1, zr, 22, 4); // Clear DN, FZ, and Rmode
4877 bfi(tmp1, zr, 8, 5); // Clear exception-control bits (8-12)
4878 bfi(tmp1, zr, 0, 2); // Clear AH:FIZ
4879 eor(tmp2, tmp1, tmp2);
4880 cbz(tmp2, OK); // Only reset FPCR if it's wrong
4881 set_fpcr(tmp1);
4897 // A null weak handle resolves to null.
4898 cbz(result, resolved);
4899
4900 // Only 64 bit platforms support GCs that require a tmp register
4901 // WeakHandle::resolve is an indirection like jweak.
4902 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4903 result, Address(result), tmp1, tmp2);
4904 bind(resolved);
4905 }
4906
4907 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) {
4908 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4909 ldr(dst, Address(rmethod, Method::const_offset()));
4910 ldr(dst, Address(dst, ConstMethod::constants_offset()));
4911 ldr(dst, Address(dst, ConstantPool::pool_holder_offset()));
4912 ldr(dst, Address(dst, mirror_offset));
4913 resolve_oop_handle(dst, tmp1, tmp2);
4914 }
4915
4916 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4917 assert_different_registers(oop, trial_klass, tmp);
4918 if (UseCompressedClassPointers) {
4919 if (UseCompactObjectHeaders) {
4920 load_nklass_compact(tmp, oop);
4921 } else {
4922 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4923 }
4924 if (CompressedKlassPointers::base() == nullptr) {
4925 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4926 return;
4927 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4928 && CompressedKlassPointers::shift() == 0) {
4929 // Only the bottom 32 bits matter
4930 cmpw(trial_klass, tmp);
4931 return;
4932 }
4933 decode_klass_not_null(tmp);
4934 } else {
4935 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4936 }
4937 cmp(trial_klass, tmp);
4938 }
4939
4940 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
4941 if (UseCompactObjectHeaders) {
4942 load_nklass_compact(tmp1, src);
4943 load_nklass_compact(tmp2, dst);
4944 cmpw(tmp1, tmp2);
4945 } else if (UseCompressedClassPointers) {
4946 ldrw(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4947 ldrw(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
4948 cmpw(tmp1, tmp2);
4949 } else {
4950 ldr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4951 ldr(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
4952 cmp(tmp1, tmp2);
4953 }
4954 }
4955
4956 void MacroAssembler::store_klass(Register dst, Register src) {
4957 // FIXME: Should this be a store release? concurrent gcs assumes
4958 // klass length is valid if klass field is not null.
4959 assert(!UseCompactObjectHeaders, "not with compact headers");
4960 if (UseCompressedClassPointers) {
4961 encode_klass_not_null(src);
4962 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4963 } else {
4964 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4965 }
4966 }
4967
4968 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4969 assert(!UseCompactObjectHeaders, "not with compact headers");
4970 if (UseCompressedClassPointers) {
4971 // Store to klass gap in destination
4972 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4973 }
4974 }
4975
4976 // Algorithm must match CompressedOops::encode.
4977 void MacroAssembler::encode_heap_oop(Register d, Register s) {
4978 #ifdef ASSERT
4979 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
4980 #endif
4981 verify_oop_msg(s, "broken oop in encode_heap_oop");
4982 if (CompressedOops::base() == nullptr) {
4983 if (CompressedOops::shift() != 0) {
4984 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4985 lsr(d, s, LogMinObjAlignmentInBytes);
4986 } else {
4987 mov(d, s);
4988 }
4989 } else {
5098 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
5099 }
5100 } else {
5101 assert (CompressedOops::base() == nullptr, "sanity");
5102 if (dst != src) {
5103 mov(dst, src);
5104 }
5105 }
5106 }
5107
5108 MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone);
5109
5110 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
5111 assert(UseCompressedClassPointers, "not using compressed class pointers");
5112 assert(Metaspace::initialized(), "metaspace not initialized yet");
5113
5114 if (_klass_decode_mode != KlassDecodeNone) {
5115 return _klass_decode_mode;
5116 }
5117
5118 if (CompressedKlassPointers::base() == nullptr) {
5119 return (_klass_decode_mode = KlassDecodeZero);
5120 }
5121
5122 if (operand_valid_for_logical_immediate(
5123 /*is32*/false, (uint64_t)CompressedKlassPointers::base())) {
5124 const uint64_t range_mask =
5125 (1ULL << log2i(CompressedKlassPointers::range())) - 1;
5126 if (((uint64_t)CompressedKlassPointers::base() & range_mask) == 0) {
5127 return (_klass_decode_mode = KlassDecodeXor);
5128 }
5129 }
5130
5131 const uint64_t shifted_base =
5132 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5133 guarantee((shifted_base & 0xffff0000ffffffff) == 0,
5134 "compressed class base bad alignment");
5135
5136 return (_klass_decode_mode = KlassDecodeMovk);
5137 }
5138
5139 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
5140 switch (klass_decode_mode()) {
5141 case KlassDecodeZero:
5142 if (CompressedKlassPointers::shift() != 0) {
5143 lsr(dst, src, CompressedKlassPointers::shift());
5144 } else {
5145 if (dst != src) mov(dst, src);
5146 }
5147 break;
5148
5149 case KlassDecodeXor:
5150 if (CompressedKlassPointers::shift() != 0) {
5151 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5152 lsr(dst, dst, CompressedKlassPointers::shift());
5153 } else {
5154 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5155 }
5156 break;
5157
5158 case KlassDecodeMovk:
5159 if (CompressedKlassPointers::shift() != 0) {
5160 ubfx(dst, src, CompressedKlassPointers::shift(), 32);
5161 } else {
5162 movw(dst, src);
5163 }
5164 break;
5165
5166 case KlassDecodeNone:
5167 ShouldNotReachHere();
5168 break;
5169 }
5170 }
5171
5172 void MacroAssembler::encode_klass_not_null(Register r) {
5173 encode_klass_not_null(r, r);
5174 }
5175
5176 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
5177 assert (UseCompressedClassPointers, "should only be used for compressed headers");
5178
5179 switch (klass_decode_mode()) {
5180 case KlassDecodeZero:
5181 if (CompressedKlassPointers::shift() != 0) {
5182 lsl(dst, src, CompressedKlassPointers::shift());
5183 } else {
5184 if (dst != src) mov(dst, src);
5185 }
5186 break;
5187
5188 case KlassDecodeXor:
5189 if (CompressedKlassPointers::shift() != 0) {
5190 lsl(dst, src, CompressedKlassPointers::shift());
5191 eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
5192 } else {
5193 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
5194 }
5195 break;
5196
5197 case KlassDecodeMovk: {
5198 const uint64_t shifted_base =
5199 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5200
5201 if (dst != src) movw(dst, src);
5202 movk(dst, shifted_base >> 32, 32);
5203
5204 if (CompressedKlassPointers::shift() != 0) {
5205 lsl(dst, dst, CompressedKlassPointers::shift());
5206 }
5207
5208 break;
5209 }
5210
5211 case KlassDecodeNone:
5212 ShouldNotReachHere();
5213 break;
5214 }
5215 }
5216
5217 void MacroAssembler::decode_klass_not_null(Register r) {
5218 decode_klass_not_null(r, r);
5219 }
5220
5221 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5222 #ifdef ASSERT
5223 {
5224 ThreadInVMfromUnknown tiv;
5225 assert (UseCompressedOops, "should only be used for compressed oops");
6772 if (src.first()->is_stack()) {
6773 if (dst.first()->is_stack()) {
6774 ldr(tmp, Address(rfp, reg2offset_in(src.first())));
6775 str(tmp, Address(sp, reg2offset_out(dst.first())));
6776 } else {
6777 ldrd(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first())));
6778 }
6779 } else if (src.first() != dst.first()) {
6780 if (src.is_single_phys_reg() && dst.is_single_phys_reg())
6781 fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
6782 else
6783 strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
6784 }
6785 }
6786
6787 // Implements lightweight-locking.
6788 //
6789 // - obj: the object to be locked
6790 // - t1, t2, t3: temporary registers, will be destroyed
6791 // - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
6792 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow) {
6793 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
6794 assert_different_registers(basic_lock, obj, t1, t2, t3, rscratch1);
6795
6796 Label push;
6797 const Register top = t1;
6798 const Register mark = t2;
6799 const Register t = t3;
6800
6801 // Preload the markWord. It is important that this is the first
6802 // instruction emitted as it is part of C1's null check semantics.
6803 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
6804
6805 if (UseObjectMonitorTable) {
6806 // Clear cache in case fast locking succeeds.
6807 str(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))));
6808 }
6809
6810 // Check if the lock-stack is full.
6811 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
6812 cmpw(top, (unsigned)LockStack::end_offset());
6813 br(Assembler::GE, slow);
6814
6815 // Check for recursion.
6816 subw(t, top, oopSize);
6817 ldr(t, Address(rthread, t));
6818 cmp(obj, t);
6819 br(Assembler::EQ, push);
6820
6821 // Check header for monitor (0b10).
6822 tst(mark, markWord::monitor_value);
6823 br(Assembler::NE, slow);
6824
6825 // Try to lock. Transition lock bits 0b01 => 0b00
6826 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
6827 orr(mark, mark, markWord::unlocked_value);
6828 eor(t, mark, markWord::unlocked_value);
6829 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
|