9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "code/compiledIC.hpp"
30 #include "compiler/disassembler.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/cardTableBarrierSet.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "interpreter/bytecodeHistogram.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "interpreter/interpreterRuntime.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "memory/universe.hpp"
41 #include "oops/accessDecorators.hpp"
42 #include "oops/compressedKlass.inline.hpp"
43 #include "oops/compressedOops.inline.hpp"
44 #include "oops/klass.inline.hpp"
45 #include "oops/oop.hpp"
46 #include "runtime/interfaceSupport.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/jniHandles.inline.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "runtime/stubRoutines.hpp"
51 #include "utilities/globalDefinitions.hpp"
52 #include "utilities/integerCast.hpp"
53 #include "utilities/powerOfTwo.hpp"
54 #ifdef COMPILER2
55 #include "opto/compile.hpp"
56 #include "opto/node.hpp"
57 #include "opto/output.hpp"
58 #endif
59
60 #ifdef PRODUCT
61 #define BLOCK_COMMENT(str) /* nothing */
62 #else
63 #define BLOCK_COMMENT(str) block_comment(str)
64 #endif
65 #define STOP(str) stop(str);
928 call_VM_leaf_base(entry_point, 1);
929 }
930
931 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
932 assert_different_registers(arg_1, c_rarg0);
933 pass_arg0(this, arg_0);
934 pass_arg1(this, arg_1);
935 call_VM_leaf_base(entry_point, 2);
936 }
937
938 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
939 Register arg_1, Register arg_2) {
940 assert_different_registers(arg_1, c_rarg0);
941 assert_different_registers(arg_2, c_rarg0, c_rarg1);
942 pass_arg0(this, arg_0);
943 pass_arg1(this, arg_1);
944 pass_arg2(this, arg_2);
945 call_VM_leaf_base(entry_point, 3);
946 }
947
948 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
949 pass_arg0(this, arg_0);
950 MacroAssembler::call_VM_leaf_base(entry_point, 1);
951 }
952
953 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
954
955 assert_different_registers(arg_0, c_rarg1);
956 pass_arg1(this, arg_1);
957 pass_arg0(this, arg_0);
958 MacroAssembler::call_VM_leaf_base(entry_point, 2);
959 }
960
961 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
962 assert_different_registers(arg_0, c_rarg1, c_rarg2);
963 assert_different_registers(arg_1, c_rarg2);
964 pass_arg2(this, arg_2);
965 pass_arg1(this, arg_1);
966 pass_arg0(this, arg_0);
967 MacroAssembler::call_VM_leaf_base(entry_point, 3);
3578 movptr(dst, Address((address)obj, rspec));
3579 } else {
3580 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3581 ld(dst, Address(dummy, rspec));
3582 }
3583 }
3584
3585 // Move a metadata address into a register.
3586 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3587 assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata");
3588 int oop_index;
3589 if (obj == nullptr) {
3590 oop_index = oop_recorder()->allocate_metadata_index(obj);
3591 } else {
3592 oop_index = oop_recorder()->find_index(obj);
3593 }
3594 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3595 movptr(dst, Address((address)obj, rspec));
3596 }
3597
3598 // Writes to stack successive pages until offset reached to check for
3599 // stack overflow + shadow pages. This clobbers tmp.
3600 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3601 assert_different_registers(tmp, size, t0);
3602 // Bang stack for total size given plus shadow page size.
3603 // Bang one page at a time because large size can bang beyond yellow and
3604 // red zones.
3605 mv(t0, (int)os::vm_page_size());
3606 Label loop;
3607 bind(loop);
3608 sub(tmp, sp, t0);
3609 subw(size, size, t0);
3610 sd(size, Address(tmp));
3611 bgtz(size, loop);
3612
3613 // Bang down shadow pages too.
3614 // At this point, (tmp-0) is the last address touched, so don't
3615 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3616 // was post-decremented.) Skip this address by starting at i=1, and
3617 // touch a few more pages below. N.B. It is important to touch all
3663 bool as_raw = (decorators & AS_RAW) != 0;
3664 if (as_raw) {
3665 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
3666 } else {
3667 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
3668 }
3669 }
3670
3671 void MacroAssembler::null_check(Register reg, int offset) {
3672 if (needs_explicit_null_check(offset)) {
3673 // provoke OS null exception if reg is null by
3674 // accessing M[reg] w/o changing any registers
3675 // NOTE: this is plenty to provoke a segv
3676 ld(zr, Address(reg, 0));
3677 } else {
3678 // nothing to do, (later) access of M[reg + offset]
3679 // will provoke OS null exception if reg is null
3680 }
3681 }
3682
3683 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3684 Address dst, Register val,
3685 Register tmp1, Register tmp2, Register tmp3) {
3686 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3687 decorators = AccessInternal::decorator_fixup(decorators, type);
3688 bool as_raw = (decorators & AS_RAW) != 0;
3689 if (as_raw) {
3690 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3691 } else {
3692 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3693 }
3694 }
3695
3696 // Algorithm must match CompressedOops::encode.
3697 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3698 verify_oop_msg(s, "broken oop in encode_heap_oop");
3699 if (CompressedOops::base() == nullptr) {
3700 if (CompressedOops::shift() != 0) {
3701 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3702 srli(d, s, LogMinObjAlignmentInBytes);
3765 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
3766 ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3767 srli(dst, dst, markWord::klass_shift);
3768 }
3769
3770 void MacroAssembler::load_narrow_klass(Register dst, Register src) {
3771 if (UseCompactObjectHeaders) {
3772 load_narrow_klass_compact(dst, src);
3773 } else {
3774 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3775 }
3776 }
3777
3778 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
3779 assert_different_registers(dst, tmp);
3780 assert_different_registers(src, tmp);
3781 load_narrow_klass(dst, src);
3782 decode_klass_not_null(dst, tmp);
3783 }
3784
3785 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
3786 // FIXME: Should this be a store release? concurrent gcs assumes
3787 // klass length is valid if klass field is not null.
3788 assert(!UseCompactObjectHeaders, "not with compact headers");
3789 encode_klass_not_null(src, tmp);
3790 sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3791
3792 }
3793
3794 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3795 assert(!UseCompactObjectHeaders, "not with compact headers");
3796 // Store to klass gap in destination
3797 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3798 }
3799
3800 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
3801 assert_different_registers(r, tmp);
3802 decode_klass_not_null(r, r, tmp);
3803 }
3804
5229 // need to save whatever non-callee save context might get clobbered
5230 // by the call to Thread::current() or, indeed, the call setup code.
5231 void MacroAssembler::get_thread(Register thread) {
5232 // save all call-clobbered regs except thread
5233 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
5234 RegSet::range(x28, x31) + ra - thread;
5235 push_reg(saved_regs, sp);
5236
5237 mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
5238 jalr(t1);
5239 if (thread != c_rarg0) {
5240 mv(thread, c_rarg0);
5241 }
5242
5243 // restore pushed registers
5244 pop_reg(saved_regs, sp);
5245 }
5246
5247 void MacroAssembler::load_byte_map_base(Register reg) {
5248 CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
5249 mv(reg, (uint64_t)ctbs->card_table_base_const());
5250 }
5251
5252 void MacroAssembler::build_frame(int framesize) {
5253 assert(framesize >= 2, "framesize must include space for FP/RA");
5254 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5255 sub(sp, sp, framesize);
5256 sd(fp, Address(sp, framesize - 2 * wordSize));
5257 sd(ra, Address(sp, framesize - wordSize));
5258 if (PreserveFramePointer) { add(fp, sp, framesize); }
5259 }
5260
5261 void MacroAssembler::remove_frame(int framesize) {
5262 assert(framesize >= 2, "framesize must include space for FP/RA");
5263 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5264 ld(fp, Address(sp, framesize - 2 * wordSize));
5265 ld(ra, Address(sp, framesize - wordSize));
5266 add(sp, sp, framesize);
5267 }
5268
5269 void MacroAssembler::reserved_stack_check() {
5270 // testing if reserved zone needs to be enabled
5271 Label no_reserved_zone_enabling;
5272
5273 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
5274 bltu(sp, t0, no_reserved_zone_enabling);
5275
5276 enter(); // RA and FP are live.
5277 mv(c_rarg0, xthread);
5278 rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
5279 leave();
5280
5281 // We have already removed our own frame.
5282 // throw_delayed_StackOverflowError will think that it's been
5283 // called by our caller.
5284 j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
5285 should_not_reach_here();
5286
5287 bind(no_reserved_zone_enabling);
5288 }
5514 is_simm12(dst.offset())) || is_simm12(value)),
5515 "invalid value and address mode combination");
5516 Address adr = add_memory_helper(dst, tmp2);
5517 assert(!adr.uses(tmp1), "invalid dst for address decrement");
5518 lwu(tmp1, adr);
5519 subw(tmp1, tmp1, value, tmp2);
5520 sw(tmp1, adr);
5521 }
5522
5523 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
5524 load_method_holder(result, method);
5525 ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
5526 }
5527
5528 void MacroAssembler::load_method_holder(Register holder, Register method) {
5529 ld(holder, Address(method, Method::const_offset())); // ConstMethod*
5530 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
5531 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
5532 }
5533
5534 // string indexof
5535 // compute index by trailing zeros
5536 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
5537 Register match_mask, Register result,
5538 Register ch2, Register tmp,
5539 bool haystack_isL) {
5540 int haystack_chr_shift = haystack_isL ? 0 : 1;
5541 srl(match_mask, match_mask, trailing_zeros);
5542 srli(match_mask, match_mask, 1);
5543 srli(tmp, trailing_zeros, LogBitsPerByte);
5544 if (!haystack_isL) andi(tmp, tmp, 0xE);
5545 add(haystack, haystack, tmp);
5546 ld(ch2, Address(haystack));
5547 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
5548 add(result, result, tmp);
5549 }
5550
5551 // string indexof
5552 // Find pattern element in src, compute match mask,
5553 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
6840 bnez(tmp1, slow, /* is_far */ true);
6841 }
6842
6843 // Check if the lock-stack is full.
6844 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
6845 mv(t, (unsigned)LockStack::end_offset());
6846 bge(top, t, slow, /* is_far */ true);
6847
6848 // Check for recursion.
6849 add(t, xthread, top);
6850 ld(t, Address(t, -oopSize));
6851 beq(obj, t, push);
6852
6853 // Check header for monitor (0b10).
6854 test_bit(t, mark, exact_log2(markWord::monitor_value));
6855 bnez(t, slow, /* is_far */ true);
6856
6857 // Try to lock. Transition lock-bits 0b01 => 0b00
6858 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
6859 ori(mark, mark, markWord::unlocked_value);
6860 xori(t, mark, markWord::unlocked_value);
6861 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
6862 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
6863 bne(mark, t, slow, /* is_far */ true);
6864
6865 bind(push);
6866 // After successful lock, push object on lock-stack.
6867 add(t, xthread, top);
6868 sd(obj, Address(t));
6869 addiw(top, top, oopSize);
6870 sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
6871 }
6872
6873 // Implements ligthweight-unlocking.
6874 //
6875 // - obj: the object to be unlocked
6876 // - tmp1, tmp2, tmp3: temporary registers
6877 // - slow: branched to if unlocking fails
6878 void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
6879 assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
|
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "ci/ciInlineKlass.hpp"
30 #include "code/compiledIC.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/barrierSetAssembler.hpp"
34 #include "gc/shared/cardTable.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "interpreter/bytecodeHistogram.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "interpreter/interpreterRuntime.hpp"
40 #include "memory/resourceArea.hpp"
41 #include "memory/universe.hpp"
42 #include "oops/accessDecorators.hpp"
43 #include "oops/compressedKlass.inline.hpp"
44 #include "oops/compressedOops.inline.hpp"
45 #include "oops/klass.inline.hpp"
46 #include "oops/oop.hpp"
47 #include "oops/resolvedFieldEntry.hpp"
48 #include "runtime/interfaceSupport.inline.hpp"
49 #include "runtime/javaThread.hpp"
50 #include "runtime/jniHandles.inline.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "utilities/globalDefinitions.hpp"
54 #include "utilities/integerCast.hpp"
55 #include "utilities/powerOfTwo.hpp"
56 #ifdef COMPILER2
57 #include "opto/compile.hpp"
58 #include "opto/node.hpp"
59 #include "opto/output.hpp"
60 #endif
61
62 #ifdef PRODUCT
63 #define BLOCK_COMMENT(str) /* nothing */
64 #else
65 #define BLOCK_COMMENT(str) block_comment(str)
66 #endif
67 #define STOP(str) stop(str);
930 call_VM_leaf_base(entry_point, 1);
931 }
932
933 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
934 assert_different_registers(arg_1, c_rarg0);
935 pass_arg0(this, arg_0);
936 pass_arg1(this, arg_1);
937 call_VM_leaf_base(entry_point, 2);
938 }
939
940 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
941 Register arg_1, Register arg_2) {
942 assert_different_registers(arg_1, c_rarg0);
943 assert_different_registers(arg_2, c_rarg0, c_rarg1);
944 pass_arg0(this, arg_0);
945 pass_arg1(this, arg_1);
946 pass_arg2(this, arg_2);
947 call_VM_leaf_base(entry_point, 3);
948 }
949
950 void MacroAssembler::super_call_VM_leaf(address entry_point) {
951 MacroAssembler::call_VM_leaf_base(entry_point, 1);
952 }
953
954 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
955 pass_arg0(this, arg_0);
956 MacroAssembler::call_VM_leaf_base(entry_point, 1);
957 }
958
959 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
960
961 assert_different_registers(arg_0, c_rarg1);
962 pass_arg1(this, arg_1);
963 pass_arg0(this, arg_0);
964 MacroAssembler::call_VM_leaf_base(entry_point, 2);
965 }
966
967 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
968 assert_different_registers(arg_0, c_rarg1, c_rarg2);
969 assert_different_registers(arg_1, c_rarg2);
970 pass_arg2(this, arg_2);
971 pass_arg1(this, arg_1);
972 pass_arg0(this, arg_0);
973 MacroAssembler::call_VM_leaf_base(entry_point, 3);
3584 movptr(dst, Address((address)obj, rspec));
3585 } else {
3586 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3587 ld(dst, Address(dummy, rspec));
3588 }
3589 }
3590
3591 // Move a metadata address into a register.
3592 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3593 assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata");
3594 int oop_index;
3595 if (obj == nullptr) {
3596 oop_index = oop_recorder()->allocate_metadata_index(obj);
3597 } else {
3598 oop_index = oop_recorder()->find_index(obj);
3599 }
3600 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3601 movptr(dst, Address((address)obj, rspec));
3602 }
3603
3604 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3605 assert_different_registers(holder_klass, index, layout_info);
3606 InlineLayoutInfo array[2];
3607 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3608 if (is_power_of_2(size)) {
3609 slli(index, index, log2i_exact(size)); // Scale index by power of 2
3610 } else {
3611 mv(layout_info, size);
3612 mul(index, index, layout_info); // Scale the index to be the entry index * array_element_size
3613 }
3614 ld(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset()));
3615 add(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3616 add(layout_info, layout_info, index);
3617 la(layout_info, Address(layout_info));
3618 }
3619
3620 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
3621 Register inline_layout_info) {
3622 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3623 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3624 }
3625
3626 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3627 ld(offset, Address(inline_klass, InlineKlass::adr_members_offset()));
3628 lwu(offset, Address(offset, InlineKlass::payload_offset_offset()));
3629 }
3630
3631 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass) {
3632 assert_different_registers(data, t0);
3633 // ((address) (void*) o) + vk->payload_offset();
3634 Register offset = (data == oop) ? t0 : data;
3635 payload_offset(inline_klass, offset);
3636 if (data == oop) {
3637 add(data, data, offset);
3638 } else {
3639 add(data, oop, offset);
3640 la(data, Address(data));
3641 }
3642 }
3643
3644 // Writes to stack successive pages until offset reached to check for
3645 // stack overflow + shadow pages. This clobbers tmp.
3646 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3647 assert_different_registers(tmp, size, t0);
3648 // Bang stack for total size given plus shadow page size.
3649 // Bang one page at a time because large size can bang beyond yellow and
3650 // red zones.
3651 mv(t0, (int)os::vm_page_size());
3652 Label loop;
3653 bind(loop);
3654 sub(tmp, sp, t0);
3655 subw(size, size, t0);
3656 sd(size, Address(tmp));
3657 bgtz(size, loop);
3658
3659 // Bang down shadow pages too.
3660 // At this point, (tmp-0) is the last address touched, so don't
3661 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3662 // was post-decremented.) Skip this address by starting at i=1, and
3663 // touch a few more pages below. N.B. It is important to touch all
3709 bool as_raw = (decorators & AS_RAW) != 0;
3710 if (as_raw) {
3711 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
3712 } else {
3713 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
3714 }
3715 }
3716
3717 void MacroAssembler::null_check(Register reg, int offset) {
3718 if (needs_explicit_null_check(offset)) {
3719 // provoke OS null exception if reg is null by
3720 // accessing M[reg] w/o changing any registers
3721 // NOTE: this is plenty to provoke a segv
3722 ld(zr, Address(reg, 0));
3723 } else {
3724 // nothing to do, (later) access of M[reg + offset]
3725 // will provoke OS null exception if reg is null
3726 }
3727 }
3728
3729 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
3730 test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3731 bnez(temp_reg, is_null_free_inline_type);
3732 }
3733
3734 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
3735 test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3736 beqz(temp_reg, not_null_free_inline_type);
3737 }
3738
3739 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
3740 test_bit(temp_reg, flags, ResolvedFieldEntry::is_flat_shift);
3741 bnez(temp_reg, is_flat);
3742 }
3743
3744 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3745 assert_different_registers(markword, t1);
3746 mv(t1, markWord::inline_type_pattern_mask);
3747 andr(markword, markword, t1);
3748 mv(t1, markWord::inline_type_pattern);
3749 beq(markword, t1, is_inline_type);
3750 }
3751
3752 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null) {
3753 assert_different_registers(tmp, t0);
3754 if (can_be_null) {
3755 beqz(object, not_inline_type);
3756 }
3757 const int is_inline_type_mask = markWord::inline_type_pattern;
3758 ld(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
3759 mv(t0, is_inline_type_mask);
3760 andr(tmp, tmp, t0);
3761 bne(tmp, t0, not_inline_type);
3762 }
3763
3764 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t tst_bit, bool jmp_set, Label& jmp_label) {
3765 assert_different_registers(temp_reg, t0);
3766 // load mark word
3767 ld(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
3768 if (!UseObjectMonitorTable) {
3769 Label test_mark_word;
3770 // check displaced
3771 test_bit(t0, temp_reg, exact_log2(markWord::unlocked_value));
3772 bnez(t0, test_mark_word);
3773 // slow path use klass prototype
3774 load_prototype_header(temp_reg, oop);
3775
3776 bind(test_mark_word);
3777 }
3778 andi(temp_reg, temp_reg, tst_bit);
3779 if (jmp_set) {
3780 bnez(temp_reg, jmp_label, /* is_far */ true);
3781 } else {
3782 beqz(temp_reg, jmp_label, /* is_far */ true);
3783 }
3784 }
3785
3786 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) {
3787 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
3788 }
3789
3790 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
3791 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
3792 }
3793
3794 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label&is_non_flat_array) {
3795 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3796 }
3797
3798 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
3799 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3800 }
3801
3802 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3803 test_bit(t0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3804 bnez(t0, is_flat_array);
3805 }
3806
3807 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3808 Address dst, Register val,
3809 Register tmp1, Register tmp2, Register tmp3) {
3810 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3811 decorators = AccessInternal::decorator_fixup(decorators, type);
3812 bool as_raw = (decorators & AS_RAW) != 0;
3813 if (as_raw) {
3814 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3815 } else {
3816 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3817 }
3818 }
3819
3820 // Algorithm must match CompressedOops::encode.
3821 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3822 verify_oop_msg(s, "broken oop in encode_heap_oop");
3823 if (CompressedOops::base() == nullptr) {
3824 if (CompressedOops::shift() != 0) {
3825 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3826 srli(d, s, LogMinObjAlignmentInBytes);
3889 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
3890 ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3891 srli(dst, dst, markWord::klass_shift);
3892 }
3893
3894 void MacroAssembler::load_narrow_klass(Register dst, Register src) {
3895 if (UseCompactObjectHeaders) {
3896 load_narrow_klass_compact(dst, src);
3897 } else {
3898 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3899 }
3900 }
3901
3902 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
3903 assert_different_registers(dst, tmp);
3904 assert_different_registers(src, tmp);
3905 load_narrow_klass(dst, src);
3906 decode_klass_not_null(dst, tmp);
3907 }
3908
3909 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
3910 load_klass(dst, src, tmp);
3911 ld(dst, Address(dst, Klass::prototype_header_offset()));
3912 }
3913
3914 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
3915 // FIXME: Should this be a store release? concurrent gcs assumes
3916 // klass length is valid if klass field is not null.
3917 assert(!UseCompactObjectHeaders, "not with compact headers");
3918 encode_klass_not_null(src, tmp);
3919 sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3920
3921 }
3922
3923 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3924 assert(!UseCompactObjectHeaders, "not with compact headers");
3925 // Store to klass gap in destination
3926 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3927 }
3928
3929 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
3930 assert_different_registers(r, tmp);
3931 decode_klass_not_null(r, r, tmp);
3932 }
3933
5358 // need to save whatever non-callee save context might get clobbered
5359 // by the call to Thread::current() or, indeed, the call setup code.
5360 void MacroAssembler::get_thread(Register thread) {
5361 // save all call-clobbered regs except thread
5362 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
5363 RegSet::range(x28, x31) + ra - thread;
5364 push_reg(saved_regs, sp);
5365
5366 mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
5367 jalr(t1);
5368 if (thread != c_rarg0) {
5369 mv(thread, c_rarg0);
5370 }
5371
5372 // restore pushed registers
5373 pop_reg(saved_regs, sp);
5374 }
5375
5376 void MacroAssembler::load_byte_map_base(Register reg) {
5377 CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
5378 // Strictly speaking the card table base isn't an address at all, and it might
5379 // even be negative. It is thus materialised as a constant.
5380 mv(reg, (uint64_t)ctbs->card_table_base_const());
5381 }
5382
5383 void MacroAssembler::build_frame(int framesize) {
5384 assert(framesize >= 2, "framesize must include space for FP/RA");
5385 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5386 sub(sp, sp, framesize);
5387 sd(fp, Address(sp, framesize - 2 * wordSize));
5388 sd(ra, Address(sp, framesize - wordSize));
5389 if (PreserveFramePointer) { add(fp, sp, framesize); }
5390 }
5391
5392 void MacroAssembler::remove_frame(int framesize) {
5393 assert(framesize >= 2, "framesize must include space for FP/RA");
5394 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5395 ld(fp, Address(sp, framesize - 2 * wordSize));
5396 ld(ra, Address(sp, framesize - wordSize));
5397 add(sp, sp, framesize);
5398 }
5399
5400 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
5401 assert(!needs_stack_repair, "unimplemented");
5402 remove_frame(initial_framesize);
5403 }
5404
5405 #ifdef COMPILER2
5406 // C2 compiled method's prolog code
5407 // Moved here from riscv.ad to support Valhalla code belows
5408 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5409 if (C->clinit_barrier_on_entry()) {
5410 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
5411
5412 Label L_skip_barrier;
5413
5414 mov_metadata(t1, C->method()->holder()->constant_encoding());
5415 clinit_barrier(t1, t0, &L_skip_barrier);
5416 far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
5417 bind(L_skip_barrier);
5418 }
5419
5420 int bangsize = C->output()->bang_size_in_bytes();
5421 if (C->output()->need_stack_bang(bangsize)) {
5422 generate_stack_overflow_check(bangsize);
5423 }
5424
5425 // n.b. frame size includes space for return pc and fp
5426 const long framesize = C->output()->frame_size_in_bytes();
5427 build_frame(framesize);
5428
5429 assert(!C->needs_stack_repair(), "unimplemented");
5430 }
5431 #endif // COMPILER2
5432
5433 // Move a value between registers/stack slots and update the reg_state
5434 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5435 Unimplemented();
5436 return false;
5437 }
5438
5439 // Read all fields from an inline type oop and store the values in registers/stack slots
5440 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5441 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5442 RegState reg_state[]) {
5443
5444 Unimplemented();
5445 return false;
5446 }
5447
5448 // Pack fields back into an inline type oop
5449 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5450 VMRegPair* from, int from_count, int& from_index, VMReg to,
5451 RegState reg_state[], Register val_array) {
5452 Unimplemented();
5453 return false;
5454 }
5455
5456 // Calculate the extra stack space required for packing or unpacking inline
5457 // args and adjust the stack pointer
5458 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5459 Unimplemented();
5460 return false;
5461 }
5462
5463 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5464 Unimplemented();
5465 return reg;
5466 }
5467
5468 void MacroAssembler::reserved_stack_check() {
5469 // testing if reserved zone needs to be enabled
5470 Label no_reserved_zone_enabling;
5471
5472 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
5473 bltu(sp, t0, no_reserved_zone_enabling);
5474
5475 enter(); // RA and FP are live.
5476 mv(c_rarg0, xthread);
5477 rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
5478 leave();
5479
5480 // We have already removed our own frame.
5481 // throw_delayed_StackOverflowError will think that it's been
5482 // called by our caller.
5483 j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
5484 should_not_reach_here();
5485
5486 bind(no_reserved_zone_enabling);
5487 }
5713 is_simm12(dst.offset())) || is_simm12(value)),
5714 "invalid value and address mode combination");
5715 Address adr = add_memory_helper(dst, tmp2);
5716 assert(!adr.uses(tmp1), "invalid dst for address decrement");
5717 lwu(tmp1, adr);
5718 subw(tmp1, tmp1, value, tmp2);
5719 sw(tmp1, adr);
5720 }
5721
5722 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
5723 load_method_holder(result, method);
5724 ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
5725 }
5726
5727 void MacroAssembler::load_method_holder(Register holder, Register method) {
5728 ld(holder, Address(method, Method::const_offset())); // ConstMethod*
5729 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
5730 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
5731 }
5732
5733 void MacroAssembler::load_metadata(Register dst, Register src) {
5734 if (UseCompactObjectHeaders) {
5735 load_narrow_klass_compact(dst, src);
5736 } else {
5737 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5738 }
5739 }
5740
5741 // string indexof
5742 // compute index by trailing zeros
5743 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
5744 Register match_mask, Register result,
5745 Register ch2, Register tmp,
5746 bool haystack_isL) {
5747 int haystack_chr_shift = haystack_isL ? 0 : 1;
5748 srl(match_mask, match_mask, trailing_zeros);
5749 srli(match_mask, match_mask, 1);
5750 srli(tmp, trailing_zeros, LogBitsPerByte);
5751 if (!haystack_isL) andi(tmp, tmp, 0xE);
5752 add(haystack, haystack, tmp);
5753 ld(ch2, Address(haystack));
5754 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
5755 add(result, result, tmp);
5756 }
5757
5758 // string indexof
5759 // Find pattern element in src, compute match mask,
5760 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
7047 bnez(tmp1, slow, /* is_far */ true);
7048 }
7049
7050 // Check if the lock-stack is full.
7051 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7052 mv(t, (unsigned)LockStack::end_offset());
7053 bge(top, t, slow, /* is_far */ true);
7054
7055 // Check for recursion.
7056 add(t, xthread, top);
7057 ld(t, Address(t, -oopSize));
7058 beq(obj, t, push);
7059
7060 // Check header for monitor (0b10).
7061 test_bit(t, mark, exact_log2(markWord::monitor_value));
7062 bnez(t, slow, /* is_far */ true);
7063
7064 // Try to lock. Transition lock-bits 0b01 => 0b00
7065 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
7066 ori(mark, mark, markWord::unlocked_value);
7067 // Mask inline_type bit such that we go to the slow path if object is an inline type
7068 andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
7069 xori(t, mark, markWord::unlocked_value);
7070 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
7071 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
7072 bne(mark, t, slow, /* is_far */ true);
7073
7074 bind(push);
7075 // After successful lock, push object on lock-stack.
7076 add(t, xthread, top);
7077 sd(obj, Address(t));
7078 addiw(top, top, oopSize);
7079 sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7080 }
7081
7082 // Implements ligthweight-unlocking.
7083 //
7084 // - obj: the object to be unlocked
7085 // - tmp1, tmp2, tmp3: temporary registers
7086 // - slow: branched to if unlocking fails
7087 void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
7088 assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
|