9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "code/compiledIC.hpp"
30 #include "compiler/disassembler.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/cardTableBarrierSet.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "interpreter/bytecodeHistogram.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "interpreter/interpreterRuntime.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "memory/universe.hpp"
41 #include "oops/accessDecorators.hpp"
42 #include "oops/compressedKlass.inline.hpp"
43 #include "oops/compressedOops.inline.hpp"
44 #include "oops/klass.inline.hpp"
45 #include "oops/oop.hpp"
46 #include "runtime/interfaceSupport.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/jniHandles.inline.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "runtime/stubRoutines.hpp"
51 #include "utilities/globalDefinitions.hpp"
52 #include "utilities/integerCast.hpp"
53 #include "utilities/powerOfTwo.hpp"
54 #ifdef COMPILER2
55 #include "opto/compile.hpp"
56 #include "opto/node.hpp"
57 #include "opto/output.hpp"
58 #endif
59
60 #ifdef PRODUCT
61 #define BLOCK_COMMENT(str) /* nothing */
62 #else
63 #define BLOCK_COMMENT(str) block_comment(str)
64 #endif
65 #define STOP(str) stop(str);
928 call_VM_leaf_base(entry_point, 1);
929 }
930
931 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
932 assert_different_registers(arg_1, c_rarg0);
933 pass_arg0(this, arg_0);
934 pass_arg1(this, arg_1);
935 call_VM_leaf_base(entry_point, 2);
936 }
937
938 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
939 Register arg_1, Register arg_2) {
940 assert_different_registers(arg_1, c_rarg0);
941 assert_different_registers(arg_2, c_rarg0, c_rarg1);
942 pass_arg0(this, arg_0);
943 pass_arg1(this, arg_1);
944 pass_arg2(this, arg_2);
945 call_VM_leaf_base(entry_point, 3);
946 }
947
948 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
949 pass_arg0(this, arg_0);
950 MacroAssembler::call_VM_leaf_base(entry_point, 1);
951 }
952
953 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
954
955 assert_different_registers(arg_0, c_rarg1);
956 pass_arg1(this, arg_1);
957 pass_arg0(this, arg_0);
958 MacroAssembler::call_VM_leaf_base(entry_point, 2);
959 }
960
961 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
962 assert_different_registers(arg_0, c_rarg1, c_rarg2);
963 assert_different_registers(arg_1, c_rarg2);
964 pass_arg2(this, arg_2);
965 pass_arg1(this, arg_1);
966 pass_arg0(this, arg_0);
967 MacroAssembler::call_VM_leaf_base(entry_point, 3);
3578 movptr(dst, Address((address)obj, rspec));
3579 } else {
3580 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3581 ld(dst, Address(dummy, rspec));
3582 }
3583 }
3584
3585 // Move a metadata address into a register.
3586 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3587 assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata");
3588 int oop_index;
3589 if (obj == nullptr) {
3590 oop_index = oop_recorder()->allocate_metadata_index(obj);
3591 } else {
3592 oop_index = oop_recorder()->find_index(obj);
3593 }
3594 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3595 movptr(dst, Address((address)obj, rspec));
3596 }
3597
3598 // Writes to stack successive pages until offset reached to check for
3599 // stack overflow + shadow pages. This clobbers tmp.
3600 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3601 assert_different_registers(tmp, size, t0);
3602 // Bang stack for total size given plus shadow page size.
3603 // Bang one page at a time because large size can bang beyond yellow and
3604 // red zones.
3605 mv(t0, (int)os::vm_page_size());
3606 Label loop;
3607 bind(loop);
3608 sub(tmp, sp, t0);
3609 subw(size, size, t0);
3610 sd(size, Address(tmp));
3611 bgtz(size, loop);
3612
3613 // Bang down shadow pages too.
3614 // At this point, (tmp-0) is the last address touched, so don't
3615 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3616 // was post-decremented.) Skip this address by starting at i=1, and
3617 // touch a few more pages below. N.B. It is important to touch all
3663 bool as_raw = (decorators & AS_RAW) != 0;
3664 if (as_raw) {
3665 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
3666 } else {
3667 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
3668 }
3669 }
3670
3671 void MacroAssembler::null_check(Register reg, int offset) {
3672 if (needs_explicit_null_check(offset)) {
3673 // provoke OS null exception if reg is null by
3674 // accessing M[reg] w/o changing any registers
3675 // NOTE: this is plenty to provoke a segv
3676 ld(zr, Address(reg, 0));
3677 } else {
3678 // nothing to do, (later) access of M[reg + offset]
3679 // will provoke OS null exception if reg is null
3680 }
3681 }
3682
3683 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3684 Address dst, Register val,
3685 Register tmp1, Register tmp2, Register tmp3) {
3686 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3687 decorators = AccessInternal::decorator_fixup(decorators, type);
3688 bool as_raw = (decorators & AS_RAW) != 0;
3689 if (as_raw) {
3690 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3691 } else {
3692 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3693 }
3694 }
3695
3696 // Algorithm must match CompressedOops::encode.
3697 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3698 verify_oop_msg(s, "broken oop in encode_heap_oop");
3699 if (CompressedOops::base() == nullptr) {
3700 if (CompressedOops::shift() != 0) {
3701 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3702 srli(d, s, LogMinObjAlignmentInBytes);
3762 }
3763
3764 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
3765 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
3766 ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3767 srli(dst, dst, markWord::klass_shift);
3768 }
3769
3770 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
3771 assert_different_registers(dst, tmp);
3772 assert_different_registers(src, tmp);
3773 if (UseCompactObjectHeaders) {
3774 load_narrow_klass_compact(dst, src);
3775 decode_klass_not_null(dst, tmp);
3776 } else {
3777 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3778 decode_klass_not_null(dst, tmp);
3779 }
3780 }
3781
3782 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
3783 // FIXME: Should this be a store release? concurrent gcs assumes
3784 // klass length is valid if klass field is not null.
3785 assert(!UseCompactObjectHeaders, "not with compact headers");
3786 encode_klass_not_null(src, tmp);
3787 sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3788
3789 }
3790
3791 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3792 assert(!UseCompactObjectHeaders, "not with compact headers");
3793 // Store to klass gap in destination
3794 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3795 }
3796
3797 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
3798 assert_different_registers(r, tmp);
3799 decode_klass_not_null(r, r, tmp);
3800 }
3801
5226 // need to save whatever non-callee save context might get clobbered
5227 // by the call to Thread::current() or, indeed, the call setup code.
5228 void MacroAssembler::get_thread(Register thread) {
5229 // save all call-clobbered regs except thread
5230 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
5231 RegSet::range(x28, x31) + ra - thread;
5232 push_reg(saved_regs, sp);
5233
5234 mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
5235 jalr(t1);
5236 if (thread != c_rarg0) {
5237 mv(thread, c_rarg0);
5238 }
5239
5240 // restore pushed registers
5241 pop_reg(saved_regs, sp);
5242 }
5243
5244 void MacroAssembler::load_byte_map_base(Register reg) {
5245 CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
5246 mv(reg, (uint64_t)ctbs->card_table_base_const());
5247 }
5248
5249 void MacroAssembler::build_frame(int framesize) {
5250 assert(framesize >= 2, "framesize must include space for FP/RA");
5251 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5252 sub(sp, sp, framesize);
5253 sd(fp, Address(sp, framesize - 2 * wordSize));
5254 sd(ra, Address(sp, framesize - wordSize));
5255 if (PreserveFramePointer) { add(fp, sp, framesize); }
5256 }
5257
5258 void MacroAssembler::remove_frame(int framesize) {
5259 assert(framesize >= 2, "framesize must include space for FP/RA");
5260 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5261 ld(fp, Address(sp, framesize - 2 * wordSize));
5262 ld(ra, Address(sp, framesize - wordSize));
5263 add(sp, sp, framesize);
5264 }
5265
5266 void MacroAssembler::reserved_stack_check() {
5267 // testing if reserved zone needs to be enabled
5268 Label no_reserved_zone_enabling;
5269
5270 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
5271 bltu(sp, t0, no_reserved_zone_enabling);
5272
5273 enter(); // RA and FP are live.
5274 mv(c_rarg0, xthread);
5275 rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
5276 leave();
5277
5278 // We have already removed our own frame.
5279 // throw_delayed_StackOverflowError will think that it's been
5280 // called by our caller.
5281 j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
5282 should_not_reach_here();
5283
5284 bind(no_reserved_zone_enabling);
5285 }
5511 is_simm12(dst.offset())) || is_simm12(value)),
5512 "invalid value and address mode combination");
5513 Address adr = add_memory_helper(dst, tmp2);
5514 assert(!adr.uses(tmp1), "invalid dst for address decrement");
5515 lwu(tmp1, adr);
5516 subw(tmp1, tmp1, value, tmp2);
5517 sw(tmp1, adr);
5518 }
5519
5520 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
5521 load_method_holder(result, method);
5522 ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
5523 }
5524
5525 void MacroAssembler::load_method_holder(Register holder, Register method) {
5526 ld(holder, Address(method, Method::const_offset())); // ConstMethod*
5527 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
5528 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
5529 }
5530
5531 // string indexof
5532 // compute index by trailing zeros
5533 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
5534 Register match_mask, Register result,
5535 Register ch2, Register tmp,
5536 bool haystack_isL) {
5537 int haystack_chr_shift = haystack_isL ? 0 : 1;
5538 srl(match_mask, match_mask, trailing_zeros);
5539 srli(match_mask, match_mask, 1);
5540 srli(tmp, trailing_zeros, LogBitsPerByte);
5541 if (!haystack_isL) andi(tmp, tmp, 0xE);
5542 add(haystack, haystack, tmp);
5543 ld(ch2, Address(haystack));
5544 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
5545 add(result, result, tmp);
5546 }
5547
5548 // string indexof
5549 // Find pattern element in src, compute match mask,
5550 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
6837 bnez(tmp1, slow, /* is_far */ true);
6838 }
6839
6840 // Check if the lock-stack is full.
6841 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
6842 mv(t, (unsigned)LockStack::end_offset());
6843 bge(top, t, slow, /* is_far */ true);
6844
6845 // Check for recursion.
6846 add(t, xthread, top);
6847 ld(t, Address(t, -oopSize));
6848 beq(obj, t, push);
6849
6850 // Check header for monitor (0b10).
6851 test_bit(t, mark, exact_log2(markWord::monitor_value));
6852 bnez(t, slow, /* is_far */ true);
6853
6854 // Try to lock. Transition lock-bits 0b01 => 0b00
6855 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
6856 ori(mark, mark, markWord::unlocked_value);
6857 xori(t, mark, markWord::unlocked_value);
6858 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
6859 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
6860 bne(mark, t, slow, /* is_far */ true);
6861
6862 bind(push);
6863 // After successful lock, push object on lock-stack.
6864 add(t, xthread, top);
6865 sd(obj, Address(t));
6866 addiw(top, top, oopSize);
6867 sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
6868 }
6869
6870 // Implements ligthweight-unlocking.
6871 //
6872 // - obj: the object to be unlocked
6873 // - tmp1, tmp2, tmp3: temporary registers
6874 // - slow: branched to if unlocking fails
6875 void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
6876 assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
|
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "ci/ciInlineKlass.hpp"
30 #include "code/compiledIC.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/barrierSetAssembler.hpp"
34 #include "gc/shared/cardTable.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "interpreter/bytecodeHistogram.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "interpreter/interpreterRuntime.hpp"
40 #include "memory/resourceArea.hpp"
41 #include "memory/universe.hpp"
42 #include "oops/accessDecorators.hpp"
43 #include "oops/compressedKlass.inline.hpp"
44 #include "oops/compressedOops.inline.hpp"
45 #include "oops/klass.inline.hpp"
46 #include "oops/oop.hpp"
47 #include "oops/resolvedFieldEntry.hpp"
48 #include "runtime/interfaceSupport.inline.hpp"
49 #include "runtime/javaThread.hpp"
50 #include "runtime/jniHandles.inline.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "utilities/globalDefinitions.hpp"
54 #include "utilities/integerCast.hpp"
55 #include "utilities/powerOfTwo.hpp"
56 #ifdef COMPILER2
57 #include "opto/compile.hpp"
58 #include "opto/node.hpp"
59 #include "opto/output.hpp"
60 #endif
61
62 #ifdef PRODUCT
63 #define BLOCK_COMMENT(str) /* nothing */
64 #else
65 #define BLOCK_COMMENT(str) block_comment(str)
66 #endif
67 #define STOP(str) stop(str);
930 call_VM_leaf_base(entry_point, 1);
931 }
932
933 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
934 assert_different_registers(arg_1, c_rarg0);
935 pass_arg0(this, arg_0);
936 pass_arg1(this, arg_1);
937 call_VM_leaf_base(entry_point, 2);
938 }
939
940 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
941 Register arg_1, Register arg_2) {
942 assert_different_registers(arg_1, c_rarg0);
943 assert_different_registers(arg_2, c_rarg0, c_rarg1);
944 pass_arg0(this, arg_0);
945 pass_arg1(this, arg_1);
946 pass_arg2(this, arg_2);
947 call_VM_leaf_base(entry_point, 3);
948 }
949
950 void MacroAssembler::super_call_VM_leaf(address entry_point) {
951 MacroAssembler::call_VM_leaf_base(entry_point, 1);
952 }
953
954 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
955 pass_arg0(this, arg_0);
956 MacroAssembler::call_VM_leaf_base(entry_point, 1);
957 }
958
959 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
960
961 assert_different_registers(arg_0, c_rarg1);
962 pass_arg1(this, arg_1);
963 pass_arg0(this, arg_0);
964 MacroAssembler::call_VM_leaf_base(entry_point, 2);
965 }
966
967 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
968 assert_different_registers(arg_0, c_rarg1, c_rarg2);
969 assert_different_registers(arg_1, c_rarg2);
970 pass_arg2(this, arg_2);
971 pass_arg1(this, arg_1);
972 pass_arg0(this, arg_0);
973 MacroAssembler::call_VM_leaf_base(entry_point, 3);
3584 movptr(dst, Address((address)obj, rspec));
3585 } else {
3586 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3587 ld(dst, Address(dummy, rspec));
3588 }
3589 }
3590
3591 // Move a metadata address into a register.
3592 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3593 assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata");
3594 int oop_index;
3595 if (obj == nullptr) {
3596 oop_index = oop_recorder()->allocate_metadata_index(obj);
3597 } else {
3598 oop_index = oop_recorder()->find_index(obj);
3599 }
3600 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3601 movptr(dst, Address((address)obj, rspec));
3602 }
3603
3604 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3605 assert_different_registers(holder_klass, index, layout_info);
3606 InlineLayoutInfo array[2];
3607 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3608 if (is_power_of_2(size)) {
3609 slli(index, index, log2i_exact(size)); // Scale index by power of 2
3610 } else {
3611 mv(layout_info, size);
3612 mul(index, index, layout_info); // Scale the index to be the entry index * array_element_size
3613 }
3614 ld(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset()));
3615 add(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3616 add(layout_info, layout_info, index);
3617 la(layout_info, Address(layout_info));
3618 }
3619
3620 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
3621 Register inline_layout_info) {
3622 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3623 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3624 }
3625
3626 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3627 ld(offset, Address(inline_klass, InlineKlass::adr_members_offset()));
3628 lwu(offset, Address(offset, InlineKlass::payload_offset_offset()));
3629 }
3630
3631 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass) {
3632 assert_different_registers(data, t0);
3633 // ((address) (void*) o) + vk->payload_offset();
3634 Register offset = (data == oop) ? t0 : data;
3635 payload_offset(inline_klass, offset);
3636 if (data == oop) {
3637 add(data, data, offset);
3638 } else {
3639 add(data, oop, offset);
3640 la(data, Address(data));
3641 }
3642 }
3643
3644 // Writes to stack successive pages until offset reached to check for
3645 // stack overflow + shadow pages. This clobbers tmp.
3646 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3647 assert_different_registers(tmp, size, t0);
3648 // Bang stack for total size given plus shadow page size.
3649 // Bang one page at a time because large size can bang beyond yellow and
3650 // red zones.
3651 mv(t0, (int)os::vm_page_size());
3652 Label loop;
3653 bind(loop);
3654 sub(tmp, sp, t0);
3655 subw(size, size, t0);
3656 sd(size, Address(tmp));
3657 bgtz(size, loop);
3658
3659 // Bang down shadow pages too.
3660 // At this point, (tmp-0) is the last address touched, so don't
3661 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3662 // was post-decremented.) Skip this address by starting at i=1, and
3663 // touch a few more pages below. N.B. It is important to touch all
3709 bool as_raw = (decorators & AS_RAW) != 0;
3710 if (as_raw) {
3711 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
3712 } else {
3713 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
3714 }
3715 }
3716
3717 void MacroAssembler::null_check(Register reg, int offset) {
3718 if (needs_explicit_null_check(offset)) {
3719 // provoke OS null exception if reg is null by
3720 // accessing M[reg] w/o changing any registers
3721 // NOTE: this is plenty to provoke a segv
3722 ld(zr, Address(reg, 0));
3723 } else {
3724 // nothing to do, (later) access of M[reg + offset]
3725 // will provoke OS null exception if reg is null
3726 }
3727 }
3728
3729 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
3730 test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3731 bnez(temp_reg, is_null_free_inline_type);
3732 }
3733
3734 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
3735 test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3736 beqz(temp_reg, not_null_free_inline_type);
3737 }
3738
3739 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
3740 test_bit(temp_reg, flags, ResolvedFieldEntry::is_flat_shift);
3741 bnez(temp_reg, is_flat);
3742 }
3743
3744 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3745 assert_different_registers(markword, t1);
3746 mv(t1, markWord::inline_type_pattern_mask);
3747 andr(markword, markword, t1);
3748 mv(t1, markWord::inline_type_pattern);
3749 beq(markword, t1, is_inline_type);
3750 }
3751
3752 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null) {
3753 assert_different_registers(tmp, t0);
3754 if (can_be_null) {
3755 beqz(object, not_inline_type);
3756 }
3757 const int is_inline_type_mask = markWord::inline_type_pattern;
3758 ld(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
3759 mv(t0, is_inline_type_mask);
3760 andr(tmp, tmp, t0);
3761 bne(tmp, t0, not_inline_type);
3762 }
3763
3764 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t tst_bit, bool jmp_set, Label& jmp_label) {
3765 assert_different_registers(temp_reg, t0);
3766 Label test_mark_word;
3767 // load mark word
3768 ld(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
3769 // check displaced
3770 test_bit(t0, temp_reg, exact_log2(markWord::unlocked_value));
3771 bnez(t0, test_mark_word);
3772 // slow path use klass prototype
3773 load_prototype_header(temp_reg, oop);
3774
3775 bind(test_mark_word);
3776 andi(temp_reg, temp_reg, tst_bit);
3777 if (jmp_set) {
3778 bnez(temp_reg, jmp_label, /* is_far */ true);
3779 } else {
3780 beqz(temp_reg, jmp_label, /* is_far */ true);
3781 }
3782 }
3783
3784 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) {
3785 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
3786 }
3787
3788 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
3789 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
3790 }
3791
3792 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label&is_non_flat_array) {
3793 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3794 }
3795
3796 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
3797 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3798 }
3799
3800 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3801 test_bit(t0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3802 bnez(t0, is_flat_array);
3803 }
3804
3805 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3806 Address dst, Register val,
3807 Register tmp1, Register tmp2, Register tmp3) {
3808 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3809 decorators = AccessInternal::decorator_fixup(decorators, type);
3810 bool as_raw = (decorators & AS_RAW) != 0;
3811 if (as_raw) {
3812 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3813 } else {
3814 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3815 }
3816 }
3817
3818 // Algorithm must match CompressedOops::encode.
3819 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3820 verify_oop_msg(s, "broken oop in encode_heap_oop");
3821 if (CompressedOops::base() == nullptr) {
3822 if (CompressedOops::shift() != 0) {
3823 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3824 srli(d, s, LogMinObjAlignmentInBytes);
3884 }
3885
3886 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
3887 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
3888 ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3889 srli(dst, dst, markWord::klass_shift);
3890 }
3891
3892 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
3893 assert_different_registers(dst, tmp);
3894 assert_different_registers(src, tmp);
3895 if (UseCompactObjectHeaders) {
3896 load_narrow_klass_compact(dst, src);
3897 decode_klass_not_null(dst, tmp);
3898 } else {
3899 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3900 decode_klass_not_null(dst, tmp);
3901 }
3902 }
3903
3904 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
3905 load_klass(dst, src, tmp);
3906 ld(dst, Address(dst, Klass::prototype_header_offset()));
3907 }
3908
3909 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
3910 // FIXME: Should this be a store release? concurrent gcs assumes
3911 // klass length is valid if klass field is not null.
3912 assert(!UseCompactObjectHeaders, "not with compact headers");
3913 encode_klass_not_null(src, tmp);
3914 sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3915
3916 }
3917
3918 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3919 assert(!UseCompactObjectHeaders, "not with compact headers");
3920 // Store to klass gap in destination
3921 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3922 }
3923
3924 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
3925 assert_different_registers(r, tmp);
3926 decode_klass_not_null(r, r, tmp);
3927 }
3928
5353 // need to save whatever non-callee save context might get clobbered
5354 // by the call to Thread::current() or, indeed, the call setup code.
5355 void MacroAssembler::get_thread(Register thread) {
5356 // save all call-clobbered regs except thread
5357 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
5358 RegSet::range(x28, x31) + ra - thread;
5359 push_reg(saved_regs, sp);
5360
5361 mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
5362 jalr(t1);
5363 if (thread != c_rarg0) {
5364 mv(thread, c_rarg0);
5365 }
5366
5367 // restore pushed registers
5368 pop_reg(saved_regs, sp);
5369 }
5370
5371 void MacroAssembler::load_byte_map_base(Register reg) {
5372 CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
5373 // Strictly speaking the card table base isn't an address at all, and it might
5374 // even be negative. It is thus materialised as a constant.
5375 mv(reg, (uint64_t)ctbs->card_table_base_const());
5376 }
5377
5378 void MacroAssembler::build_frame(int framesize) {
5379 assert(framesize >= 2, "framesize must include space for FP/RA");
5380 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5381 sub(sp, sp, framesize);
5382 sd(fp, Address(sp, framesize - 2 * wordSize));
5383 sd(ra, Address(sp, framesize - wordSize));
5384 if (PreserveFramePointer) { add(fp, sp, framesize); }
5385 }
5386
5387 void MacroAssembler::remove_frame(int framesize) {
5388 assert(framesize >= 2, "framesize must include space for FP/RA");
5389 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5390 ld(fp, Address(sp, framesize - 2 * wordSize));
5391 ld(ra, Address(sp, framesize - wordSize));
5392 add(sp, sp, framesize);
5393 }
5394
5395 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
5396 assert(!needs_stack_repair, "unimplemented");
5397 remove_frame(initial_framesize);
5398 }
5399
5400 #ifdef COMPILER2
5401 // C2 compiled method's prolog code
5402 // Moved here from riscv.ad to support Valhalla code belows
5403 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5404 if (C->clinit_barrier_on_entry()) {
5405 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
5406
5407 Label L_skip_barrier;
5408
5409 mov_metadata(t1, C->method()->holder()->constant_encoding());
5410 clinit_barrier(t1, t0, &L_skip_barrier);
5411 far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
5412 bind(L_skip_barrier);
5413 }
5414
5415 int bangsize = C->output()->bang_size_in_bytes();
5416 if (C->output()->need_stack_bang(bangsize)) {
5417 generate_stack_overflow_check(bangsize);
5418 }
5419
5420 // n.b. frame size includes space for return pc and fp
5421 const long framesize = C->output()->frame_size_in_bytes();
5422 build_frame(framesize);
5423
5424 assert(!C->needs_stack_repair(), "unimplemented");
5425 }
5426 #endif // COMPILER2
5427
5428 // Move a value between registers/stack slots and update the reg_state
5429 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5430 Unimplemented();
5431 return false;
5432 }
5433
5434 // Read all fields from an inline type oop and store the values in registers/stack slots
5435 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5436 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5437 RegState reg_state[]) {
5438
5439 Unimplemented();
5440 return false;
5441 }
5442
5443 // Pack fields back into an inline type oop
5444 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5445 VMRegPair* from, int from_count, int& from_index, VMReg to,
5446 RegState reg_state[], Register val_array) {
5447 Unimplemented();
5448 return false;
5449 }
5450
5451 // Calculate the extra stack space required for packing or unpacking inline
5452 // args and adjust the stack pointer
5453 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5454 Unimplemented();
5455 return false;
5456 }
5457
5458 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5459 Unimplemented();
5460 return reg;
5461 }
5462
5463 void MacroAssembler::reserved_stack_check() {
5464 // testing if reserved zone needs to be enabled
5465 Label no_reserved_zone_enabling;
5466
5467 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
5468 bltu(sp, t0, no_reserved_zone_enabling);
5469
5470 enter(); // RA and FP are live.
5471 mv(c_rarg0, xthread);
5472 rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
5473 leave();
5474
5475 // We have already removed our own frame.
5476 // throw_delayed_StackOverflowError will think that it's been
5477 // called by our caller.
5478 j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
5479 should_not_reach_here();
5480
5481 bind(no_reserved_zone_enabling);
5482 }
5708 is_simm12(dst.offset())) || is_simm12(value)),
5709 "invalid value and address mode combination");
5710 Address adr = add_memory_helper(dst, tmp2);
5711 assert(!adr.uses(tmp1), "invalid dst for address decrement");
5712 lwu(tmp1, adr);
5713 subw(tmp1, tmp1, value, tmp2);
5714 sw(tmp1, adr);
5715 }
5716
5717 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
5718 load_method_holder(result, method);
5719 ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
5720 }
5721
5722 void MacroAssembler::load_method_holder(Register holder, Register method) {
5723 ld(holder, Address(method, Method::const_offset())); // ConstMethod*
5724 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
5725 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
5726 }
5727
5728 void MacroAssembler::load_metadata(Register dst, Register src) {
5729 if (UseCompactObjectHeaders) {
5730 load_narrow_klass_compact(dst, src);
5731 } else {
5732 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5733 }
5734 }
5735
5736 // string indexof
5737 // compute index by trailing zeros
5738 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
5739 Register match_mask, Register result,
5740 Register ch2, Register tmp,
5741 bool haystack_isL) {
5742 int haystack_chr_shift = haystack_isL ? 0 : 1;
5743 srl(match_mask, match_mask, trailing_zeros);
5744 srli(match_mask, match_mask, 1);
5745 srli(tmp, trailing_zeros, LogBitsPerByte);
5746 if (!haystack_isL) andi(tmp, tmp, 0xE);
5747 add(haystack, haystack, tmp);
5748 ld(ch2, Address(haystack));
5749 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
5750 add(result, result, tmp);
5751 }
5752
5753 // string indexof
5754 // Find pattern element in src, compute match mask,
5755 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
7042 bnez(tmp1, slow, /* is_far */ true);
7043 }
7044
7045 // Check if the lock-stack is full.
7046 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7047 mv(t, (unsigned)LockStack::end_offset());
7048 bge(top, t, slow, /* is_far */ true);
7049
7050 // Check for recursion.
7051 add(t, xthread, top);
7052 ld(t, Address(t, -oopSize));
7053 beq(obj, t, push);
7054
7055 // Check header for monitor (0b10).
7056 test_bit(t, mark, exact_log2(markWord::monitor_value));
7057 bnez(t, slow, /* is_far */ true);
7058
7059 // Try to lock. Transition lock-bits 0b01 => 0b00
7060 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
7061 ori(mark, mark, markWord::unlocked_value);
7062 // Mask inline_type bit such that we go to the slow path if object is an inline type
7063 andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
7064 xori(t, mark, markWord::unlocked_value);
7065 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
7066 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
7067 bne(mark, t, slow, /* is_far */ true);
7068
7069 bind(push);
7070 // After successful lock, push object on lock-stack.
7071 add(t, xthread, top);
7072 sd(obj, Address(t));
7073 addiw(top, top, oopSize);
7074 sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7075 }
7076
7077 // Implements ligthweight-unlocking.
7078 //
7079 // - obj: the object to be unlocked
7080 // - tmp1, tmp2, tmp3: temporary registers
7081 // - slow: branched to if unlocking fails
7082 void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
7083 assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
|