29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "oops/methodData.hpp"
31 #include "opto/c2_MacroAssembler.hpp"
32 #include "opto/intrinsicnode.hpp"
33 #include "opto/output.hpp"
34 #include "opto/opcodes.hpp"
35 #include "opto/subnode.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 // C2 compiled method's prolog code.
49 void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
50
51 // WARNING: Initial instruction MUST be 5 bytes or longer so that
52 // NativeJump::patch_verified_entry will be able to patch out the entry
53 // code safely. The push to verify stack depth is ok at 5 bytes,
54 // the frame allocation can be either 3 or 6 bytes. So if we don't do
55 // stack bang then we must use the 6 byte frame allocation even if
56 // we have no frame. :-(
57 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
58
59 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
60 // Remove word for return addr
61 framesize -= wordSize;
62 stack_bang_size -= wordSize;
63
64 // Calls to C2R adapters often do not accept exceptional returns.
65 // We require that their callers must bang for them. But be careful, because
66 // some VM calls (such as call site linkage) can use several kilobytes of
67 // stack. But the stack safety zone should account for that.
68 // See bugs 4446381, 4468289, 4497237.
69 if (stack_bang_size > 0) {
82 // Create frame
83 if (framesize) {
84 subptr(rsp, framesize);
85 }
86 } else {
87 // Create frame (force generation of a 4 byte immediate value)
88 subptr_imm32(rsp, framesize);
89
90 // Save RBP register now.
91 framesize -= wordSize;
92 movptr(Address(rsp, framesize), rbp);
93 // Save caller's stack pointer into RBP if the frame pointer is preserved.
94 if (PreserveFramePointer) {
95 movptr(rbp, rsp);
96 if (framesize > 0) {
97 addptr(rbp, framesize);
98 }
99 }
100 }
101
102 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
103 framesize -= wordSize;
104 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
105 }
106
107 #ifndef _LP64
108 // If method sets FPU control word do it now
109 if (fp_mode_24b) {
110 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
111 }
112 if (UseSSE >= 2 && VerifyFPU) {
113 verify_FPU(0, "FPU stack must be clean on entry");
114 }
115 #endif
116
117 #ifdef ASSERT
118 if (VerifyStackAtCalls) {
119 Label L;
120 push(rax);
121 mov(rax, rsp);
122 andptr(rax, StackAlignmentInBytes-1);
123 cmpptr(rax, StackAlignmentInBytes-wordSize);
124 pop(rax);
125 jcc(Assembler::equal, L);
126 STOP("Stack is not properly aligned!");
127 bind(L);
128 }
129 #endif
130
131 if (!is_stub) {
132 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
133 #ifdef _LP64
134 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
135 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
136 Label dummy_slow_path;
137 Label dummy_continuation;
138 Label* slow_path = &dummy_slow_path;
139 Label* continuation = &dummy_continuation;
140 if (!Compile::current()->output()->in_scratch_emit_size()) {
141 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
142 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
143 Compile::current()->output()->add_stub(stub);
144 slow_path = &stub->entry();
145 continuation = &stub->continuation();
146 }
147 bs->nmethod_entry_barrier(this, slow_path, continuation);
148 }
149 #else
150 // Don't bother with out-of-line nmethod entry barrier stub for x86_32.
151 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
152 #endif
153 }
154 }
155
156 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
157 switch (vlen_in_bytes) {
158 case 4: // fall-through
159 case 8: // fall-through
160 case 16: return Assembler::AVX_128bit;
161 case 32: return Assembler::AVX_256bit;
162 case 64: return Assembler::AVX_512bit;
163
164 default: {
165 ShouldNotReachHere();
166 return Assembler::AVX_NoVec;
167 }
168 }
169 }
170
171 #if INCLUDE_RTM_OPT
172
173 // Update rtm_counters based on abort status
591
592 #if INCLUDE_RTM_OPT
593 if (UseRTMForStackLocks && use_rtm) {
594 assert(LockingMode != LM_MONITOR, "LockingMode == 0 (LM_MONITOR) and +UseRTMForStackLocks are mutually exclusive");
595 rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
596 stack_rtm_counters, method_data, profile_rtm,
597 DONE_LABEL, IsInflated);
598 }
599 #endif // INCLUDE_RTM_OPT
600
601 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
602 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
603 jcc(Assembler::notZero, IsInflated);
604
605 if (LockingMode == LM_MONITOR) {
606 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
607 testptr(objReg, objReg);
608 } else if (LockingMode == LM_LEGACY) {
609 // Attempt stack-locking ...
610 orptr (tmpReg, markWord::unlocked_value);
611 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
612 lock();
613 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
614 jcc(Assembler::equal, COUNT); // Success
615
616 // Recursive locking.
617 // The object is stack-locked: markword contains stack pointer to BasicLock.
618 // Locked by current thread if difference with current SP is less than one page.
619 subptr(tmpReg, rsp);
620 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
621 andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
622 movptr(Address(boxReg, 0), tmpReg);
623 } else {
624 assert(LockingMode == LM_LIGHTWEIGHT, "");
625 lightweight_lock(objReg, tmpReg, thread, scrReg, NO_COUNT);
626 jmp(COUNT);
627 }
628 jmp(DONE_LABEL);
629
630 bind(IsInflated);
|
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "oops/methodData.hpp"
31 #include "opto/c2_MacroAssembler.hpp"
32 #include "opto/intrinsicnode.hpp"
33 #include "opto/output.hpp"
34 #include "opto/opcodes.hpp"
35 #include "opto/subnode.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 // C2 compiled method's prolog code.
49 void C2_MacroAssembler::verified_entry(Compile* C, int sp_inc) {
50 if (C->clinit_barrier_on_entry()) {
51 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
52 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
53
54 Label L_skip_barrier;
55 Register klass = rscratch1;
56
57 mov_metadata(klass, C->method()->holder()->constant_encoding());
58 clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
59
60 jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
61
62 bind(L_skip_barrier);
63 }
64
65 int framesize = C->output()->frame_size_in_bytes();
66 int bangsize = C->output()->bang_size_in_bytes();
67 bool fp_mode_24b = false;
68 int stack_bang_size = C->output()->need_stack_bang(bangsize) ? bangsize : 0;
69
70 // WARNING: Initial instruction MUST be 5 bytes or longer so that
71 // NativeJump::patch_verified_entry will be able to patch out the entry
72 // code safely. The push to verify stack depth is ok at 5 bytes,
73 // the frame allocation can be either 3 or 6 bytes. So if we don't do
74 // stack bang then we must use the 6 byte frame allocation even if
75 // we have no frame. :-(
76 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
77
78 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
79 // Remove word for return addr
80 framesize -= wordSize;
81 stack_bang_size -= wordSize;
82
83 // Calls to C2R adapters often do not accept exceptional returns.
84 // We require that their callers must bang for them. But be careful, because
85 // some VM calls (such as call site linkage) can use several kilobytes of
86 // stack. But the stack safety zone should account for that.
87 // See bugs 4446381, 4468289, 4497237.
88 if (stack_bang_size > 0) {
101 // Create frame
102 if (framesize) {
103 subptr(rsp, framesize);
104 }
105 } else {
106 // Create frame (force generation of a 4 byte immediate value)
107 subptr_imm32(rsp, framesize);
108
109 // Save RBP register now.
110 framesize -= wordSize;
111 movptr(Address(rsp, framesize), rbp);
112 // Save caller's stack pointer into RBP if the frame pointer is preserved.
113 if (PreserveFramePointer) {
114 movptr(rbp, rsp);
115 if (framesize > 0) {
116 addptr(rbp, framesize);
117 }
118 }
119 }
120
121 if (C->needs_stack_repair()) {
122 // Save stack increment just below the saved rbp (also account for fixed framesize and rbp)
123 assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
124 movptr(Address(rsp, framesize - wordSize), sp_inc + framesize + wordSize);
125 }
126
127 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
128 framesize -= wordSize;
129 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
130 }
131
132 #ifndef _LP64
133 // If method sets FPU control word do it now
134 if (fp_mode_24b) {
135 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
136 }
137 if (UseSSE >= 2 && VerifyFPU) {
138 verify_FPU(0, "FPU stack must be clean on entry");
139 }
140 #endif
141
142 #ifdef ASSERT
143 if (VerifyStackAtCalls) {
144 Label L;
145 push(rax);
146 mov(rax, rsp);
147 andptr(rax, StackAlignmentInBytes-1);
148 cmpptr(rax, StackAlignmentInBytes-wordSize);
149 pop(rax);
150 jcc(Assembler::equal, L);
151 STOP("Stack is not properly aligned!");
152 bind(L);
153 }
154 #endif
155 }
156
157 void C2_MacroAssembler::entry_barrier() {
158 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
159 #ifdef _LP64
160 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
161 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
162 Label dummy_slow_path;
163 Label dummy_continuation;
164 Label* slow_path = &dummy_slow_path;
165 Label* continuation = &dummy_continuation;
166 if (!Compile::current()->output()->in_scratch_emit_size()) {
167 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
168 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
169 Compile::current()->output()->add_stub(stub);
170 slow_path = &stub->entry();
171 continuation = &stub->continuation();
172 }
173 bs->nmethod_entry_barrier(this, slow_path, continuation);
174 }
175 #else
176 // Don't bother with out-of-line nmethod entry barrier stub for x86_32.
177 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
178 #endif
179 }
180
181 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
182 switch (vlen_in_bytes) {
183 case 4: // fall-through
184 case 8: // fall-through
185 case 16: return Assembler::AVX_128bit;
186 case 32: return Assembler::AVX_256bit;
187 case 64: return Assembler::AVX_512bit;
188
189 default: {
190 ShouldNotReachHere();
191 return Assembler::AVX_NoVec;
192 }
193 }
194 }
195
196 #if INCLUDE_RTM_OPT
197
198 // Update rtm_counters based on abort status
616
617 #if INCLUDE_RTM_OPT
618 if (UseRTMForStackLocks && use_rtm) {
619 assert(LockingMode != LM_MONITOR, "LockingMode == 0 (LM_MONITOR) and +UseRTMForStackLocks are mutually exclusive");
620 rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
621 stack_rtm_counters, method_data, profile_rtm,
622 DONE_LABEL, IsInflated);
623 }
624 #endif // INCLUDE_RTM_OPT
625
626 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
627 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
628 jcc(Assembler::notZero, IsInflated);
629
630 if (LockingMode == LM_MONITOR) {
631 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
632 testptr(objReg, objReg);
633 } else if (LockingMode == LM_LEGACY) {
634 // Attempt stack-locking ...
635 orptr (tmpReg, markWord::unlocked_value);
636 if (EnableValhalla) {
637 // Mask inline_type bit such that we go to the slow path if object is an inline type
638 andptr(tmpReg, ~((int) markWord::inline_type_bit_in_place));
639 }
640 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
641 lock();
642 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
643 jcc(Assembler::equal, COUNT); // Success
644
645 // Recursive locking.
646 // The object is stack-locked: markword contains stack pointer to BasicLock.
647 // Locked by current thread if difference with current SP is less than one page.
648 subptr(tmpReg, rsp);
649 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
650 andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
651 movptr(Address(boxReg, 0), tmpReg);
652 } else {
653 assert(LockingMode == LM_LIGHTWEIGHT, "");
654 lightweight_lock(objReg, tmpReg, thread, scrReg, NO_COUNT);
655 jmp(COUNT);
656 }
657 jmp(DONE_LABEL);
658
659 bind(IsInflated);
|