32 #include "opto/output.hpp"
33 #include "opto/opcodes.hpp"
34 #include "opto/subnode.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/powerOfTwo.hpp"
41 #include "utilities/sizes.hpp"
42
43 #ifdef PRODUCT
44 #define BLOCK_COMMENT(str) /* nothing */
45 #define STOP(error) stop(error)
46 #else
47 #define BLOCK_COMMENT(str) block_comment(str)
48 #define STOP(error) block_comment(error); stop(error)
49 #endif
50
51 // C2 compiled method's prolog code.
52 void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
53
54 // WARNING: Initial instruction MUST be 5 bytes or longer so that
55 // NativeJump::patch_verified_entry will be able to patch out the entry
56 // code safely. The push to verify stack depth is ok at 5 bytes,
57 // the frame allocation can be either 3 or 6 bytes. So if we don't do
58 // stack bang then we must use the 6 byte frame allocation even if
59 // we have no frame. :-(
60 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
61
62 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
63 // Remove word for return addr
64 framesize -= wordSize;
65 stack_bang_size -= wordSize;
66
67 // Calls to C2R adapters often do not accept exceptional returns.
68 // We require that their callers must bang for them. But be careful, because
69 // some VM calls (such as call site linkage) can use several kilobytes of
70 // stack. But the stack safety zone should account for that.
71 // See bugs 4446381, 4468289, 4497237.
72 if (stack_bang_size > 0) {
85 // Create frame
86 if (framesize) {
87 subptr(rsp, framesize);
88 }
89 } else {
90 // Create frame (force generation of a 4 byte immediate value)
91 subptr_imm32(rsp, framesize);
92
93 // Save RBP register now.
94 framesize -= wordSize;
95 movptr(Address(rsp, framesize), rbp);
96 // Save caller's stack pointer into RBP if the frame pointer is preserved.
97 if (PreserveFramePointer) {
98 movptr(rbp, rsp);
99 if (framesize > 0) {
100 addptr(rbp, framesize);
101 }
102 }
103 }
104
105 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
106 framesize -= wordSize;
107 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
108 }
109
110 #ifndef _LP64
111 // If method sets FPU control word do it now
112 if (fp_mode_24b) {
113 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
114 }
115 if (UseSSE >= 2 && VerifyFPU) {
116 verify_FPU(0, "FPU stack must be clean on entry");
117 }
118 #endif
119
120 #ifdef ASSERT
121 if (VerifyStackAtCalls) {
122 Label L;
123 push(rax);
124 mov(rax, rsp);
125 andptr(rax, StackAlignmentInBytes-1);
126 cmpptr(rax, StackAlignmentInBytes-wordSize);
127 pop(rax);
128 jcc(Assembler::equal, L);
129 STOP("Stack is not properly aligned!");
130 bind(L);
131 }
132 #endif
133
134 if (!is_stub) {
135 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
136 #ifdef _LP64
137 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
138 Label dummy_slow_path;
139 Label dummy_continuation;
140 Label* slow_path = &dummy_slow_path;
141 Label* continuation = &dummy_continuation;
142 if (!Compile::current()->output()->in_scratch_emit_size()) {
143 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
144 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
145 Compile::current()->output()->add_stub(stub);
146 slow_path = &stub->entry();
147 continuation = &stub->continuation();
148 }
149 bs->nmethod_entry_barrier(this, slow_path, continuation);
150 #else
151 // Don't bother with out-of-line nmethod entry barrier stub for x86_32.
152 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
153 #endif
154 }
155 }
156
157 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
158 switch (vlen_in_bytes) {
159 case 4: // fall-through
160 case 8: // fall-through
161 case 16: return Assembler::AVX_128bit;
162 case 32: return Assembler::AVX_256bit;
163 case 64: return Assembler::AVX_512bit;
164
165 default: {
166 ShouldNotReachHere();
167 return Assembler::AVX_NoVec;
168 }
169 }
170 }
171
172 // fast_lock and fast_unlock used by C2
173
174 // Because the transitions from emitted code to the runtime
272
273 Label IsInflated, DONE_LABEL, NO_COUNT, COUNT;
274
275 if (DiagnoseSyncOnValueBasedClasses != 0) {
276 load_klass(tmpReg, objReg, scrReg);
277 testb(Address(tmpReg, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
278 jcc(Assembler::notZero, DONE_LABEL);
279 }
280
281 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
282 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
283 jcc(Assembler::notZero, IsInflated);
284
285 if (LockingMode == LM_MONITOR) {
286 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
287 testptr(objReg, objReg);
288 } else {
289 assert(LockingMode == LM_LEGACY, "must be");
290 // Attempt stack-locking ...
291 orptr (tmpReg, markWord::unlocked_value);
292 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
293 lock();
294 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
295 jcc(Assembler::equal, COUNT); // Success
296
297 // Recursive locking.
298 // The object is stack-locked: markword contains stack pointer to BasicLock.
299 // Locked by current thread if difference with current SP is less than one page.
300 subptr(tmpReg, rsp);
301 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
302 andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
303 movptr(Address(boxReg, 0), tmpReg);
304 }
305 jmp(DONE_LABEL);
306
307 bind(IsInflated);
308 // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
309
310 #ifndef _LP64
311 // Just take slow path to avoid dealing with 64 bit atomic instructions here.
|
32 #include "opto/output.hpp"
33 #include "opto/opcodes.hpp"
34 #include "opto/subnode.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/powerOfTwo.hpp"
41 #include "utilities/sizes.hpp"
42
43 #ifdef PRODUCT
44 #define BLOCK_COMMENT(str) /* nothing */
45 #define STOP(error) stop(error)
46 #else
47 #define BLOCK_COMMENT(str) block_comment(str)
48 #define STOP(error) block_comment(error); stop(error)
49 #endif
50
51 // C2 compiled method's prolog code.
52 void C2_MacroAssembler::verified_entry(Compile* C, int sp_inc) {
53 if (C->clinit_barrier_on_entry()) {
54 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
55 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
56
57 Label L_skip_barrier;
58 Register klass = rscratch1;
59
60 mov_metadata(klass, C->method()->holder()->constant_encoding());
61 clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
62
63 jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
64
65 bind(L_skip_barrier);
66 }
67
68 int framesize = C->output()->frame_size_in_bytes();
69 int bangsize = C->output()->bang_size_in_bytes();
70 bool fp_mode_24b = false;
71 int stack_bang_size = C->output()->need_stack_bang(bangsize) ? bangsize : 0;
72
73 // WARNING: Initial instruction MUST be 5 bytes or longer so that
74 // NativeJump::patch_verified_entry will be able to patch out the entry
75 // code safely. The push to verify stack depth is ok at 5 bytes,
76 // the frame allocation can be either 3 or 6 bytes. So if we don't do
77 // stack bang then we must use the 6 byte frame allocation even if
78 // we have no frame. :-(
79 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
80
81 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
82 // Remove word for return addr
83 framesize -= wordSize;
84 stack_bang_size -= wordSize;
85
86 // Calls to C2R adapters often do not accept exceptional returns.
87 // We require that their callers must bang for them. But be careful, because
88 // some VM calls (such as call site linkage) can use several kilobytes of
89 // stack. But the stack safety zone should account for that.
90 // See bugs 4446381, 4468289, 4497237.
91 if (stack_bang_size > 0) {
104 // Create frame
105 if (framesize) {
106 subptr(rsp, framesize);
107 }
108 } else {
109 // Create frame (force generation of a 4 byte immediate value)
110 subptr_imm32(rsp, framesize);
111
112 // Save RBP register now.
113 framesize -= wordSize;
114 movptr(Address(rsp, framesize), rbp);
115 // Save caller's stack pointer into RBP if the frame pointer is preserved.
116 if (PreserveFramePointer) {
117 movptr(rbp, rsp);
118 if (framesize > 0) {
119 addptr(rbp, framesize);
120 }
121 }
122 }
123
124 if (C->needs_stack_repair()) {
125 // Save stack increment just below the saved rbp (also account for fixed framesize and rbp)
126 assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
127 movptr(Address(rsp, framesize - wordSize), sp_inc + framesize + wordSize);
128 }
129
130 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
131 framesize -= wordSize;
132 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
133 }
134
135 #ifndef _LP64
136 // If method sets FPU control word do it now
137 if (fp_mode_24b) {
138 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
139 }
140 if (UseSSE >= 2 && VerifyFPU) {
141 verify_FPU(0, "FPU stack must be clean on entry");
142 }
143 #endif
144
145 #ifdef ASSERT
146 if (VerifyStackAtCalls) {
147 Label L;
148 push(rax);
149 mov(rax, rsp);
150 andptr(rax, StackAlignmentInBytes-1);
151 cmpptr(rax, StackAlignmentInBytes-wordSize);
152 pop(rax);
153 jcc(Assembler::equal, L);
154 STOP("Stack is not properly aligned!");
155 bind(L);
156 }
157 #endif
158 }
159
160 void C2_MacroAssembler::entry_barrier() {
161 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
162 #ifdef _LP64
163 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
164 Label dummy_slow_path;
165 Label dummy_continuation;
166 Label* slow_path = &dummy_slow_path;
167 Label* continuation = &dummy_continuation;
168 if (!Compile::current()->output()->in_scratch_emit_size()) {
169 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
170 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
171 Compile::current()->output()->add_stub(stub);
172 slow_path = &stub->entry();
173 continuation = &stub->continuation();
174 }
175 bs->nmethod_entry_barrier(this, slow_path, continuation);
176 #else
177 // Don't bother with out-of-line nmethod entry barrier stub for x86_32.
178 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
179 #endif
180 }
181
182 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
183 switch (vlen_in_bytes) {
184 case 4: // fall-through
185 case 8: // fall-through
186 case 16: return Assembler::AVX_128bit;
187 case 32: return Assembler::AVX_256bit;
188 case 64: return Assembler::AVX_512bit;
189
190 default: {
191 ShouldNotReachHere();
192 return Assembler::AVX_NoVec;
193 }
194 }
195 }
196
197 // fast_lock and fast_unlock used by C2
198
199 // Because the transitions from emitted code to the runtime
297
298 Label IsInflated, DONE_LABEL, NO_COUNT, COUNT;
299
300 if (DiagnoseSyncOnValueBasedClasses != 0) {
301 load_klass(tmpReg, objReg, scrReg);
302 testb(Address(tmpReg, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
303 jcc(Assembler::notZero, DONE_LABEL);
304 }
305
306 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
307 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
308 jcc(Assembler::notZero, IsInflated);
309
310 if (LockingMode == LM_MONITOR) {
311 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
312 testptr(objReg, objReg);
313 } else {
314 assert(LockingMode == LM_LEGACY, "must be");
315 // Attempt stack-locking ...
316 orptr (tmpReg, markWord::unlocked_value);
317 if (EnableValhalla) {
318 // Mask inline_type bit such that we go to the slow path if object is an inline type
319 andptr(tmpReg, ~((int) markWord::inline_type_bit_in_place));
320 }
321 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
322 lock();
323 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
324 jcc(Assembler::equal, COUNT); // Success
325
326 // Recursive locking.
327 // The object is stack-locked: markword contains stack pointer to BasicLock.
328 // Locked by current thread if difference with current SP is less than one page.
329 subptr(tmpReg, rsp);
330 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
331 andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
332 movptr(Address(boxReg, 0), tmpReg);
333 }
334 jmp(DONE_LABEL);
335
336 bind(IsInflated);
337 // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
338
339 #ifndef _LP64
340 // Just take slow path to avoid dealing with 64 bit atomic instructions here.
|