32 #include "opto/output.hpp"
33 #include "opto/opcodes.hpp"
34 #include "opto/subnode.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/powerOfTwo.hpp"
41 #include "utilities/sizes.hpp"
42
43 #ifdef PRODUCT
44 #define BLOCK_COMMENT(str) /* nothing */
45 #define STOP(error) stop(error)
46 #else
47 #define BLOCK_COMMENT(str) block_comment(str)
48 #define STOP(error) block_comment(error); stop(error)
49 #endif
50
51 // C2 compiled method's prolog code.
52 void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
53 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
54
55 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
56 // Remove word for return addr
57 framesize -= wordSize;
58 stack_bang_size -= wordSize;
59
60 // Calls to C2R adapters often do not accept exceptional returns.
61 // We require that their callers must bang for them. But be careful, because
62 // some VM calls (such as call site linkage) can use several kilobytes of
63 // stack. But the stack safety zone should account for that.
64 // See bugs 4446381, 4468289, 4497237.
65 if (stack_bang_size > 0) {
66 generate_stack_overflow_check(stack_bang_size);
67
68 // We always push rbp, so that on return to interpreter rbp, will be
69 // restored correctly and we can correct the stack.
70 push(rbp);
71 // Save caller's stack pointer into RBP if the frame pointer is preserved.
72 if (PreserveFramePointer) {
77
78 // Create frame
79 if (framesize) {
80 subptr(rsp, framesize);
81 }
82 } else {
83 subptr(rsp, framesize);
84
85 // Save RBP register now.
86 framesize -= wordSize;
87 movptr(Address(rsp, framesize), rbp);
88 // Save caller's stack pointer into RBP if the frame pointer is preserved.
89 if (PreserveFramePointer) {
90 movptr(rbp, rsp);
91 if (framesize > 0) {
92 addptr(rbp, framesize);
93 }
94 }
95 }
96
97 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
98 framesize -= wordSize;
99 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
100 }
101
102 #ifdef ASSERT
103 if (VerifyStackAtCalls) {
104 Label L;
105 push(rax);
106 mov(rax, rsp);
107 andptr(rax, StackAlignmentInBytes-1);
108 cmpptr(rax, StackAlignmentInBytes-wordSize);
109 pop(rax);
110 jcc(Assembler::equal, L);
111 STOP("Stack is not properly aligned!");
112 bind(L);
113 }
114 #endif
115
116 if (!is_stub) {
117 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
118 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
119 Label dummy_slow_path;
120 Label dummy_continuation;
121 Label* slow_path = &dummy_slow_path;
122 Label* continuation = &dummy_continuation;
123 if (!Compile::current()->output()->in_scratch_emit_size()) {
124 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
125 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
126 Compile::current()->output()->add_stub(stub);
127 slow_path = &stub->entry();
128 continuation = &stub->continuation();
129 }
130 bs->nmethod_entry_barrier(this, slow_path, continuation);
131 }
132 }
133
134 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
135 switch (vlen_in_bytes) {
136 case 4: // fall-through
137 case 8: // fall-through
138 case 16: return Assembler::AVX_128bit;
139 case 32: return Assembler::AVX_256bit;
140 case 64: return Assembler::AVX_512bit;
141
142 default: {
143 ShouldNotReachHere();
144 return Assembler::AVX_NoVec;
145 }
146 }
147 }
148
149 // fast_lock and fast_unlock used by C2
150
151 // Because the transitions from emitted code to the runtime
249
250 Label IsInflated, DONE_LABEL, NO_COUNT, COUNT;
251
252 if (DiagnoseSyncOnValueBasedClasses != 0) {
253 load_klass(tmpReg, objReg, scrReg);
254 testb(Address(tmpReg, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
255 jcc(Assembler::notZero, DONE_LABEL);
256 }
257
258 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
259 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
260 jcc(Assembler::notZero, IsInflated);
261
262 if (LockingMode == LM_MONITOR) {
263 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
264 testptr(objReg, objReg);
265 } else {
266 assert(LockingMode == LM_LEGACY, "must be");
267 // Attempt stack-locking ...
268 orptr (tmpReg, markWord::unlocked_value);
269 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
270 lock();
271 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
272 jcc(Assembler::equal, COUNT); // Success
273
274 // Recursive locking.
275 // The object is stack-locked: markword contains stack pointer to BasicLock.
276 // Locked by current thread if difference with current SP is less than one page.
277 subptr(tmpReg, rsp);
278 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
279 andptr(tmpReg, (int32_t) (7 - (int)os::vm_page_size()) );
280 movptr(Address(boxReg, 0), tmpReg);
281 }
282 jmp(DONE_LABEL);
283
284 bind(IsInflated);
285 // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
286
287 // Unconditionally set box->_displaced_header = markWord::unused_mark().
288 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
|
32 #include "opto/output.hpp"
33 #include "opto/opcodes.hpp"
34 #include "opto/subnode.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/checkedCast.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/powerOfTwo.hpp"
41 #include "utilities/sizes.hpp"
42
43 #ifdef PRODUCT
44 #define BLOCK_COMMENT(str) /* nothing */
45 #define STOP(error) stop(error)
46 #else
47 #define BLOCK_COMMENT(str) block_comment(str)
48 #define STOP(error) block_comment(error); stop(error)
49 #endif
50
51 // C2 compiled method's prolog code.
52 void C2_MacroAssembler::verified_entry(Compile* C, int sp_inc) {
53 if (C->clinit_barrier_on_entry()) {
54 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
55 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
56
57 Label L_skip_barrier;
58 Register klass = rscratch1;
59
60 mov_metadata(klass, C->method()->holder()->constant_encoding());
61 clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
62
63 jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
64
65 bind(L_skip_barrier);
66 }
67
68 int framesize = C->output()->frame_size_in_bytes();
69 int bangsize = C->output()->bang_size_in_bytes();
70 bool fp_mode_24b = false;
71 int stack_bang_size = C->output()->need_stack_bang(bangsize) ? bangsize : 0;
72
73 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
74
75 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
76 // Remove word for return addr
77 framesize -= wordSize;
78 stack_bang_size -= wordSize;
79
80 // Calls to C2R adapters often do not accept exceptional returns.
81 // We require that their callers must bang for them. But be careful, because
82 // some VM calls (such as call site linkage) can use several kilobytes of
83 // stack. But the stack safety zone should account for that.
84 // See bugs 4446381, 4468289, 4497237.
85 if (stack_bang_size > 0) {
86 generate_stack_overflow_check(stack_bang_size);
87
88 // We always push rbp, so that on return to interpreter rbp, will be
89 // restored correctly and we can correct the stack.
90 push(rbp);
91 // Save caller's stack pointer into RBP if the frame pointer is preserved.
92 if (PreserveFramePointer) {
97
98 // Create frame
99 if (framesize) {
100 subptr(rsp, framesize);
101 }
102 } else {
103 subptr(rsp, framesize);
104
105 // Save RBP register now.
106 framesize -= wordSize;
107 movptr(Address(rsp, framesize), rbp);
108 // Save caller's stack pointer into RBP if the frame pointer is preserved.
109 if (PreserveFramePointer) {
110 movptr(rbp, rsp);
111 if (framesize > 0) {
112 addptr(rbp, framesize);
113 }
114 }
115 }
116
117 if (C->needs_stack_repair()) {
118 // Save stack increment just below the saved rbp (also account for fixed framesize and rbp)
119 assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
120 movptr(Address(rsp, framesize - wordSize), sp_inc + framesize + wordSize);
121 }
122
123 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
124 framesize -= wordSize;
125 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
126 }
127
128 #ifdef ASSERT
129 if (VerifyStackAtCalls) {
130 Label L;
131 push(rax);
132 mov(rax, rsp);
133 andptr(rax, StackAlignmentInBytes-1);
134 cmpptr(rax, StackAlignmentInBytes-wordSize);
135 pop(rax);
136 jcc(Assembler::equal, L);
137 STOP("Stack is not properly aligned!");
138 bind(L);
139 }
140 #endif
141 }
142
143 void C2_MacroAssembler::entry_barrier() {
144 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
145 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
146 Label dummy_slow_path;
147 Label dummy_continuation;
148 Label* slow_path = &dummy_slow_path;
149 Label* continuation = &dummy_continuation;
150 if (!Compile::current()->output()->in_scratch_emit_size()) {
151 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
152 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
153 Compile::current()->output()->add_stub(stub);
154 slow_path = &stub->entry();
155 continuation = &stub->continuation();
156 }
157 bs->nmethod_entry_barrier(this, slow_path, continuation);
158 }
159
160 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
161 switch (vlen_in_bytes) {
162 case 4: // fall-through
163 case 8: // fall-through
164 case 16: return Assembler::AVX_128bit;
165 case 32: return Assembler::AVX_256bit;
166 case 64: return Assembler::AVX_512bit;
167
168 default: {
169 ShouldNotReachHere();
170 return Assembler::AVX_NoVec;
171 }
172 }
173 }
174
175 // fast_lock and fast_unlock used by C2
176
177 // Because the transitions from emitted code to the runtime
275
276 Label IsInflated, DONE_LABEL, NO_COUNT, COUNT;
277
278 if (DiagnoseSyncOnValueBasedClasses != 0) {
279 load_klass(tmpReg, objReg, scrReg);
280 testb(Address(tmpReg, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
281 jcc(Assembler::notZero, DONE_LABEL);
282 }
283
284 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
285 testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
286 jcc(Assembler::notZero, IsInflated);
287
288 if (LockingMode == LM_MONITOR) {
289 // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
290 testptr(objReg, objReg);
291 } else {
292 assert(LockingMode == LM_LEGACY, "must be");
293 // Attempt stack-locking ...
294 orptr (tmpReg, markWord::unlocked_value);
295 if (EnableValhalla) {
296 // Mask inline_type bit such that we go to the slow path if object is an inline type
297 andptr(tmpReg, ~((int) markWord::inline_type_bit_in_place));
298 }
299 movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
300 lock();
301 cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
302 jcc(Assembler::equal, COUNT); // Success
303
304 // Recursive locking.
305 // The object is stack-locked: markword contains stack pointer to BasicLock.
306 // Locked by current thread if difference with current SP is less than one page.
307 subptr(tmpReg, rsp);
308 // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
309 andptr(tmpReg, (int32_t) (7 - (int)os::vm_page_size()) );
310 movptr(Address(boxReg, 0), tmpReg);
311 }
312 jmp(DONE_LABEL);
313
314 bind(IsInflated);
315 // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
316
317 // Unconditionally set box->_displaced_header = markWord::unused_mark().
318 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
|