17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "c1/c1_MacroAssembler.hpp"
26 #include "c1/c1_Runtime1.hpp"
27 #include "code/compiledIC.hpp"
28 #include "compiler/compilerDefinitions.inline.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/collectedHeap.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "interpreter/interpreter.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "runtime/basicLock.hpp"
37 #include "runtime/globals.hpp"
38 #include "runtime/os.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubRoutines.hpp"
41 #include "utilities/checkedCast.hpp"
42 #include "utilities/globalDefinitions.hpp"
43
44 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
45 const int aligned_mask = BytesPerWord -1;
46 const int hdr_offset = oopDesc::mark_offset_in_bytes();
47 assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
48 assert_different_registers(hdr, obj, disp_hdr, tmp);
49 int null_check_offset = -1;
50
51 verify_oop(obj);
52
53 // save object being locked into the BasicObjectLock
54 movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
55
56 null_check_offset = offset();
57
58 if (LockingMode == LM_LIGHTWEIGHT) {
59 lightweight_lock(disp_hdr, obj, hdr, tmp, slow_case);
60 } else if (LockingMode == LM_LEGACY) {
61 Label done;
62
63 if (DiagnoseSyncOnValueBasedClasses != 0) {
64 load_klass(hdr, obj, rscratch1);
65 testb(Address(hdr, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
66 jcc(Assembler::notZero, slow_case);
67 }
68
69 // Load object header
70 movptr(hdr, Address(obj, hdr_offset));
71 // and mark it as unlocked
72 orptr(hdr, markWord::unlocked_value);
73 // save unlocked object header into the displaced header location on the stack
74 movptr(Address(disp_hdr, 0), hdr);
75 // test if object header is still the same (i.e. unlocked), and if so, store the
76 // displaced header address in the object header - if it is not the same, get the
77 // object header instead
78 MacroAssembler::lock(); // must be immediately before cmpxchg!
79 cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
80 // if the object header was the same, we're done
81 jcc(Assembler::equal, done);
82 // if the object header was not the same, it is now in the hdr register
83 // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
84 //
85 // 1) (hdr & aligned_mask) == 0
86 // 2) rsp <= hdr
87 // 3) hdr <= rsp + page_size
88 //
89 // these 3 tests can be done by evaluating the following expression:
90 //
91 // (hdr - rsp) & (aligned_mask - page_size)
92 //
140 jcc(Assembler::notEqual, slow_case);
141 // done
142 bind(done);
143 dec_held_monitor_count();
144 }
145 }
146
147
148 // Defines obj, preserves var_size_in_bytes
149 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
150 if (UseTLAB) {
151 tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
152 } else {
153 jmp(slow_case);
154 }
155 }
156
157
158 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
159 assert_different_registers(obj, klass, len, t1, t2);
160 if (UseCompactObjectHeaders) {
161 movptr(t1, Address(klass, Klass::prototype_header_offset()));
162 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
163 } else if (UseCompressedClassPointers) { // Take care not to kill klass
164 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
165 movptr(t1, klass);
166 encode_klass_not_null(t1, rscratch1);
167 movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
168 } else {
169 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
170 movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
171 }
172
173 if (len->is_valid()) {
174 movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
175 int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
176 if (!is_aligned(base_offset, BytesPerWord)) {
177 assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
178 // Clear gap/first 4 bytes following the length field.
179 xorl(t1, t1);
180 movl(Address(obj, base_offset), t1);
181 }
182 } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
183 xorptr(t1, t1);
184 store_klass_gap(obj, t1);
185 }
186 }
187
188
189 // preserves obj, destroys len_in_bytes
280
281 initialize_header(obj, klass, len, t1, t2);
282
283 // clear rest of allocated space
284 if (zero_array) {
285 const Register len_zero = len;
286 // Align-up to word boundary, because we clear the 4 bytes potentially
287 // following the length field in initialize_header().
288 int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
289 initialize_body(obj, arr_size, base_offset, len_zero);
290 }
291
292 if (CURRENT_ENV->dtrace_alloc_probes()) {
293 assert(obj == rax, "must be");
294 call(RuntimeAddress(Runtime1::entry_for(C1StubId::dtrace_object_alloc_id)));
295 }
296
297 verify_oop(obj);
298 }
299
300 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
301 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
302 // Make sure there is enough stack space for this method's activation.
303 // Note that we do this before doing an enter(). This matches the
304 // ordering of C2's stack overflow check / rsp decrement and allows
305 // the SharedRuntime stack overflow handling to be consistent
306 // between the two compilers.
307 generate_stack_overflow_check(bang_size_in_bytes);
308
309 push(rbp);
310 if (PreserveFramePointer) {
311 mov(rbp, rsp);
312 }
313 decrement(rsp, frame_size_in_bytes); // does not emit code for frame_size == 0
314
315 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
316 // C1 code is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub
317 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
318 }
319
320
321 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
322 increment(rsp, frame_size_in_bytes); // Does not emit code for frame_size == 0
323 pop(rbp);
324 }
325
326
327 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
328 if (breakAtEntry) int3();
329 // build frame
330 }
331
332 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) {
333 // rbp, + 0: link
334 // + 1: return address
335 // + 2: argument with offset 0
336 // + 3: argument with offset 1
337 // + 4: ...
338
339 movptr(reg, Address(rbp, (offset_in_words + 2) * BytesPerWord));
340 }
341
342 #ifndef PRODUCT
343
344 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
345 if (!VerifyOops) return;
346 verify_oop_addr(Address(rsp, stack_offset));
347 }
348
349 void C1_MacroAssembler::verify_not_null_oop(Register r) {
350 if (!VerifyOops) return;
351 Label not_null;
|
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "c1/c1_MacroAssembler.hpp"
26 #include "c1/c1_Runtime1.hpp"
27 #include "code/compiledIC.hpp"
28 #include "compiler/compilerDefinitions.inline.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/collectedHeap.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "interpreter/interpreter.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "runtime/basicLock.hpp"
37 #include "runtime/frame.inline.hpp"
38 #include "runtime/globals.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "utilities/checkedCast.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
46 const int aligned_mask = BytesPerWord -1;
47 const int hdr_offset = oopDesc::mark_offset_in_bytes();
48 assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
49 assert_different_registers(hdr, obj, disp_hdr, tmp);
50 int null_check_offset = -1;
51
52 verify_oop(obj);
53
54 // save object being locked into the BasicObjectLock
55 movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
56
57 null_check_offset = offset();
58
59 if (LockingMode == LM_LIGHTWEIGHT) {
60 lightweight_lock(disp_hdr, obj, hdr, tmp, slow_case);
61 } else if (LockingMode == LM_LEGACY) {
62 Label done;
63
64 if (DiagnoseSyncOnValueBasedClasses != 0) {
65 load_klass(hdr, obj, rscratch1);
66 testb(Address(hdr, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
67 jcc(Assembler::notZero, slow_case);
68 }
69
70 // Load object header
71 movptr(hdr, Address(obj, hdr_offset));
72 // and mark it as unlocked
73 orptr(hdr, markWord::unlocked_value);
74 if (EnableValhalla) {
75 // Mask inline_type bit such that we go to the slow path if object is an inline type
76 andptr(hdr, ~((int) markWord::inline_type_bit_in_place));
77 }
78 // save unlocked object header into the displaced header location on the stack
79 movptr(Address(disp_hdr, 0), hdr);
80 // test if object header is still the same (i.e. unlocked), and if so, store the
81 // displaced header address in the object header - if it is not the same, get the
82 // object header instead
83 MacroAssembler::lock(); // must be immediately before cmpxchg!
84 cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
85 // if the object header was the same, we're done
86 jcc(Assembler::equal, done);
87 // if the object header was not the same, it is now in the hdr register
88 // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
89 //
90 // 1) (hdr & aligned_mask) == 0
91 // 2) rsp <= hdr
92 // 3) hdr <= rsp + page_size
93 //
94 // these 3 tests can be done by evaluating the following expression:
95 //
96 // (hdr - rsp) & (aligned_mask - page_size)
97 //
145 jcc(Assembler::notEqual, slow_case);
146 // done
147 bind(done);
148 dec_held_monitor_count();
149 }
150 }
151
152
153 // Defines obj, preserves var_size_in_bytes
154 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
155 if (UseTLAB) {
156 tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
157 } else {
158 jmp(slow_case);
159 }
160 }
161
162
163 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
164 assert_different_registers(obj, klass, len, t1, t2);
165 if (UseCompactObjectHeaders || EnableValhalla) {
166 movptr(t1, Address(klass, Klass::prototype_header_offset()));
167 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
168 } else {
169 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
170 }
171 if (UseCompressedClassPointers) { // Take care not to kill klass
172 movptr(t1, klass);
173 encode_klass_not_null(t1, rscratch1);
174 movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
175 } else if (!UseCompactObjectHeaders) {
176 movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
177 }
178
179 if (len->is_valid()) {
180 movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
181 int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
182 if (!is_aligned(base_offset, BytesPerWord)) {
183 assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
184 // Clear gap/first 4 bytes following the length field.
185 xorl(t1, t1);
186 movl(Address(obj, base_offset), t1);
187 }
188 } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
189 xorptr(t1, t1);
190 store_klass_gap(obj, t1);
191 }
192 }
193
194
195 // preserves obj, destroys len_in_bytes
286
287 initialize_header(obj, klass, len, t1, t2);
288
289 // clear rest of allocated space
290 if (zero_array) {
291 const Register len_zero = len;
292 // Align-up to word boundary, because we clear the 4 bytes potentially
293 // following the length field in initialize_header().
294 int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
295 initialize_body(obj, arr_size, base_offset, len_zero);
296 }
297
298 if (CURRENT_ENV->dtrace_alloc_probes()) {
299 assert(obj == rax, "must be");
300 call(RuntimeAddress(Runtime1::entry_for(C1StubId::dtrace_object_alloc_id)));
301 }
302
303 verify_oop(obj);
304 }
305
306 void C1_MacroAssembler::build_frame_helper(int frame_size_in_bytes, int sp_offset_for_orig_pc, int sp_inc, bool reset_orig_pc, bool needs_stack_repair) {
307 push(rbp);
308 if (PreserveFramePointer) {
309 mov(rbp, rsp);
310 }
311 decrement(rsp, frame_size_in_bytes);
312
313 if (needs_stack_repair) {
314 // Save stack increment (also account for fixed framesize and rbp)
315 assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
316 int real_frame_size = sp_inc + frame_size_in_bytes + wordSize;
317 movptr(Address(rsp, frame_size_in_bytes - wordSize), real_frame_size);
318 }
319 if (reset_orig_pc) {
320 // Zero orig_pc to detect deoptimization during buffering in the entry points
321 movptr(Address(rsp, sp_offset_for_orig_pc), 0);
322 }
323 }
324
325 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes, int sp_offset_for_orig_pc, bool needs_stack_repair, bool has_scalarized_args, Label* verified_inline_entry_label) {
326 // Make sure there is enough stack space for this method's activation.
327 // Note that we do this before doing an enter(). This matches the
328 // ordering of C2's stack overflow check / rsp decrement and allows
329 // the SharedRuntime stack overflow handling to be consistent
330 // between the two compilers.
331 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
332 generate_stack_overflow_check(bang_size_in_bytes);
333
334 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, 0, has_scalarized_args, needs_stack_repair);
335
336 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
337 // C1 code is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub
338 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
339
340 if (verified_inline_entry_label != nullptr) {
341 // Jump here from the scalarized entry points that already created the frame.
342 bind(*verified_inline_entry_label);
343 }
344 }
345
346 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
347 if (breakAtEntry) int3();
348 // build frame
349 }
350
351 int C1_MacroAssembler::scalarized_entry(const CompiledEntrySignature* ces, int frame_size_in_bytes, int bang_size_in_bytes, int sp_offset_for_orig_pc, Label& verified_inline_entry_label, bool is_inline_ro_entry) {
352 assert(InlineTypePassFieldsAsArgs, "sanity");
353 // Make sure there is enough stack space for this method's activation.
354 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
355 generate_stack_overflow_check(bang_size_in_bytes);
356
357 GrowableArray<SigEntry>* sig = ces->sig();
358 GrowableArray<SigEntry>* sig_cc = is_inline_ro_entry ? ces->sig_cc_ro() : ces->sig_cc();
359 VMRegPair* regs = ces->regs();
360 VMRegPair* regs_cc = is_inline_ro_entry ? ces->regs_cc_ro() : ces->regs_cc();
361 int args_on_stack = ces->args_on_stack();
362 int args_on_stack_cc = is_inline_ro_entry ? ces->args_on_stack_cc_ro() : ces->args_on_stack_cc();
363
364 assert(sig->length() <= sig_cc->length(), "Zero-sized inline class not allowed!");
365 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, sig_cc->length());
366 int args_passed = sig->length();
367 int args_passed_cc = SigEntry::fill_sig_bt(sig_cc, sig_bt);
368
369 // Create a temp frame so we can call into the runtime. It must be properly set up to accommodate GC.
370 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, 0, true, ces->c1_needs_stack_repair());
371
372 // The runtime call might safepoint, make sure nmethod entry barrier is executed
373 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
374 // C1 code is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub
375 bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */);
376
377 // FIXME -- call runtime only if we cannot in-line allocate all the incoming inline type args.
378 movptr(rbx, (intptr_t)(ces->method()));
379 if (is_inline_ro_entry) {
380 call(RuntimeAddress(Runtime1::entry_for(C1StubId::buffer_inline_args_no_receiver_id)));
381 } else {
382 call(RuntimeAddress(Runtime1::entry_for(C1StubId::buffer_inline_args_id)));
383 }
384 int rt_call_offset = offset();
385
386 // Remove the temp frame
387 addptr(rsp, frame_size_in_bytes);
388 pop(rbp);
389
390 // Check if we need to extend the stack for packing
391 int sp_inc = 0;
392 if (args_on_stack > args_on_stack_cc) {
393 sp_inc = extend_stack_for_inline_args(args_on_stack);
394 }
395
396 shuffle_inline_args(true, is_inline_ro_entry, sig_cc,
397 args_passed_cc, args_on_stack_cc, regs_cc, // from
398 args_passed, args_on_stack, regs, // to
399 sp_inc, rax);
400
401 // Create the real frame. Below jump will then skip over the stack banging and frame
402 // setup code in the verified_inline_entry (which has a different real_frame_size).
403 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, sp_inc, false, ces->c1_needs_stack_repair());
404
405 jmp(verified_inline_entry_label);
406 return rt_call_offset;
407 }
408
409 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) {
410 // rbp, + 0: link
411 // + 1: return address
412 // + 2: argument with offset 0
413 // + 3: argument with offset 1
414 // + 4: ...
415
416 movptr(reg, Address(rbp, (offset_in_words + 2) * BytesPerWord));
417 }
418
419 #ifndef PRODUCT
420
421 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
422 if (!VerifyOops) return;
423 verify_oop_addr(Address(rsp, stack_offset));
424 }
425
426 void C1_MacroAssembler::verify_not_null_oop(Register r) {
427 if (!VerifyOops) return;
428 Label not_null;
|