1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456
457 Label start;
458 __ bind(start);
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
468 "out of bounds read in post-call NOP check");
469 __ end_a_stub();
470
471 return entry_offset;
472 }
473
474 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
475 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
476 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
477 assert(result->fpu() == 0, "result must already be on TOS");
478 }
479 if (InlineTypeReturnedAsFields) {
480 #ifndef _LP64
481 Unimplemented();
482 #endif
483 // Check if we are returning an non-null inline type and load its fields into registers
484 ciType* return_type = compilation()->method()->return_type();
485 if (return_type->is_inlinetype()) {
486 ciInlineKlass* vk = return_type->as_inline_klass();
487 if (vk->can_be_returned_as_fields()) {
488 address unpack_handler = vk->unpack_handler();
489 assert(unpack_handler != nullptr, "must be");
490 __ call(RuntimeAddress(unpack_handler));
491 }
492 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
493 Label skip;
494 Label not_null;
495 __ testptr(rax, rax);
496 __ jcc(Assembler::notZero, not_null);
497 // Returned value is null, zero all return registers because they may belong to oop fields
498 __ xorq(j_rarg1, j_rarg1);
499 __ xorq(j_rarg2, j_rarg2);
500 __ xorq(j_rarg3, j_rarg3);
501 __ xorq(j_rarg4, j_rarg4);
502 __ xorq(j_rarg5, j_rarg5);
503 __ jmp(skip);
504 __ bind(not_null);
505
506 // Check if we are returning an non-null inline type and load its fields into registers
507 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
508
509 // Load fields from a buffered value with an inline class specific handler
510 __ load_klass(rdi, rax, rscratch1);
511 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
512 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
513 // Unpack handler can be null if inline type is not scalarizable in returns
514 __ testptr(rdi, rdi);
515 __ jcc(Assembler::zero, skip);
516 __ call(rdi);
517
518 __ bind(skip);
519 }
520 // At this point, rax points to the value object (for interpreter or C1 caller).
521 // The fields of the object are copied into registers (for C2 caller).
522 }
523
524 // Pop the stack before the safepoint code
525 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
526
527 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
528 __ reserved_stack_check();
529 }
530
531 // Note: we do not need to round double result; float result has the right precision
532 // the poll sets the condition code, but no data registers
533
534 code_stub->set_safepoint_offset(__ offset());
535 __ relocate(relocInfo::poll_return_type);
536 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
537 __ ret(0);
538 }
539
540
541 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
542 return (__ store_inline_type_fields_to_buf(vk, false));
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 guarantee(info != nullptr, "Shouldn't be null");
547 int offset = __ offset();
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 add_debug_info_for_branch(info);
551 __ relocate(relocInfo::poll_type);
552 address pre_pc = __ pc();
553 __ testl(rax, Address(poll_addr, 0));
554 address post_pc = __ pc();
555 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
556 return offset;
557 }
558
559
560 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
561 if (from_reg != to_reg) __ mov(to_reg, from_reg);
562 }
563
564 void LIR_Assembler::swap_reg(Register a, Register b) {
565 __ xchgptr(a, b);
566 }
567
568
569 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
570 assert(src->is_constant(), "should not call otherwise");
571 assert(dest->is_register(), "should not call otherwise");
572 LIR_Const* c = src->as_constant_ptr();
573
574 switch (c->type()) {
575 case T_INT: {
576 assert(patch_code == lir_patch_none, "no patching handled here");
577 __ movl(dest->as_register(), c->as_jint());
578 break;
579 }
580
581 case T_ADDRESS: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movptr(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_LONG: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
590 break;
591 }
592
593 case T_OBJECT: {
594 if (patch_code != lir_patch_none) {
595 jobject2reg_with_patching(dest->as_register(), info);
596 } else {
597 __ movoop(dest->as_register(), c->as_jobject());
598 }
599 break;
600 }
601
602 case T_METADATA: {
603 if (patch_code != lir_patch_none) {
604 klass2reg_with_patching(dest->as_register(), info);
605 } else {
606 __ mov_metadata(dest->as_register(), c->as_metadata());
607 }
608 break;
609 }
610
611 case T_FLOAT: {
612 if (dest->is_single_xmm()) {
613 if (UseAVX <= 2 && c->is_zero_float()) {
614 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
615 } else {
616 __ movflt(dest->as_xmm_float_reg(),
617 InternalAddress(float_constant(c->as_jfloat())));
618 }
619 } else {
620 ShouldNotReachHere();
621 }
622 break;
623 }
624
625 case T_DOUBLE: {
626 if (dest->is_double_xmm()) {
627 if (UseAVX <= 2 && c->is_zero_double()) {
628 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
629 } else {
630 __ movdbl(dest->as_xmm_double_reg(),
631 InternalAddress(double_constant(c->as_jdouble())));
632 }
633 } else {
634 ShouldNotReachHere();
635 }
636 break;
637 }
638
639 default:
640 ShouldNotReachHere();
641 }
642 }
643
644 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
645 assert(src->is_constant(), "should not call otherwise");
646 assert(dest->is_stack(), "should not call otherwise");
647 LIR_Const* c = src->as_constant_ptr();
648
649 switch (c->type()) {
650 case T_INT: // fall through
651 case T_FLOAT:
652 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
653 break;
654
655 case T_ADDRESS:
656 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
657 break;
658
659 case T_OBJECT:
660 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
661 break;
662
663 case T_LONG: // fall through
664 case T_DOUBLE:
665 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
666 lo_word_offset_in_bytes),
667 (intptr_t)c->as_jlong_bits(),
668 rscratch1);
669 break;
670
671 default:
672 ShouldNotReachHere();
673 }
674 }
675
676 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
677 assert(src->is_constant(), "should not call otherwise");
678 assert(dest->is_address(), "should not call otherwise");
679 LIR_Const* c = src->as_constant_ptr();
680 LIR_Address* addr = dest->as_address_ptr();
681
682 int null_check_here = code_offset();
683 switch (type) {
684 case T_INT: // fall through
685 case T_FLOAT:
686 __ movl(as_Address(addr), c->as_jint_bits());
687 break;
688
689 case T_ADDRESS:
690 __ movptr(as_Address(addr), c->as_jint_bits());
691 break;
692
693 case T_OBJECT: // fall through
694 case T_ARRAY:
695 if (c->as_jobject() == nullptr) {
696 if (UseCompressedOops && !wide) {
697 __ movl(as_Address(addr), NULL_WORD);
698 } else {
699 __ xorptr(rscratch1, rscratch1);
700 null_check_here = code_offset();
701 __ movptr(as_Address(addr), rscratch1);
702 }
703 } else {
704 if (is_literal_address(addr)) {
705 ShouldNotReachHere();
706 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
707 } else {
708 __ movoop(rscratch1, c->as_jobject());
709 if (UseCompressedOops && !wide) {
710 __ encode_heap_oop(rscratch1);
711 null_check_here = code_offset();
712 __ movl(as_Address_lo(addr), rscratch1);
713 } else {
714 null_check_here = code_offset();
715 __ movptr(as_Address_lo(addr), rscratch1);
716 }
717 }
718 }
719 break;
720
721 case T_LONG: // fall through
722 case T_DOUBLE:
723 if (is_literal_address(addr)) {
724 ShouldNotReachHere();
725 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
726 } else {
727 __ movptr(r10, (intptr_t)c->as_jlong_bits());
728 null_check_here = code_offset();
729 __ movptr(as_Address_lo(addr), r10);
730 }
731 break;
732
733 case T_BOOLEAN: // fall through
734 case T_BYTE:
735 __ movb(as_Address(addr), c->as_jint() & 0xFF);
736 break;
737
738 case T_CHAR: // fall through
739 case T_SHORT:
740 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
741 break;
742
743 default:
744 ShouldNotReachHere();
745 };
746
747 if (info != nullptr) {
748 add_debug_info_for_null_check(null_check_here, info);
749 }
750 }
751
752
753 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
754 assert(src->is_register(), "should not call otherwise");
755 assert(dest->is_register(), "should not call otherwise");
756
757 // move between cpu-registers
758 if (dest->is_single_cpu()) {
759 if (src->type() == T_LONG) {
760 // Can do LONG -> OBJECT
761 move_regs(src->as_register_lo(), dest->as_register());
762 return;
763 }
764 assert(src->is_single_cpu(), "must match");
765 if (src->type() == T_OBJECT) {
766 __ verify_oop(src->as_register());
767 }
768 move_regs(src->as_register(), dest->as_register());
769
770 } else if (dest->is_double_cpu()) {
771 if (is_reference_type(src->type())) {
772 // Surprising to me but we can see move of a long to t_object
773 __ verify_oop(src->as_register());
774 move_regs(src->as_register(), dest->as_register_lo());
775 return;
776 }
777 assert(src->is_double_cpu(), "must match");
778 Register f_lo = src->as_register_lo();
779 Register f_hi = src->as_register_hi();
780 Register t_lo = dest->as_register_lo();
781 Register t_hi = dest->as_register_hi();
782 assert(f_hi == f_lo, "must be same");
783 assert(t_hi == t_lo, "must be same");
784 move_regs(f_lo, t_lo);
785
786 // move between xmm-registers
787 } else if (dest->is_single_xmm()) {
788 assert(src->is_single_xmm(), "must match");
789 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
790 } else if (dest->is_double_xmm()) {
791 assert(src->is_double_xmm(), "must match");
792 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
793
794 } else {
795 ShouldNotReachHere();
796 }
797 }
798
799 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
800 assert(src->is_register(), "should not call otherwise");
801 assert(dest->is_stack(), "should not call otherwise");
802
803 if (src->is_single_cpu()) {
804 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
805 if (is_reference_type(type)) {
806 __ verify_oop(src->as_register());
807 __ movptr (dst, src->as_register());
808 } else if (type == T_METADATA || type == T_ADDRESS) {
809 __ movptr (dst, src->as_register());
810 } else {
811 __ movl (dst, src->as_register());
812 }
813
814 } else if (src->is_double_cpu()) {
815 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
816 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
817 __ movptr (dstLO, src->as_register_lo());
818
819 } else if (src->is_single_xmm()) {
820 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
821 __ movflt(dst_addr, src->as_xmm_float_reg());
822
823 } else if (src->is_double_xmm()) {
824 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
825 __ movdbl(dst_addr, src->as_xmm_double_reg());
826
827 } else {
828 ShouldNotReachHere();
829 }
830 }
831
832
833 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
834 LIR_Address* to_addr = dest->as_address_ptr();
835 PatchingStub* patch = nullptr;
836 Register compressed_src = rscratch1;
837
838 if (is_reference_type(type)) {
839 __ verify_oop(src->as_register());
840 if (UseCompressedOops && !wide) {
841 __ movptr(compressed_src, src->as_register());
842 __ encode_heap_oop(compressed_src);
843 if (patch_code != lir_patch_none) {
844 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
845 }
846 }
847 }
848
849 if (patch_code != lir_patch_none) {
850 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
851 Address toa = as_Address(to_addr);
852 assert(toa.disp() != 0, "must have");
853 }
854
855 int null_check_here = code_offset();
856 switch (type) {
857 case T_FLOAT: {
858 assert(src->is_single_xmm(), "not a float");
859 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
860 break;
861 }
862
863 case T_DOUBLE: {
864 assert(src->is_double_xmm(), "not a double");
865 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
866 break;
867 }
868
869 case T_ARRAY: // fall through
870 case T_OBJECT: // fall through
871 if (UseCompressedOops && !wide) {
872 __ movl(as_Address(to_addr), compressed_src);
873 } else {
874 __ movptr(as_Address(to_addr), src->as_register());
875 }
876 break;
877 case T_ADDRESS:
878 __ movptr(as_Address(to_addr), src->as_register());
879 break;
880 case T_INT:
881 __ movl(as_Address(to_addr), src->as_register());
882 break;
883
884 case T_LONG: {
885 Register from_lo = src->as_register_lo();
886 Register from_hi = src->as_register_hi();
887 __ movptr(as_Address_lo(to_addr), from_lo);
888 break;
889 }
890
891 case T_BYTE: // fall through
892 case T_BOOLEAN: {
893 Register src_reg = src->as_register();
894 Address dst_addr = as_Address(to_addr);
895 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
896 __ movb(dst_addr, src_reg);
897 break;
898 }
899
900 case T_CHAR: // fall through
901 case T_SHORT:
902 __ movw(as_Address(to_addr), src->as_register());
903 break;
904
905 default:
906 ShouldNotReachHere();
907 }
908 if (info != nullptr) {
909 add_debug_info_for_null_check(null_check_here, info);
910 }
911
912 if (patch_code != lir_patch_none) {
913 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
914 }
915 }
916
917
918 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
919 assert(src->is_stack(), "should not call otherwise");
920 assert(dest->is_register(), "should not call otherwise");
921
922 if (dest->is_single_cpu()) {
923 if (is_reference_type(type)) {
924 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
925 __ verify_oop(dest->as_register());
926 } else if (type == T_METADATA || type == T_ADDRESS) {
927 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
928 } else {
929 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
930 }
931
932 } else if (dest->is_double_cpu()) {
933 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
934 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
935 __ movptr(dest->as_register_lo(), src_addr_LO);
936
937 } else if (dest->is_single_xmm()) {
938 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
939 __ movflt(dest->as_xmm_float_reg(), src_addr);
940
941 } else if (dest->is_double_xmm()) {
942 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
943 __ movdbl(dest->as_xmm_double_reg(), src_addr);
944
945 } else {
946 ShouldNotReachHere();
947 }
948 }
949
950
951 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
952 if (src->is_single_stack()) {
953 if (is_reference_type(type)) {
954 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
955 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
956 } else {
957 //no pushl on 64bits
958 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
959 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
960 }
961
962 } else if (src->is_double_stack()) {
963 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
964 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
965
966 } else {
967 ShouldNotReachHere();
968 }
969 }
970
971
972 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
973 assert(src->is_address(), "should not call otherwise");
974 assert(dest->is_register(), "should not call otherwise");
975
976 LIR_Address* addr = src->as_address_ptr();
977 Address from_addr = as_Address(addr);
978
979 if (addr->base()->type() == T_OBJECT) {
980 __ verify_oop(addr->base()->as_pointer_register());
981 }
982
983 switch (type) {
984 case T_BOOLEAN: // fall through
985 case T_BYTE: // fall through
986 case T_CHAR: // fall through
987 case T_SHORT:
988 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
989 // on pre P6 processors we may get partial register stalls
990 // so blow away the value of to_rinfo before loading a
991 // partial word into it. Do it here so that it precedes
992 // the potential patch point below.
993 __ xorptr(dest->as_register(), dest->as_register());
994 }
995 break;
996 default:
997 break;
998 }
999
1000 PatchingStub* patch = nullptr;
1001 if (patch_code != lir_patch_none) {
1002 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1003 assert(from_addr.disp() != 0, "must have");
1004 }
1005 if (info != nullptr) {
1006 add_debug_info_for_null_check_here(info);
1007 }
1008
1009 switch (type) {
1010 case T_FLOAT: {
1011 if (dest->is_single_xmm()) {
1012 __ movflt(dest->as_xmm_float_reg(), from_addr);
1013 } else {
1014 ShouldNotReachHere();
1015 }
1016 break;
1017 }
1018
1019 case T_DOUBLE: {
1020 if (dest->is_double_xmm()) {
1021 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1022 } else {
1023 ShouldNotReachHere();
1024 }
1025 break;
1026 }
1027
1028 case T_OBJECT: // fall through
1029 case T_ARRAY: // fall through
1030 if (UseCompressedOops && !wide) {
1031 __ movl(dest->as_register(), from_addr);
1032 } else {
1033 __ movptr(dest->as_register(), from_addr);
1034 }
1035 break;
1036
1037 case T_ADDRESS:
1038 __ movptr(dest->as_register(), from_addr);
1039 break;
1040 case T_INT:
1041 __ movl(dest->as_register(), from_addr);
1042 break;
1043
1044 case T_LONG: {
1045 Register to_lo = dest->as_register_lo();
1046 Register to_hi = dest->as_register_hi();
1047 __ movptr(to_lo, as_Address_lo(addr));
1048 break;
1049 }
1050
1051 case T_BOOLEAN: // fall through
1052 case T_BYTE: {
1053 Register dest_reg = dest->as_register();
1054 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1055 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1056 __ movsbl(dest_reg, from_addr);
1057 } else {
1058 __ movb(dest_reg, from_addr);
1059 __ shll(dest_reg, 24);
1060 __ sarl(dest_reg, 24);
1061 }
1062 break;
1063 }
1064
1065 case T_CHAR: {
1066 Register dest_reg = dest->as_register();
1067 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1068 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1069 __ movzwl(dest_reg, from_addr);
1070 } else {
1071 __ movw(dest_reg, from_addr);
1072 }
1073 break;
1074 }
1075
1076 case T_SHORT: {
1077 Register dest_reg = dest->as_register();
1078 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1079 __ movswl(dest_reg, from_addr);
1080 } else {
1081 __ movw(dest_reg, from_addr);
1082 __ shll(dest_reg, 16);
1083 __ sarl(dest_reg, 16);
1084 }
1085 break;
1086 }
1087
1088 default:
1089 ShouldNotReachHere();
1090 }
1091
1092 if (patch != nullptr) {
1093 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1094 }
1095
1096 if (is_reference_type(type)) {
1097 if (UseCompressedOops && !wide) {
1098 __ decode_heap_oop(dest->as_register());
1099 }
1100
1101 __ verify_oop(dest->as_register());
1102 }
1103 }
1104
1105
1106 NEEDS_CLEANUP; // This could be static?
1107 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1108 int elem_size = type2aelembytes(type);
1109 switch (elem_size) {
1110 case 1: return Address::times_1;
1111 case 2: return Address::times_2;
1112 case 4: return Address::times_4;
1113 case 8: return Address::times_8;
1114 }
1115 ShouldNotReachHere();
1116 return Address::no_scale;
1117 }
1118
1119
1120 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1121 switch (op->code()) {
1122 case lir_idiv:
1123 case lir_irem:
1124 arithmetic_idiv(op->code(),
1125 op->in_opr1(),
1126 op->in_opr2(),
1127 op->in_opr3(),
1128 op->result_opr(),
1129 op->info());
1130 break;
1131 case lir_fmad:
1132 __ fmad(op->result_opr()->as_xmm_double_reg(),
1133 op->in_opr1()->as_xmm_double_reg(),
1134 op->in_opr2()->as_xmm_double_reg(),
1135 op->in_opr3()->as_xmm_double_reg());
1136 break;
1137 case lir_fmaf:
1138 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1139 op->in_opr1()->as_xmm_float_reg(),
1140 op->in_opr2()->as_xmm_float_reg(),
1141 op->in_opr3()->as_xmm_float_reg());
1142 break;
1143 default: ShouldNotReachHere(); break;
1144 }
1145 }
1146
1147 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1148 #ifdef ASSERT
1149 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1150 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1151 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1152 #endif
1153
1154 if (op->cond() == lir_cond_always) {
1155 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1156 __ jmp (*(op->label()));
1157 } else {
1158 Assembler::Condition acond = Assembler::zero;
1159 if (op->code() == lir_cond_float_branch) {
1160 assert(op->ublock() != nullptr, "must have unordered successor");
1161 __ jcc(Assembler::parity, *(op->ublock()->label()));
1162 switch(op->cond()) {
1163 case lir_cond_equal: acond = Assembler::equal; break;
1164 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1165 case lir_cond_less: acond = Assembler::below; break;
1166 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1167 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1168 case lir_cond_greater: acond = Assembler::above; break;
1169 default: ShouldNotReachHere();
1170 }
1171 } else {
1172 switch (op->cond()) {
1173 case lir_cond_equal: acond = Assembler::equal; break;
1174 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1175 case lir_cond_less: acond = Assembler::less; break;
1176 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1177 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1178 case lir_cond_greater: acond = Assembler::greater; break;
1179 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1180 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1181 default: ShouldNotReachHere();
1182 }
1183 }
1184 __ jcc(acond,*(op->label()));
1185 }
1186 }
1187
1188 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1189 LIR_Opr src = op->in_opr();
1190 LIR_Opr dest = op->result_opr();
1191
1192 switch (op->bytecode()) {
1193 case Bytecodes::_i2l:
1194 __ movl2ptr(dest->as_register_lo(), src->as_register());
1195 break;
1196
1197 case Bytecodes::_l2i:
1198 __ movl(dest->as_register(), src->as_register_lo());
1199 break;
1200
1201 case Bytecodes::_i2b:
1202 move_regs(src->as_register(), dest->as_register());
1203 __ sign_extend_byte(dest->as_register());
1204 break;
1205
1206 case Bytecodes::_i2c:
1207 move_regs(src->as_register(), dest->as_register());
1208 __ andl(dest->as_register(), 0xFFFF);
1209 break;
1210
1211 case Bytecodes::_i2s:
1212 move_regs(src->as_register(), dest->as_register());
1213 __ sign_extend_short(dest->as_register());
1214 break;
1215
1216 case Bytecodes::_f2d:
1217 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1218 break;
1219
1220 case Bytecodes::_d2f:
1221 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1222 break;
1223
1224 case Bytecodes::_i2f:
1225 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1226 break;
1227
1228 case Bytecodes::_i2d:
1229 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1230 break;
1231
1232 case Bytecodes::_l2f:
1233 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1234 break;
1235
1236 case Bytecodes::_l2d:
1237 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1238 break;
1239
1240 case Bytecodes::_f2i:
1241 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1242 break;
1243
1244 case Bytecodes::_d2i:
1245 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1246 break;
1247
1248 case Bytecodes::_f2l:
1249 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1250 break;
1251
1252 case Bytecodes::_d2l:
1253 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1254 break;
1255
1256 default: ShouldNotReachHere();
1257 }
1258 }
1259
1260 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1261 if (op->init_check()) {
1262 add_debug_info_for_null_check_here(op->stub()->info());
1263 // init_state needs acquire, but x86 is TSO, and so we are already good.
1264 __ cmpb(Address(op->klass()->as_register(),
1265 InstanceKlass::init_state_offset()),
1266 InstanceKlass::fully_initialized);
1267 __ jcc(Assembler::notEqual, *op->stub()->entry());
1268 }
1269 __ allocate_object(op->obj()->as_register(),
1270 op->tmp1()->as_register(),
1271 op->tmp2()->as_register(),
1272 op->header_size(),
1273 op->object_size(),
1274 op->klass()->as_register(),
1275 *op->stub()->entry());
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1280 Register len = op->len()->as_register();
1281 __ movslq(len, len);
1282
1283 if (UseSlowPath || op->always_slow_path() ||
1284 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1285 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1286 __ jmp(*op->stub()->entry());
1287 } else {
1288 Register tmp1 = op->tmp1()->as_register();
1289 Register tmp2 = op->tmp2()->as_register();
1290 Register tmp3 = op->tmp3()->as_register();
1291 if (len == tmp1) {
1292 tmp1 = tmp3;
1293 } else if (len == tmp2) {
1294 tmp2 = tmp3;
1295 } else if (len == tmp3) {
1296 // everything is ok
1297 } else {
1298 __ mov(tmp3, len);
1299 }
1300 __ allocate_array(op->obj()->as_register(),
1301 len,
1302 tmp1,
1303 tmp2,
1304 arrayOopDesc::base_offset_in_bytes(op->type()),
1305 array_element_size(op->type()),
1306 op->klass()->as_register(),
1307 *op->stub()->entry(),
1308 op->zero_array());
1309 }
1310 __ bind(*op->stub()->continuation());
1311 }
1312
1313 void LIR_Assembler::type_profile_helper(Register mdo,
1314 ciMethodData *md, ciProfileData *data,
1315 Register recv) {
1316 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1317 __ profile_receiver_type(recv, mdo, mdp_offset);
1318 }
1319
1320 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1321 // we always need a stub for the failure case.
1322 CodeStub* stub = op->stub();
1323 Register obj = op->object()->as_register();
1324 Register k_RInfo = op->tmp1()->as_register();
1325 Register klass_RInfo = op->tmp2()->as_register();
1326 Register dst = op->result_opr()->as_register();
1327 ciKlass* k = op->klass();
1328 Register Rtmp1 = noreg;
1329 Register tmp_load_klass = rscratch1;
1330
1331 // check if it needs to be profiled
1332 ciMethodData* md = nullptr;
1333 ciProfileData* data = nullptr;
1334
1335 if (op->should_profile()) {
1336 ciMethod* method = op->profiled_method();
1337 assert(method != nullptr, "Should have method");
1338 int bci = op->profiled_bci();
1339 md = method->method_data_or_null();
1340 assert(md != nullptr, "Sanity");
1341 data = md->bci_to_data(bci);
1342 assert(data != nullptr, "need data for type check");
1343 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1344 }
1345 Label* success_target = success;
1346 Label* failure_target = failure;
1347
1348 if (obj == k_RInfo) {
1349 k_RInfo = dst;
1350 } else if (obj == klass_RInfo) {
1351 klass_RInfo = dst;
1352 }
1353 if (k->is_loaded() && !UseCompressedClassPointers) {
1354 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1355 } else {
1356 Rtmp1 = op->tmp3()->as_register();
1357 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1358 }
1359
1360 assert_different_registers(obj, k_RInfo, klass_RInfo);
1361
1362 if (op->need_null_check()) {
1363 __ testptr(obj, obj);
1364 if (op->should_profile()) {
1365 Label not_null;
1366 Register mdo = klass_RInfo;
1367 __ mov_metadata(mdo, md->constant_encoding());
1368 __ jccb(Assembler::notEqual, not_null);
1369 // Object is null; update MDO and exit
1370 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1371 int header_bits = BitData::null_seen_byte_constant();
1372 __ orb(data_addr, header_bits);
1373 __ jmp(*obj_is_null);
1374 __ bind(not_null);
1375
1376 Register recv = k_RInfo;
1377 __ load_klass(recv, obj, tmp_load_klass);
1378 type_profile_helper(mdo, md, data, recv);
1379 } else {
1380 __ jcc(Assembler::equal, *obj_is_null);
1381 }
1382 }
1383
1384 if (!k->is_loaded()) {
1385 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1386 } else {
1387 __ mov_metadata(k_RInfo, k->constant_encoding());
1388 }
1389 __ verify_oop(obj);
1390
1391 if (op->fast_check()) {
1392 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1393 // get object class
1394 // not a safepoint as obj null check happens earlier
1395 if (UseCompressedClassPointers) {
1396 __ load_klass(Rtmp1, obj, tmp_load_klass);
1397 __ cmpptr(k_RInfo, Rtmp1);
1398 } else {
1399 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1400 }
1401 __ jcc(Assembler::notEqual, *failure_target);
1402 // successful cast, fall through to profile or jump
1403 } else {
1404 // get object class
1405 // not a safepoint as obj null check happens earlier
1406 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1407 if (k->is_loaded()) {
1408 // See if we get an immediate positive hit
1409 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1410 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // See if we get an immediate positive hit
1415 __ jcc(Assembler::equal, *success_target);
1416 // check for self
1417 if (k->is_loaded() && k->is_obj_array_klass()) {
1418 // For a direct pointer comparison, we need the refined array klass pointer
1419 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1420 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1421 __ cmpptr(klass_RInfo, tmp_load_klass);
1422 } else {
1423 __ cmpptr(klass_RInfo, k_RInfo);
1424 }
1425 __ jcc(Assembler::equal, *success_target);
1426
1427 __ push_ppx(klass_RInfo);
1428 __ push_ppx(k_RInfo);
1429 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1430 __ pop_ppx(klass_RInfo);
1431 __ pop_ppx(klass_RInfo);
1432 // result is a boolean
1433 __ testl(klass_RInfo, klass_RInfo);
1434 __ jcc(Assembler::equal, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 }
1437 } else {
1438 // perform the fast part of the checking logic
1439 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1440 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1441 __ push_ppx(klass_RInfo);
1442 __ push_ppx(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1444 __ pop_ppx(klass_RInfo);
1445 __ pop_ppx(k_RInfo);
1446 // result is a boolean
1447 __ testl(k_RInfo, k_RInfo);
1448 __ jcc(Assembler::equal, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 }
1452 __ jmp(*success);
1453 }
1454
1455
1456 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1457 Register tmp_load_klass = rscratch1;
1458 LIR_Code code = op->code();
1459 if (code == lir_store_check) {
1460 Register value = op->object()->as_register();
1461 Register array = op->array()->as_register();
1462 Register k_RInfo = op->tmp1()->as_register();
1463 Register klass_RInfo = op->tmp2()->as_register();
1464 Register Rtmp1 = op->tmp3()->as_register();
1465
1466 CodeStub* stub = op->stub();
1467
1468 // check if it needs to be profiled
1469 ciMethodData* md = nullptr;
1470 ciProfileData* data = nullptr;
1471
1472 if (op->should_profile()) {
1473 ciMethod* method = op->profiled_method();
1474 assert(method != nullptr, "Should have method");
1475 int bci = op->profiled_bci();
1476 md = method->method_data_or_null();
1477 assert(md != nullptr, "Sanity");
1478 data = md->bci_to_data(bci);
1479 assert(data != nullptr, "need data for type check");
1480 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1481 }
1482 Label done;
1483 Label* success_target = &done;
1484 Label* failure_target = stub->entry();
1485
1486 __ testptr(value, value);
1487 if (op->should_profile()) {
1488 Label not_null;
1489 Register mdo = klass_RInfo;
1490 __ mov_metadata(mdo, md->constant_encoding());
1491 __ jccb(Assembler::notEqual, not_null);
1492 // Object is null; update MDO and exit
1493 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1494 int header_bits = BitData::null_seen_byte_constant();
1495 __ orb(data_addr, header_bits);
1496 __ jmp(done);
1497 __ bind(not_null);
1498
1499 Register recv = k_RInfo;
1500 __ load_klass(recv, value, tmp_load_klass);
1501 type_profile_helper(mdo, md, data, recv);
1502 } else {
1503 __ jcc(Assembler::equal, done);
1504 }
1505
1506 add_debug_info_for_null_check_here(op->info_for_exception());
1507 __ load_klass(k_RInfo, array, tmp_load_klass);
1508 __ load_klass(klass_RInfo, value, tmp_load_klass);
1509
1510 // get instance klass (it's already uncompressed)
1511 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1512 // perform the fast part of the checking logic
1513 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1514 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1515 __ push_ppx(klass_RInfo);
1516 __ push_ppx(k_RInfo);
1517 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1518 __ pop_ppx(klass_RInfo);
1519 __ pop_ppx(k_RInfo);
1520 // result is a boolean
1521 __ testl(k_RInfo, k_RInfo);
1522 __ jcc(Assembler::equal, *failure_target);
1523 // fall through to the success case
1524
1525 __ bind(done);
1526 } else
1527 if (code == lir_checkcast) {
1528 Register obj = op->object()->as_register();
1529 Register dst = op->result_opr()->as_register();
1530 Label success;
1531 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1532 __ bind(success);
1533 if (dst != obj) {
1534 __ mov(dst, obj);
1535 }
1536 } else
1537 if (code == lir_instanceof) {
1538 Register obj = op->object()->as_register();
1539 Register dst = op->result_opr()->as_register();
1540 Label success, failure, done;
1541 emit_typecheck_helper(op, &success, &failure, &failure);
1542 __ bind(failure);
1543 __ xorptr(dst, dst);
1544 __ jmpb(done);
1545 __ bind(success);
1546 __ movptr(dst, 1);
1547 __ bind(done);
1548 } else {
1549 ShouldNotReachHere();
1550 }
1551
1552 }
1553
1554 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1555 // We are loading/storing from/to an array that *may* be a flat array (the
1556 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1557 // If this array is a flat array, take the slow path.
1558 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1559 if (!op->value()->is_illegal()) {
1560 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1561 // The array is not a flat array, but it might be null-free. If we are storing
1562 // a null into a null-free array, take the slow path (which will throw NPE).
1563 Label skip;
1564 __ cmpptr(op->value()->as_register(), NULL_WORD);
1565 __ jcc(Assembler::notEqual, skip);
1566 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1567 __ bind(skip);
1568 }
1569 }
1570
1571 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1572 // We are storing into an array that *may* be null-free (the declared type is
1573 // Object[], abstract[], interface[] or VT.ref[]).
1574 Label test_mark_word;
1575 Register tmp = op->tmp()->as_register();
1576 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1577 __ testl(tmp, markWord::unlocked_value);
1578 __ jccb(Assembler::notZero, test_mark_word);
1579 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1580 __ bind(test_mark_word);
1581 __ testl(tmp, markWord::null_free_array_bit_in_place);
1582 }
1583
1584 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1585 Label L_oops_equal;
1586 Label L_oops_not_equal;
1587 Label L_end;
1588
1589 Register left = op->left()->as_register();
1590 Register right = op->right()->as_register();
1591
1592 __ cmpptr(left, right);
1593 __ jcc(Assembler::equal, L_oops_equal);
1594
1595 // (1) Null check -- if one of the operands is null, the other must not be null (because
1596 // the two references are not equal), so they are not substitutable,
1597 // FIXME: do null check only if the operand is nullable
1598 __ testptr(left, right);
1599 __ jcc(Assembler::zero, L_oops_not_equal);
1600
1601 ciKlass* left_klass = op->left_klass();
1602 ciKlass* right_klass = op->right_klass();
1603
1604 // (2) Inline type check -- if either of the operands is not a inline type,
1605 // they are not substitutable. We do this only if we are not sure that the
1606 // operands are inline type
1607 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1608 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1609 Register tmp1 = op->tmp1()->as_register();
1610 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1611 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1612 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1613 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1614 __ jcc(Assembler::notEqual, L_oops_not_equal);
1615 }
1616
1617 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1618 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1619 // No need to load klass -- the operands are statically known to be the same inline klass.
1620 __ jmp(*op->stub()->entry());
1621 } else {
1622 Register left_klass_op = op->left_klass_op()->as_register();
1623 Register right_klass_op = op->right_klass_op()->as_register();
1624
1625 if (UseCompressedClassPointers) {
1626 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1627 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1628 __ cmpl(left_klass_op, right_klass_op);
1629 } else {
1630 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1631 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1632 __ cmpptr(left_klass_op, right_klass_op);
1633 }
1634
1635 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1636 // fall through to L_oops_not_equal
1637 }
1638
1639 __ bind(L_oops_not_equal);
1640 move(op->not_equal_result(), op->result_opr());
1641 __ jmp(L_end);
1642
1643 __ bind(L_oops_equal);
1644 move(op->equal_result(), op->result_opr());
1645 __ jmp(L_end);
1646
1647 // We've returned from the stub. RAX contains 0x0 IFF the two
1648 // operands are not substitutable. (Don't compare against 0x1 in case the
1649 // C compiler is naughty)
1650 __ bind(*op->stub()->continuation());
1651 __ cmpl(rax, 0);
1652 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1653 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1654 // fall-through
1655 __ bind(L_end);
1656 }
1657
1658 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1659 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1660 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1661 Register newval = op->new_value()->as_register();
1662 Register cmpval = op->cmp_value()->as_register();
1663 assert(cmpval == rax, "wrong register");
1664 assert(newval != noreg, "new val must be register");
1665 assert(cmpval != newval, "cmp and new values must be in different registers");
1666 assert(cmpval != addr, "cmp and addr must be in different registers");
1667 assert(newval != addr, "new value and addr must be in different registers");
1668
1669 if (op->code() == lir_cas_obj) {
1670 if (UseCompressedOops) {
1671 __ encode_heap_oop(cmpval);
1672 __ mov(rscratch1, newval);
1673 __ encode_heap_oop(rscratch1);
1674 __ lock();
1675 // cmpval (rax) is implicitly used by this instruction
1676 __ cmpxchgl(rscratch1, Address(addr, 0));
1677 } else {
1678 __ lock();
1679 __ cmpxchgptr(newval, Address(addr, 0));
1680 }
1681 } else {
1682 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1683 __ lock();
1684 __ cmpxchgl(newval, Address(addr, 0));
1685 }
1686 } else if (op->code() == lir_cas_long) {
1687 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1688 Register newval = op->new_value()->as_register_lo();
1689 Register cmpval = op->cmp_value()->as_register_lo();
1690 assert(cmpval == rax, "wrong register");
1691 assert(newval != noreg, "new val must be register");
1692 assert(cmpval != newval, "cmp and new values must be in different registers");
1693 assert(cmpval != addr, "cmp and addr must be in different registers");
1694 assert(newval != addr, "new value and addr must be in different registers");
1695 __ lock();
1696 __ cmpxchgq(newval, Address(addr, 0));
1697 } else {
1698 Unimplemented();
1699 }
1700 }
1701
1702 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1703 assert(dst->is_cpu_register(), "must be");
1704 assert(dst->type() == src->type(), "must be");
1705
1706 if (src->is_cpu_register()) {
1707 reg2reg(src, dst);
1708 } else if (src->is_stack()) {
1709 stack2reg(src, dst, dst->type());
1710 } else if (src->is_constant()) {
1711 const2reg(src, dst, lir_patch_none, nullptr);
1712 } else {
1713 ShouldNotReachHere();
1714 }
1715 }
1716
1717 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1718 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1719 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1720
1721 Assembler::Condition acond, ncond;
1722 switch (condition) {
1723 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1724 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1725 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1726 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1727 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1728 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1729 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1730 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1731 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1732 ShouldNotReachHere();
1733 }
1734
1735 if (opr1->is_cpu_register()) {
1736 reg2reg(opr1, result);
1737 } else if (opr1->is_stack()) {
1738 stack2reg(opr1, result, result->type());
1739 } else if (opr1->is_constant()) {
1740 const2reg(opr1, result, lir_patch_none, nullptr);
1741 } else {
1742 ShouldNotReachHere();
1743 }
1744
1745 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1746 // optimized version that does not require a branch
1747 if (opr2->is_single_cpu()) {
1748 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1749 __ cmov(ncond, result->as_register(), opr2->as_register());
1750 } else if (opr2->is_double_cpu()) {
1751 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1752 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1753 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1754 } else if (opr2->is_single_stack()) {
1755 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1756 } else if (opr2->is_double_stack()) {
1757 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1758 } else {
1759 ShouldNotReachHere();
1760 }
1761
1762 } else {
1763 Label skip;
1764 __ jccb(acond, skip);
1765 if (opr2->is_cpu_register()) {
1766 reg2reg(opr2, result);
1767 } else if (opr2->is_stack()) {
1768 stack2reg(opr2, result, result->type());
1769 } else if (opr2->is_constant()) {
1770 const2reg(opr2, result, lir_patch_none, nullptr);
1771 } else {
1772 ShouldNotReachHere();
1773 }
1774 __ bind(skip);
1775 }
1776 }
1777
1778
1779 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1780 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1781
1782 if (left->is_single_cpu()) {
1783 assert(left == dest, "left and dest must be equal");
1784 Register lreg = left->as_register();
1785
1786 if (right->is_single_cpu()) {
1787 // cpu register - cpu register
1788 Register rreg = right->as_register();
1789 switch (code) {
1790 case lir_add: __ addl (lreg, rreg); break;
1791 case lir_sub: __ subl (lreg, rreg); break;
1792 case lir_mul: __ imull(lreg, rreg); break;
1793 default: ShouldNotReachHere();
1794 }
1795
1796 } else if (right->is_stack()) {
1797 // cpu register - stack
1798 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1799 switch (code) {
1800 case lir_add: __ addl(lreg, raddr); break;
1801 case lir_sub: __ subl(lreg, raddr); break;
1802 default: ShouldNotReachHere();
1803 }
1804
1805 } else if (right->is_constant()) {
1806 // cpu register - constant
1807 jint c = right->as_constant_ptr()->as_jint();
1808 switch (code) {
1809 case lir_add: {
1810 __ incrementl(lreg, c);
1811 break;
1812 }
1813 case lir_sub: {
1814 __ decrementl(lreg, c);
1815 break;
1816 }
1817 default: ShouldNotReachHere();
1818 }
1819
1820 } else {
1821 ShouldNotReachHere();
1822 }
1823
1824 } else if (left->is_double_cpu()) {
1825 assert(left == dest, "left and dest must be equal");
1826 Register lreg_lo = left->as_register_lo();
1827 Register lreg_hi = left->as_register_hi();
1828
1829 if (right->is_double_cpu()) {
1830 // cpu register - cpu register
1831 Register rreg_lo = right->as_register_lo();
1832 Register rreg_hi = right->as_register_hi();
1833 assert_different_registers(lreg_lo, rreg_lo);
1834 switch (code) {
1835 case lir_add:
1836 __ addptr(lreg_lo, rreg_lo);
1837 break;
1838 case lir_sub:
1839 __ subptr(lreg_lo, rreg_lo);
1840 break;
1841 case lir_mul:
1842 __ imulq(lreg_lo, rreg_lo);
1843 break;
1844 default:
1845 ShouldNotReachHere();
1846 }
1847
1848 } else if (right->is_constant()) {
1849 // cpu register - constant
1850 jlong c = right->as_constant_ptr()->as_jlong_bits();
1851 __ movptr(r10, (intptr_t) c);
1852 switch (code) {
1853 case lir_add:
1854 __ addptr(lreg_lo, r10);
1855 break;
1856 case lir_sub:
1857 __ subptr(lreg_lo, r10);
1858 break;
1859 default:
1860 ShouldNotReachHere();
1861 }
1862
1863 } else {
1864 ShouldNotReachHere();
1865 }
1866
1867 } else if (left->is_single_xmm()) {
1868 assert(left == dest, "left and dest must be equal");
1869 XMMRegister lreg = left->as_xmm_float_reg();
1870
1871 if (right->is_single_xmm()) {
1872 XMMRegister rreg = right->as_xmm_float_reg();
1873 switch (code) {
1874 case lir_add: __ addss(lreg, rreg); break;
1875 case lir_sub: __ subss(lreg, rreg); break;
1876 case lir_mul: __ mulss(lreg, rreg); break;
1877 case lir_div: __ divss(lreg, rreg); break;
1878 default: ShouldNotReachHere();
1879 }
1880 } else {
1881 Address raddr;
1882 if (right->is_single_stack()) {
1883 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1884 } else if (right->is_constant()) {
1885 // hack for now
1886 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1887 } else {
1888 ShouldNotReachHere();
1889 }
1890 switch (code) {
1891 case lir_add: __ addss(lreg, raddr); break;
1892 case lir_sub: __ subss(lreg, raddr); break;
1893 case lir_mul: __ mulss(lreg, raddr); break;
1894 case lir_div: __ divss(lreg, raddr); break;
1895 default: ShouldNotReachHere();
1896 }
1897 }
1898
1899 } else if (left->is_double_xmm()) {
1900 assert(left == dest, "left and dest must be equal");
1901
1902 XMMRegister lreg = left->as_xmm_double_reg();
1903 if (right->is_double_xmm()) {
1904 XMMRegister rreg = right->as_xmm_double_reg();
1905 switch (code) {
1906 case lir_add: __ addsd(lreg, rreg); break;
1907 case lir_sub: __ subsd(lreg, rreg); break;
1908 case lir_mul: __ mulsd(lreg, rreg); break;
1909 case lir_div: __ divsd(lreg, rreg); break;
1910 default: ShouldNotReachHere();
1911 }
1912 } else {
1913 Address raddr;
1914 if (right->is_double_stack()) {
1915 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1916 } else if (right->is_constant()) {
1917 // hack for now
1918 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1919 } else {
1920 ShouldNotReachHere();
1921 }
1922 switch (code) {
1923 case lir_add: __ addsd(lreg, raddr); break;
1924 case lir_sub: __ subsd(lreg, raddr); break;
1925 case lir_mul: __ mulsd(lreg, raddr); break;
1926 case lir_div: __ divsd(lreg, raddr); break;
1927 default: ShouldNotReachHere();
1928 }
1929 }
1930
1931 } else if (left->is_single_stack() || left->is_address()) {
1932 assert(left == dest, "left and dest must be equal");
1933
1934 Address laddr;
1935 if (left->is_single_stack()) {
1936 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1937 } else if (left->is_address()) {
1938 laddr = as_Address(left->as_address_ptr());
1939 } else {
1940 ShouldNotReachHere();
1941 }
1942
1943 if (right->is_single_cpu()) {
1944 Register rreg = right->as_register();
1945 switch (code) {
1946 case lir_add: __ addl(laddr, rreg); break;
1947 case lir_sub: __ subl(laddr, rreg); break;
1948 default: ShouldNotReachHere();
1949 }
1950 } else if (right->is_constant()) {
1951 jint c = right->as_constant_ptr()->as_jint();
1952 switch (code) {
1953 case lir_add: {
1954 __ incrementl(laddr, c);
1955 break;
1956 }
1957 case lir_sub: {
1958 __ decrementl(laddr, c);
1959 break;
1960 }
1961 default: ShouldNotReachHere();
1962 }
1963 } else {
1964 ShouldNotReachHere();
1965 }
1966
1967 } else {
1968 ShouldNotReachHere();
1969 }
1970 }
1971
1972
1973 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1974 if (value->is_double_xmm()) {
1975 switch(code) {
1976 case lir_abs :
1977 {
1978 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1979 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1980 }
1981 assert(!tmp->is_valid(), "do not need temporary");
1982 __ andpd(dest->as_xmm_double_reg(),
1983 ExternalAddress((address)double_signmask_pool),
1984 rscratch1);
1985 }
1986 break;
1987
1988 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1989 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1990 default : ShouldNotReachHere();
1991 }
1992
1993 } else if (code == lir_f2hf) {
1994 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1995 } else if (code == lir_hf2f) {
1996 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1997 } else {
1998 Unimplemented();
1999 }
2000 }
2001
2002 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2003 // assert(left->destroys_register(), "check");
2004 if (left->is_single_cpu()) {
2005 Register reg = left->as_register();
2006 if (right->is_constant()) {
2007 int val = right->as_constant_ptr()->as_jint();
2008 switch (code) {
2009 case lir_logic_and: __ andl (reg, val); break;
2010 case lir_logic_or: __ orl (reg, val); break;
2011 case lir_logic_xor: __ xorl (reg, val); break;
2012 default: ShouldNotReachHere();
2013 }
2014 } else if (right->is_stack()) {
2015 // added support for stack operands
2016 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2017 switch (code) {
2018 case lir_logic_and: __ andl (reg, raddr); break;
2019 case lir_logic_or: __ orl (reg, raddr); break;
2020 case lir_logic_xor: __ xorl (reg, raddr); break;
2021 default: ShouldNotReachHere();
2022 }
2023 } else {
2024 Register rright = right->as_register();
2025 switch (code) {
2026 case lir_logic_and: __ andptr (reg, rright); break;
2027 case lir_logic_or : __ orptr (reg, rright); break;
2028 case lir_logic_xor: __ xorptr (reg, rright); break;
2029 default: ShouldNotReachHere();
2030 }
2031 }
2032 move_regs(reg, dst->as_register());
2033 } else {
2034 Register l_lo = left->as_register_lo();
2035 Register l_hi = left->as_register_hi();
2036 if (right->is_constant()) {
2037 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2038 switch (code) {
2039 case lir_logic_and:
2040 __ andq(l_lo, rscratch1);
2041 break;
2042 case lir_logic_or:
2043 __ orq(l_lo, rscratch1);
2044 break;
2045 case lir_logic_xor:
2046 __ xorq(l_lo, rscratch1);
2047 break;
2048 default: ShouldNotReachHere();
2049 }
2050 } else {
2051 Register r_lo;
2052 if (is_reference_type(right->type())) {
2053 r_lo = right->as_register();
2054 } else {
2055 r_lo = right->as_register_lo();
2056 }
2057 switch (code) {
2058 case lir_logic_and:
2059 __ andptr(l_lo, r_lo);
2060 break;
2061 case lir_logic_or:
2062 __ orptr(l_lo, r_lo);
2063 break;
2064 case lir_logic_xor:
2065 __ xorptr(l_lo, r_lo);
2066 break;
2067 default: ShouldNotReachHere();
2068 }
2069 }
2070
2071 Register dst_lo = dst->as_register_lo();
2072 Register dst_hi = dst->as_register_hi();
2073
2074 move_regs(l_lo, dst_lo);
2075 }
2076 }
2077
2078
2079 // we assume that rax, and rdx can be overwritten
2080 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2081
2082 assert(left->is_single_cpu(), "left must be register");
2083 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2084 assert(result->is_single_cpu(), "result must be register");
2085
2086 // assert(left->destroys_register(), "check");
2087 // assert(right->destroys_register(), "check");
2088
2089 Register lreg = left->as_register();
2090 Register dreg = result->as_register();
2091
2092 if (right->is_constant()) {
2093 jint divisor = right->as_constant_ptr()->as_jint();
2094 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2095 if (code == lir_idiv) {
2096 assert(lreg == rax, "must be rax,");
2097 assert(temp->as_register() == rdx, "tmp register must be rdx");
2098 __ cdql(); // sign extend into rdx:rax
2099 if (divisor == 2) {
2100 __ subl(lreg, rdx);
2101 } else {
2102 __ andl(rdx, divisor - 1);
2103 __ addl(lreg, rdx);
2104 }
2105 __ sarl(lreg, log2i_exact(divisor));
2106 move_regs(lreg, dreg);
2107 } else if (code == lir_irem) {
2108 Label done;
2109 __ mov(dreg, lreg);
2110 __ andl(dreg, 0x80000000 | (divisor - 1));
2111 __ jcc(Assembler::positive, done);
2112 __ decrement(dreg);
2113 __ orl(dreg, ~(divisor - 1));
2114 __ increment(dreg);
2115 __ bind(done);
2116 } else {
2117 ShouldNotReachHere();
2118 }
2119 } else {
2120 Register rreg = right->as_register();
2121 assert(lreg == rax, "left register must be rax,");
2122 assert(rreg != rdx, "right register must not be rdx");
2123 assert(temp->as_register() == rdx, "tmp register must be rdx");
2124
2125 move_regs(lreg, rax);
2126
2127 int idivl_offset = __ corrected_idivl(rreg);
2128 if (ImplicitDiv0Checks) {
2129 add_debug_info_for_div0(idivl_offset, info);
2130 }
2131 if (code == lir_irem) {
2132 move_regs(rdx, dreg); // result is in rdx
2133 } else {
2134 move_regs(rax, dreg);
2135 }
2136 }
2137 }
2138
2139
2140 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2141 if (opr1->is_single_cpu()) {
2142 Register reg1 = opr1->as_register();
2143 if (opr2->is_single_cpu()) {
2144 // cpu register - cpu register
2145 if (is_reference_type(opr1->type())) {
2146 __ cmpoop(reg1, opr2->as_register());
2147 } else {
2148 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2149 __ cmpl(reg1, opr2->as_register());
2150 }
2151 } else if (opr2->is_stack()) {
2152 // cpu register - stack
2153 if (is_reference_type(opr1->type())) {
2154 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2155 } else {
2156 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2157 }
2158 } else if (opr2->is_constant()) {
2159 // cpu register - constant
2160 LIR_Const* c = opr2->as_constant_ptr();
2161 if (c->type() == T_INT) {
2162 jint i = c->as_jint();
2163 if (i == 0) {
2164 __ testl(reg1, reg1);
2165 } else {
2166 __ cmpl(reg1, i);
2167 }
2168 } else if (c->type() == T_METADATA) {
2169 // All we need for now is a comparison with null for equality.
2170 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2171 Metadata* m = c->as_metadata();
2172 if (m == nullptr) {
2173 __ testptr(reg1, reg1);
2174 } else {
2175 ShouldNotReachHere();
2176 }
2177 } else if (is_reference_type(c->type())) {
2178 // In 64bit oops are single register
2179 jobject o = c->as_jobject();
2180 if (o == nullptr) {
2181 __ testptr(reg1, reg1);
2182 } else {
2183 __ cmpoop(reg1, o, rscratch1);
2184 }
2185 } else {
2186 fatal("unexpected type: %s", basictype_to_str(c->type()));
2187 }
2188 // cpu register - address
2189 } else if (opr2->is_address()) {
2190 if (op->info() != nullptr) {
2191 add_debug_info_for_null_check_here(op->info());
2192 }
2193 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2194 } else {
2195 ShouldNotReachHere();
2196 }
2197
2198 } else if(opr1->is_double_cpu()) {
2199 Register xlo = opr1->as_register_lo();
2200 Register xhi = opr1->as_register_hi();
2201 if (opr2->is_double_cpu()) {
2202 __ cmpptr(xlo, opr2->as_register_lo());
2203 } else if (opr2->is_constant()) {
2204 // cpu register - constant 0
2205 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2206 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2207 } else {
2208 ShouldNotReachHere();
2209 }
2210
2211 } else if (opr1->is_single_xmm()) {
2212 XMMRegister reg1 = opr1->as_xmm_float_reg();
2213 if (opr2->is_single_xmm()) {
2214 // xmm register - xmm register
2215 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2216 } else if (opr2->is_stack()) {
2217 // xmm register - stack
2218 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2219 } else if (opr2->is_constant()) {
2220 // xmm register - constant
2221 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2222 } else if (opr2->is_address()) {
2223 // xmm register - address
2224 if (op->info() != nullptr) {
2225 add_debug_info_for_null_check_here(op->info());
2226 }
2227 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2228 } else {
2229 ShouldNotReachHere();
2230 }
2231
2232 } else if (opr1->is_double_xmm()) {
2233 XMMRegister reg1 = opr1->as_xmm_double_reg();
2234 if (opr2->is_double_xmm()) {
2235 // xmm register - xmm register
2236 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2237 } else if (opr2->is_stack()) {
2238 // xmm register - stack
2239 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2240 } else if (opr2->is_constant()) {
2241 // xmm register - constant
2242 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2243 } else if (opr2->is_address()) {
2244 // xmm register - address
2245 if (op->info() != nullptr) {
2246 add_debug_info_for_null_check_here(op->info());
2247 }
2248 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2249 } else {
2250 ShouldNotReachHere();
2251 }
2252
2253 } else if (opr1->is_address() && opr2->is_constant()) {
2254 LIR_Const* c = opr2->as_constant_ptr();
2255 if (is_reference_type(c->type())) {
2256 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2257 __ movoop(rscratch1, c->as_jobject());
2258 }
2259 if (op->info() != nullptr) {
2260 add_debug_info_for_null_check_here(op->info());
2261 }
2262 // special case: address - constant
2263 LIR_Address* addr = opr1->as_address_ptr();
2264 if (c->type() == T_INT) {
2265 __ cmpl(as_Address(addr), c->as_jint());
2266 } else if (is_reference_type(c->type())) {
2267 // %%% Make this explode if addr isn't reachable until we figure out a
2268 // better strategy by giving noreg as the temp for as_Address
2269 __ cmpoop(rscratch1, as_Address(addr, noreg));
2270 } else {
2271 ShouldNotReachHere();
2272 }
2273
2274 } else {
2275 ShouldNotReachHere();
2276 }
2277 }
2278
2279 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2280 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2281 if (left->is_single_xmm()) {
2282 assert(right->is_single_xmm(), "must match");
2283 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2284 } else if (left->is_double_xmm()) {
2285 assert(right->is_double_xmm(), "must match");
2286 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2287
2288 } else {
2289 ShouldNotReachHere();
2290 }
2291 } else {
2292 assert(code == lir_cmp_l2i, "check");
2293 Label done;
2294 Register dest = dst->as_register();
2295 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2296 __ movl(dest, -1);
2297 __ jccb(Assembler::less, done);
2298 __ setb(Assembler::notZero, dest);
2299 __ movzbl(dest, dest);
2300 __ bind(done);
2301 }
2302 }
2303
2304
2305 void LIR_Assembler::align_call(LIR_Code code) {
2306 // make sure that the displacement word of the call ends up word aligned
2307 int offset = __ offset();
2308 switch (code) {
2309 case lir_static_call:
2310 case lir_optvirtual_call:
2311 case lir_dynamic_call:
2312 offset += NativeCall::displacement_offset;
2313 break;
2314 case lir_icvirtual_call:
2315 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2316 break;
2317 default: ShouldNotReachHere();
2318 }
2319 __ align(BytesPerWord, offset);
2320 }
2321
2322
2323 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2324 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2325 "must be aligned");
2326 __ call(AddressLiteral(op->addr(), rtype));
2327 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2328 __ post_call_nop();
2329 }
2330
2331
2332 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2333 __ ic_call(op->addr());
2334 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2335 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2336 "must be aligned");
2337 __ post_call_nop();
2338 }
2339
2340
2341 void LIR_Assembler::emit_static_call_stub() {
2342 address call_pc = __ pc();
2343 address stub = __ start_a_stub(call_stub_size());
2344 if (stub == nullptr) {
2345 bailout("static call stub overflow");
2346 return;
2347 }
2348
2349 int start = __ offset();
2350
2351 // make sure that the displacement word of the call ends up word aligned
2352 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2353 __ relocate(static_stub_Relocation::spec(call_pc));
2354 __ mov_metadata(rbx, (Metadata*)nullptr);
2355 // must be set to -1 at code generation time
2356 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2357 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2358 __ jump(RuntimeAddress(__ pc()));
2359
2360 assert(__ offset() - start <= call_stub_size(), "stub too big");
2361 __ end_a_stub();
2362 }
2363
2364
2365 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2366 assert(exceptionOop->as_register() == rax, "must match");
2367 assert(exceptionPC->as_register() == rdx, "must match");
2368
2369 // exception object is not added to oop map by LinearScan
2370 // (LinearScan assumes that no oops are in fixed registers)
2371 info->add_register_oop(exceptionOop);
2372 StubId unwind_id;
2373
2374 // get current pc information
2375 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2376 int pc_for_athrow_offset = __ offset();
2377 InternalAddress pc_for_athrow(__ pc());
2378 __ lea(exceptionPC->as_register(), pc_for_athrow);
2379 add_call_info(pc_for_athrow_offset, info); // for exception handler
2380
2381 __ verify_not_null_oop(rax);
2382 // search an exception handler (rax: exception oop, rdx: throwing pc)
2383 if (compilation()->has_fpu_code()) {
2384 unwind_id = StubId::c1_handle_exception_id;
2385 } else {
2386 unwind_id = StubId::c1_handle_exception_nofpu_id;
2387 }
2388 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2389
2390 // enough room for two byte trap
2391 __ nop();
2392 }
2393
2394
2395 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2396 assert(exceptionOop->as_register() == rax, "must match");
2397
2398 __ jmp(_unwind_handler_entry);
2399 }
2400
2401
2402 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2403
2404 // optimized version for linear scan:
2405 // * count must be already in ECX (guaranteed by LinearScan)
2406 // * left and dest must be equal
2407 // * tmp must be unused
2408 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2409 assert(left == dest, "left and dest must be equal");
2410 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2411
2412 if (left->is_single_cpu()) {
2413 Register value = left->as_register();
2414 assert(value != SHIFT_count, "left cannot be ECX");
2415
2416 switch (code) {
2417 case lir_shl: __ shll(value); break;
2418 case lir_shr: __ sarl(value); break;
2419 case lir_ushr: __ shrl(value); break;
2420 default: ShouldNotReachHere();
2421 }
2422 } else if (left->is_double_cpu()) {
2423 Register lo = left->as_register_lo();
2424 Register hi = left->as_register_hi();
2425 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2426 switch (code) {
2427 case lir_shl: __ shlptr(lo); break;
2428 case lir_shr: __ sarptr(lo); break;
2429 case lir_ushr: __ shrptr(lo); break;
2430 default: ShouldNotReachHere();
2431 }
2432 } else {
2433 ShouldNotReachHere();
2434 }
2435 }
2436
2437
2438 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2439 if (dest->is_single_cpu()) {
2440 // first move left into dest so that left is not destroyed by the shift
2441 Register value = dest->as_register();
2442 count = count & 0x1F; // Java spec
2443
2444 move_regs(left->as_register(), value);
2445 switch (code) {
2446 case lir_shl: __ shll(value, count); break;
2447 case lir_shr: __ sarl(value, count); break;
2448 case lir_ushr: __ shrl(value, count); break;
2449 default: ShouldNotReachHere();
2450 }
2451 } else if (dest->is_double_cpu()) {
2452 // first move left into dest so that left is not destroyed by the shift
2453 Register value = dest->as_register_lo();
2454 count = count & 0x1F; // Java spec
2455
2456 move_regs(left->as_register_lo(), value);
2457 switch (code) {
2458 case lir_shl: __ shlptr(value, count); break;
2459 case lir_shr: __ sarptr(value, count); break;
2460 case lir_ushr: __ shrptr(value, count); break;
2461 default: ShouldNotReachHere();
2462 }
2463 } else {
2464 ShouldNotReachHere();
2465 }
2466 }
2467
2468
2469 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2470 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2471 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2472 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2473 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2474 }
2475
2476
2477 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2478 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2479 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2480 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2481 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2482 }
2483
2484
2485 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2486 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2487 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2488 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2489 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2490 }
2491
2492
2493 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2494 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2495 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2496 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2497 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2498 }
2499
2500
2501 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2502 if (null_check) {
2503 __ testptr(obj, obj);
2504 __ jcc(Assembler::zero, *slow_path->entry());
2505 }
2506 if (is_dest) {
2507 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2508 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2509 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2510 } else {
2511 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2512 }
2513 }
2514
2515
2516 // This code replaces a call to arraycopy; no exception may
2517 // be thrown in this code, they must be thrown in the System.arraycopy
2518 // activation frame; we could save some checks if this would not be the case
2519 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2520 ciArrayKlass* default_type = op->expected_type();
2521 Register src = op->src()->as_register();
2522 Register dst = op->dst()->as_register();
2523 Register src_pos = op->src_pos()->as_register();
2524 Register dst_pos = op->dst_pos()->as_register();
2525 Register length = op->length()->as_register();
2526 Register tmp = op->tmp()->as_register();
2527 Register tmp_load_klass = rscratch1;
2528 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2529
2530 CodeStub* stub = op->stub();
2531 int flags = op->flags();
2532 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2533 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2534
2535 if (flags & LIR_OpArrayCopy::always_slow_path) {
2536 __ jmp(*stub->entry());
2537 __ bind(*stub->continuation());
2538 return;
2539 }
2540
2541 // if we don't know anything, just go through the generic arraycopy
2542 if (default_type == nullptr) {
2543 // save outgoing arguments on stack in case call to System.arraycopy is needed
2544 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2545 // for interpreter calling conventions. Now we have to do it in new style conventions.
2546 // For the moment until C1 gets the new register allocator I just force all the
2547 // args to the right place (except the register args) and then on the back side
2548 // reload the register args properly if we go slow path. Yuck
2549
2550 // These are proper for the calling convention
2551 store_parameter(length, 2);
2552 store_parameter(dst_pos, 1);
2553 store_parameter(dst, 0);
2554
2555 // these are just temporary placements until we need to reload
2556 store_parameter(src_pos, 3);
2557 store_parameter(src, 4);
2558
2559 address copyfunc_addr = StubRoutines::generic_arraycopy();
2560 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2561
2562 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2563 // The arguments are in java calling convention so we can trivially shift them to C
2564 // convention
2565 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2566 __ mov(c_rarg0, j_rarg0);
2567 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2568 __ mov(c_rarg1, j_rarg1);
2569 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2570 __ mov(c_rarg2, j_rarg2);
2571 assert_different_registers(c_rarg3, j_rarg4);
2572 __ mov(c_rarg3, j_rarg3);
2573 #ifdef _WIN64
2574 // Allocate abi space for args but be sure to keep stack aligned
2575 __ subptr(rsp, 6*wordSize);
2576 store_parameter(j_rarg4, 4);
2577 #ifndef PRODUCT
2578 if (PrintC1Statistics) {
2579 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2580 }
2581 #endif
2582 __ call(RuntimeAddress(copyfunc_addr));
2583 __ addptr(rsp, 6*wordSize);
2584 #else
2585 __ mov(c_rarg4, j_rarg4);
2586 #ifndef PRODUCT
2587 if (PrintC1Statistics) {
2588 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2589 }
2590 #endif
2591 __ call(RuntimeAddress(copyfunc_addr));
2592 #endif // _WIN64
2593
2594 __ testl(rax, rax);
2595 __ jcc(Assembler::equal, *stub->continuation());
2596
2597 __ mov(tmp, rax);
2598 __ xorl(tmp, -1);
2599
2600 // Reload values from the stack so they are where the stub
2601 // expects them.
2602 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2603 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2604 __ movptr (length, Address(rsp, 2*BytesPerWord));
2605 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2606 __ movptr (src, Address(rsp, 4*BytesPerWord));
2607
2608 __ subl(length, tmp);
2609 __ addl(src_pos, tmp);
2610 __ addl(dst_pos, tmp);
2611 __ jmp(*stub->entry());
2612
2613 __ bind(*stub->continuation());
2614 return;
2615 }
2616
2617 // Handle inline type arrays
2618 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2619 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2620 }
2621 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2622 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2623 }
2624
2625 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2626
2627 int elem_size = type2aelembytes(basic_type);
2628 Address::ScaleFactor scale;
2629
2630 switch (elem_size) {
2631 case 1 :
2632 scale = Address::times_1;
2633 break;
2634 case 2 :
2635 scale = Address::times_2;
2636 break;
2637 case 4 :
2638 scale = Address::times_4;
2639 break;
2640 case 8 :
2641 scale = Address::times_8;
2642 break;
2643 default:
2644 scale = Address::no_scale;
2645 ShouldNotReachHere();
2646 }
2647
2648 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2649 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2650
2651 // length and pos's are all sign extended at this point on 64bit
2652
2653 // test for null
2654 if (flags & LIR_OpArrayCopy::src_null_check) {
2655 __ testptr(src, src);
2656 __ jcc(Assembler::zero, *stub->entry());
2657 }
2658 if (flags & LIR_OpArrayCopy::dst_null_check) {
2659 __ testptr(dst, dst);
2660 __ jcc(Assembler::zero, *stub->entry());
2661 }
2662
2663 // If the compiler was not able to prove that exact type of the source or the destination
2664 // of the arraycopy is an array type, check at runtime if the source or the destination is
2665 // an instance type.
2666 if (flags & LIR_OpArrayCopy::type_check) {
2667 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2668 __ load_klass(tmp, dst, tmp_load_klass);
2669 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2670 __ jcc(Assembler::greaterEqual, *stub->entry());
2671 }
2672
2673 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2674 __ load_klass(tmp, src, tmp_load_klass);
2675 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2676 __ jcc(Assembler::greaterEqual, *stub->entry());
2677 }
2678 }
2679
2680 // check if negative
2681 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2682 __ testl(src_pos, src_pos);
2683 __ jcc(Assembler::less, *stub->entry());
2684 }
2685 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2686 __ testl(dst_pos, dst_pos);
2687 __ jcc(Assembler::less, *stub->entry());
2688 }
2689
2690 if (flags & LIR_OpArrayCopy::src_range_check) {
2691 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2692 __ cmpl(tmp, src_length_addr);
2693 __ jcc(Assembler::above, *stub->entry());
2694 }
2695 if (flags & LIR_OpArrayCopy::dst_range_check) {
2696 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2697 __ cmpl(tmp, dst_length_addr);
2698 __ jcc(Assembler::above, *stub->entry());
2699 }
2700
2701 if (flags & LIR_OpArrayCopy::length_positive_check) {
2702 __ testl(length, length);
2703 __ jcc(Assembler::less, *stub->entry());
2704 }
2705
2706 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2707 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2708
2709 if (flags & LIR_OpArrayCopy::type_check) {
2710 // We don't know the array types are compatible
2711 if (basic_type != T_OBJECT) {
2712 // Simple test for basic type arrays
2713 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2714 __ jcc(Assembler::notEqual, *stub->entry());
2715 } else {
2716 // For object arrays, if src is a sub class of dst then we can
2717 // safely do the copy.
2718 Label cont, slow;
2719
2720 __ push_ppx(src);
2721 __ push_ppx(dst);
2722
2723 __ load_klass(src, src, tmp_load_klass);
2724 __ load_klass(dst, dst, tmp_load_klass);
2725
2726 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2727
2728 __ push_ppx(src);
2729 __ push_ppx(dst);
2730 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2731 __ pop_ppx(dst);
2732 __ pop_ppx(src);
2733
2734 __ testl(src, src);
2735 __ jcc(Assembler::notEqual, cont);
2736
2737 __ bind(slow);
2738 __ pop_ppx(dst);
2739 __ pop_ppx(src);
2740
2741 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2742 if (copyfunc_addr != nullptr) { // use stub if available
2743 // src is not a sub class of dst so we have to do a
2744 // per-element check.
2745
2746 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2747 if ((flags & mask) != mask) {
2748 // Check that at least both of them object arrays.
2749 assert(flags & mask, "one of the two should be known to be an object array");
2750
2751 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2752 __ load_klass(tmp, src, tmp_load_klass);
2753 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2754 __ load_klass(tmp, dst, tmp_load_klass);
2755 }
2756 int lh_offset = in_bytes(Klass::layout_helper_offset());
2757 Address klass_lh_addr(tmp, lh_offset);
2758 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2759 __ cmpl(klass_lh_addr, objArray_lh);
2760 __ jcc(Assembler::notEqual, *stub->entry());
2761 }
2762
2763 // Spill because stubs can use any register they like and it's
2764 // easier to restore just those that we care about.
2765 store_parameter(dst, 0);
2766 store_parameter(dst_pos, 1);
2767 store_parameter(length, 2);
2768 store_parameter(src_pos, 3);
2769 store_parameter(src, 4);
2770
2771 __ movl2ptr(length, length); //higher 32bits must be null
2772
2773 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2774 assert_different_registers(c_rarg0, dst, dst_pos, length);
2775 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2776 assert_different_registers(c_rarg1, dst, length);
2777
2778 __ mov(c_rarg2, length);
2779 assert_different_registers(c_rarg2, dst);
2780
2781 #ifdef _WIN64
2782 // Allocate abi space for args but be sure to keep stack aligned
2783 __ subptr(rsp, 6*wordSize);
2784 __ load_klass(c_rarg3, dst, tmp_load_klass);
2785 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2786 store_parameter(c_rarg3, 4);
2787 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2788 __ call(RuntimeAddress(copyfunc_addr));
2789 __ addptr(rsp, 6*wordSize);
2790 #else
2791 __ load_klass(c_rarg4, dst, tmp_load_klass);
2792 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2793 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2794 __ call(RuntimeAddress(copyfunc_addr));
2795 #endif
2796
2797 #ifndef PRODUCT
2798 if (PrintC1Statistics) {
2799 Label failed;
2800 __ testl(rax, rax);
2801 __ jcc(Assembler::notZero, failed);
2802 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2803 __ bind(failed);
2804 }
2805 #endif
2806
2807 __ testl(rax, rax);
2808 __ jcc(Assembler::zero, *stub->continuation());
2809
2810 #ifndef PRODUCT
2811 if (PrintC1Statistics) {
2812 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2813 }
2814 #endif
2815
2816 __ mov(tmp, rax);
2817
2818 __ xorl(tmp, -1);
2819
2820 // Restore previously spilled arguments
2821 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2822 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2823 __ movptr (length, Address(rsp, 2*BytesPerWord));
2824 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2825 __ movptr (src, Address(rsp, 4*BytesPerWord));
2826
2827
2828 __ subl(length, tmp);
2829 __ addl(src_pos, tmp);
2830 __ addl(dst_pos, tmp);
2831 }
2832
2833 __ jmp(*stub->entry());
2834
2835 __ bind(cont);
2836 __ pop(dst);
2837 __ pop(src);
2838 }
2839 }
2840
2841 #ifdef ASSERT
2842 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2843 // Sanity check the known type with the incoming class. For the
2844 // primitive case the types must match exactly with src.klass and
2845 // dst.klass each exactly matching the default type. For the
2846 // object array case, if no type check is needed then either the
2847 // dst type is exactly the expected type and the src type is a
2848 // subtype which we can't check or src is the same array as dst
2849 // but not necessarily exactly of type default_type.
2850 Label known_ok, halt;
2851
2852 __ mov_metadata(tmp, default_type->constant_encoding());
2853 if (UseCompressedClassPointers) {
2854 __ encode_klass_not_null(tmp, rscratch1);
2855 }
2856
2857 if (basic_type != T_OBJECT) {
2858 __ cmp_klass(tmp, dst, tmp2);
2859 __ jcc(Assembler::notEqual, halt);
2860 __ cmp_klass(tmp, src, tmp2);
2861 __ jcc(Assembler::equal, known_ok);
2862 } else {
2863 __ cmp_klass(tmp, dst, tmp2);
2864 __ jcc(Assembler::equal, known_ok);
2865 __ cmpptr(src, dst);
2866 __ jcc(Assembler::equal, known_ok);
2867 }
2868 __ bind(halt);
2869 __ stop("incorrect type information in arraycopy");
2870 __ bind(known_ok);
2871 }
2872 #endif
2873
2874 #ifndef PRODUCT
2875 if (PrintC1Statistics) {
2876 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2877 }
2878 #endif
2879
2880 assert_different_registers(c_rarg0, dst, dst_pos, length);
2881 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2882 assert_different_registers(c_rarg1, length);
2883 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2884 __ mov(c_rarg2, length);
2885
2886 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2887 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2888 const char *name;
2889 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2890 __ call_VM_leaf(entry, 0);
2891
2892 if (stub != nullptr) {
2893 __ bind(*stub->continuation());
2894 }
2895 }
2896
2897 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2898 assert(op->crc()->is_single_cpu(), "crc must be register");
2899 assert(op->val()->is_single_cpu(), "byte value must be register");
2900 assert(op->result_opr()->is_single_cpu(), "result must be register");
2901 Register crc = op->crc()->as_register();
2902 Register val = op->val()->as_register();
2903 Register res = op->result_opr()->as_register();
2904
2905 assert_different_registers(val, crc, res);
2906
2907 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2908 __ notl(crc); // ~crc
2909 __ update_byte_crc32(crc, val, res);
2910 __ notl(crc); // ~crc
2911 __ mov(res, crc);
2912 }
2913
2914 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2915 Register obj = op->obj_opr()->as_register(); // may not be an oop
2916 Register hdr = op->hdr_opr()->as_register();
2917 Register lock = op->lock_opr()->as_register();
2918 if (op->code() == lir_lock) {
2919 Register tmp = op->scratch_opr()->as_register();
2920 // add debug info for NullPointerException only if one is possible
2921 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2922 if (op->info() != nullptr) {
2923 add_debug_info_for_null_check(null_check_offset, op->info());
2924 }
2925 // done
2926 } else if (op->code() == lir_unlock) {
2927 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2928 } else {
2929 Unimplemented();
2930 }
2931 __ bind(*op->stub()->continuation());
2932 }
2933
2934 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2935 Register obj = op->obj()->as_pointer_register();
2936 Register result = op->result_opr()->as_pointer_register();
2937
2938 CodeEmitInfo* info = op->info();
2939 if (info != nullptr) {
2940 add_debug_info_for_null_check_here(info);
2941 }
2942
2943 __ load_klass(result, obj, rscratch1);
2944 }
2945
2946 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2947 ciMethod* method = op->profiled_method();
2948 int bci = op->profiled_bci();
2949 ciMethod* callee = op->profiled_callee();
2950 Register tmp_load_klass = rscratch1;
2951
2952 // Update counter for all call types
2953 ciMethodData* md = method->method_data_or_null();
2954 assert(md != nullptr, "Sanity");
2955 ciProfileData* data = md->bci_to_data(bci);
2956 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2957 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2958 Register mdo = op->mdo()->as_register();
2959 __ mov_metadata(mdo, md->constant_encoding());
2960 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2961 // Perform additional virtual call profiling for invokevirtual and
2962 // invokeinterface bytecodes
2963 if (op->should_profile_receiver_type()) {
2964 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2965 Register recv = op->recv()->as_register();
2966 assert_different_registers(mdo, recv);
2967 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2968 ciKlass* known_klass = op->known_holder();
2969 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2970 // We know the type that will be seen at this call site; we can
2971 // statically update the MethodData* rather than needing to do
2972 // dynamic tests on the receiver type.
2973 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2974 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2975 ciKlass* receiver = vc_data->receiver(i);
2976 if (known_klass->equals(receiver)) {
2977 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2978 __ addptr(data_addr, DataLayout::counter_increment);
2979 return;
2980 }
2981 }
2982 // Receiver type is not found in profile data.
2983 // Fall back to runtime helper to handle the rest at runtime.
2984 __ mov_metadata(recv, known_klass->constant_encoding());
2985 } else {
2986 __ load_klass(recv, recv, tmp_load_klass);
2987 }
2988 type_profile_helper(mdo, md, data, recv);
2989 } else {
2990 // Static call
2991 __ addptr(counter_addr, DataLayout::counter_increment);
2992 }
2993 }
2994
2995 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2996 Register obj = op->obj()->as_register();
2997 Register tmp = op->tmp()->as_pointer_register();
2998 Register tmp_load_klass = rscratch1;
2999 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3000 ciKlass* exact_klass = op->exact_klass();
3001 intptr_t current_klass = op->current_klass();
3002 bool not_null = op->not_null();
3003 bool no_conflict = op->no_conflict();
3004
3005 Label update, next, none;
3006
3007 bool do_null = !not_null;
3008 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3009 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3010
3011 assert(do_null || do_update, "why are we here?");
3012 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3013
3014 __ verify_oop(obj);
3015
3016 #ifdef ASSERT
3017 if (obj == tmp) {
3018 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3019 } else {
3020 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3021 }
3022 #endif
3023 if (do_null) {
3024 __ testptr(obj, obj);
3025 __ jccb(Assembler::notZero, update);
3026 if (!TypeEntries::was_null_seen(current_klass)) {
3027 __ testptr(mdo_addr, TypeEntries::null_seen);
3028 #ifndef ASSERT
3029 __ jccb(Assembler::notZero, next); // already set
3030 #else
3031 __ jcc(Assembler::notZero, next); // already set
3032 #endif
3033 // atomic update to prevent overwriting Klass* with 0
3034 __ lock();
3035 __ orptr(mdo_addr, TypeEntries::null_seen);
3036 }
3037 if (do_update) {
3038 #ifndef ASSERT
3039 __ jmpb(next);
3040 }
3041 #else
3042 __ jmp(next);
3043 }
3044 } else {
3045 __ testptr(obj, obj);
3046 __ jcc(Assembler::notZero, update);
3047 __ stop("unexpected null obj");
3048 #endif
3049 }
3050
3051 __ bind(update);
3052
3053 if (do_update) {
3054 #ifdef ASSERT
3055 if (exact_klass != nullptr) {
3056 Label ok;
3057 __ load_klass(tmp, obj, tmp_load_klass);
3058 __ push_ppx(tmp);
3059 __ mov_metadata(tmp, exact_klass->constant_encoding());
3060 __ cmpptr(tmp, Address(rsp, 0));
3061 __ jcc(Assembler::equal, ok);
3062 __ stop("exact klass and actual klass differ");
3063 __ bind(ok);
3064 __ pop_ppx(tmp);
3065 }
3066 #endif
3067 if (!no_conflict) {
3068 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3069 if (exact_klass != nullptr) {
3070 __ mov_metadata(tmp, exact_klass->constant_encoding());
3071 } else {
3072 __ load_klass(tmp, obj, tmp_load_klass);
3073 }
3074 __ mov(rscratch1, tmp); // save original value before XOR
3075 __ xorptr(tmp, mdo_addr);
3076 __ testptr(tmp, TypeEntries::type_klass_mask);
3077 // klass seen before, nothing to do. The unknown bit may have been
3078 // set already but no need to check.
3079 __ jccb(Assembler::zero, next);
3080
3081 __ testptr(tmp, TypeEntries::type_unknown);
3082 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3083
3084 if (TypeEntries::is_type_none(current_klass)) {
3085 __ testptr(mdo_addr, TypeEntries::type_mask);
3086 __ jccb(Assembler::zero, none);
3087 // There is a chance that the checks above (re-reading profiling
3088 // data from memory) fail if another thread has just set the
3089 // profiling to this obj's klass
3090 __ mov(tmp, rscratch1); // get back original value before XOR
3091 __ xorptr(tmp, mdo_addr);
3092 __ testptr(tmp, TypeEntries::type_klass_mask);
3093 __ jccb(Assembler::zero, next);
3094 }
3095 } else {
3096 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3097 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3098
3099 __ testptr(mdo_addr, TypeEntries::type_unknown);
3100 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3101 }
3102
3103 // different than before. Cannot keep accurate profile.
3104 __ orptr(mdo_addr, TypeEntries::type_unknown);
3105
3106 if (TypeEntries::is_type_none(current_klass)) {
3107 __ jmpb(next);
3108
3109 __ bind(none);
3110 // first time here. Set profile type.
3111 __ movptr(mdo_addr, tmp);
3112 #ifdef ASSERT
3113 __ andptr(tmp, TypeEntries::type_klass_mask);
3114 __ verify_klass_ptr(tmp);
3115 #endif
3116 }
3117 } else {
3118 // There's a single possible klass at this profile point
3119 assert(exact_klass != nullptr, "should be");
3120 if (TypeEntries::is_type_none(current_klass)) {
3121 __ mov_metadata(tmp, exact_klass->constant_encoding());
3122 __ xorptr(tmp, mdo_addr);
3123 __ testptr(tmp, TypeEntries::type_klass_mask);
3124 #ifdef ASSERT
3125 __ jcc(Assembler::zero, next);
3126
3127 {
3128 Label ok;
3129 __ push_ppx(tmp);
3130 __ testptr(mdo_addr, TypeEntries::type_mask);
3131 __ jcc(Assembler::zero, ok);
3132 // may have been set by another thread
3133 __ mov_metadata(tmp, exact_klass->constant_encoding());
3134 __ xorptr(tmp, mdo_addr);
3135 __ testptr(tmp, TypeEntries::type_mask);
3136 __ jcc(Assembler::zero, ok);
3137
3138 __ stop("unexpected profiling mismatch");
3139 __ bind(ok);
3140 __ pop_ppx(tmp);
3141 }
3142 #else
3143 __ jccb(Assembler::zero, next);
3144 #endif
3145 // first time here. Set profile type.
3146 __ movptr(mdo_addr, tmp);
3147 #ifdef ASSERT
3148 __ andptr(tmp, TypeEntries::type_klass_mask);
3149 __ verify_klass_ptr(tmp);
3150 #endif
3151 } else {
3152 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3153 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3154
3155 __ testptr(mdo_addr, TypeEntries::type_unknown);
3156 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3157
3158 __ orptr(mdo_addr, TypeEntries::type_unknown);
3159 }
3160 }
3161 }
3162 __ bind(next);
3163 }
3164
3165 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3166 Register obj = op->obj()->as_register();
3167 Register tmp = op->tmp()->as_pointer_register();
3168 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3169 bool not_null = op->not_null();
3170 int flag = op->flag();
3171
3172 Label not_inline_type;
3173 if (!not_null) {
3174 __ testptr(obj, obj);
3175 __ jccb(Assembler::zero, not_inline_type);
3176 }
3177
3178 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3179
3180 __ orb(mdo_addr, flag);
3181
3182 __ bind(not_inline_type);
3183 }
3184
3185
3186 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3187 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3188 }
3189
3190
3191 void LIR_Assembler::align_backward_branch_target() {
3192 __ align(BytesPerWord);
3193 }
3194
3195
3196 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3197 if (left->is_single_cpu()) {
3198 __ negl(left->as_register());
3199 move_regs(left->as_register(), dest->as_register());
3200
3201 } else if (left->is_double_cpu()) {
3202 Register lo = left->as_register_lo();
3203 Register dst = dest->as_register_lo();
3204 __ movptr(dst, lo);
3205 __ negptr(dst);
3206
3207 } else if (dest->is_single_xmm()) {
3208 assert(!tmp->is_valid(), "do not need temporary");
3209 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3210 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3211 }
3212 __ xorps(dest->as_xmm_float_reg(),
3213 ExternalAddress((address)float_signflip_pool),
3214 rscratch1);
3215 } else if (dest->is_double_xmm()) {
3216 assert(!tmp->is_valid(), "do not need temporary");
3217 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3218 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3219 }
3220 __ xorpd(dest->as_xmm_double_reg(),
3221 ExternalAddress((address)double_signflip_pool),
3222 rscratch1);
3223 } else {
3224 ShouldNotReachHere();
3225 }
3226 }
3227
3228
3229 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3230 assert(src->is_address(), "must be an address");
3231 assert(dest->is_register(), "must be a register");
3232
3233 PatchingStub* patch = nullptr;
3234 if (patch_code != lir_patch_none) {
3235 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3236 }
3237
3238 Register reg = dest->as_pointer_register();
3239 LIR_Address* addr = src->as_address_ptr();
3240 __ lea(reg, as_Address(addr));
3241
3242 if (patch != nullptr) {
3243 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3244 }
3245 }
3246
3247
3248
3249 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3250 assert(!tmp->is_valid(), "don't need temporary");
3251 __ call(RuntimeAddress(dest));
3252 if (info != nullptr) {
3253 add_call_info_here(info);
3254 }
3255 __ post_call_nop();
3256 }
3257
3258
3259 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3260 assert(type == T_LONG, "only for volatile long fields");
3261
3262 if (info != nullptr) {
3263 add_debug_info_for_null_check_here(info);
3264 }
3265
3266 if (src->is_double_xmm()) {
3267 if (dest->is_double_cpu()) {
3268 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3269 } else if (dest->is_double_stack()) {
3270 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3271 } else if (dest->is_address()) {
3272 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3273 } else {
3274 ShouldNotReachHere();
3275 }
3276
3277 } else if (dest->is_double_xmm()) {
3278 if (src->is_double_stack()) {
3279 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3280 } else if (src->is_address()) {
3281 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3282 } else {
3283 ShouldNotReachHere();
3284 }
3285
3286 } else {
3287 ShouldNotReachHere();
3288 }
3289 }
3290
3291 #ifdef ASSERT
3292 // emit run-time assertion
3293 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3294 assert(op->code() == lir_assert, "must be");
3295
3296 if (op->in_opr1()->is_valid()) {
3297 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3298 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3299 } else {
3300 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3301 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3302 }
3303
3304 Label ok;
3305 if (op->condition() != lir_cond_always) {
3306 Assembler::Condition acond = Assembler::zero;
3307 switch (op->condition()) {
3308 case lir_cond_equal: acond = Assembler::equal; break;
3309 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3310 case lir_cond_less: acond = Assembler::less; break;
3311 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3312 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3313 case lir_cond_greater: acond = Assembler::greater; break;
3314 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3315 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3316 default: ShouldNotReachHere();
3317 }
3318 __ jcc(acond, ok);
3319 }
3320 if (op->halt()) {
3321 const char* str = __ code_string(op->msg());
3322 __ stop(str);
3323 } else {
3324 breakpoint();
3325 }
3326 __ bind(ok);
3327 }
3328 #endif
3329
3330 void LIR_Assembler::membar() {
3331 // QQQ sparc TSO uses this,
3332 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3333 }
3334
3335 void LIR_Assembler::membar_acquire() {
3336 // No x86 machines currently require load fences
3337 }
3338
3339 void LIR_Assembler::membar_release() {
3340 // No x86 machines currently require store fences
3341 }
3342
3343 void LIR_Assembler::membar_loadload() {
3344 // no-op
3345 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3346 }
3347
3348 void LIR_Assembler::membar_storestore() {
3349 // no-op
3350 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3351 }
3352
3353 void LIR_Assembler::membar_loadstore() {
3354 // no-op
3355 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3356 }
3357
3358 void LIR_Assembler::membar_storeload() {
3359 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3360 }
3361
3362 void LIR_Assembler::on_spin_wait() {
3363 __ pause ();
3364 }
3365
3366 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3367 assert(result_reg->is_register(), "check");
3368 __ mov(result_reg->as_register(), r15_thread);
3369 }
3370
3371 void LIR_Assembler::check_orig_pc() {
3372 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3373 }
3374
3375 void LIR_Assembler::peephole(LIR_List*) {
3376 // do nothing for now
3377 }
3378
3379 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3380 assert(data == dest, "xchg/xadd uses only 2 operands");
3381
3382 if (data->type() == T_INT) {
3383 if (code == lir_xadd) {
3384 __ lock();
3385 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3386 } else {
3387 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3388 }
3389 } else if (data->is_oop()) {
3390 assert (code == lir_xchg, "xadd for oops");
3391 Register obj = data->as_register();
3392 if (UseCompressedOops) {
3393 __ encode_heap_oop(obj);
3394 __ xchgl(obj, as_Address(src->as_address_ptr()));
3395 __ decode_heap_oop(obj);
3396 } else {
3397 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3398 }
3399 } else if (data->type() == T_LONG) {
3400 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3401 if (code == lir_xadd) {
3402 __ lock();
3403 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3404 } else {
3405 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3406 }
3407 } else {
3408 ShouldNotReachHere();
3409 }
3410 }
3411
3412 #undef __