1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
63 operand[0] = lo;
64 operand[1] = hi;
65 return operand;
66 }
67
68 // Buffer for 128-bits masks used by SSE instructions.
69 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
70
71 // Static initialization during VM startup.
72 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
73 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
74 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
75 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
76
77 #if INCLUDE_CDS
78 // publish external addresses defined in this file
79 void LIR_Assembler::init_AOTAddressTable(GrowableArray<address>& external_addresses) {
80 #define ADD(addr) external_addresses.append((address)(addr));
81 ADD(float_signmask_pool);
82 ADD(double_signmask_pool);
83 ADD(float_signflip_pool);
84 ADD(double_signflip_pool);
85 #undef ADD
86 }
87 #endif // INCLUDE_CDS
88
89 NEEDS_CLEANUP // remove this definitions ?
90 const Register SYNC_header = rax; // synchronization header
91 const Register SHIFT_count = rcx; // where count for shift operations must be
92
93 #define __ _masm->
94
95 static void select_different_registers(Register preserve,
96 Register extra,
97 Register &tmp1,
98 Register &tmp2,
99 Register &tmp3) {
100 if (tmp1 == preserve) {
101 assert_different_registers(tmp1, tmp2, tmp3, extra);
102 tmp1 = extra;
103 } else if (tmp2 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp2 = extra;
106 } else if (tmp3 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp3 = extra;
109 }
110 assert_different_registers(preserve, tmp1, tmp2, tmp3);
111 }
112
113
114
115 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
116 if (opr->is_constant()) {
117 LIR_Const* constant = opr->as_constant_ptr();
118 switch (constant->type()) {
119 case T_INT: {
120 return true;
121 }
122
123 default:
124 return false;
125 }
126 }
127 return false;
128 }
129
130
131 LIR_Opr LIR_Assembler::receiverOpr() {
132 return FrameMap::receiver_opr;
133 }
134
135 LIR_Opr LIR_Assembler::osrBufferPointer() {
136 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
137 }
138
139 //--------------fpu register translations-----------------------
140
141
142 address LIR_Assembler::float_constant(float f) {
143 address const_addr = __ float_constant(f);
144 if (const_addr == nullptr) {
145 bailout("const section overflow");
146 return __ code()->consts()->start();
147 } else {
148 return const_addr;
149 }
150 }
151
152
153 address LIR_Assembler::double_constant(double d) {
154 address const_addr = __ double_constant(d);
155 if (const_addr == nullptr) {
156 bailout("const section overflow");
157 return __ code()->consts()->start();
158 } else {
159 return const_addr;
160 }
161 }
162
163 void LIR_Assembler::breakpoint() {
164 __ int3();
165 }
166
167 void LIR_Assembler::push(LIR_Opr opr) {
168 if (opr->is_single_cpu()) {
169 __ push_reg(opr->as_register());
170 } else if (opr->is_double_cpu()) {
171 __ push_reg(opr->as_register_lo());
172 } else if (opr->is_stack()) {
173 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
174 } else if (opr->is_constant()) {
175 LIR_Const* const_opr = opr->as_constant_ptr();
176 if (const_opr->type() == T_OBJECT) {
177 __ push_oop(const_opr->as_jobject(), rscratch1);
178 } else if (const_opr->type() == T_INT) {
179 __ push_jint(const_opr->as_jint());
180 } else {
181 ShouldNotReachHere();
182 }
183
184 } else {
185 ShouldNotReachHere();
186 }
187 }
188
189 void LIR_Assembler::pop(LIR_Opr opr) {
190 if (opr->is_single_cpu()) {
191 __ pop_reg(opr->as_register());
192 } else {
193 ShouldNotReachHere();
194 }
195 }
196
197 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
198 return addr->base()->is_illegal() && addr->index()->is_illegal();
199 }
200
201 //-------------------------------------------
202
203 Address LIR_Assembler::as_Address(LIR_Address* addr) {
204 return as_Address(addr, rscratch1);
205 }
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
208 if (addr->base()->is_illegal()) {
209 assert(addr->index()->is_illegal(), "must be illegal too");
210 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
211 if (! __ reachable(laddr)) {
212 __ movptr(tmp, laddr.addr());
213 Address res(tmp, 0);
214 return res;
215 } else {
216 return __ as_Address(laddr);
217 }
218 }
219
220 Register base = addr->base()->as_pointer_register();
221
222 if (addr->index()->is_illegal()) {
223 return Address( base, addr->disp());
224 } else if (addr->index()->is_cpu_register()) {
225 Register index = addr->index()->as_pointer_register();
226 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
227 } else if (addr->index()->is_constant()) {
228 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
229 assert(Assembler::is_simm32(addr_offset), "must be");
230
231 return Address(base, addr_offset);
232 } else {
233 Unimplemented();
234 return Address();
235 }
236 }
237
238
239 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
240 Address base = as_Address(addr);
241 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
242 }
243
244
245 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
246 return as_Address(addr);
247 }
248
249
250 void LIR_Assembler::osr_entry() {
251 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
252 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
253 ValueStack* entry_state = osr_entry->state();
254 int number_of_locks = entry_state->locks_size();
255
256 // we jump here if osr happens with the interpreter
257 // state set up to continue at the beginning of the
258 // loop that triggered osr - in particular, we have
259 // the following registers setup:
260 //
261 // rcx: osr buffer
262 //
263
264 // build frame
265 ciMethod* m = compilation()->method();
266 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
267
268 // OSR buffer is
269 //
270 // locals[nlocals-1..0]
271 // monitors[0..number_of_locks]
272 //
273 // locals is a direct copy of the interpreter frame so in the osr buffer
274 // so first slot in the local array is the last local from the interpreter
275 // and last slot is local[0] (receiver) from the interpreter
276 //
277 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
278 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
279 // in the interpreter frame (the method lock if a sync method)
280
281 // Initialize monitors in the compiled activation.
282 // rcx: pointer to osr buffer
283 //
284 // All other registers are dead at this point and the locals will be
285 // copied into place by code emitted in the IR.
286
287 Register OSR_buf = osrBufferPointer()->as_pointer_register();
288 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
289 int monitor_offset = BytesPerWord * method()->max_locals() +
290 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
291 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
292 // the OSR buffer using 2 word entries: first the lock and then
293 // the oop.
294 for (int i = 0; i < number_of_locks; i++) {
295 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
296 #ifdef ASSERT
297 // verify the interpreter's monitor has a non-null object
298 {
299 Label L;
300 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
301 __ jcc(Assembler::notZero, L);
302 __ stop("locked object is null");
303 __ bind(L);
304 }
305 #endif
306 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
307 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
308 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
309 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
310 }
311 }
312 }
313
314
315 // inline cache check; done before the frame is built.
316 int LIR_Assembler::check_icache() {
317 return __ ic_check(CodeEntryAlignment);
318 }
319
320 void LIR_Assembler::clinit_barrier(ciMethod* method) {
321 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
322 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
323
324 Label L_skip_barrier;
325 Register klass = rscratch1;
326
327 __ mov_metadata(klass, method->holder()->constant_encoding());
328 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
329
330 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
331
332 __ bind(L_skip_barrier);
333 }
334
335 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
336 jobject o = nullptr;
337 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
338 __ movoop(reg, o);
339 patching_epilog(patch, lir_patch_normal, reg, info);
340 }
341
342 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
343 Metadata* o = nullptr;
344 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
345 __ mov_metadata(reg, o);
346 patching_epilog(patch, lir_patch_normal, reg, info);
347 }
348
349 // This specifies the rsp decrement needed to build the frame
350 int LIR_Assembler::initial_frame_size_in_bytes() const {
351 // if rounding, must let FrameMap know!
352
353 // The frame_map records size in slots (32bit word)
354
355 // subtract two words to account for return address and link
356 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
357 }
358
359
360 int LIR_Assembler::emit_exception_handler() {
361 // generate code for exception handler
362 address handler_base = __ start_a_stub(exception_handler_size());
363 if (handler_base == nullptr) {
364 // not enough space left for the handler
365 bailout("exception handler overflow");
366 return -1;
367 }
368
369 int offset = code_offset();
370
371 // the exception oop and pc are in rax, and rdx
372 // no other registers need to be preserved, so invalidate them
373 __ invalidate_registers(false, true, true, false, true, true);
374
375 // check that there is really an exception
376 __ verify_not_null_oop(rax);
377
378 // search an exception handler (rax: exception oop, rdx: throwing pc)
379 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
380 __ should_not_reach_here();
381 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
382 __ end_a_stub();
383
384 return offset;
385 }
386
387
388 // Emit the code to remove the frame from the stack in the exception
389 // unwind path.
390 int LIR_Assembler::emit_unwind_handler() {
391 #ifndef PRODUCT
392 if (CommentedAssembly) {
393 _masm->block_comment("Unwind handler");
394 }
395 #endif
396
397 int offset = code_offset();
398
399 // Fetch the exception from TLS and clear out exception related thread state
400 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
401 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
402 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
403
404 __ bind(_unwind_handler_entry);
405 __ verify_not_null_oop(rax);
406 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
407 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
408 }
409
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
451 int offset = code_offset();
452
453 Label start;
454 __ bind(start);
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning a non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning a non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
561 __ xchgptr(a, b);
562 }
563
564
565 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
566 assert(src->is_constant(), "should not call otherwise");
567 assert(dest->is_register(), "should not call otherwise");
568 LIR_Const* c = src->as_constant_ptr();
569
570 switch (c->type()) {
571 case T_INT: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movl(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_ADDRESS: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 __ movptr(dest->as_register(), c->as_jint());
580 break;
581 }
582
583 case T_LONG: {
584 #if INCLUDE_CDS
585 if (AOTCodeCache::is_on_for_dump()) {
586 address b = c->as_pointer();
587 if (b == (address)ThreadIdentifier::unsafe_offset()) {
588 __ lea(dest->as_register_lo(), ExternalAddress(b));
589 break;
590 }
591 }
592 #endif
593 assert(patch_code == lir_patch_none, "no patching handled here");
594 #if INCLUDE_CDS
595 if (AOTCodeCache::is_on_for_dump()) {
596 address b = c->as_pointer();
597 if (b == (address)ThreadIdentifier::unsafe_offset()) {
598 __ lea(dest->as_register_lo(), ExternalAddress(b));
599 break;
600 }
601 if (AOTRuntimeConstants::contains(b)) {
602 __ load_aotrc_address(dest->as_register_lo(), b);
603 break;
604 }
605 }
606 #endif
607 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
608 break;
609 }
610
611 case T_OBJECT: {
612 if (patch_code != lir_patch_none) {
613 jobject2reg_with_patching(dest->as_register(), info);
614 } else {
615 __ movoop(dest->as_register(), c->as_jobject());
616 }
617 break;
618 }
619
620 case T_METADATA: {
621 if (patch_code != lir_patch_none) {
622 klass2reg_with_patching(dest->as_register(), info);
623 } else {
624 __ mov_metadata(dest->as_register(), c->as_metadata());
625 }
626 break;
627 }
628
629 case T_FLOAT: {
630 if (dest->is_single_xmm()) {
631 if (UseAVX <= 2 && c->is_zero_float()) {
632 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
633 } else {
634 __ movflt(dest->as_xmm_float_reg(),
635 InternalAddress(float_constant(c->as_jfloat())));
636 }
637 } else {
638 ShouldNotReachHere();
639 }
640 break;
641 }
642
643 case T_DOUBLE: {
644 if (dest->is_double_xmm()) {
645 if (UseAVX <= 2 && c->is_zero_double()) {
646 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
647 } else {
648 __ movdbl(dest->as_xmm_double_reg(),
649 InternalAddress(double_constant(c->as_jdouble())));
650 }
651 } else {
652 ShouldNotReachHere();
653 }
654 break;
655 }
656
657 default:
658 ShouldNotReachHere();
659 }
660 }
661
662 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
663 assert(src->is_constant(), "should not call otherwise");
664 assert(dest->is_stack(), "should not call otherwise");
665 LIR_Const* c = src->as_constant_ptr();
666
667 switch (c->type()) {
668 case T_INT: // fall through
669 case T_FLOAT:
670 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
671 break;
672
673 case T_ADDRESS:
674 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
675 break;
676
677 case T_OBJECT:
678 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
679 break;
680
681 case T_LONG: // fall through
682 case T_DOUBLE:
683 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
684 lo_word_offset_in_bytes),
685 (intptr_t)c->as_jlong_bits(),
686 rscratch1);
687 break;
688
689 default:
690 ShouldNotReachHere();
691 }
692 }
693
694 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
695 assert(src->is_constant(), "should not call otherwise");
696 assert(dest->is_address(), "should not call otherwise");
697 LIR_Const* c = src->as_constant_ptr();
698 LIR_Address* addr = dest->as_address_ptr();
699
700 int null_check_here = code_offset();
701 switch (type) {
702 case T_INT: // fall through
703 case T_FLOAT:
704 __ movl(as_Address(addr), c->as_jint_bits());
705 break;
706
707 case T_ADDRESS:
708 __ movptr(as_Address(addr), c->as_jint_bits());
709 break;
710
711 case T_OBJECT: // fall through
712 case T_ARRAY:
713 if (c->as_jobject() == nullptr) {
714 if (UseCompressedOops && !wide) {
715 __ movl(as_Address(addr), NULL_WORD);
716 } else {
717 __ xorptr(rscratch1, rscratch1);
718 null_check_here = code_offset();
719 __ movptr(as_Address(addr), rscratch1);
720 }
721 } else {
722 if (is_literal_address(addr)) {
723 ShouldNotReachHere();
724 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
725 } else {
726 __ movoop(rscratch1, c->as_jobject());
727 if (UseCompressedOops && !wide) {
728 __ encode_heap_oop(rscratch1);
729 null_check_here = code_offset();
730 __ movl(as_Address_lo(addr), rscratch1);
731 } else {
732 null_check_here = code_offset();
733 __ movptr(as_Address_lo(addr), rscratch1);
734 }
735 }
736 }
737 break;
738
739 case T_LONG: // fall through
740 case T_DOUBLE:
741 if (is_literal_address(addr)) {
742 ShouldNotReachHere();
743 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
744 } else {
745 __ movptr(r10, (intptr_t)c->as_jlong_bits());
746 null_check_here = code_offset();
747 __ movptr(as_Address_lo(addr), r10);
748 }
749 break;
750
751 case T_BOOLEAN: // fall through
752 case T_BYTE:
753 __ movb(as_Address(addr), c->as_jint() & 0xFF);
754 break;
755
756 case T_CHAR: // fall through
757 case T_SHORT:
758 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
759 break;
760
761 default:
762 ShouldNotReachHere();
763 };
764
765 if (info != nullptr) {
766 add_debug_info_for_null_check(null_check_here, info);
767 }
768 }
769
770
771 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
772 assert(src->is_register(), "should not call otherwise");
773 assert(dest->is_register(), "should not call otherwise");
774
775 // move between cpu-registers
776 if (dest->is_single_cpu()) {
777 if (src->type() == T_LONG) {
778 // Can do LONG -> OBJECT
779 move_regs(src->as_register_lo(), dest->as_register());
780 return;
781 }
782 assert(src->is_single_cpu(), "must match");
783 if (src->type() == T_OBJECT) {
784 __ verify_oop(src->as_register());
785 }
786 move_regs(src->as_register(), dest->as_register());
787
788 } else if (dest->is_double_cpu()) {
789 if (is_reference_type(src->type())) {
790 // Surprising to me but we can see move of a long to t_object
791 __ verify_oop(src->as_register());
792 move_regs(src->as_register(), dest->as_register_lo());
793 return;
794 }
795 assert(src->is_double_cpu(), "must match");
796 Register f_lo = src->as_register_lo();
797 Register f_hi = src->as_register_hi();
798 Register t_lo = dest->as_register_lo();
799 Register t_hi = dest->as_register_hi();
800 assert(f_hi == f_lo, "must be same");
801 assert(t_hi == t_lo, "must be same");
802 move_regs(f_lo, t_lo);
803
804 // move between xmm-registers
805 } else if (dest->is_single_xmm()) {
806 assert(src->is_single_xmm(), "must match");
807 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
808 } else if (dest->is_double_xmm()) {
809 assert(src->is_double_xmm(), "must match");
810 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
811
812 } else {
813 ShouldNotReachHere();
814 }
815 }
816
817 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
818 assert(src->is_register(), "should not call otherwise");
819 assert(dest->is_stack(), "should not call otherwise");
820
821 if (src->is_single_cpu()) {
822 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
823 if (is_reference_type(type)) {
824 __ verify_oop(src->as_register());
825 __ movptr (dst, src->as_register());
826 } else if (type == T_METADATA || type == T_ADDRESS) {
827 __ movptr (dst, src->as_register());
828 } else {
829 __ movl (dst, src->as_register());
830 }
831
832 } else if (src->is_double_cpu()) {
833 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
834 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
835 __ movptr (dstLO, src->as_register_lo());
836
837 } else if (src->is_single_xmm()) {
838 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
839 __ movflt(dst_addr, src->as_xmm_float_reg());
840
841 } else if (src->is_double_xmm()) {
842 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
843 __ movdbl(dst_addr, src->as_xmm_double_reg());
844
845 } else {
846 ShouldNotReachHere();
847 }
848 }
849
850
851 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
852 LIR_Address* to_addr = dest->as_address_ptr();
853 PatchingStub* patch = nullptr;
854 Register compressed_src = rscratch1;
855
856 if (is_reference_type(type)) {
857 __ verify_oop(src->as_register());
858 if (UseCompressedOops && !wide) {
859 __ movptr(compressed_src, src->as_register());
860 __ encode_heap_oop(compressed_src);
861 if (patch_code != lir_patch_none) {
862 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
863 }
864 }
865 }
866
867 if (patch_code != lir_patch_none) {
868 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
869 Address toa = as_Address(to_addr);
870 assert(toa.disp() != 0, "must have");
871 }
872
873 int null_check_here = code_offset();
874 switch (type) {
875 case T_FLOAT: {
876 assert(src->is_single_xmm(), "not a float");
877 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
878 break;
879 }
880
881 case T_DOUBLE: {
882 assert(src->is_double_xmm(), "not a double");
883 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
884 break;
885 }
886
887 case T_ARRAY: // fall through
888 case T_OBJECT: // fall through
889 if (UseCompressedOops && !wide) {
890 __ movl(as_Address(to_addr), compressed_src);
891 } else {
892 __ movptr(as_Address(to_addr), src->as_register());
893 }
894 break;
895 case T_ADDRESS:
896 __ movptr(as_Address(to_addr), src->as_register());
897 break;
898 case T_INT:
899 __ movl(as_Address(to_addr), src->as_register());
900 break;
901
902 case T_LONG: {
903 Register from_lo = src->as_register_lo();
904 Register from_hi = src->as_register_hi();
905 __ movptr(as_Address_lo(to_addr), from_lo);
906 break;
907 }
908
909 case T_BYTE: // fall through
910 case T_BOOLEAN: {
911 Register src_reg = src->as_register();
912 Address dst_addr = as_Address(to_addr);
913 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
914 __ movb(dst_addr, src_reg);
915 break;
916 }
917
918 case T_CHAR: // fall through
919 case T_SHORT:
920 __ movw(as_Address(to_addr), src->as_register());
921 break;
922
923 default:
924 ShouldNotReachHere();
925 }
926 if (info != nullptr) {
927 add_debug_info_for_null_check(null_check_here, info);
928 }
929
930 if (patch_code != lir_patch_none) {
931 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
932 }
933 }
934
935
936 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
937 assert(src->is_stack(), "should not call otherwise");
938 assert(dest->is_register(), "should not call otherwise");
939
940 if (dest->is_single_cpu()) {
941 if (is_reference_type(type)) {
942 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
943 __ verify_oop(dest->as_register());
944 } else if (type == T_METADATA || type == T_ADDRESS) {
945 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
946 } else {
947 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
948 }
949
950 } else if (dest->is_double_cpu()) {
951 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
952 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
953 __ movptr(dest->as_register_lo(), src_addr_LO);
954
955 } else if (dest->is_single_xmm()) {
956 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
957 __ movflt(dest->as_xmm_float_reg(), src_addr);
958
959 } else if (dest->is_double_xmm()) {
960 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
961 __ movdbl(dest->as_xmm_double_reg(), src_addr);
962
963 } else {
964 ShouldNotReachHere();
965 }
966 }
967
968
969 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
970 if (src->is_single_stack()) {
971 if (is_reference_type(type)) {
972 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
973 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
974 } else {
975 //no pushl on 64bits
976 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
977 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
978 }
979
980 } else if (src->is_double_stack()) {
981 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
982 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
983
984 } else {
985 ShouldNotReachHere();
986 }
987 }
988
989
990 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
991 assert(src->is_address(), "should not call otherwise");
992 assert(dest->is_register(), "should not call otherwise");
993
994 LIR_Address* addr = src->as_address_ptr();
995 Address from_addr = as_Address(addr);
996
997 if (addr->base()->type() == T_OBJECT) {
998 __ verify_oop(addr->base()->as_pointer_register());
999 }
1000
1001 switch (type) {
1002 case T_BOOLEAN: // fall through
1003 case T_BYTE: // fall through
1004 case T_CHAR: // fall through
1005 case T_SHORT:
1006 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1007 // on pre P6 processors we may get partial register stalls
1008 // so blow away the value of to_rinfo before loading a
1009 // partial word into it. Do it here so that it precedes
1010 // the potential patch point below.
1011 __ xorptr(dest->as_register(), dest->as_register());
1012 }
1013 break;
1014 default:
1015 break;
1016 }
1017
1018 PatchingStub* patch = nullptr;
1019 if (patch_code != lir_patch_none) {
1020 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1021 assert(from_addr.disp() != 0, "must have");
1022 }
1023 if (info != nullptr) {
1024 add_debug_info_for_null_check_here(info);
1025 }
1026
1027 switch (type) {
1028 case T_FLOAT: {
1029 if (dest->is_single_xmm()) {
1030 __ movflt(dest->as_xmm_float_reg(), from_addr);
1031 } else {
1032 ShouldNotReachHere();
1033 }
1034 break;
1035 }
1036
1037 case T_DOUBLE: {
1038 if (dest->is_double_xmm()) {
1039 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1040 } else {
1041 ShouldNotReachHere();
1042 }
1043 break;
1044 }
1045
1046 case T_OBJECT: // fall through
1047 case T_ARRAY: // fall through
1048 if (UseCompressedOops && !wide) {
1049 __ movl(dest->as_register(), from_addr);
1050 } else {
1051 __ movptr(dest->as_register(), from_addr);
1052 }
1053 break;
1054
1055 case T_ADDRESS:
1056 __ movptr(dest->as_register(), from_addr);
1057 break;
1058 case T_INT:
1059 __ movl(dest->as_register(), from_addr);
1060 break;
1061
1062 case T_LONG: {
1063 Register to_lo = dest->as_register_lo();
1064 Register to_hi = dest->as_register_hi();
1065 __ movptr(to_lo, as_Address_lo(addr));
1066 break;
1067 }
1068
1069 case T_BOOLEAN: // fall through
1070 case T_BYTE: {
1071 Register dest_reg = dest->as_register();
1072 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1073 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1074 __ movsbl(dest_reg, from_addr);
1075 } else {
1076 __ movb(dest_reg, from_addr);
1077 __ shll(dest_reg, 24);
1078 __ sarl(dest_reg, 24);
1079 }
1080 break;
1081 }
1082
1083 case T_CHAR: {
1084 Register dest_reg = dest->as_register();
1085 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1086 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1087 __ movzwl(dest_reg, from_addr);
1088 } else {
1089 __ movw(dest_reg, from_addr);
1090 }
1091 break;
1092 }
1093
1094 case T_SHORT: {
1095 Register dest_reg = dest->as_register();
1096 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1097 __ movswl(dest_reg, from_addr);
1098 } else {
1099 __ movw(dest_reg, from_addr);
1100 __ shll(dest_reg, 16);
1101 __ sarl(dest_reg, 16);
1102 }
1103 break;
1104 }
1105
1106 default:
1107 ShouldNotReachHere();
1108 }
1109
1110 if (patch != nullptr) {
1111 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1112 }
1113
1114 if (is_reference_type(type)) {
1115 if (UseCompressedOops && !wide) {
1116 __ decode_heap_oop(dest->as_register());
1117 }
1118
1119 __ verify_oop(dest->as_register());
1120 }
1121 }
1122
1123
1124 NEEDS_CLEANUP; // This could be static?
1125 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1126 int elem_size = type2aelembytes(type);
1127 switch (elem_size) {
1128 case 1: return Address::times_1;
1129 case 2: return Address::times_2;
1130 case 4: return Address::times_4;
1131 case 8: return Address::times_8;
1132 }
1133 ShouldNotReachHere();
1134 return Address::no_scale;
1135 }
1136
1137
1138 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1139 switch (op->code()) {
1140 case lir_idiv:
1141 case lir_irem:
1142 arithmetic_idiv(op->code(),
1143 op->in_opr1(),
1144 op->in_opr2(),
1145 op->in_opr3(),
1146 op->result_opr(),
1147 op->info());
1148 break;
1149 case lir_fmad:
1150 __ fmad(op->result_opr()->as_xmm_double_reg(),
1151 op->in_opr1()->as_xmm_double_reg(),
1152 op->in_opr2()->as_xmm_double_reg(),
1153 op->in_opr3()->as_xmm_double_reg());
1154 break;
1155 case lir_fmaf:
1156 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1157 op->in_opr1()->as_xmm_float_reg(),
1158 op->in_opr2()->as_xmm_float_reg(),
1159 op->in_opr3()->as_xmm_float_reg());
1160 break;
1161 default: ShouldNotReachHere(); break;
1162 }
1163 }
1164
1165 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1166 #ifdef ASSERT
1167 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1168 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1169 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1170 #endif
1171
1172 if (op->cond() == lir_cond_always) {
1173 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1174 __ jmp (*(op->label()));
1175 } else {
1176 Assembler::Condition acond = Assembler::zero;
1177 if (op->code() == lir_cond_float_branch) {
1178 assert(op->ublock() != nullptr, "must have unordered successor");
1179 __ jcc(Assembler::parity, *(op->ublock()->label()));
1180 switch(op->cond()) {
1181 case lir_cond_equal: acond = Assembler::equal; break;
1182 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1183 case lir_cond_less: acond = Assembler::below; break;
1184 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1185 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1186 case lir_cond_greater: acond = Assembler::above; break;
1187 default: ShouldNotReachHere();
1188 }
1189 } else {
1190 switch (op->cond()) {
1191 case lir_cond_equal: acond = Assembler::equal; break;
1192 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1193 case lir_cond_less: acond = Assembler::less; break;
1194 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1195 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1196 case lir_cond_greater: acond = Assembler::greater; break;
1197 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1198 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1199 default: ShouldNotReachHere();
1200 }
1201 }
1202 __ jcc(acond,*(op->label()));
1203 }
1204 }
1205
1206 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1207 LIR_Opr src = op->in_opr();
1208 LIR_Opr dest = op->result_opr();
1209
1210 switch (op->bytecode()) {
1211 case Bytecodes::_i2l:
1212 __ movl2ptr(dest->as_register_lo(), src->as_register());
1213 break;
1214
1215 case Bytecodes::_l2i:
1216 __ movl(dest->as_register(), src->as_register_lo());
1217 break;
1218
1219 case Bytecodes::_i2b:
1220 move_regs(src->as_register(), dest->as_register());
1221 __ sign_extend_byte(dest->as_register());
1222 break;
1223
1224 case Bytecodes::_i2c:
1225 move_regs(src->as_register(), dest->as_register());
1226 __ andl(dest->as_register(), 0xFFFF);
1227 break;
1228
1229 case Bytecodes::_i2s:
1230 move_regs(src->as_register(), dest->as_register());
1231 __ sign_extend_short(dest->as_register());
1232 break;
1233
1234 case Bytecodes::_f2d:
1235 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1236 break;
1237
1238 case Bytecodes::_d2f:
1239 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1240 break;
1241
1242 case Bytecodes::_i2f:
1243 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1244 break;
1245
1246 case Bytecodes::_i2d:
1247 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1248 break;
1249
1250 case Bytecodes::_l2f:
1251 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1252 break;
1253
1254 case Bytecodes::_l2d:
1255 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1256 break;
1257
1258 case Bytecodes::_f2i:
1259 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1260 break;
1261
1262 case Bytecodes::_d2i:
1263 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1264 break;
1265
1266 case Bytecodes::_f2l:
1267 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1268 break;
1269
1270 case Bytecodes::_d2l:
1271 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1272 break;
1273
1274 default: ShouldNotReachHere();
1275 }
1276 }
1277
1278 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1279 if (op->init_check()) {
1280 add_debug_info_for_null_check_here(op->stub()->info());
1281 // init_state needs acquire, but x86 is TSO, and so we are already good.
1282 __ cmpb(Address(op->klass()->as_register(),
1283 InstanceKlass::init_state_offset()),
1284 InstanceKlass::fully_initialized);
1285 __ jcc(Assembler::notEqual, *op->stub()->entry());
1286 }
1287 __ allocate_object(op->obj()->as_register(),
1288 op->tmp1()->as_register(),
1289 op->tmp2()->as_register(),
1290 op->header_size(),
1291 op->object_size(),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298 Register len = op->len()->as_register();
1299 __ movslq(len, len);
1300
1301 if (UseSlowPath || op->always_slow_path() ||
1302 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1304 __ jmp(*op->stub()->entry());
1305 } else {
1306 Register tmp1 = op->tmp1()->as_register();
1307 Register tmp2 = op->tmp2()->as_register();
1308 Register tmp3 = op->tmp3()->as_register();
1309 if (len == tmp1) {
1310 tmp1 = tmp3;
1311 } else if (len == tmp2) {
1312 tmp2 = tmp3;
1313 } else if (len == tmp3) {
1314 // everything is ok
1315 } else {
1316 __ mov(tmp3, len);
1317 }
1318 __ allocate_array(op->obj()->as_register(),
1319 len,
1320 tmp1,
1321 tmp2,
1322 arrayOopDesc::base_offset_in_bytes(op->type()),
1323 array_element_size(op->type()),
1324 op->klass()->as_register(),
1325 *op->stub()->entry(),
1326 op->zero_array());
1327 }
1328 __ bind(*op->stub()->continuation());
1329 }
1330
1331 void LIR_Assembler::type_profile_helper(Register mdo,
1332 ciMethodData *md, ciProfileData *data,
1333 Register recv) {
1334 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1335 __ profile_receiver_type(recv, mdo, mdp_offset);
1336 }
1337
1338 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1339 // we always need a stub for the failure case.
1340 CodeStub* stub = op->stub();
1341 Register obj = op->object()->as_register();
1342 Register k_RInfo = op->tmp1()->as_register();
1343 Register klass_RInfo = op->tmp2()->as_register();
1344 Register dst = op->result_opr()->as_register();
1345 ciKlass* k = op->klass();
1346 Register Rtmp1 = noreg;
1347 Register tmp_load_klass = rscratch1;
1348
1349 // check if it needs to be profiled
1350 ciMethodData* md = nullptr;
1351 ciProfileData* data = nullptr;
1352
1353 if (op->should_profile()) {
1354 ciMethod* method = op->profiled_method();
1355 assert(method != nullptr, "Should have method");
1356 int bci = op->profiled_bci();
1357 md = method->method_data_or_null();
1358 assert(md != nullptr, "Sanity");
1359 data = md->bci_to_data(bci);
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373
1374 assert_different_registers(obj, k_RInfo, klass_RInfo);
1375
1376 if (op->need_null_check()) {
1377 __ testptr(obj, obj);
1378 if (op->should_profile()) {
1379 Label not_null;
1380 Register mdo = klass_RInfo;
1381 __ mov_metadata(mdo, md->constant_encoding());
1382 __ jccb(Assembler::notEqual, not_null);
1383 // Object is null; update MDO and exit
1384 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385 int header_bits = BitData::null_seen_byte_constant();
1386 __ orb(data_addr, header_bits);
1387 __ jmp(*obj_is_null);
1388 __ bind(not_null);
1389
1390 Register recv = k_RInfo;
1391 __ load_klass(recv, obj, tmp_load_klass);
1392 type_profile_helper(mdo, md, data, recv);
1393 } else {
1394 __ jcc(Assembler::equal, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(Rtmp1, obj, tmp_load_klass);
1410 __ cmpptr(k_RInfo, Rtmp1);
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 if (!k_refined->is_loaded()) {
1431 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1432 return;
1433 }
1434 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1435 __ cmpptr(klass_RInfo, tmp_load_klass);
1436 } else {
1437 __ cmpptr(klass_RInfo, k_RInfo);
1438 }
1439 __ jcc(Assembler::equal, *success_target);
1440
1441 __ push_ppx(klass_RInfo);
1442 __ push_ppx(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1444 __ pop_ppx(klass_RInfo);
1445 __ pop_ppx(klass_RInfo);
1446 // result is a boolean
1447 __ testl(klass_RInfo, klass_RInfo);
1448 __ jcc(Assembler::equal, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 } else {
1452 // perform the fast part of the checking logic
1453 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1454 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1455 __ push_ppx(klass_RInfo);
1456 __ push_ppx(k_RInfo);
1457 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1458 __ pop_ppx(klass_RInfo);
1459 __ pop_ppx(k_RInfo);
1460 // result is a boolean
1461 __ testl(k_RInfo, k_RInfo);
1462 __ jcc(Assembler::equal, *failure_target);
1463 // successful cast, fall through to profile or jump
1464 }
1465 }
1466 __ jmp(*success);
1467 }
1468
1469
1470 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1471 Register tmp_load_klass = rscratch1;
1472 LIR_Code code = op->code();
1473 if (code == lir_store_check) {
1474 Register value = op->object()->as_register();
1475 Register array = op->array()->as_register();
1476 Register k_RInfo = op->tmp1()->as_register();
1477 Register klass_RInfo = op->tmp2()->as_register();
1478 Register Rtmp1 = op->tmp3()->as_register();
1479
1480 CodeStub* stub = op->stub();
1481
1482 // check if it needs to be profiled
1483 ciMethodData* md = nullptr;
1484 ciProfileData* data = nullptr;
1485
1486 if (op->should_profile()) {
1487 ciMethod* method = op->profiled_method();
1488 assert(method != nullptr, "Should have method");
1489 int bci = op->profiled_bci();
1490 md = method->method_data_or_null();
1491 assert(md != nullptr, "Sanity");
1492 data = md->bci_to_data(bci);
1493 assert(data != nullptr, "need data for type check");
1494 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1495 }
1496 Label done;
1497 Label* success_target = &done;
1498 Label* failure_target = stub->entry();
1499
1500 __ testptr(value, value);
1501 if (op->should_profile()) {
1502 Label not_null;
1503 Register mdo = klass_RInfo;
1504 __ mov_metadata(mdo, md->constant_encoding());
1505 __ jccb(Assembler::notEqual, not_null);
1506 // Object is null; update MDO and exit
1507 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1508 int header_bits = BitData::null_seen_byte_constant();
1509 __ orb(data_addr, header_bits);
1510 __ jmp(done);
1511 __ bind(not_null);
1512
1513 Register recv = k_RInfo;
1514 __ load_klass(recv, value, tmp_load_klass);
1515 type_profile_helper(mdo, md, data, recv);
1516 } else {
1517 __ jcc(Assembler::equal, done);
1518 }
1519
1520 add_debug_info_for_null_check_here(op->info_for_exception());
1521 __ load_klass(k_RInfo, array, tmp_load_klass);
1522 __ load_klass(klass_RInfo, value, tmp_load_klass);
1523
1524 // get instance klass (it's already uncompressed)
1525 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1526 // perform the fast part of the checking logic
1527 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1528 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1529 __ push_ppx(klass_RInfo);
1530 __ push_ppx(k_RInfo);
1531 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1532 __ pop_ppx(klass_RInfo);
1533 __ pop_ppx(k_RInfo);
1534 // result is a boolean
1535 __ testl(k_RInfo, k_RInfo);
1536 __ jcc(Assembler::equal, *failure_target);
1537 // fall through to the success case
1538
1539 __ bind(done);
1540 } else
1541 if (code == lir_checkcast) {
1542 Register obj = op->object()->as_register();
1543 Register dst = op->result_opr()->as_register();
1544 Label success;
1545 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1546 __ bind(success);
1547 if (dst != obj) {
1548 __ mov(dst, obj);
1549 }
1550 } else
1551 if (code == lir_instanceof) {
1552 Register obj = op->object()->as_register();
1553 Register dst = op->result_opr()->as_register();
1554 Label success, failure, done;
1555 emit_typecheck_helper(op, &success, &failure, &failure);
1556 __ bind(failure);
1557 __ xorptr(dst, dst);
1558 __ jmpb(done);
1559 __ bind(success);
1560 __ movptr(dst, 1);
1561 __ bind(done);
1562 } else {
1563 ShouldNotReachHere();
1564 }
1565
1566 }
1567
1568 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1569 // We are loading/storing from/to an array that *may* be a flat array (the
1570 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1571 // If this array is a flat array, take the slow path.
1572 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1573 }
1574
1575 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1576 // We are storing into an array that *may* be null-free (the declared type is
1577 // Object[], abstract[], interface[] or VT.ref[]).
1578 Label test_mark_word;
1579 Register tmp = op->tmp()->as_register();
1580 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1581 __ testl(tmp, markWord::unlocked_value);
1582 __ jccb(Assembler::notZero, test_mark_word);
1583 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1584 __ bind(test_mark_word);
1585 __ testl(tmp, markWord::null_free_array_bit_in_place);
1586 }
1587
1588 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1589 Label L_oops_equal;
1590 Label L_oops_not_equal;
1591 Label L_end;
1592
1593 Register left = op->left()->as_register();
1594 Register right = op->right()->as_register();
1595
1596 __ cmpptr(left, right);
1597 __ jcc(Assembler::equal, L_oops_equal);
1598
1599 // (1) Null check -- if one of the operands is null, the other must not be null (because
1600 // the two references are not equal), so they are not substitutable,
1601 __ testptr(left, left);
1602 __ jcc(Assembler::zero, L_oops_not_equal);
1603 __ testptr(right, right);
1604 __ jcc(Assembler::zero, L_oops_not_equal);
1605
1606 ciKlass* left_klass = op->left_klass();
1607 ciKlass* right_klass = op->right_klass();
1608
1609 // (2) Inline type check -- if either of the operands is not an inline type,
1610 // they are not substitutable. We do this only if we are not sure that the
1611 // operands are inline type
1612 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1613 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1614 Register tmp = op->tmp1()->as_register();
1615 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1616 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1617 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1619 __ jcc(Assembler::notEqual, L_oops_not_equal);
1620 }
1621
1622 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1623 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1624 // No need to load klass -- the operands are statically known to be the same inline klass.
1625 __ jmp(*op->stub()->entry());
1626 } else {
1627 Register tmp1 = op->tmp1()->as_register();
1628 Register tmp2 = op->tmp2()->as_register();
1629 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1630 __ jmp (*op->stub()->entry()); // -> do slow check
1631 } else {
1632 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1633 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1634 }
1635 // fall through to L_oops_not_equal
1636 }
1637
1638 __ bind(L_oops_not_equal);
1639 move(op->not_equal_result(), op->result_opr());
1640 __ jmp(L_end);
1641
1642 __ bind(L_oops_equal);
1643 move(op->equal_result(), op->result_opr());
1644 __ jmp(L_end);
1645
1646 // We've returned from the stub. RAX contains 0x0 IFF the two
1647 // operands are not substitutable. (Don't compare against 0x1 in case the
1648 // C compiler is naughty)
1649 __ bind(*op->stub()->continuation());
1650 __ cmpl(rax, 0);
1651 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1652 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1653 // fall-through
1654 __ bind(L_end);
1655 }
1656
1657 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1658 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1659 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1660 Register newval = op->new_value()->as_register();
1661 Register cmpval = op->cmp_value()->as_register();
1662 assert(cmpval == rax, "wrong register");
1663 assert(newval != noreg, "new val must be register");
1664 assert(cmpval != newval, "cmp and new values must be in different registers");
1665 assert(cmpval != addr, "cmp and addr must be in different registers");
1666 assert(newval != addr, "new value and addr must be in different registers");
1667
1668 if (op->code() == lir_cas_obj) {
1669 if (UseCompressedOops) {
1670 __ encode_heap_oop(cmpval);
1671 __ mov(rscratch1, newval);
1672 __ encode_heap_oop(rscratch1);
1673 __ lock();
1674 // cmpval (rax) is implicitly used by this instruction
1675 __ cmpxchgl(rscratch1, Address(addr, 0));
1676 } else {
1677 __ lock();
1678 __ cmpxchgptr(newval, Address(addr, 0));
1679 }
1680 } else {
1681 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1682 __ lock();
1683 __ cmpxchgl(newval, Address(addr, 0));
1684 }
1685 } else if (op->code() == lir_cas_long) {
1686 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1687 Register newval = op->new_value()->as_register_lo();
1688 Register cmpval = op->cmp_value()->as_register_lo();
1689 assert(cmpval == rax, "wrong register");
1690 assert(newval != noreg, "new val must be register");
1691 assert(cmpval != newval, "cmp and new values must be in different registers");
1692 assert(cmpval != addr, "cmp and addr must be in different registers");
1693 assert(newval != addr, "new value and addr must be in different registers");
1694 __ lock();
1695 __ cmpxchgq(newval, Address(addr, 0));
1696 } else {
1697 Unimplemented();
1698 }
1699 }
1700
1701 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1702 assert(dst->is_cpu_register(), "must be");
1703 assert(dst->type() == src->type(), "must be");
1704
1705 if (src->is_cpu_register()) {
1706 reg2reg(src, dst);
1707 } else if (src->is_stack()) {
1708 stack2reg(src, dst, dst->type());
1709 } else if (src->is_constant()) {
1710 const2reg(src, dst, lir_patch_none, nullptr);
1711 } else {
1712 ShouldNotReachHere();
1713 }
1714 }
1715
1716 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1717 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1718 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1719
1720 Assembler::Condition acond, ncond;
1721 switch (condition) {
1722 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1723 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1724 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1725 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1726 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1727 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1728 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1729 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1730 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1731 ShouldNotReachHere();
1732 }
1733
1734 if (opr1->is_cpu_register()) {
1735 reg2reg(opr1, result);
1736 } else if (opr1->is_stack()) {
1737 stack2reg(opr1, result, result->type());
1738 } else if (opr1->is_constant()) {
1739 const2reg(opr1, result, lir_patch_none, nullptr);
1740 } else {
1741 ShouldNotReachHere();
1742 }
1743
1744 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1745 // optimized version that does not require a branch
1746 if (opr2->is_single_cpu()) {
1747 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1748 __ cmov(ncond, result->as_register(), opr2->as_register());
1749 } else if (opr2->is_double_cpu()) {
1750 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1751 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1752 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1753 } else if (opr2->is_single_stack()) {
1754 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1755 } else if (opr2->is_double_stack()) {
1756 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1757 } else {
1758 ShouldNotReachHere();
1759 }
1760
1761 } else {
1762 Label skip;
1763 __ jccb(acond, skip);
1764 if (opr2->is_cpu_register()) {
1765 reg2reg(opr2, result);
1766 } else if (opr2->is_stack()) {
1767 stack2reg(opr2, result, result->type());
1768 } else if (opr2->is_constant()) {
1769 const2reg(opr2, result, lir_patch_none, nullptr);
1770 } else {
1771 ShouldNotReachHere();
1772 }
1773 __ bind(skip);
1774 }
1775 }
1776
1777
1778 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1779 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1780
1781 if (left->is_single_cpu()) {
1782 assert(left == dest, "left and dest must be equal");
1783 Register lreg = left->as_register();
1784
1785 if (right->is_single_cpu()) {
1786 // cpu register - cpu register
1787 Register rreg = right->as_register();
1788 switch (code) {
1789 case lir_add: __ addl (lreg, rreg); break;
1790 case lir_sub: __ subl (lreg, rreg); break;
1791 case lir_mul: __ imull(lreg, rreg); break;
1792 default: ShouldNotReachHere();
1793 }
1794
1795 } else if (right->is_stack()) {
1796 // cpu register - stack
1797 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1798 switch (code) {
1799 case lir_add: __ addl(lreg, raddr); break;
1800 case lir_sub: __ subl(lreg, raddr); break;
1801 default: ShouldNotReachHere();
1802 }
1803
1804 } else if (right->is_constant()) {
1805 // cpu register - constant
1806 jint c = right->as_constant_ptr()->as_jint();
1807 switch (code) {
1808 case lir_add: {
1809 __ incrementl(lreg, c);
1810 break;
1811 }
1812 case lir_sub: {
1813 __ decrementl(lreg, c);
1814 break;
1815 }
1816 default: ShouldNotReachHere();
1817 }
1818
1819 } else {
1820 ShouldNotReachHere();
1821 }
1822
1823 } else if (left->is_double_cpu()) {
1824 assert(left == dest, "left and dest must be equal");
1825 Register lreg_lo = left->as_register_lo();
1826 Register lreg_hi = left->as_register_hi();
1827
1828 if (right->is_double_cpu()) {
1829 // cpu register - cpu register
1830 Register rreg_lo = right->as_register_lo();
1831 Register rreg_hi = right->as_register_hi();
1832 assert_different_registers(lreg_lo, rreg_lo);
1833 switch (code) {
1834 case lir_add:
1835 __ addptr(lreg_lo, rreg_lo);
1836 break;
1837 case lir_sub:
1838 __ subptr(lreg_lo, rreg_lo);
1839 break;
1840 case lir_mul:
1841 __ imulq(lreg_lo, rreg_lo);
1842 break;
1843 default:
1844 ShouldNotReachHere();
1845 }
1846
1847 } else if (right->is_constant()) {
1848 // cpu register - constant
1849 jlong c = right->as_constant_ptr()->as_jlong_bits();
1850 __ movptr(r10, (intptr_t) c);
1851 switch (code) {
1852 case lir_add:
1853 __ addptr(lreg_lo, r10);
1854 break;
1855 case lir_sub:
1856 __ subptr(lreg_lo, r10);
1857 break;
1858 default:
1859 ShouldNotReachHere();
1860 }
1861
1862 } else {
1863 ShouldNotReachHere();
1864 }
1865
1866 } else if (left->is_single_xmm()) {
1867 assert(left == dest, "left and dest must be equal");
1868 XMMRegister lreg = left->as_xmm_float_reg();
1869
1870 if (right->is_single_xmm()) {
1871 XMMRegister rreg = right->as_xmm_float_reg();
1872 switch (code) {
1873 case lir_add: __ addss(lreg, rreg); break;
1874 case lir_sub: __ subss(lreg, rreg); break;
1875 case lir_mul: __ mulss(lreg, rreg); break;
1876 case lir_div: __ divss(lreg, rreg); break;
1877 default: ShouldNotReachHere();
1878 }
1879 } else {
1880 Address raddr;
1881 if (right->is_single_stack()) {
1882 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1883 } else if (right->is_constant()) {
1884 // hack for now
1885 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1886 } else {
1887 ShouldNotReachHere();
1888 }
1889 switch (code) {
1890 case lir_add: __ addss(lreg, raddr); break;
1891 case lir_sub: __ subss(lreg, raddr); break;
1892 case lir_mul: __ mulss(lreg, raddr); break;
1893 case lir_div: __ divss(lreg, raddr); break;
1894 default: ShouldNotReachHere();
1895 }
1896 }
1897
1898 } else if (left->is_double_xmm()) {
1899 assert(left == dest, "left and dest must be equal");
1900
1901 XMMRegister lreg = left->as_xmm_double_reg();
1902 if (right->is_double_xmm()) {
1903 XMMRegister rreg = right->as_xmm_double_reg();
1904 switch (code) {
1905 case lir_add: __ addsd(lreg, rreg); break;
1906 case lir_sub: __ subsd(lreg, rreg); break;
1907 case lir_mul: __ mulsd(lreg, rreg); break;
1908 case lir_div: __ divsd(lreg, rreg); break;
1909 default: ShouldNotReachHere();
1910 }
1911 } else {
1912 Address raddr;
1913 if (right->is_double_stack()) {
1914 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1915 } else if (right->is_constant()) {
1916 // hack for now
1917 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1918 } else {
1919 ShouldNotReachHere();
1920 }
1921 switch (code) {
1922 case lir_add: __ addsd(lreg, raddr); break;
1923 case lir_sub: __ subsd(lreg, raddr); break;
1924 case lir_mul: __ mulsd(lreg, raddr); break;
1925 case lir_div: __ divsd(lreg, raddr); break;
1926 default: ShouldNotReachHere();
1927 }
1928 }
1929
1930 } else if (left->is_single_stack() || left->is_address()) {
1931 assert(left == dest, "left and dest must be equal");
1932
1933 Address laddr;
1934 if (left->is_single_stack()) {
1935 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1936 } else if (left->is_address()) {
1937 laddr = as_Address(left->as_address_ptr());
1938 } else {
1939 ShouldNotReachHere();
1940 }
1941
1942 if (right->is_single_cpu()) {
1943 Register rreg = right->as_register();
1944 switch (code) {
1945 case lir_add: __ addl(laddr, rreg); break;
1946 case lir_sub: __ subl(laddr, rreg); break;
1947 default: ShouldNotReachHere();
1948 }
1949 } else if (right->is_constant()) {
1950 jint c = right->as_constant_ptr()->as_jint();
1951 switch (code) {
1952 case lir_add: {
1953 __ incrementl(laddr, c);
1954 break;
1955 }
1956 case lir_sub: {
1957 __ decrementl(laddr, c);
1958 break;
1959 }
1960 default: ShouldNotReachHere();
1961 }
1962 } else {
1963 ShouldNotReachHere();
1964 }
1965
1966 } else {
1967 ShouldNotReachHere();
1968 }
1969 }
1970
1971
1972 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1973 if (value->is_double_xmm()) {
1974 switch(code) {
1975 case lir_abs :
1976 {
1977 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1978 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1979 }
1980 assert(!tmp->is_valid(), "do not need temporary");
1981 __ andpd(dest->as_xmm_double_reg(),
1982 ExternalAddress((address)double_signmask_pool),
1983 rscratch1);
1984 }
1985 break;
1986
1987 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1988 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1989 default : ShouldNotReachHere();
1990 }
1991
1992 } else if (code == lir_f2hf) {
1993 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1994 } else if (code == lir_hf2f) {
1995 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1996 } else {
1997 Unimplemented();
1998 }
1999 }
2000
2001 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2002 // assert(left->destroys_register(), "check");
2003 if (left->is_single_cpu()) {
2004 Register reg = left->as_register();
2005 if (right->is_constant()) {
2006 int val = right->as_constant_ptr()->as_jint();
2007 switch (code) {
2008 case lir_logic_and: __ andl (reg, val); break;
2009 case lir_logic_or: __ orl (reg, val); break;
2010 case lir_logic_xor: __ xorl (reg, val); break;
2011 default: ShouldNotReachHere();
2012 }
2013 } else if (right->is_stack()) {
2014 // added support for stack operands
2015 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2016 switch (code) {
2017 case lir_logic_and: __ andl (reg, raddr); break;
2018 case lir_logic_or: __ orl (reg, raddr); break;
2019 case lir_logic_xor: __ xorl (reg, raddr); break;
2020 default: ShouldNotReachHere();
2021 }
2022 } else {
2023 Register rright = right->as_register();
2024 switch (code) {
2025 case lir_logic_and: __ andptr (reg, rright); break;
2026 case lir_logic_or : __ orptr (reg, rright); break;
2027 case lir_logic_xor: __ xorptr (reg, rright); break;
2028 default: ShouldNotReachHere();
2029 }
2030 }
2031 move_regs(reg, dst->as_register());
2032 } else {
2033 Register l_lo = left->as_register_lo();
2034 Register l_hi = left->as_register_hi();
2035 if (right->is_constant()) {
2036 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2037 switch (code) {
2038 case lir_logic_and:
2039 __ andq(l_lo, rscratch1);
2040 break;
2041 case lir_logic_or:
2042 __ orq(l_lo, rscratch1);
2043 break;
2044 case lir_logic_xor:
2045 __ xorq(l_lo, rscratch1);
2046 break;
2047 default: ShouldNotReachHere();
2048 }
2049 } else {
2050 Register r_lo;
2051 if (is_reference_type(right->type())) {
2052 r_lo = right->as_register();
2053 } else {
2054 r_lo = right->as_register_lo();
2055 }
2056 switch (code) {
2057 case lir_logic_and:
2058 __ andptr(l_lo, r_lo);
2059 break;
2060 case lir_logic_or:
2061 __ orptr(l_lo, r_lo);
2062 break;
2063 case lir_logic_xor:
2064 __ xorptr(l_lo, r_lo);
2065 break;
2066 default: ShouldNotReachHere();
2067 }
2068 }
2069
2070 Register dst_lo = dst->as_register_lo();
2071 Register dst_hi = dst->as_register_hi();
2072
2073 move_regs(l_lo, dst_lo);
2074 }
2075 }
2076
2077
2078 // we assume that rax, and rdx can be overwritten
2079 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2080
2081 assert(left->is_single_cpu(), "left must be register");
2082 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2083 assert(result->is_single_cpu(), "result must be register");
2084
2085 // assert(left->destroys_register(), "check");
2086 // assert(right->destroys_register(), "check");
2087
2088 Register lreg = left->as_register();
2089 Register dreg = result->as_register();
2090
2091 if (right->is_constant()) {
2092 jint divisor = right->as_constant_ptr()->as_jint();
2093 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2094 if (code == lir_idiv) {
2095 assert(lreg == rax, "must be rax,");
2096 assert(temp->as_register() == rdx, "tmp register must be rdx");
2097 __ cdql(); // sign extend into rdx:rax
2098 if (divisor == 2) {
2099 __ subl(lreg, rdx);
2100 } else {
2101 __ andl(rdx, divisor - 1);
2102 __ addl(lreg, rdx);
2103 }
2104 __ sarl(lreg, log2i_exact(divisor));
2105 move_regs(lreg, dreg);
2106 } else if (code == lir_irem) {
2107 Label done;
2108 __ mov(dreg, lreg);
2109 __ andl(dreg, 0x80000000 | (divisor - 1));
2110 __ jcc(Assembler::positive, done);
2111 __ decrement(dreg);
2112 __ orl(dreg, ~(divisor - 1));
2113 __ increment(dreg);
2114 __ bind(done);
2115 } else {
2116 ShouldNotReachHere();
2117 }
2118 } else {
2119 Register rreg = right->as_register();
2120 assert(lreg == rax, "left register must be rax,");
2121 assert(rreg != rdx, "right register must not be rdx");
2122 assert(temp->as_register() == rdx, "tmp register must be rdx");
2123
2124 move_regs(lreg, rax);
2125
2126 int idivl_offset = __ corrected_idivl(rreg);
2127 if (ImplicitDiv0Checks) {
2128 add_debug_info_for_div0(idivl_offset, info);
2129 }
2130 if (code == lir_irem) {
2131 move_regs(rdx, dreg); // result is in rdx
2132 } else {
2133 move_regs(rax, dreg);
2134 }
2135 }
2136 }
2137
2138
2139 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2140 if (opr1->is_single_cpu()) {
2141 Register reg1 = opr1->as_register();
2142 if (opr2->is_single_cpu()) {
2143 // cpu register - cpu register
2144 if (is_reference_type(opr1->type())) {
2145 __ cmpoop(reg1, opr2->as_register());
2146 } else {
2147 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2148 __ cmpl(reg1, opr2->as_register());
2149 }
2150 } else if (opr2->is_stack()) {
2151 // cpu register - stack
2152 if (is_reference_type(opr1->type())) {
2153 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2154 } else {
2155 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2156 }
2157 } else if (opr2->is_constant()) {
2158 // cpu register - constant
2159 LIR_Const* c = opr2->as_constant_ptr();
2160 if (c->type() == T_INT) {
2161 jint i = c->as_jint();
2162 if (i == 0) {
2163 __ testl(reg1, reg1);
2164 } else {
2165 __ cmpl(reg1, i);
2166 }
2167 } else if (c->type() == T_METADATA) {
2168 // All we need for now is a comparison with null for equality.
2169 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2170 Metadata* m = c->as_metadata();
2171 if (m == nullptr) {
2172 __ testptr(reg1, reg1);
2173 } else {
2174 ShouldNotReachHere();
2175 }
2176 } else if (is_reference_type(c->type())) {
2177 // In 64bit oops are single register
2178 jobject o = c->as_jobject();
2179 if (o == nullptr) {
2180 __ testptr(reg1, reg1);
2181 } else {
2182 __ cmpoop(reg1, o, rscratch1);
2183 }
2184 } else {
2185 fatal("unexpected type: %s", basictype_to_str(c->type()));
2186 }
2187 // cpu register - address
2188 } else if (opr2->is_address()) {
2189 if (op->info() != nullptr) {
2190 add_debug_info_for_null_check_here(op->info());
2191 }
2192 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2193 } else {
2194 ShouldNotReachHere();
2195 }
2196
2197 } else if(opr1->is_double_cpu()) {
2198 Register xlo = opr1->as_register_lo();
2199 Register xhi = opr1->as_register_hi();
2200 if (opr2->is_double_cpu()) {
2201 __ cmpptr(xlo, opr2->as_register_lo());
2202 } else if (opr2->is_constant()) {
2203 // cpu register - constant 0
2204 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2205 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2206 } else {
2207 ShouldNotReachHere();
2208 }
2209
2210 } else if (opr1->is_single_xmm()) {
2211 XMMRegister reg1 = opr1->as_xmm_float_reg();
2212 if (opr2->is_single_xmm()) {
2213 // xmm register - xmm register
2214 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2215 } else if (opr2->is_stack()) {
2216 // xmm register - stack
2217 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2218 } else if (opr2->is_constant()) {
2219 // xmm register - constant
2220 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2221 } else if (opr2->is_address()) {
2222 // xmm register - address
2223 if (op->info() != nullptr) {
2224 add_debug_info_for_null_check_here(op->info());
2225 }
2226 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2227 } else {
2228 ShouldNotReachHere();
2229 }
2230
2231 } else if (opr1->is_double_xmm()) {
2232 XMMRegister reg1 = opr1->as_xmm_double_reg();
2233 if (opr2->is_double_xmm()) {
2234 // xmm register - xmm register
2235 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2236 } else if (opr2->is_stack()) {
2237 // xmm register - stack
2238 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2239 } else if (opr2->is_constant()) {
2240 // xmm register - constant
2241 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2242 } else if (opr2->is_address()) {
2243 // xmm register - address
2244 if (op->info() != nullptr) {
2245 add_debug_info_for_null_check_here(op->info());
2246 }
2247 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2248 } else {
2249 ShouldNotReachHere();
2250 }
2251
2252 } else if (opr1->is_address() && opr2->is_constant()) {
2253 LIR_Const* c = opr2->as_constant_ptr();
2254 if (is_reference_type(c->type())) {
2255 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2256 __ movoop(rscratch1, c->as_jobject());
2257 }
2258 if (op->info() != nullptr) {
2259 add_debug_info_for_null_check_here(op->info());
2260 }
2261 // special case: address - constant
2262 LIR_Address* addr = opr1->as_address_ptr();
2263 if (c->type() == T_INT) {
2264 __ cmpl(as_Address(addr), c->as_jint());
2265 } else if (is_reference_type(c->type())) {
2266 // %%% Make this explode if addr isn't reachable until we figure out a
2267 // better strategy by giving noreg as the temp for as_Address
2268 __ cmpoop(rscratch1, as_Address(addr, noreg));
2269 } else {
2270 ShouldNotReachHere();
2271 }
2272
2273 } else {
2274 ShouldNotReachHere();
2275 }
2276 }
2277
2278 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2279 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2280 if (left->is_single_xmm()) {
2281 assert(right->is_single_xmm(), "must match");
2282 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2283 } else if (left->is_double_xmm()) {
2284 assert(right->is_double_xmm(), "must match");
2285 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2286
2287 } else {
2288 ShouldNotReachHere();
2289 }
2290 } else {
2291 assert(code == lir_cmp_l2i, "check");
2292 Label done;
2293 Register dest = dst->as_register();
2294 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2295 __ movl(dest, -1);
2296 __ jccb(Assembler::less, done);
2297 __ setb(Assembler::notZero, dest);
2298 __ movzbl(dest, dest);
2299 __ bind(done);
2300 }
2301 }
2302
2303
2304 void LIR_Assembler::align_call(LIR_Code code) {
2305 // make sure that the displacement word of the call ends up word aligned
2306 int offset = __ offset();
2307 switch (code) {
2308 case lir_static_call:
2309 case lir_optvirtual_call:
2310 case lir_dynamic_call:
2311 offset += NativeCall::displacement_offset;
2312 break;
2313 case lir_icvirtual_call:
2314 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2315 break;
2316 default: ShouldNotReachHere();
2317 }
2318 __ align(BytesPerWord, offset);
2319 }
2320
2321
2322 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2323 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2324 "must be aligned");
2325 __ call(AddressLiteral(op->addr(), rtype));
2326 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2327 __ post_call_nop();
2328 }
2329
2330
2331 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2332 __ ic_call(op->addr());
2333 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2334 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2335 "must be aligned");
2336 __ post_call_nop();
2337 }
2338
2339
2340 void LIR_Assembler::emit_static_call_stub() {
2341 address call_pc = __ pc();
2342 address stub = __ start_a_stub(call_stub_size());
2343 if (stub == nullptr) {
2344 bailout("static call stub overflow");
2345 return;
2346 }
2347
2348 int start = __ offset();
2349
2350 // make sure that the displacement word of the call ends up word aligned
2351 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2352 __ relocate(static_stub_Relocation::spec(call_pc));
2353 __ mov_metadata(rbx, (Metadata*)nullptr);
2354 // must be set to -1 at code generation time
2355 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2356 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2357 __ jump(RuntimeAddress(__ pc()));
2358
2359 assert(__ offset() - start <= call_stub_size(), "stub too big");
2360 __ end_a_stub();
2361 }
2362
2363
2364 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2365 assert(exceptionOop->as_register() == rax, "must match");
2366 assert(exceptionPC->as_register() == rdx, "must match");
2367
2368 // exception object is not added to oop map by LinearScan
2369 // (LinearScan assumes that no oops are in fixed registers)
2370 info->add_register_oop(exceptionOop);
2371 StubId unwind_id;
2372
2373 // get current pc information
2374 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2375 int pc_for_athrow_offset = __ offset();
2376 InternalAddress pc_for_athrow(__ pc());
2377 __ lea(exceptionPC->as_register(), pc_for_athrow);
2378 add_call_info(pc_for_athrow_offset, info); // for exception handler
2379
2380 __ verify_not_null_oop(rax);
2381 // search an exception handler (rax: exception oop, rdx: throwing pc)
2382 if (compilation()->has_fpu_code()) {
2383 unwind_id = StubId::c1_handle_exception_id;
2384 } else {
2385 unwind_id = StubId::c1_handle_exception_nofpu_id;
2386 }
2387 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2388
2389 // enough room for two byte trap
2390 __ nop();
2391 }
2392
2393
2394 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2395 assert(exceptionOop->as_register() == rax, "must match");
2396
2397 __ jmp(_unwind_handler_entry);
2398 }
2399
2400
2401 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2402
2403 // optimized version for linear scan:
2404 // * count must be already in ECX (guaranteed by LinearScan)
2405 // * left and dest must be equal
2406 // * tmp must be unused
2407 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2408 assert(left == dest, "left and dest must be equal");
2409 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2410
2411 if (left->is_single_cpu()) {
2412 Register value = left->as_register();
2413 assert(value != SHIFT_count, "left cannot be ECX");
2414
2415 switch (code) {
2416 case lir_shl: __ shll(value); break;
2417 case lir_shr: __ sarl(value); break;
2418 case lir_ushr: __ shrl(value); break;
2419 default: ShouldNotReachHere();
2420 }
2421 } else if (left->is_double_cpu()) {
2422 Register lo = left->as_register_lo();
2423 Register hi = left->as_register_hi();
2424 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2425 switch (code) {
2426 case lir_shl: __ shlptr(lo); break;
2427 case lir_shr: __ sarptr(lo); break;
2428 case lir_ushr: __ shrptr(lo); break;
2429 default: ShouldNotReachHere();
2430 }
2431 } else {
2432 ShouldNotReachHere();
2433 }
2434 }
2435
2436
2437 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2438 if (dest->is_single_cpu()) {
2439 // first move left into dest so that left is not destroyed by the shift
2440 Register value = dest->as_register();
2441 count = count & 0x1F; // Java spec
2442
2443 move_regs(left->as_register(), value);
2444 switch (code) {
2445 case lir_shl: __ shll(value, count); break;
2446 case lir_shr: __ sarl(value, count); break;
2447 case lir_ushr: __ shrl(value, count); break;
2448 default: ShouldNotReachHere();
2449 }
2450 } else if (dest->is_double_cpu()) {
2451 // first move left into dest so that left is not destroyed by the shift
2452 Register value = dest->as_register_lo();
2453 count = count & 0x1F; // Java spec
2454
2455 move_regs(left->as_register_lo(), value);
2456 switch (code) {
2457 case lir_shl: __ shlptr(value, count); break;
2458 case lir_shr: __ sarptr(value, count); break;
2459 case lir_ushr: __ shrptr(value, count); break;
2460 default: ShouldNotReachHere();
2461 }
2462 } else {
2463 ShouldNotReachHere();
2464 }
2465 }
2466
2467
2468 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2469 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2470 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2471 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2472 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2473 }
2474
2475
2476 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2477 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2478 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2479 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2480 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2481 }
2482
2483
2484 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2485 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2486 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2487 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2488 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2489 }
2490
2491
2492 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2493 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2497 }
2498
2499
2500 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2501 if (null_check) {
2502 __ testptr(obj, obj);
2503 __ jcc(Assembler::zero, *slow_path->entry());
2504 }
2505 if (is_dest) {
2506 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2507 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2508 } else {
2509 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2510 }
2511 }
2512
2513
2514 // This code replaces a call to arraycopy; no exception may
2515 // be thrown in this code, they must be thrown in the System.arraycopy
2516 // activation frame; we could save some checks if this would not be the case
2517 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2518 ciArrayKlass* default_type = op->expected_type();
2519 Register src = op->src()->as_register();
2520 Register dst = op->dst()->as_register();
2521 Register src_pos = op->src_pos()->as_register();
2522 Register dst_pos = op->dst_pos()->as_register();
2523 Register length = op->length()->as_register();
2524 Register tmp = op->tmp()->as_register();
2525 Register tmp_load_klass = rscratch1;
2526 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2527
2528 CodeStub* stub = op->stub();
2529 int flags = op->flags();
2530 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2531 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2532
2533 if (flags & LIR_OpArrayCopy::always_slow_path) {
2534 __ jmp(*stub->entry());
2535 __ bind(*stub->continuation());
2536 return;
2537 }
2538
2539 // if we don't know anything, just go through the generic arraycopy
2540 if (default_type == nullptr) {
2541 // save outgoing arguments on stack in case call to System.arraycopy is needed
2542 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2543 // for interpreter calling conventions. Now we have to do it in new style conventions.
2544 // For the moment until C1 gets the new register allocator I just force all the
2545 // args to the right place (except the register args) and then on the back side
2546 // reload the register args properly if we go slow path. Yuck
2547
2548 // These are proper for the calling convention
2549 store_parameter(length, 2);
2550 store_parameter(dst_pos, 1);
2551 store_parameter(dst, 0);
2552
2553 // these are just temporary placements until we need to reload
2554 store_parameter(src_pos, 3);
2555 store_parameter(src, 4);
2556
2557 address copyfunc_addr = StubRoutines::generic_arraycopy();
2558 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2559
2560 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2561 // The arguments are in java calling convention so we can trivially shift them to C
2562 // convention
2563 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2564 __ mov(c_rarg0, j_rarg0);
2565 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2566 __ mov(c_rarg1, j_rarg1);
2567 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2568 __ mov(c_rarg2, j_rarg2);
2569 assert_different_registers(c_rarg3, j_rarg4);
2570 __ mov(c_rarg3, j_rarg3);
2571 #ifdef _WIN64
2572 // Allocate abi space for args but be sure to keep stack aligned
2573 __ subptr(rsp, 6*wordSize);
2574 store_parameter(j_rarg4, 4);
2575 #ifndef PRODUCT
2576 if (PrintC1Statistics) {
2577 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2578 }
2579 #endif
2580 __ call(RuntimeAddress(copyfunc_addr));
2581 __ addptr(rsp, 6*wordSize);
2582 #else
2583 __ mov(c_rarg4, j_rarg4);
2584 #ifndef PRODUCT
2585 if (PrintC1Statistics) {
2586 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2587 }
2588 #endif
2589 __ call(RuntimeAddress(copyfunc_addr));
2590 #endif // _WIN64
2591
2592 __ testl(rax, rax);
2593 __ jcc(Assembler::equal, *stub->continuation());
2594
2595 __ mov(tmp, rax);
2596 __ xorl(tmp, -1);
2597
2598 // Reload values from the stack so they are where the stub
2599 // expects them.
2600 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2601 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2602 __ movptr (length, Address(rsp, 2*BytesPerWord));
2603 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2604 __ movptr (src, Address(rsp, 4*BytesPerWord));
2605
2606 __ subl(length, tmp);
2607 __ addl(src_pos, tmp);
2608 __ addl(dst_pos, tmp);
2609 __ jmp(*stub->entry());
2610
2611 __ bind(*stub->continuation());
2612 return;
2613 }
2614
2615 // Handle inline type arrays
2616 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2617 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2618 }
2619 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2620 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2621 }
2622
2623 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2624
2625 int elem_size = type2aelembytes(basic_type);
2626 Address::ScaleFactor scale;
2627
2628 switch (elem_size) {
2629 case 1 :
2630 scale = Address::times_1;
2631 break;
2632 case 2 :
2633 scale = Address::times_2;
2634 break;
2635 case 4 :
2636 scale = Address::times_4;
2637 break;
2638 case 8 :
2639 scale = Address::times_8;
2640 break;
2641 default:
2642 scale = Address::no_scale;
2643 ShouldNotReachHere();
2644 }
2645
2646 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2647 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2648
2649 // length and pos's are all sign extended at this point on 64bit
2650
2651 // test for null
2652 if (flags & LIR_OpArrayCopy::src_null_check) {
2653 __ testptr(src, src);
2654 __ jcc(Assembler::zero, *stub->entry());
2655 }
2656 if (flags & LIR_OpArrayCopy::dst_null_check) {
2657 __ testptr(dst, dst);
2658 __ jcc(Assembler::zero, *stub->entry());
2659 }
2660
2661 // If the compiler was not able to prove that exact type of the source or the destination
2662 // of the arraycopy is an array type, check at runtime if the source or the destination is
2663 // an instance type.
2664 if (flags & LIR_OpArrayCopy::type_check) {
2665 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2666 __ load_klass(tmp, dst, tmp_load_klass);
2667 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2668 __ jcc(Assembler::greaterEqual, *stub->entry());
2669 }
2670
2671 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2672 __ load_klass(tmp, src, tmp_load_klass);
2673 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2674 __ jcc(Assembler::greaterEqual, *stub->entry());
2675 }
2676 }
2677
2678 // check if negative
2679 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2680 __ testl(src_pos, src_pos);
2681 __ jcc(Assembler::less, *stub->entry());
2682 }
2683 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2684 __ testl(dst_pos, dst_pos);
2685 __ jcc(Assembler::less, *stub->entry());
2686 }
2687
2688 if (flags & LIR_OpArrayCopy::src_range_check) {
2689 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2690 __ cmpl(tmp, src_length_addr);
2691 __ jcc(Assembler::above, *stub->entry());
2692 }
2693 if (flags & LIR_OpArrayCopy::dst_range_check) {
2694 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2695 __ cmpl(tmp, dst_length_addr);
2696 __ jcc(Assembler::above, *stub->entry());
2697 }
2698
2699 if (flags & LIR_OpArrayCopy::length_positive_check) {
2700 __ testl(length, length);
2701 __ jcc(Assembler::less, *stub->entry());
2702 }
2703
2704 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2705 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2706
2707 if (flags & LIR_OpArrayCopy::type_check) {
2708 // We don't know the array types are compatible
2709 if (basic_type != T_OBJECT) {
2710 // Simple test for basic type arrays
2711 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2712 __ jcc(Assembler::notEqual, *stub->entry());
2713 } else {
2714 // For object arrays, if src is a sub class of dst then we can
2715 // safely do the copy.
2716 Label cont, slow;
2717
2718 __ push_ppx(src);
2719 __ push_ppx(dst);
2720
2721 __ load_klass(src, src, tmp_load_klass);
2722 __ load_klass(dst, dst, tmp_load_klass);
2723
2724 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2725
2726 __ push_ppx(src);
2727 __ push_ppx(dst);
2728 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2729 __ pop_ppx(dst);
2730 __ pop_ppx(src);
2731
2732 __ testl(src, src);
2733 __ jcc(Assembler::notEqual, cont);
2734
2735 __ bind(slow);
2736 __ pop_ppx(dst);
2737 __ pop_ppx(src);
2738
2739 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2740 if (copyfunc_addr != nullptr) { // use stub if available
2741 // src is not a sub class of dst so we have to do a
2742 // per-element check.
2743
2744 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2745 if ((flags & mask) != mask) {
2746 // Check that at least both of them object arrays.
2747 assert(flags & mask, "one of the two should be known to be an object array");
2748
2749 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2750 __ load_klass(tmp, src, tmp_load_klass);
2751 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2752 __ load_klass(tmp, dst, tmp_load_klass);
2753 }
2754 int lh_offset = in_bytes(Klass::layout_helper_offset());
2755 Address klass_lh_addr(tmp, lh_offset);
2756 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2757 __ cmpl(klass_lh_addr, objArray_lh);
2758 __ jcc(Assembler::notEqual, *stub->entry());
2759 }
2760
2761 // Spill because stubs can use any register they like and it's
2762 // easier to restore just those that we care about.
2763 store_parameter(dst, 0);
2764 store_parameter(dst_pos, 1);
2765 store_parameter(length, 2);
2766 store_parameter(src_pos, 3);
2767 store_parameter(src, 4);
2768
2769 __ movl2ptr(length, length); //higher 32bits must be null
2770
2771 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2772 assert_different_registers(c_rarg0, dst, dst_pos, length);
2773 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2774 assert_different_registers(c_rarg1, dst, length);
2775
2776 __ mov(c_rarg2, length);
2777 assert_different_registers(c_rarg2, dst);
2778
2779 #ifdef _WIN64
2780 // Allocate abi space for args but be sure to keep stack aligned
2781 __ subptr(rsp, 6*wordSize);
2782 __ load_klass(c_rarg3, dst, tmp_load_klass);
2783 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2784 store_parameter(c_rarg3, 4);
2785 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2786 __ call(RuntimeAddress(copyfunc_addr));
2787 __ addptr(rsp, 6*wordSize);
2788 #else
2789 __ load_klass(c_rarg4, dst, tmp_load_klass);
2790 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2791 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2792 __ call(RuntimeAddress(copyfunc_addr));
2793 #endif
2794
2795 #ifndef PRODUCT
2796 if (PrintC1Statistics) {
2797 Label failed;
2798 __ testl(rax, rax);
2799 __ jcc(Assembler::notZero, failed);
2800 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2801 __ bind(failed);
2802 }
2803 #endif
2804
2805 __ testl(rax, rax);
2806 __ jcc(Assembler::zero, *stub->continuation());
2807
2808 #ifndef PRODUCT
2809 if (PrintC1Statistics) {
2810 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2811 }
2812 #endif
2813
2814 __ mov(tmp, rax);
2815
2816 __ xorl(tmp, -1);
2817
2818 // Restore previously spilled arguments
2819 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2820 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2821 __ movptr (length, Address(rsp, 2*BytesPerWord));
2822 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2823 __ movptr (src, Address(rsp, 4*BytesPerWord));
2824
2825
2826 __ subl(length, tmp);
2827 __ addl(src_pos, tmp);
2828 __ addl(dst_pos, tmp);
2829 }
2830
2831 __ jmp(*stub->entry());
2832
2833 __ bind(cont);
2834 __ pop(dst);
2835 __ pop(src);
2836 }
2837 }
2838
2839 #ifdef ASSERT
2840 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2841 // Sanity check the known type with the incoming class. For the
2842 // primitive case the types must match exactly with src.klass and
2843 // dst.klass each exactly matching the default type. For the
2844 // object array case, if no type check is needed then either the
2845 // dst type is exactly the expected type and the src type is a
2846 // subtype which we can't check or src is the same array as dst
2847 // but not necessarily exactly of type default_type.
2848 Label known_ok, halt;
2849 __ mov_metadata(tmp, default_type->constant_encoding());
2850 __ encode_klass_not_null(tmp, rscratch1);
2851
2852 if (basic_type != T_OBJECT) {
2853 __ cmp_klass(tmp, dst, tmp2);
2854 __ jcc(Assembler::notEqual, halt);
2855 __ cmp_klass(tmp, src, tmp2);
2856 __ jcc(Assembler::equal, known_ok);
2857 } else {
2858 __ cmp_klass(tmp, dst, tmp2);
2859 __ jcc(Assembler::equal, known_ok);
2860 __ cmpptr(src, dst);
2861 __ jcc(Assembler::equal, known_ok);
2862 }
2863 __ bind(halt);
2864 __ stop("incorrect type information in arraycopy");
2865 __ bind(known_ok);
2866 }
2867 #endif
2868
2869 #ifndef PRODUCT
2870 if (PrintC1Statistics) {
2871 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2872 }
2873 #endif
2874
2875 assert_different_registers(c_rarg0, dst, dst_pos, length);
2876 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2877 assert_different_registers(c_rarg1, length);
2878 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2879 __ mov(c_rarg2, length);
2880
2881 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2882 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2883 const char *name;
2884 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2885 __ call_VM_leaf(entry, 0);
2886
2887 if (stub != nullptr) {
2888 __ bind(*stub->continuation());
2889 }
2890 }
2891
2892 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2893 assert(op->crc()->is_single_cpu(), "crc must be register");
2894 assert(op->val()->is_single_cpu(), "byte value must be register");
2895 assert(op->result_opr()->is_single_cpu(), "result must be register");
2896 Register crc = op->crc()->as_register();
2897 Register val = op->val()->as_register();
2898 Register res = op->result_opr()->as_register();
2899
2900 assert_different_registers(val, crc, res);
2901
2902 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2903 __ notl(crc); // ~crc
2904 __ update_byte_crc32(crc, val, res);
2905 __ notl(crc); // ~crc
2906 __ mov(res, crc);
2907 }
2908
2909 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2910 Register obj = op->obj_opr()->as_register(); // may not be an oop
2911 Register hdr = op->hdr_opr()->as_register();
2912 Register lock = op->lock_opr()->as_register();
2913 if (op->code() == lir_lock) {
2914 Register tmp = op->scratch_opr()->as_register();
2915 // add debug info for NullPointerException only if one is possible
2916 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2917 if (op->info() != nullptr) {
2918 add_debug_info_for_null_check(null_check_offset, op->info());
2919 }
2920 // done
2921 } else if (op->code() == lir_unlock) {
2922 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2923 } else {
2924 Unimplemented();
2925 }
2926 __ bind(*op->stub()->continuation());
2927 }
2928
2929 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2930 Register obj = op->obj()->as_pointer_register();
2931 Register result = op->result_opr()->as_pointer_register();
2932
2933 CodeEmitInfo* info = op->info();
2934 if (info != nullptr) {
2935 add_debug_info_for_null_check_here(info);
2936 }
2937
2938 __ load_klass(result, obj, rscratch1);
2939 }
2940
2941 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2942 ciMethod* method = op->profiled_method();
2943 int bci = op->profiled_bci();
2944 ciMethod* callee = op->profiled_callee();
2945 Register tmp_load_klass = rscratch1;
2946
2947 // Update counter for all call types
2948 ciMethodData* md = method->method_data_or_null();
2949 assert(md != nullptr, "Sanity");
2950 ciProfileData* data = md->bci_to_data(bci);
2951 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2952 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2953 Register mdo = op->mdo()->as_register();
2954 __ mov_metadata(mdo, md->constant_encoding());
2955 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2956 // Perform additional virtual call profiling for invokevirtual and
2957 // invokeinterface bytecodes
2958 if (op->should_profile_receiver_type()) {
2959 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2960 Register recv = op->recv()->as_register();
2961 assert_different_registers(mdo, recv);
2962 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2963 ciKlass* known_klass = op->known_holder();
2964 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2965 // We know the type that will be seen at this call site; we can
2966 // statically update the MethodData* rather than needing to do
2967 // dynamic tests on the receiver type.
2968 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2969 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2970 ciKlass* receiver = vc_data->receiver(i);
2971 if (known_klass->equals(receiver)) {
2972 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2973 __ addptr(data_addr, DataLayout::counter_increment);
2974 return;
2975 }
2976 }
2977 // Receiver type is not found in profile data.
2978 // Fall back to runtime helper to handle the rest at runtime.
2979 __ mov_metadata(recv, known_klass->constant_encoding());
2980 } else {
2981 __ load_klass(recv, recv, tmp_load_klass);
2982 }
2983 type_profile_helper(mdo, md, data, recv);
2984 } else {
2985 // Static call
2986 __ addptr(counter_addr, DataLayout::counter_increment);
2987 }
2988 }
2989
2990 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2991 Register obj = op->obj()->as_register();
2992 Register tmp = op->tmp()->as_pointer_register();
2993 Register tmp_load_klass = rscratch1;
2994 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2995 ciKlass* exact_klass = op->exact_klass();
2996 intptr_t current_klass = op->current_klass();
2997 bool not_null = op->not_null();
2998 bool no_conflict = op->no_conflict();
2999
3000 Label update, next, none;
3001
3002 bool do_null = !not_null;
3003 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3004 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3005
3006 assert(do_null || do_update, "why are we here?");
3007 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3008
3009 __ verify_oop(obj);
3010
3011 #ifdef ASSERT
3012 if (obj == tmp) {
3013 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3014 } else {
3015 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3016 }
3017 #endif
3018 if (do_null) {
3019 __ testptr(obj, obj);
3020 __ jccb(Assembler::notZero, update);
3021 if (!TypeEntries::was_null_seen(current_klass)) {
3022 __ testptr(mdo_addr, TypeEntries::null_seen);
3023 #ifndef ASSERT
3024 __ jccb(Assembler::notZero, next); // already set
3025 #else
3026 __ jcc(Assembler::notZero, next); // already set
3027 #endif
3028 // atomic update to prevent overwriting Klass* with 0
3029 __ lock();
3030 __ orptr(mdo_addr, TypeEntries::null_seen);
3031 }
3032 if (do_update) {
3033 #ifndef ASSERT
3034 __ jmpb(next);
3035 }
3036 #else
3037 __ jmp(next);
3038 }
3039 } else {
3040 __ testptr(obj, obj);
3041 __ jcc(Assembler::notZero, update);
3042 __ stop("unexpected null obj");
3043 #endif
3044 }
3045
3046 __ bind(update);
3047
3048 if (do_update) {
3049 #ifdef ASSERT
3050 if (exact_klass != nullptr) {
3051 Label ok;
3052 __ load_klass(tmp, obj, tmp_load_klass);
3053 __ push_ppx(tmp);
3054 __ mov_metadata(tmp, exact_klass->constant_encoding());
3055 __ cmpptr(tmp, Address(rsp, 0));
3056 __ jcc(Assembler::equal, ok);
3057 __ stop("exact klass and actual klass differ");
3058 __ bind(ok);
3059 __ pop_ppx(tmp);
3060 }
3061 #endif
3062 if (!no_conflict) {
3063 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3064 if (exact_klass != nullptr) {
3065 __ mov_metadata(tmp, exact_klass->constant_encoding());
3066 } else {
3067 __ load_klass(tmp, obj, tmp_load_klass);
3068 }
3069 __ mov(rscratch1, tmp); // save original value before XOR
3070 __ xorptr(tmp, mdo_addr);
3071 __ testptr(tmp, TypeEntries::type_klass_mask);
3072 // klass seen before, nothing to do. The unknown bit may have been
3073 // set already but no need to check.
3074 __ jccb(Assembler::zero, next);
3075
3076 __ testptr(tmp, TypeEntries::type_unknown);
3077 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3078
3079 if (TypeEntries::is_type_none(current_klass)) {
3080 __ testptr(mdo_addr, TypeEntries::type_mask);
3081 __ jccb(Assembler::zero, none);
3082 // There is a chance that the checks above (re-reading profiling
3083 // data from memory) fail if another thread has just set the
3084 // profiling to this obj's klass
3085 __ mov(tmp, rscratch1); // get back original value before XOR
3086 __ xorptr(tmp, mdo_addr);
3087 __ testptr(tmp, TypeEntries::type_klass_mask);
3088 __ jccb(Assembler::zero, next);
3089 }
3090 } else {
3091 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3092 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3093
3094 __ testptr(mdo_addr, TypeEntries::type_unknown);
3095 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3096 }
3097
3098 // different than before. Cannot keep accurate profile.
3099 __ orptr(mdo_addr, TypeEntries::type_unknown);
3100
3101 if (TypeEntries::is_type_none(current_klass)) {
3102 __ jmpb(next);
3103
3104 __ bind(none);
3105 // first time here. Set profile type.
3106 __ movptr(mdo_addr, tmp);
3107 #ifdef ASSERT
3108 __ andptr(tmp, TypeEntries::type_klass_mask);
3109 __ verify_klass_ptr(tmp);
3110 #endif
3111 }
3112 } else {
3113 // There's a single possible klass at this profile point
3114 assert(exact_klass != nullptr, "should be");
3115 if (TypeEntries::is_type_none(current_klass)) {
3116 __ mov_metadata(tmp, exact_klass->constant_encoding());
3117 __ xorptr(tmp, mdo_addr);
3118 __ testptr(tmp, TypeEntries::type_klass_mask);
3119 #ifdef ASSERT
3120 __ jcc(Assembler::zero, next);
3121
3122 {
3123 Label ok;
3124 __ push_ppx(tmp);
3125 __ testptr(mdo_addr, TypeEntries::type_mask);
3126 __ jcc(Assembler::zero, ok);
3127 // may have been set by another thread
3128 __ mov_metadata(tmp, exact_klass->constant_encoding());
3129 __ xorptr(tmp, mdo_addr);
3130 __ testptr(tmp, TypeEntries::type_mask);
3131 __ jcc(Assembler::zero, ok);
3132
3133 __ stop("unexpected profiling mismatch");
3134 __ bind(ok);
3135 __ pop_ppx(tmp);
3136 }
3137 #else
3138 __ jccb(Assembler::zero, next);
3139 #endif
3140 // first time here. Set profile type.
3141 __ movptr(mdo_addr, tmp);
3142 #ifdef ASSERT
3143 __ andptr(tmp, TypeEntries::type_klass_mask);
3144 __ verify_klass_ptr(tmp);
3145 #endif
3146 } else {
3147 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3148 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3149
3150 __ testptr(mdo_addr, TypeEntries::type_unknown);
3151 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3152
3153 __ orptr(mdo_addr, TypeEntries::type_unknown);
3154 }
3155 }
3156 }
3157 __ bind(next);
3158 }
3159
3160 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3161 Register obj = op->obj()->as_register();
3162 Register tmp = op->tmp()->as_pointer_register();
3163 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3164 bool not_null = op->not_null();
3165 int flag = op->flag();
3166
3167 Label not_inline_type;
3168 if (!not_null) {
3169 __ testptr(obj, obj);
3170 __ jccb(Assembler::zero, not_inline_type);
3171 }
3172
3173 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3174
3175 __ orb(mdo_addr, flag);
3176
3177 __ bind(not_inline_type);
3178 }
3179
3180
3181 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3182 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3183 }
3184
3185
3186 void LIR_Assembler::align_backward_branch_target() {
3187 __ align(BytesPerWord);
3188 }
3189
3190
3191 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3192 if (left->is_single_cpu()) {
3193 __ negl(left->as_register());
3194 move_regs(left->as_register(), dest->as_register());
3195
3196 } else if (left->is_double_cpu()) {
3197 Register lo = left->as_register_lo();
3198 Register dst = dest->as_register_lo();
3199 __ movptr(dst, lo);
3200 __ negptr(dst);
3201
3202 } else if (dest->is_single_xmm()) {
3203 assert(!tmp->is_valid(), "do not need temporary");
3204 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3205 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3206 }
3207 __ xorps(dest->as_xmm_float_reg(),
3208 ExternalAddress((address)float_signflip_pool),
3209 rscratch1);
3210 } else if (dest->is_double_xmm()) {
3211 assert(!tmp->is_valid(), "do not need temporary");
3212 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3213 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3214 }
3215 __ xorpd(dest->as_xmm_double_reg(),
3216 ExternalAddress((address)double_signflip_pool),
3217 rscratch1);
3218 } else {
3219 ShouldNotReachHere();
3220 }
3221 }
3222
3223
3224 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3225 assert(src->is_address(), "must be an address");
3226 assert(dest->is_register(), "must be a register");
3227
3228 PatchingStub* patch = nullptr;
3229 if (patch_code != lir_patch_none) {
3230 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3231 }
3232
3233 Register reg = dest->as_pointer_register();
3234 LIR_Address* addr = src->as_address_ptr();
3235 __ lea(reg, as_Address(addr));
3236
3237 if (patch != nullptr) {
3238 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3239 }
3240 }
3241
3242
3243
3244 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3245 assert(!tmp->is_valid(), "don't need temporary");
3246 __ call(RuntimeAddress(dest));
3247 if (info != nullptr) {
3248 add_call_info_here(info);
3249 }
3250 __ post_call_nop();
3251 }
3252
3253
3254 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3255 assert(type == T_LONG, "only for volatile long fields");
3256
3257 if (info != nullptr) {
3258 add_debug_info_for_null_check_here(info);
3259 }
3260
3261 if (src->is_double_xmm()) {
3262 if (dest->is_double_cpu()) {
3263 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3264 } else if (dest->is_double_stack()) {
3265 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3266 } else if (dest->is_address()) {
3267 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3268 } else {
3269 ShouldNotReachHere();
3270 }
3271
3272 } else if (dest->is_double_xmm()) {
3273 if (src->is_double_stack()) {
3274 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3275 } else if (src->is_address()) {
3276 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3277 } else {
3278 ShouldNotReachHere();
3279 }
3280
3281 } else {
3282 ShouldNotReachHere();
3283 }
3284 }
3285
3286 #ifdef ASSERT
3287 // emit run-time assertion
3288 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3289 assert(op->code() == lir_assert, "must be");
3290
3291 if (op->in_opr1()->is_valid()) {
3292 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3293 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3294 } else {
3295 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3296 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3297 }
3298
3299 Label ok;
3300 if (op->condition() != lir_cond_always) {
3301 Assembler::Condition acond = Assembler::zero;
3302 switch (op->condition()) {
3303 case lir_cond_equal: acond = Assembler::equal; break;
3304 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3305 case lir_cond_less: acond = Assembler::less; break;
3306 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3307 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3308 case lir_cond_greater: acond = Assembler::greater; break;
3309 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3310 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3311 default: ShouldNotReachHere();
3312 }
3313 __ jcc(acond, ok);
3314 }
3315 if (op->halt()) {
3316 const char* str = __ code_string(op->msg());
3317 __ stop(str);
3318 } else {
3319 breakpoint();
3320 }
3321 __ bind(ok);
3322 }
3323 #endif
3324
3325 void LIR_Assembler::membar() {
3326 // QQQ sparc TSO uses this,
3327 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3328 }
3329
3330 void LIR_Assembler::membar_acquire() {
3331 // No x86 machines currently require load fences
3332 }
3333
3334 void LIR_Assembler::membar_release() {
3335 // No x86 machines currently require store fences
3336 }
3337
3338 void LIR_Assembler::membar_loadload() {
3339 // no-op
3340 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3341 }
3342
3343 void LIR_Assembler::membar_storestore() {
3344 // no-op
3345 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3346 }
3347
3348 void LIR_Assembler::membar_loadstore() {
3349 // no-op
3350 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3351 }
3352
3353 void LIR_Assembler::membar_storeload() {
3354 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3355 }
3356
3357 void LIR_Assembler::on_spin_wait() {
3358 __ pause ();
3359 }
3360
3361 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3362 assert(result_reg->is_register(), "check");
3363 __ mov(result_reg->as_register(), r15_thread);
3364 }
3365
3366 void LIR_Assembler::check_orig_pc() {
3367 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3368 }
3369
3370 void LIR_Assembler::peephole(LIR_List*) {
3371 // do nothing for now
3372 }
3373
3374 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3375 assert(data == dest, "xchg/xadd uses only 2 operands");
3376
3377 if (data->type() == T_INT) {
3378 if (code == lir_xadd) {
3379 __ lock();
3380 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3381 } else {
3382 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3383 }
3384 } else if (data->is_oop()) {
3385 assert (code == lir_xchg, "xadd for oops");
3386 Register obj = data->as_register();
3387 if (UseCompressedOops) {
3388 __ encode_heap_oop(obj);
3389 __ xchgl(obj, as_Address(src->as_address_ptr()));
3390 __ decode_heap_oop(obj);
3391 } else {
3392 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3393 }
3394 } else if (data->type() == T_LONG) {
3395 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3396 if (code == lir_xadd) {
3397 __ lock();
3398 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3399 } else {
3400 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3401 }
3402 } else {
3403 ShouldNotReachHere();
3404 }
3405 }
3406
3407 #undef __