1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "ci/ciUtilities.hpp"
36 #include "code/aotCodeCache.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456
457 Label start;
458 __ bind(start);
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
468 "out of bounds read in post-call NOP check");
469 __ end_a_stub();
470
471 return entry_offset;
472 }
473
474 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
475 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
476 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
477 assert(result->fpu() == 0, "result must already be on TOS");
478 }
479
480 // Pop the stack before the safepoint code
481 __ remove_frame(initial_frame_size_in_bytes());
482
483 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
484 __ reserved_stack_check();
485 }
486
487 // Note: we do not need to round double result; float result has the right precision
488 // the poll sets the condition code, but no data registers
489
490 code_stub->set_safepoint_offset(__ offset());
491 __ relocate(relocInfo::poll_return_type);
492 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
493 __ ret(0);
494 }
495
496
497 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
498 guarantee(info != nullptr, "Shouldn't be null");
499 int offset = __ offset();
500 const Register poll_addr = rscratch1;
501 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
502 add_debug_info_for_branch(info);
503 __ relocate(relocInfo::poll_type);
504 address pre_pc = __ pc();
505 __ testl(rax, Address(poll_addr, 0));
506 address post_pc = __ pc();
507 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
508 return offset;
509 }
510
511
512 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
513 if (from_reg != to_reg) __ mov(to_reg, from_reg);
514 }
515
516 void LIR_Assembler::swap_reg(Register a, Register b) {
517 __ xchgptr(a, b);
518 }
519
520
521 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
522 assert(src->is_constant(), "should not call otherwise");
523 assert(dest->is_register(), "should not call otherwise");
524 LIR_Const* c = src->as_constant_ptr();
525
526 switch (c->type()) {
527 case T_INT: {
528 assert(patch_code == lir_patch_none, "no patching handled here");
529 __ movl(dest->as_register(), c->as_jint());
530 break;
531 }
532
533 case T_ADDRESS: {
534 assert(patch_code == lir_patch_none, "no patching handled here");
535 __ movptr(dest->as_register(), c->as_jint());
536 break;
537 }
538
539 case T_LONG: {
540 assert(patch_code == lir_patch_none, "no patching handled here");
541 #if INCLUDE_CDS
542 if (AOTCodeCache::is_on_for_dump()) {
543 address b = c->as_pointer();
544 if (b == (address)ThreadIdentifier::unsafe_offset()) {
545 __ lea(dest->as_register_lo(), ExternalAddress(b));
546 break;
547 }
548 if (AOTRuntimeConstants::contains(b)) {
549 __ load_aotrc_address(dest->as_register_lo(), b);
550 break;
551 }
552 }
553 #endif
554 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
555 break;
556 }
557
558 case T_OBJECT: {
559 if (patch_code != lir_patch_none) {
560 jobject2reg_with_patching(dest->as_register(), info);
561 } else {
562 __ movoop(dest->as_register(), c->as_jobject());
563 }
564 break;
565 }
566
567 case T_METADATA: {
568 if (patch_code != lir_patch_none) {
569 klass2reg_with_patching(dest->as_register(), info);
570 } else {
571 __ mov_metadata(dest->as_register(), c->as_metadata());
572 }
573 break;
574 }
575
576 case T_FLOAT: {
577 if (dest->is_single_xmm()) {
578 if (UseAVX <= 2 && c->is_zero_float()) {
579 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
580 } else {
581 __ movflt(dest->as_xmm_float_reg(),
582 InternalAddress(float_constant(c->as_jfloat())));
583 }
584 } else {
585 ShouldNotReachHere();
586 }
587 break;
588 }
589
590 case T_DOUBLE: {
591 if (dest->is_double_xmm()) {
592 if (UseAVX <= 2 && c->is_zero_double()) {
593 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
594 } else {
595 __ movdbl(dest->as_xmm_double_reg(),
596 InternalAddress(double_constant(c->as_jdouble())));
597 }
598 } else {
599 ShouldNotReachHere();
600 }
601 break;
602 }
603
604 default:
605 ShouldNotReachHere();
606 }
607 }
608
609 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
610 assert(src->is_constant(), "should not call otherwise");
611 assert(dest->is_stack(), "should not call otherwise");
612 LIR_Const* c = src->as_constant_ptr();
613
614 switch (c->type()) {
615 case T_INT: // fall through
616 case T_FLOAT:
617 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
618 break;
619
620 case T_ADDRESS:
621 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
622 break;
623
624 case T_OBJECT:
625 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
626 break;
627
628 case T_LONG: // fall through
629 case T_DOUBLE:
630 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
631 lo_word_offset_in_bytes),
632 (intptr_t)c->as_jlong_bits(),
633 rscratch1);
634 break;
635
636 default:
637 ShouldNotReachHere();
638 }
639 }
640
641 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
642 assert(src->is_constant(), "should not call otherwise");
643 assert(dest->is_address(), "should not call otherwise");
644 LIR_Const* c = src->as_constant_ptr();
645 LIR_Address* addr = dest->as_address_ptr();
646
647 int null_check_here = code_offset();
648 switch (type) {
649 case T_INT: // fall through
650 case T_FLOAT:
651 __ movl(as_Address(addr), c->as_jint_bits());
652 break;
653
654 case T_ADDRESS:
655 __ movptr(as_Address(addr), c->as_jint_bits());
656 break;
657
658 case T_OBJECT: // fall through
659 case T_ARRAY:
660 if (c->as_jobject() == nullptr) {
661 if (UseCompressedOops && !wide) {
662 __ movl(as_Address(addr), NULL_WORD);
663 } else {
664 __ xorptr(rscratch1, rscratch1);
665 null_check_here = code_offset();
666 __ movptr(as_Address(addr), rscratch1);
667 }
668 } else {
669 if (is_literal_address(addr)) {
670 ShouldNotReachHere();
671 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
672 } else {
673 __ movoop(rscratch1, c->as_jobject());
674 if (UseCompressedOops && !wide) {
675 __ encode_heap_oop(rscratch1);
676 null_check_here = code_offset();
677 __ movl(as_Address_lo(addr), rscratch1);
678 } else {
679 null_check_here = code_offset();
680 __ movptr(as_Address_lo(addr), rscratch1);
681 }
682 }
683 }
684 break;
685
686 case T_LONG: // fall through
687 case T_DOUBLE:
688 if (is_literal_address(addr)) {
689 ShouldNotReachHere();
690 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
691 } else {
692 __ movptr(r10, (intptr_t)c->as_jlong_bits());
693 null_check_here = code_offset();
694 __ movptr(as_Address_lo(addr), r10);
695 }
696 break;
697
698 case T_BOOLEAN: // fall through
699 case T_BYTE:
700 __ movb(as_Address(addr), c->as_jint() & 0xFF);
701 break;
702
703 case T_CHAR: // fall through
704 case T_SHORT:
705 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
706 break;
707
708 default:
709 ShouldNotReachHere();
710 };
711
712 if (info != nullptr) {
713 add_debug_info_for_null_check(null_check_here, info);
714 }
715 }
716
717
718 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
719 assert(src->is_register(), "should not call otherwise");
720 assert(dest->is_register(), "should not call otherwise");
721
722 // move between cpu-registers
723 if (dest->is_single_cpu()) {
724 if (src->type() == T_LONG) {
725 // Can do LONG -> OBJECT
726 move_regs(src->as_register_lo(), dest->as_register());
727 return;
728 }
729 assert(src->is_single_cpu(), "must match");
730 if (src->type() == T_OBJECT) {
731 __ verify_oop(src->as_register());
732 }
733 move_regs(src->as_register(), dest->as_register());
734
735 } else if (dest->is_double_cpu()) {
736 if (is_reference_type(src->type())) {
737 // Surprising to me but we can see move of a long to t_object
738 __ verify_oop(src->as_register());
739 move_regs(src->as_register(), dest->as_register_lo());
740 return;
741 }
742 assert(src->is_double_cpu(), "must match");
743 Register f_lo = src->as_register_lo();
744 Register f_hi = src->as_register_hi();
745 Register t_lo = dest->as_register_lo();
746 Register t_hi = dest->as_register_hi();
747 assert(f_hi == f_lo, "must be same");
748 assert(t_hi == t_lo, "must be same");
749 move_regs(f_lo, t_lo);
750
751 // move between xmm-registers
752 } else if (dest->is_single_xmm()) {
753 assert(src->is_single_xmm(), "must match");
754 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
755 } else if (dest->is_double_xmm()) {
756 assert(src->is_double_xmm(), "must match");
757 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
758
759 } else {
760 ShouldNotReachHere();
761 }
762 }
763
764 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
765 assert(src->is_register(), "should not call otherwise");
766 assert(dest->is_stack(), "should not call otherwise");
767
768 if (src->is_single_cpu()) {
769 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
770 if (is_reference_type(type)) {
771 __ verify_oop(src->as_register());
772 __ movptr (dst, src->as_register());
773 } else if (type == T_METADATA || type == T_ADDRESS) {
774 __ movptr (dst, src->as_register());
775 } else {
776 __ movl (dst, src->as_register());
777 }
778
779 } else if (src->is_double_cpu()) {
780 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
781 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
782 __ movptr (dstLO, src->as_register_lo());
783
784 } else if (src->is_single_xmm()) {
785 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
786 __ movflt(dst_addr, src->as_xmm_float_reg());
787
788 } else if (src->is_double_xmm()) {
789 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
790 __ movdbl(dst_addr, src->as_xmm_double_reg());
791
792 } else {
793 ShouldNotReachHere();
794 }
795 }
796
797
798 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
799 LIR_Address* to_addr = dest->as_address_ptr();
800 PatchingStub* patch = nullptr;
801 Register compressed_src = rscratch1;
802
803 if (is_reference_type(type)) {
804 __ verify_oop(src->as_register());
805 if (UseCompressedOops && !wide) {
806 __ movptr(compressed_src, src->as_register());
807 __ encode_heap_oop(compressed_src);
808 if (patch_code != lir_patch_none) {
809 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
810 }
811 }
812 }
813
814 if (patch_code != lir_patch_none) {
815 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
816 Address toa = as_Address(to_addr);
817 assert(toa.disp() != 0, "must have");
818 }
819
820 int null_check_here = code_offset();
821 switch (type) {
822 case T_FLOAT: {
823 assert(src->is_single_xmm(), "not a float");
824 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
825 break;
826 }
827
828 case T_DOUBLE: {
829 assert(src->is_double_xmm(), "not a double");
830 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
831 break;
832 }
833
834 case T_ARRAY: // fall through
835 case T_OBJECT: // fall through
836 if (UseCompressedOops && !wide) {
837 __ movl(as_Address(to_addr), compressed_src);
838 } else {
839 __ movptr(as_Address(to_addr), src->as_register());
840 }
841 break;
842 case T_ADDRESS:
843 __ movptr(as_Address(to_addr), src->as_register());
844 break;
845 case T_INT:
846 __ movl(as_Address(to_addr), src->as_register());
847 break;
848
849 case T_LONG: {
850 Register from_lo = src->as_register_lo();
851 Register from_hi = src->as_register_hi();
852 __ movptr(as_Address_lo(to_addr), from_lo);
853 break;
854 }
855
856 case T_BYTE: // fall through
857 case T_BOOLEAN: {
858 Register src_reg = src->as_register();
859 Address dst_addr = as_Address(to_addr);
860 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
861 __ movb(dst_addr, src_reg);
862 break;
863 }
864
865 case T_CHAR: // fall through
866 case T_SHORT:
867 __ movw(as_Address(to_addr), src->as_register());
868 break;
869
870 default:
871 ShouldNotReachHere();
872 }
873 if (info != nullptr) {
874 add_debug_info_for_null_check(null_check_here, info);
875 }
876
877 if (patch_code != lir_patch_none) {
878 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
879 }
880 }
881
882
883 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
884 assert(src->is_stack(), "should not call otherwise");
885 assert(dest->is_register(), "should not call otherwise");
886
887 if (dest->is_single_cpu()) {
888 if (is_reference_type(type)) {
889 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
890 __ verify_oop(dest->as_register());
891 } else if (type == T_METADATA || type == T_ADDRESS) {
892 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
893 } else {
894 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
895 }
896
897 } else if (dest->is_double_cpu()) {
898 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
899 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
900 __ movptr(dest->as_register_lo(), src_addr_LO);
901
902 } else if (dest->is_single_xmm()) {
903 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
904 __ movflt(dest->as_xmm_float_reg(), src_addr);
905
906 } else if (dest->is_double_xmm()) {
907 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
908 __ movdbl(dest->as_xmm_double_reg(), src_addr);
909
910 } else {
911 ShouldNotReachHere();
912 }
913 }
914
915
916 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
917 if (src->is_single_stack()) {
918 if (is_reference_type(type)) {
919 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
920 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
921 } else {
922 //no pushl on 64bits
923 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
924 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
925 }
926
927 } else if (src->is_double_stack()) {
928 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
929 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
930
931 } else {
932 ShouldNotReachHere();
933 }
934 }
935
936
937 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
938 assert(src->is_address(), "should not call otherwise");
939 assert(dest->is_register(), "should not call otherwise");
940
941 LIR_Address* addr = src->as_address_ptr();
942 Address from_addr = as_Address(addr);
943
944 if (addr->base()->type() == T_OBJECT) {
945 __ verify_oop(addr->base()->as_pointer_register());
946 }
947
948 switch (type) {
949 case T_BOOLEAN: // fall through
950 case T_BYTE: // fall through
951 case T_CHAR: // fall through
952 case T_SHORT:
953 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
954 // on pre P6 processors we may get partial register stalls
955 // so blow away the value of to_rinfo before loading a
956 // partial word into it. Do it here so that it precedes
957 // the potential patch point below.
958 __ xorptr(dest->as_register(), dest->as_register());
959 }
960 break;
961 default:
962 break;
963 }
964
965 PatchingStub* patch = nullptr;
966 if (patch_code != lir_patch_none) {
967 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
968 assert(from_addr.disp() != 0, "must have");
969 }
970 if (info != nullptr) {
971 add_debug_info_for_null_check_here(info);
972 }
973
974 switch (type) {
975 case T_FLOAT: {
976 if (dest->is_single_xmm()) {
977 __ movflt(dest->as_xmm_float_reg(), from_addr);
978 } else {
979 ShouldNotReachHere();
980 }
981 break;
982 }
983
984 case T_DOUBLE: {
985 if (dest->is_double_xmm()) {
986 __ movdbl(dest->as_xmm_double_reg(), from_addr);
987 } else {
988 ShouldNotReachHere();
989 }
990 break;
991 }
992
993 case T_OBJECT: // fall through
994 case T_ARRAY: // fall through
995 if (UseCompressedOops && !wide) {
996 __ movl(dest->as_register(), from_addr);
997 } else {
998 __ movptr(dest->as_register(), from_addr);
999 }
1000 break;
1001
1002 case T_ADDRESS:
1003 __ movptr(dest->as_register(), from_addr);
1004 break;
1005 case T_INT:
1006 __ movl(dest->as_register(), from_addr);
1007 break;
1008
1009 case T_LONG: {
1010 Register to_lo = dest->as_register_lo();
1011 Register to_hi = dest->as_register_hi();
1012 __ movptr(to_lo, as_Address_lo(addr));
1013 break;
1014 }
1015
1016 case T_BOOLEAN: // fall through
1017 case T_BYTE: {
1018 Register dest_reg = dest->as_register();
1019 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1020 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1021 __ movsbl(dest_reg, from_addr);
1022 } else {
1023 __ movb(dest_reg, from_addr);
1024 __ shll(dest_reg, 24);
1025 __ sarl(dest_reg, 24);
1026 }
1027 break;
1028 }
1029
1030 case T_CHAR: {
1031 Register dest_reg = dest->as_register();
1032 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1033 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1034 __ movzwl(dest_reg, from_addr);
1035 } else {
1036 __ movw(dest_reg, from_addr);
1037 }
1038 break;
1039 }
1040
1041 case T_SHORT: {
1042 Register dest_reg = dest->as_register();
1043 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1044 __ movswl(dest_reg, from_addr);
1045 } else {
1046 __ movw(dest_reg, from_addr);
1047 __ shll(dest_reg, 16);
1048 __ sarl(dest_reg, 16);
1049 }
1050 break;
1051 }
1052
1053 default:
1054 ShouldNotReachHere();
1055 }
1056
1057 if (patch != nullptr) {
1058 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1059 }
1060
1061 if (is_reference_type(type)) {
1062 if (UseCompressedOops && !wide) {
1063 __ decode_heap_oop(dest->as_register());
1064 }
1065
1066 __ verify_oop(dest->as_register());
1067 }
1068 }
1069
1070
1071 NEEDS_CLEANUP; // This could be static?
1072 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1073 int elem_size = type2aelembytes(type);
1074 switch (elem_size) {
1075 case 1: return Address::times_1;
1076 case 2: return Address::times_2;
1077 case 4: return Address::times_4;
1078 case 8: return Address::times_8;
1079 }
1080 ShouldNotReachHere();
1081 return Address::no_scale;
1082 }
1083
1084
1085 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1086 switch (op->code()) {
1087 case lir_idiv:
1088 case lir_irem:
1089 arithmetic_idiv(op->code(),
1090 op->in_opr1(),
1091 op->in_opr2(),
1092 op->in_opr3(),
1093 op->result_opr(),
1094 op->info());
1095 break;
1096 case lir_fmad:
1097 __ fmad(op->result_opr()->as_xmm_double_reg(),
1098 op->in_opr1()->as_xmm_double_reg(),
1099 op->in_opr2()->as_xmm_double_reg(),
1100 op->in_opr3()->as_xmm_double_reg());
1101 break;
1102 case lir_fmaf:
1103 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1104 op->in_opr1()->as_xmm_float_reg(),
1105 op->in_opr2()->as_xmm_float_reg(),
1106 op->in_opr3()->as_xmm_float_reg());
1107 break;
1108 default: ShouldNotReachHere(); break;
1109 }
1110 }
1111
1112 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1113 #ifdef ASSERT
1114 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1115 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1116 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1117 #endif
1118
1119 if (op->cond() == lir_cond_always) {
1120 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1121 __ jmp (*(op->label()));
1122 } else {
1123 Assembler::Condition acond = Assembler::zero;
1124 if (op->code() == lir_cond_float_branch) {
1125 assert(op->ublock() != nullptr, "must have unordered successor");
1126 __ jcc(Assembler::parity, *(op->ublock()->label()));
1127 switch(op->cond()) {
1128 case lir_cond_equal: acond = Assembler::equal; break;
1129 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1130 case lir_cond_less: acond = Assembler::below; break;
1131 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1132 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1133 case lir_cond_greater: acond = Assembler::above; break;
1134 default: ShouldNotReachHere();
1135 }
1136 } else {
1137 switch (op->cond()) {
1138 case lir_cond_equal: acond = Assembler::equal; break;
1139 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1140 case lir_cond_less: acond = Assembler::less; break;
1141 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1142 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1143 case lir_cond_greater: acond = Assembler::greater; break;
1144 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1145 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1146 default: ShouldNotReachHere();
1147 }
1148 }
1149 __ jcc(acond,*(op->label()));
1150 }
1151 }
1152
1153 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1154 LIR_Opr src = op->in_opr();
1155 LIR_Opr dest = op->result_opr();
1156
1157 switch (op->bytecode()) {
1158 case Bytecodes::_i2l:
1159 __ movl2ptr(dest->as_register_lo(), src->as_register());
1160 break;
1161
1162 case Bytecodes::_l2i:
1163 __ movl(dest->as_register(), src->as_register_lo());
1164 break;
1165
1166 case Bytecodes::_i2b:
1167 move_regs(src->as_register(), dest->as_register());
1168 __ sign_extend_byte(dest->as_register());
1169 break;
1170
1171 case Bytecodes::_i2c:
1172 move_regs(src->as_register(), dest->as_register());
1173 __ andl(dest->as_register(), 0xFFFF);
1174 break;
1175
1176 case Bytecodes::_i2s:
1177 move_regs(src->as_register(), dest->as_register());
1178 __ sign_extend_short(dest->as_register());
1179 break;
1180
1181 case Bytecodes::_f2d:
1182 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1183 break;
1184
1185 case Bytecodes::_d2f:
1186 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1187 break;
1188
1189 case Bytecodes::_i2f:
1190 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1191 break;
1192
1193 case Bytecodes::_i2d:
1194 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1195 break;
1196
1197 case Bytecodes::_l2f:
1198 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1199 break;
1200
1201 case Bytecodes::_l2d:
1202 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1203 break;
1204
1205 case Bytecodes::_f2i:
1206 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1207 break;
1208
1209 case Bytecodes::_d2i:
1210 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1211 break;
1212
1213 case Bytecodes::_f2l:
1214 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1215 break;
1216
1217 case Bytecodes::_d2l:
1218 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1219 break;
1220
1221 default: ShouldNotReachHere();
1222 }
1223 }
1224
1225 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1226 if (op->init_check()) {
1227 add_debug_info_for_null_check_here(op->stub()->info());
1228 // init_state needs acquire, but x86 is TSO, and so we are already good.
1229 __ cmpb(Address(op->klass()->as_register(),
1230 InstanceKlass::init_state_offset()),
1231 InstanceKlass::fully_initialized);
1232 __ jcc(Assembler::notEqual, *op->stub()->entry());
1233 }
1234 __ allocate_object(op->obj()->as_register(),
1235 op->tmp1()->as_register(),
1236 op->tmp2()->as_register(),
1237 op->header_size(),
1238 op->object_size(),
1239 op->klass()->as_register(),
1240 *op->stub()->entry());
1241 __ bind(*op->stub()->continuation());
1242 }
1243
1244 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1245 Register len = op->len()->as_register();
1246 __ movslq(len, len);
1247
1248 if (UseSlowPath ||
1249 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1250 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1251 __ jmp(*op->stub()->entry());
1252 } else {
1253 Register tmp1 = op->tmp1()->as_register();
1254 Register tmp2 = op->tmp2()->as_register();
1255 Register tmp3 = op->tmp3()->as_register();
1256 if (len == tmp1) {
1257 tmp1 = tmp3;
1258 } else if (len == tmp2) {
1259 tmp2 = tmp3;
1260 } else if (len == tmp3) {
1261 // everything is ok
1262 } else {
1263 __ mov(tmp3, len);
1264 }
1265 __ allocate_array(op->obj()->as_register(),
1266 len,
1267 tmp1,
1268 tmp2,
1269 arrayOopDesc::base_offset_in_bytes(op->type()),
1270 array_element_size(op->type()),
1271 op->klass()->as_register(),
1272 *op->stub()->entry(),
1273 op->zero_array());
1274 }
1275 __ bind(*op->stub()->continuation());
1276 }
1277
1278 void LIR_Assembler::type_profile_helper(Register mdo,
1279 ciMethodData *md, ciProfileData *data,
1280 Register recv) {
1281 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1282 __ profile_receiver_type(recv, mdo, mdp_offset);
1283 }
1284
1285 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1286 // we always need a stub for the failure case.
1287 CodeStub* stub = op->stub();
1288 Register obj = op->object()->as_register();
1289 Register k_RInfo = op->tmp1()->as_register();
1290 Register klass_RInfo = op->tmp2()->as_register();
1291 Register dst = op->result_opr()->as_register();
1292 ciKlass* k = op->klass();
1293 Register Rtmp1 = noreg;
1294 Register tmp_load_klass = rscratch1;
1295
1296 // check if it needs to be profiled
1297 ciMethodData* md = nullptr;
1298 ciProfileData* data = nullptr;
1299
1300 if (op->should_profile()) {
1301 ciMethod* method = op->profiled_method();
1302 assert(method != nullptr, "Should have method");
1303 int bci = op->profiled_bci();
1304 md = method->method_data_or_null();
1305 assert(md != nullptr, "Sanity");
1306 data = md->bci_to_data(bci);
1307 assert(data != nullptr, "need data for type check");
1308 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1309 }
1310 Label* success_target = success;
1311 Label* failure_target = failure;
1312
1313 if (obj == k_RInfo) {
1314 k_RInfo = dst;
1315 } else if (obj == klass_RInfo) {
1316 klass_RInfo = dst;
1317 }
1318 if (k->is_loaded() && !UseCompressedClassPointers) {
1319 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1320 } else {
1321 Rtmp1 = op->tmp3()->as_register();
1322 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1323 }
1324
1325 assert_different_registers(obj, k_RInfo, klass_RInfo);
1326
1327 __ testptr(obj, obj);
1328 if (op->should_profile()) {
1329 Label not_null;
1330 Register mdo = klass_RInfo;
1331 __ mov_metadata(mdo, md->constant_encoding());
1332 __ jccb(Assembler::notEqual, not_null);
1333 // Object is null; update MDO and exit
1334 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1335 int header_bits = BitData::null_seen_byte_constant();
1336 __ orb(data_addr, header_bits);
1337 __ jmp(*obj_is_null);
1338 __ bind(not_null);
1339
1340 Register recv = k_RInfo;
1341 __ load_klass(recv, obj, tmp_load_klass);
1342 type_profile_helper(mdo, md, data, recv);
1343 } else {
1344 __ jcc(Assembler::equal, *obj_is_null);
1345 }
1346
1347 if (!k->is_loaded()) {
1348 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1349 } else {
1350 __ mov_metadata(k_RInfo, k->constant_encoding());
1351 }
1352 __ verify_oop(obj);
1353
1354 if (op->fast_check()) {
1355 // get object class
1356 // not a safepoint as obj null check happens earlier
1357 if (UseCompressedClassPointers) {
1358 __ load_klass(Rtmp1, obj, tmp_load_klass);
1359 __ cmpptr(k_RInfo, Rtmp1);
1360 } else {
1361 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1362 }
1363 __ jcc(Assembler::notEqual, *failure_target);
1364 // successful cast, fall through to profile or jump
1365 } else {
1366 // get object class
1367 // not a safepoint as obj null check happens earlier
1368 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1369 if (k->is_loaded()) {
1370 // See if we get an immediate positive hit
1371 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1372 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1373 __ jcc(Assembler::notEqual, *failure_target);
1374 // successful cast, fall through to profile or jump
1375 } else {
1376 // See if we get an immediate positive hit
1377 __ jcc(Assembler::equal, *success_target);
1378 // check for self
1379 __ cmpptr(klass_RInfo, k_RInfo);
1380 __ jcc(Assembler::equal, *success_target);
1381
1382 __ push_ppx(klass_RInfo);
1383 __ push_ppx(k_RInfo);
1384 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1385 __ pop_ppx(klass_RInfo);
1386 __ pop_ppx(klass_RInfo);
1387 // result is a boolean
1388 __ testl(klass_RInfo, klass_RInfo);
1389 __ jcc(Assembler::equal, *failure_target);
1390 // successful cast, fall through to profile or jump
1391 }
1392 } else {
1393 // perform the fast part of the checking logic
1394 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1395 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1396 __ push_ppx(klass_RInfo);
1397 __ push_ppx(k_RInfo);
1398 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1399 __ pop_ppx(klass_RInfo);
1400 __ pop_ppx(k_RInfo);
1401 // result is a boolean
1402 __ testl(k_RInfo, k_RInfo);
1403 __ jcc(Assembler::equal, *failure_target);
1404 // successful cast, fall through to profile or jump
1405 }
1406 }
1407 __ jmp(*success);
1408 }
1409
1410
1411 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1412 Register tmp_load_klass = rscratch1;
1413 LIR_Code code = op->code();
1414 if (code == lir_store_check) {
1415 Register value = op->object()->as_register();
1416 Register array = op->array()->as_register();
1417 Register k_RInfo = op->tmp1()->as_register();
1418 Register klass_RInfo = op->tmp2()->as_register();
1419 Register Rtmp1 = op->tmp3()->as_register();
1420
1421 CodeStub* stub = op->stub();
1422
1423 // check if it needs to be profiled
1424 ciMethodData* md = nullptr;
1425 ciProfileData* data = nullptr;
1426
1427 if (op->should_profile()) {
1428 ciMethod* method = op->profiled_method();
1429 assert(method != nullptr, "Should have method");
1430 int bci = op->profiled_bci();
1431 md = method->method_data_or_null();
1432 assert(md != nullptr, "Sanity");
1433 data = md->bci_to_data(bci);
1434 assert(data != nullptr, "need data for type check");
1435 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1436 }
1437 Label done;
1438 Label* success_target = &done;
1439 Label* failure_target = stub->entry();
1440
1441 __ testptr(value, value);
1442 if (op->should_profile()) {
1443 Label not_null;
1444 Register mdo = klass_RInfo;
1445 __ mov_metadata(mdo, md->constant_encoding());
1446 __ jccb(Assembler::notEqual, not_null);
1447 // Object is null; update MDO and exit
1448 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1449 int header_bits = BitData::null_seen_byte_constant();
1450 __ orb(data_addr, header_bits);
1451 __ jmp(done);
1452 __ bind(not_null);
1453
1454 Register recv = k_RInfo;
1455 __ load_klass(recv, value, tmp_load_klass);
1456 type_profile_helper(mdo, md, data, recv);
1457 } else {
1458 __ jcc(Assembler::equal, done);
1459 }
1460
1461 add_debug_info_for_null_check_here(op->info_for_exception());
1462 __ load_klass(k_RInfo, array, tmp_load_klass);
1463 __ load_klass(klass_RInfo, value, tmp_load_klass);
1464
1465 // get instance klass (it's already uncompressed)
1466 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1467 // perform the fast part of the checking logic
1468 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1469 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1470 __ push_ppx(klass_RInfo);
1471 __ push_ppx(k_RInfo);
1472 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1473 __ pop_ppx(klass_RInfo);
1474 __ pop_ppx(k_RInfo);
1475 // result is a boolean
1476 __ testl(k_RInfo, k_RInfo);
1477 __ jcc(Assembler::equal, *failure_target);
1478 // fall through to the success case
1479
1480 __ bind(done);
1481 } else
1482 if (code == lir_checkcast) {
1483 Register obj = op->object()->as_register();
1484 Register dst = op->result_opr()->as_register();
1485 Label success;
1486 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1487 __ bind(success);
1488 if (dst != obj) {
1489 __ mov(dst, obj);
1490 }
1491 } else
1492 if (code == lir_instanceof) {
1493 Register obj = op->object()->as_register();
1494 Register dst = op->result_opr()->as_register();
1495 Label success, failure, done;
1496 emit_typecheck_helper(op, &success, &failure, &failure);
1497 __ bind(failure);
1498 __ xorptr(dst, dst);
1499 __ jmpb(done);
1500 __ bind(success);
1501 __ movptr(dst, 1);
1502 __ bind(done);
1503 } else {
1504 ShouldNotReachHere();
1505 }
1506
1507 }
1508
1509
1510 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1511 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1512 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1513 Register newval = op->new_value()->as_register();
1514 Register cmpval = op->cmp_value()->as_register();
1515 assert(cmpval == rax, "wrong register");
1516 assert(newval != noreg, "new val must be register");
1517 assert(cmpval != newval, "cmp and new values must be in different registers");
1518 assert(cmpval != addr, "cmp and addr must be in different registers");
1519 assert(newval != addr, "new value and addr must be in different registers");
1520
1521 if (op->code() == lir_cas_obj) {
1522 if (UseCompressedOops) {
1523 __ encode_heap_oop(cmpval);
1524 __ mov(rscratch1, newval);
1525 __ encode_heap_oop(rscratch1);
1526 __ lock();
1527 // cmpval (rax) is implicitly used by this instruction
1528 __ cmpxchgl(rscratch1, Address(addr, 0));
1529 } else {
1530 __ lock();
1531 __ cmpxchgptr(newval, Address(addr, 0));
1532 }
1533 } else {
1534 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1535 __ lock();
1536 __ cmpxchgl(newval, Address(addr, 0));
1537 }
1538 } else if (op->code() == lir_cas_long) {
1539 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1540 Register newval = op->new_value()->as_register_lo();
1541 Register cmpval = op->cmp_value()->as_register_lo();
1542 assert(cmpval == rax, "wrong register");
1543 assert(newval != noreg, "new val must be register");
1544 assert(cmpval != newval, "cmp and new values must be in different registers");
1545 assert(cmpval != addr, "cmp and addr must be in different registers");
1546 assert(newval != addr, "new value and addr must be in different registers");
1547 __ lock();
1548 __ cmpxchgq(newval, Address(addr, 0));
1549 } else {
1550 Unimplemented();
1551 }
1552 }
1553
1554 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1555 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1556 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1557
1558 Assembler::Condition acond, ncond;
1559 switch (condition) {
1560 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1561 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1562 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1563 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1564 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1565 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1566 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1567 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1568 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1569 ShouldNotReachHere();
1570 }
1571
1572 if (opr1->is_cpu_register()) {
1573 reg2reg(opr1, result);
1574 } else if (opr1->is_stack()) {
1575 stack2reg(opr1, result, result->type());
1576 } else if (opr1->is_constant()) {
1577 const2reg(opr1, result, lir_patch_none, nullptr);
1578 } else {
1579 ShouldNotReachHere();
1580 }
1581
1582 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1583 // optimized version that does not require a branch
1584 if (opr2->is_single_cpu()) {
1585 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1586 __ cmov(ncond, result->as_register(), opr2->as_register());
1587 } else if (opr2->is_double_cpu()) {
1588 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1589 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1590 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1591 } else if (opr2->is_single_stack()) {
1592 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1593 } else if (opr2->is_double_stack()) {
1594 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1595 } else {
1596 ShouldNotReachHere();
1597 }
1598
1599 } else {
1600 Label skip;
1601 __ jccb(acond, skip);
1602 if (opr2->is_cpu_register()) {
1603 reg2reg(opr2, result);
1604 } else if (opr2->is_stack()) {
1605 stack2reg(opr2, result, result->type());
1606 } else if (opr2->is_constant()) {
1607 const2reg(opr2, result, lir_patch_none, nullptr);
1608 } else {
1609 ShouldNotReachHere();
1610 }
1611 __ bind(skip);
1612 }
1613 }
1614
1615
1616 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1617 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1618
1619 if (left->is_single_cpu()) {
1620 assert(left == dest, "left and dest must be equal");
1621 Register lreg = left->as_register();
1622
1623 if (right->is_single_cpu()) {
1624 // cpu register - cpu register
1625 Register rreg = right->as_register();
1626 switch (code) {
1627 case lir_add: __ addl (lreg, rreg); break;
1628 case lir_sub: __ subl (lreg, rreg); break;
1629 case lir_mul: __ imull(lreg, rreg); break;
1630 default: ShouldNotReachHere();
1631 }
1632
1633 } else if (right->is_stack()) {
1634 // cpu register - stack
1635 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1636 switch (code) {
1637 case lir_add: __ addl(lreg, raddr); break;
1638 case lir_sub: __ subl(lreg, raddr); break;
1639 default: ShouldNotReachHere();
1640 }
1641
1642 } else if (right->is_constant()) {
1643 // cpu register - constant
1644 jint c = right->as_constant_ptr()->as_jint();
1645 switch (code) {
1646 case lir_add: {
1647 __ incrementl(lreg, c);
1648 break;
1649 }
1650 case lir_sub: {
1651 __ decrementl(lreg, c);
1652 break;
1653 }
1654 default: ShouldNotReachHere();
1655 }
1656
1657 } else {
1658 ShouldNotReachHere();
1659 }
1660
1661 } else if (left->is_double_cpu()) {
1662 assert(left == dest, "left and dest must be equal");
1663 Register lreg_lo = left->as_register_lo();
1664 Register lreg_hi = left->as_register_hi();
1665
1666 if (right->is_double_cpu()) {
1667 // cpu register - cpu register
1668 Register rreg_lo = right->as_register_lo();
1669 Register rreg_hi = right->as_register_hi();
1670 assert_different_registers(lreg_lo, rreg_lo);
1671 switch (code) {
1672 case lir_add:
1673 __ addptr(lreg_lo, rreg_lo);
1674 break;
1675 case lir_sub:
1676 __ subptr(lreg_lo, rreg_lo);
1677 break;
1678 case lir_mul:
1679 __ imulq(lreg_lo, rreg_lo);
1680 break;
1681 default:
1682 ShouldNotReachHere();
1683 }
1684
1685 } else if (right->is_constant()) {
1686 // cpu register - constant
1687 jlong c = right->as_constant_ptr()->as_jlong_bits();
1688 __ movptr(r10, (intptr_t) c);
1689 switch (code) {
1690 case lir_add:
1691 __ addptr(lreg_lo, r10);
1692 break;
1693 case lir_sub:
1694 __ subptr(lreg_lo, r10);
1695 break;
1696 default:
1697 ShouldNotReachHere();
1698 }
1699
1700 } else {
1701 ShouldNotReachHere();
1702 }
1703
1704 } else if (left->is_single_xmm()) {
1705 assert(left == dest, "left and dest must be equal");
1706 XMMRegister lreg = left->as_xmm_float_reg();
1707
1708 if (right->is_single_xmm()) {
1709 XMMRegister rreg = right->as_xmm_float_reg();
1710 switch (code) {
1711 case lir_add: __ addss(lreg, rreg); break;
1712 case lir_sub: __ subss(lreg, rreg); break;
1713 case lir_mul: __ mulss(lreg, rreg); break;
1714 case lir_div: __ divss(lreg, rreg); break;
1715 default: ShouldNotReachHere();
1716 }
1717 } else {
1718 Address raddr;
1719 if (right->is_single_stack()) {
1720 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1721 } else if (right->is_constant()) {
1722 // hack for now
1723 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1724 } else {
1725 ShouldNotReachHere();
1726 }
1727 switch (code) {
1728 case lir_add: __ addss(lreg, raddr); break;
1729 case lir_sub: __ subss(lreg, raddr); break;
1730 case lir_mul: __ mulss(lreg, raddr); break;
1731 case lir_div: __ divss(lreg, raddr); break;
1732 default: ShouldNotReachHere();
1733 }
1734 }
1735
1736 } else if (left->is_double_xmm()) {
1737 assert(left == dest, "left and dest must be equal");
1738
1739 XMMRegister lreg = left->as_xmm_double_reg();
1740 if (right->is_double_xmm()) {
1741 XMMRegister rreg = right->as_xmm_double_reg();
1742 switch (code) {
1743 case lir_add: __ addsd(lreg, rreg); break;
1744 case lir_sub: __ subsd(lreg, rreg); break;
1745 case lir_mul: __ mulsd(lreg, rreg); break;
1746 case lir_div: __ divsd(lreg, rreg); break;
1747 default: ShouldNotReachHere();
1748 }
1749 } else {
1750 Address raddr;
1751 if (right->is_double_stack()) {
1752 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1753 } else if (right->is_constant()) {
1754 // hack for now
1755 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1756 } else {
1757 ShouldNotReachHere();
1758 }
1759 switch (code) {
1760 case lir_add: __ addsd(lreg, raddr); break;
1761 case lir_sub: __ subsd(lreg, raddr); break;
1762 case lir_mul: __ mulsd(lreg, raddr); break;
1763 case lir_div: __ divsd(lreg, raddr); break;
1764 default: ShouldNotReachHere();
1765 }
1766 }
1767
1768 } else if (left->is_single_stack() || left->is_address()) {
1769 assert(left == dest, "left and dest must be equal");
1770
1771 Address laddr;
1772 if (left->is_single_stack()) {
1773 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1774 } else if (left->is_address()) {
1775 laddr = as_Address(left->as_address_ptr());
1776 } else {
1777 ShouldNotReachHere();
1778 }
1779
1780 if (right->is_single_cpu()) {
1781 Register rreg = right->as_register();
1782 switch (code) {
1783 case lir_add: __ addl(laddr, rreg); break;
1784 case lir_sub: __ subl(laddr, rreg); break;
1785 default: ShouldNotReachHere();
1786 }
1787 } else if (right->is_constant()) {
1788 jint c = right->as_constant_ptr()->as_jint();
1789 switch (code) {
1790 case lir_add: {
1791 __ incrementl(laddr, c);
1792 break;
1793 }
1794 case lir_sub: {
1795 __ decrementl(laddr, c);
1796 break;
1797 }
1798 default: ShouldNotReachHere();
1799 }
1800 } else {
1801 ShouldNotReachHere();
1802 }
1803
1804 } else {
1805 ShouldNotReachHere();
1806 }
1807 }
1808
1809
1810 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1811 if (value->is_double_xmm()) {
1812 switch(code) {
1813 case lir_abs :
1814 {
1815 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1816 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1817 }
1818 assert(!tmp->is_valid(), "do not need temporary");
1819 __ andpd(dest->as_xmm_double_reg(),
1820 ExternalAddress(LIR_Assembler::double_signmask_pool),
1821 rscratch1);
1822 }
1823 break;
1824
1825 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1826 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1827 default : ShouldNotReachHere();
1828 }
1829
1830 } else if (code == lir_f2hf) {
1831 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1832 } else if (code == lir_hf2f) {
1833 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1834 } else {
1835 Unimplemented();
1836 }
1837 }
1838
1839 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1840 // assert(left->destroys_register(), "check");
1841 if (left->is_single_cpu()) {
1842 Register reg = left->as_register();
1843 if (right->is_constant()) {
1844 int val = right->as_constant_ptr()->as_jint();
1845 switch (code) {
1846 case lir_logic_and: __ andl (reg, val); break;
1847 case lir_logic_or: __ orl (reg, val); break;
1848 case lir_logic_xor: __ xorl (reg, val); break;
1849 default: ShouldNotReachHere();
1850 }
1851 } else if (right->is_stack()) {
1852 // added support for stack operands
1853 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1854 switch (code) {
1855 case lir_logic_and: __ andl (reg, raddr); break;
1856 case lir_logic_or: __ orl (reg, raddr); break;
1857 case lir_logic_xor: __ xorl (reg, raddr); break;
1858 default: ShouldNotReachHere();
1859 }
1860 } else {
1861 Register rright = right->as_register();
1862 switch (code) {
1863 case lir_logic_and: __ andptr (reg, rright); break;
1864 case lir_logic_or : __ orptr (reg, rright); break;
1865 case lir_logic_xor: __ xorptr (reg, rright); break;
1866 default: ShouldNotReachHere();
1867 }
1868 }
1869 move_regs(reg, dst->as_register());
1870 } else {
1871 Register l_lo = left->as_register_lo();
1872 Register l_hi = left->as_register_hi();
1873 if (right->is_constant()) {
1874 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1875 switch (code) {
1876 case lir_logic_and:
1877 __ andq(l_lo, rscratch1);
1878 break;
1879 case lir_logic_or:
1880 __ orq(l_lo, rscratch1);
1881 break;
1882 case lir_logic_xor:
1883 __ xorq(l_lo, rscratch1);
1884 break;
1885 default: ShouldNotReachHere();
1886 }
1887 } else {
1888 Register r_lo;
1889 if (is_reference_type(right->type())) {
1890 r_lo = right->as_register();
1891 } else {
1892 r_lo = right->as_register_lo();
1893 }
1894 switch (code) {
1895 case lir_logic_and:
1896 __ andptr(l_lo, r_lo);
1897 break;
1898 case lir_logic_or:
1899 __ orptr(l_lo, r_lo);
1900 break;
1901 case lir_logic_xor:
1902 __ xorptr(l_lo, r_lo);
1903 break;
1904 default: ShouldNotReachHere();
1905 }
1906 }
1907
1908 Register dst_lo = dst->as_register_lo();
1909 Register dst_hi = dst->as_register_hi();
1910
1911 move_regs(l_lo, dst_lo);
1912 }
1913 }
1914
1915
1916 // we assume that rax, and rdx can be overwritten
1917 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1918
1919 assert(left->is_single_cpu(), "left must be register");
1920 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1921 assert(result->is_single_cpu(), "result must be register");
1922
1923 // assert(left->destroys_register(), "check");
1924 // assert(right->destroys_register(), "check");
1925
1926 Register lreg = left->as_register();
1927 Register dreg = result->as_register();
1928
1929 if (right->is_constant()) {
1930 jint divisor = right->as_constant_ptr()->as_jint();
1931 assert(divisor > 0 && is_power_of_2(divisor), "must be");
1932 if (code == lir_idiv) {
1933 assert(lreg == rax, "must be rax,");
1934 assert(temp->as_register() == rdx, "tmp register must be rdx");
1935 __ cdql(); // sign extend into rdx:rax
1936 if (divisor == 2) {
1937 __ subl(lreg, rdx);
1938 } else {
1939 __ andl(rdx, divisor - 1);
1940 __ addl(lreg, rdx);
1941 }
1942 __ sarl(lreg, log2i_exact(divisor));
1943 move_regs(lreg, dreg);
1944 } else if (code == lir_irem) {
1945 Label done;
1946 __ mov(dreg, lreg);
1947 __ andl(dreg, 0x80000000 | (divisor - 1));
1948 __ jcc(Assembler::positive, done);
1949 __ decrement(dreg);
1950 __ orl(dreg, ~(divisor - 1));
1951 __ increment(dreg);
1952 __ bind(done);
1953 } else {
1954 ShouldNotReachHere();
1955 }
1956 } else {
1957 Register rreg = right->as_register();
1958 assert(lreg == rax, "left register must be rax,");
1959 assert(rreg != rdx, "right register must not be rdx");
1960 assert(temp->as_register() == rdx, "tmp register must be rdx");
1961
1962 move_regs(lreg, rax);
1963
1964 int idivl_offset = __ corrected_idivl(rreg);
1965 if (ImplicitDiv0Checks) {
1966 add_debug_info_for_div0(idivl_offset, info);
1967 }
1968 if (code == lir_irem) {
1969 move_regs(rdx, dreg); // result is in rdx
1970 } else {
1971 move_regs(rax, dreg);
1972 }
1973 }
1974 }
1975
1976
1977 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1978 if (opr1->is_single_cpu()) {
1979 Register reg1 = opr1->as_register();
1980 if (opr2->is_single_cpu()) {
1981 // cpu register - cpu register
1982 if (is_reference_type(opr1->type())) {
1983 __ cmpoop(reg1, opr2->as_register());
1984 } else {
1985 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1986 __ cmpl(reg1, opr2->as_register());
1987 }
1988 } else if (opr2->is_stack()) {
1989 // cpu register - stack
1990 if (is_reference_type(opr1->type())) {
1991 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1992 } else {
1993 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1994 }
1995 } else if (opr2->is_constant()) {
1996 // cpu register - constant
1997 LIR_Const* c = opr2->as_constant_ptr();
1998 if (c->type() == T_INT) {
1999 jint i = c->as_jint();
2000 if (i == 0) {
2001 __ testl(reg1, reg1);
2002 } else {
2003 __ cmpl(reg1, i);
2004 }
2005 } else if (c->type() == T_METADATA) {
2006 // All we need for now is a comparison with null for equality.
2007 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2008 Metadata* m = c->as_metadata();
2009 if (m == nullptr) {
2010 __ testptr(reg1, reg1);
2011 } else {
2012 ShouldNotReachHere();
2013 }
2014 } else if (is_reference_type(c->type())) {
2015 // In 64bit oops are single register
2016 jobject o = c->as_jobject();
2017 if (o == nullptr) {
2018 __ testptr(reg1, reg1);
2019 } else {
2020 __ cmpoop(reg1, o, rscratch1);
2021 }
2022 } else {
2023 fatal("unexpected type: %s", basictype_to_str(c->type()));
2024 }
2025 // cpu register - address
2026 } else if (opr2->is_address()) {
2027 if (op->info() != nullptr) {
2028 add_debug_info_for_null_check_here(op->info());
2029 }
2030 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2031 } else {
2032 ShouldNotReachHere();
2033 }
2034
2035 } else if(opr1->is_double_cpu()) {
2036 Register xlo = opr1->as_register_lo();
2037 Register xhi = opr1->as_register_hi();
2038 if (opr2->is_double_cpu()) {
2039 __ cmpptr(xlo, opr2->as_register_lo());
2040 } else if (opr2->is_constant()) {
2041 // cpu register - constant 0
2042 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2043 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2044 } else {
2045 ShouldNotReachHere();
2046 }
2047
2048 } else if (opr1->is_single_xmm()) {
2049 XMMRegister reg1 = opr1->as_xmm_float_reg();
2050 if (opr2->is_single_xmm()) {
2051 // xmm register - xmm register
2052 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2053 } else if (opr2->is_stack()) {
2054 // xmm register - stack
2055 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2056 } else if (opr2->is_constant()) {
2057 // xmm register - constant
2058 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2059 } else if (opr2->is_address()) {
2060 // xmm register - address
2061 if (op->info() != nullptr) {
2062 add_debug_info_for_null_check_here(op->info());
2063 }
2064 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2065 } else {
2066 ShouldNotReachHere();
2067 }
2068
2069 } else if (opr1->is_double_xmm()) {
2070 XMMRegister reg1 = opr1->as_xmm_double_reg();
2071 if (opr2->is_double_xmm()) {
2072 // xmm register - xmm register
2073 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2074 } else if (opr2->is_stack()) {
2075 // xmm register - stack
2076 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2077 } else if (opr2->is_constant()) {
2078 // xmm register - constant
2079 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2080 } else if (opr2->is_address()) {
2081 // xmm register - address
2082 if (op->info() != nullptr) {
2083 add_debug_info_for_null_check_here(op->info());
2084 }
2085 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2086 } else {
2087 ShouldNotReachHere();
2088 }
2089
2090 } else if (opr1->is_address() && opr2->is_constant()) {
2091 LIR_Const* c = opr2->as_constant_ptr();
2092 if (is_reference_type(c->type())) {
2093 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2094 __ movoop(rscratch1, c->as_jobject());
2095 }
2096 if (op->info() != nullptr) {
2097 add_debug_info_for_null_check_here(op->info());
2098 }
2099 // special case: address - constant
2100 LIR_Address* addr = opr1->as_address_ptr();
2101 if (c->type() == T_INT) {
2102 __ cmpl(as_Address(addr), c->as_jint());
2103 } else if (is_reference_type(c->type())) {
2104 // %%% Make this explode if addr isn't reachable until we figure out a
2105 // better strategy by giving noreg as the temp for as_Address
2106 __ cmpoop(rscratch1, as_Address(addr, noreg));
2107 } else {
2108 ShouldNotReachHere();
2109 }
2110
2111 } else {
2112 ShouldNotReachHere();
2113 }
2114 }
2115
2116 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2117 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2118 if (left->is_single_xmm()) {
2119 assert(right->is_single_xmm(), "must match");
2120 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2121 } else if (left->is_double_xmm()) {
2122 assert(right->is_double_xmm(), "must match");
2123 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2124
2125 } else {
2126 ShouldNotReachHere();
2127 }
2128 } else {
2129 assert(code == lir_cmp_l2i, "check");
2130 Label done;
2131 Register dest = dst->as_register();
2132 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2133 __ movl(dest, -1);
2134 __ jccb(Assembler::less, done);
2135 __ setb(Assembler::notZero, dest);
2136 __ movzbl(dest, dest);
2137 __ bind(done);
2138 }
2139 }
2140
2141
2142 void LIR_Assembler::align_call(LIR_Code code) {
2143 // make sure that the displacement word of the call ends up word aligned
2144 int offset = __ offset();
2145 switch (code) {
2146 case lir_static_call:
2147 case lir_optvirtual_call:
2148 case lir_dynamic_call:
2149 offset += NativeCall::displacement_offset;
2150 break;
2151 case lir_icvirtual_call:
2152 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2153 break;
2154 default: ShouldNotReachHere();
2155 }
2156 __ align(BytesPerWord, offset);
2157 }
2158
2159
2160 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2161 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2162 "must be aligned");
2163 __ call(AddressLiteral(op->addr(), rtype));
2164 add_call_info(code_offset(), op->info());
2165 __ post_call_nop();
2166 }
2167
2168
2169 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2170 __ ic_call(op->addr());
2171 add_call_info(code_offset(), op->info());
2172 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2173 "must be aligned");
2174 __ post_call_nop();
2175 }
2176
2177
2178 void LIR_Assembler::emit_static_call_stub() {
2179 address call_pc = __ pc();
2180 address stub = __ start_a_stub(call_stub_size());
2181 if (stub == nullptr) {
2182 bailout("static call stub overflow");
2183 return;
2184 }
2185
2186 int start = __ offset();
2187
2188 // make sure that the displacement word of the call ends up word aligned
2189 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2190 __ relocate(static_stub_Relocation::spec(call_pc));
2191 __ mov_metadata(rbx, (Metadata*)nullptr);
2192 // must be set to -1 at code generation time
2193 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2194 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2195 __ jump(RuntimeAddress(__ pc()));
2196
2197 assert(__ offset() - start <= call_stub_size(), "stub too big");
2198 __ end_a_stub();
2199 }
2200
2201
2202 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2203 assert(exceptionOop->as_register() == rax, "must match");
2204 assert(exceptionPC->as_register() == rdx, "must match");
2205
2206 // exception object is not added to oop map by LinearScan
2207 // (LinearScan assumes that no oops are in fixed registers)
2208 info->add_register_oop(exceptionOop);
2209 StubId unwind_id;
2210
2211 // get current pc information
2212 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2213 int pc_for_athrow_offset = __ offset();
2214 InternalAddress pc_for_athrow(__ pc());
2215 __ lea(exceptionPC->as_register(), pc_for_athrow);
2216 add_call_info(pc_for_athrow_offset, info); // for exception handler
2217
2218 __ verify_not_null_oop(rax);
2219 // search an exception handler (rax: exception oop, rdx: throwing pc)
2220 if (compilation()->has_fpu_code()) {
2221 unwind_id = StubId::c1_handle_exception_id;
2222 } else {
2223 unwind_id = StubId::c1_handle_exception_nofpu_id;
2224 }
2225 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2226
2227 // enough room for two byte trap
2228 __ nop();
2229 }
2230
2231
2232 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2233 assert(exceptionOop->as_register() == rax, "must match");
2234
2235 __ jmp(_unwind_handler_entry);
2236 }
2237
2238
2239 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2240
2241 // optimized version for linear scan:
2242 // * count must be already in ECX (guaranteed by LinearScan)
2243 // * left and dest must be equal
2244 // * tmp must be unused
2245 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2246 assert(left == dest, "left and dest must be equal");
2247 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2248
2249 if (left->is_single_cpu()) {
2250 Register value = left->as_register();
2251 assert(value != SHIFT_count, "left cannot be ECX");
2252
2253 switch (code) {
2254 case lir_shl: __ shll(value); break;
2255 case lir_shr: __ sarl(value); break;
2256 case lir_ushr: __ shrl(value); break;
2257 default: ShouldNotReachHere();
2258 }
2259 } else if (left->is_double_cpu()) {
2260 Register lo = left->as_register_lo();
2261 Register hi = left->as_register_hi();
2262 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2263 switch (code) {
2264 case lir_shl: __ shlptr(lo); break;
2265 case lir_shr: __ sarptr(lo); break;
2266 case lir_ushr: __ shrptr(lo); break;
2267 default: ShouldNotReachHere();
2268 }
2269 } else {
2270 ShouldNotReachHere();
2271 }
2272 }
2273
2274
2275 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2276 if (dest->is_single_cpu()) {
2277 // first move left into dest so that left is not destroyed by the shift
2278 Register value = dest->as_register();
2279 count = count & 0x1F; // Java spec
2280
2281 move_regs(left->as_register(), value);
2282 switch (code) {
2283 case lir_shl: __ shll(value, count); break;
2284 case lir_shr: __ sarl(value, count); break;
2285 case lir_ushr: __ shrl(value, count); break;
2286 default: ShouldNotReachHere();
2287 }
2288 } else if (dest->is_double_cpu()) {
2289 // first move left into dest so that left is not destroyed by the shift
2290 Register value = dest->as_register_lo();
2291 count = count & 0x1F; // Java spec
2292
2293 move_regs(left->as_register_lo(), value);
2294 switch (code) {
2295 case lir_shl: __ shlptr(value, count); break;
2296 case lir_shr: __ sarptr(value, count); break;
2297 case lir_ushr: __ shrptr(value, count); break;
2298 default: ShouldNotReachHere();
2299 }
2300 } else {
2301 ShouldNotReachHere();
2302 }
2303 }
2304
2305
2306 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2307 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2308 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2309 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2310 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2311 }
2312
2313
2314 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2315 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2316 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2317 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2318 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2319 }
2320
2321
2322 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2323 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2324 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2325 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2326 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2327 }
2328
2329
2330 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2331 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2332 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2333 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2334 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2335 }
2336
2337
2338 // This code replaces a call to arraycopy; no exception may
2339 // be thrown in this code, they must be thrown in the System.arraycopy
2340 // activation frame; we could save some checks if this would not be the case
2341 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2342 ciArrayKlass* default_type = op->expected_type();
2343 Register src = op->src()->as_register();
2344 Register dst = op->dst()->as_register();
2345 Register src_pos = op->src_pos()->as_register();
2346 Register dst_pos = op->dst_pos()->as_register();
2347 Register length = op->length()->as_register();
2348 Register tmp = op->tmp()->as_register();
2349 Register tmp_load_klass = rscratch1;
2350 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2351
2352 CodeStub* stub = op->stub();
2353 int flags = op->flags();
2354 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2355 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2356
2357 // if we don't know anything, just go through the generic arraycopy
2358 if (default_type == nullptr) {
2359 // save outgoing arguments on stack in case call to System.arraycopy is needed
2360 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2361 // for interpreter calling conventions. Now we have to do it in new style conventions.
2362 // For the moment until C1 gets the new register allocator I just force all the
2363 // args to the right place (except the register args) and then on the back side
2364 // reload the register args properly if we go slow path. Yuck
2365
2366 // These are proper for the calling convention
2367 store_parameter(length, 2);
2368 store_parameter(dst_pos, 1);
2369 store_parameter(dst, 0);
2370
2371 // these are just temporary placements until we need to reload
2372 store_parameter(src_pos, 3);
2373 store_parameter(src, 4);
2374
2375 address copyfunc_addr = StubRoutines::generic_arraycopy();
2376 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2377
2378 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2379 // The arguments are in java calling convention so we can trivially shift them to C
2380 // convention
2381 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2382 __ mov(c_rarg0, j_rarg0);
2383 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2384 __ mov(c_rarg1, j_rarg1);
2385 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2386 __ mov(c_rarg2, j_rarg2);
2387 assert_different_registers(c_rarg3, j_rarg4);
2388 __ mov(c_rarg3, j_rarg3);
2389 #ifdef _WIN64
2390 // Allocate abi space for args but be sure to keep stack aligned
2391 __ subptr(rsp, 6*wordSize);
2392 store_parameter(j_rarg4, 4);
2393 #ifndef PRODUCT
2394 if (PrintC1Statistics) {
2395 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2396 }
2397 #endif
2398 __ call(RuntimeAddress(copyfunc_addr));
2399 __ addptr(rsp, 6*wordSize);
2400 #else
2401 __ mov(c_rarg4, j_rarg4);
2402 #ifndef PRODUCT
2403 if (PrintC1Statistics) {
2404 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2405 }
2406 #endif
2407 __ call(RuntimeAddress(copyfunc_addr));
2408 #endif // _WIN64
2409
2410 __ testl(rax, rax);
2411 __ jcc(Assembler::equal, *stub->continuation());
2412
2413 __ mov(tmp, rax);
2414 __ xorl(tmp, -1);
2415
2416 // Reload values from the stack so they are where the stub
2417 // expects them.
2418 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2419 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2420 __ movptr (length, Address(rsp, 2*BytesPerWord));
2421 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2422 __ movptr (src, Address(rsp, 4*BytesPerWord));
2423
2424 __ subl(length, tmp);
2425 __ addl(src_pos, tmp);
2426 __ addl(dst_pos, tmp);
2427 __ jmp(*stub->entry());
2428
2429 __ bind(*stub->continuation());
2430 return;
2431 }
2432
2433 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2434
2435 int elem_size = type2aelembytes(basic_type);
2436 Address::ScaleFactor scale;
2437
2438 switch (elem_size) {
2439 case 1 :
2440 scale = Address::times_1;
2441 break;
2442 case 2 :
2443 scale = Address::times_2;
2444 break;
2445 case 4 :
2446 scale = Address::times_4;
2447 break;
2448 case 8 :
2449 scale = Address::times_8;
2450 break;
2451 default:
2452 scale = Address::no_scale;
2453 ShouldNotReachHere();
2454 }
2455
2456 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2457 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2458
2459 // length and pos's are all sign extended at this point on 64bit
2460
2461 // test for null
2462 if (flags & LIR_OpArrayCopy::src_null_check) {
2463 __ testptr(src, src);
2464 __ jcc(Assembler::zero, *stub->entry());
2465 }
2466 if (flags & LIR_OpArrayCopy::dst_null_check) {
2467 __ testptr(dst, dst);
2468 __ jcc(Assembler::zero, *stub->entry());
2469 }
2470
2471 // If the compiler was not able to prove that exact type of the source or the destination
2472 // of the arraycopy is an array type, check at runtime if the source or the destination is
2473 // an instance type.
2474 if (flags & LIR_OpArrayCopy::type_check) {
2475 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2476 __ load_klass(tmp, dst, tmp_load_klass);
2477 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2478 __ jcc(Assembler::greaterEqual, *stub->entry());
2479 }
2480
2481 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2482 __ load_klass(tmp, src, tmp_load_klass);
2483 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2484 __ jcc(Assembler::greaterEqual, *stub->entry());
2485 }
2486 }
2487
2488 // check if negative
2489 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2490 __ testl(src_pos, src_pos);
2491 __ jcc(Assembler::less, *stub->entry());
2492 }
2493 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2494 __ testl(dst_pos, dst_pos);
2495 __ jcc(Assembler::less, *stub->entry());
2496 }
2497
2498 if (flags & LIR_OpArrayCopy::src_range_check) {
2499 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2500 __ cmpl(tmp, src_length_addr);
2501 __ jcc(Assembler::above, *stub->entry());
2502 }
2503 if (flags & LIR_OpArrayCopy::dst_range_check) {
2504 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2505 __ cmpl(tmp, dst_length_addr);
2506 __ jcc(Assembler::above, *stub->entry());
2507 }
2508
2509 if (flags & LIR_OpArrayCopy::length_positive_check) {
2510 __ testl(length, length);
2511 __ jcc(Assembler::less, *stub->entry());
2512 }
2513
2514 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2515 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2516
2517 if (flags & LIR_OpArrayCopy::type_check) {
2518 // We don't know the array types are compatible
2519 if (basic_type != T_OBJECT) {
2520 // Simple test for basic type arrays
2521 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2522 __ jcc(Assembler::notEqual, *stub->entry());
2523 } else {
2524 // For object arrays, if src is a sub class of dst then we can
2525 // safely do the copy.
2526 Label cont, slow;
2527
2528 __ push_ppx(src);
2529 __ push_ppx(dst);
2530
2531 __ load_klass(src, src, tmp_load_klass);
2532 __ load_klass(dst, dst, tmp_load_klass);
2533
2534 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2535
2536 __ push_ppx(src);
2537 __ push_ppx(dst);
2538 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2539 __ pop_ppx(dst);
2540 __ pop_ppx(src);
2541
2542 __ testl(src, src);
2543 __ jcc(Assembler::notEqual, cont);
2544
2545 __ bind(slow);
2546 __ pop_ppx(dst);
2547 __ pop_ppx(src);
2548
2549 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2550 if (copyfunc_addr != nullptr) { // use stub if available
2551 // src is not a sub class of dst so we have to do a
2552 // per-element check.
2553
2554 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2555 if ((flags & mask) != mask) {
2556 // Check that at least both of them object arrays.
2557 assert(flags & mask, "one of the two should be known to be an object array");
2558
2559 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2560 __ load_klass(tmp, src, tmp_load_klass);
2561 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2562 __ load_klass(tmp, dst, tmp_load_klass);
2563 }
2564 int lh_offset = in_bytes(Klass::layout_helper_offset());
2565 Address klass_lh_addr(tmp, lh_offset);
2566 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2567 __ cmpl(klass_lh_addr, objArray_lh);
2568 __ jcc(Assembler::notEqual, *stub->entry());
2569 }
2570
2571 // Spill because stubs can use any register they like and it's
2572 // easier to restore just those that we care about.
2573 store_parameter(dst, 0);
2574 store_parameter(dst_pos, 1);
2575 store_parameter(length, 2);
2576 store_parameter(src_pos, 3);
2577 store_parameter(src, 4);
2578
2579 __ movl2ptr(length, length); //higher 32bits must be null
2580
2581 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2582 assert_different_registers(c_rarg0, dst, dst_pos, length);
2583 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2584 assert_different_registers(c_rarg1, dst, length);
2585
2586 __ mov(c_rarg2, length);
2587 assert_different_registers(c_rarg2, dst);
2588
2589 #ifdef _WIN64
2590 // Allocate abi space for args but be sure to keep stack aligned
2591 __ subptr(rsp, 6*wordSize);
2592 __ load_klass(c_rarg3, dst, tmp_load_klass);
2593 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2594 store_parameter(c_rarg3, 4);
2595 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2596 __ call(RuntimeAddress(copyfunc_addr));
2597 __ addptr(rsp, 6*wordSize);
2598 #else
2599 __ load_klass(c_rarg4, dst, tmp_load_klass);
2600 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2601 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2602 __ call(RuntimeAddress(copyfunc_addr));
2603 #endif
2604
2605 #ifndef PRODUCT
2606 if (PrintC1Statistics) {
2607 Label failed;
2608 __ testl(rax, rax);
2609 __ jcc(Assembler::notZero, failed);
2610 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2611 __ bind(failed);
2612 }
2613 #endif
2614
2615 __ testl(rax, rax);
2616 __ jcc(Assembler::zero, *stub->continuation());
2617
2618 #ifndef PRODUCT
2619 if (PrintC1Statistics) {
2620 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2621 }
2622 #endif
2623
2624 __ mov(tmp, rax);
2625
2626 __ xorl(tmp, -1);
2627
2628 // Restore previously spilled arguments
2629 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2630 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2631 __ movptr (length, Address(rsp, 2*BytesPerWord));
2632 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2633 __ movptr (src, Address(rsp, 4*BytesPerWord));
2634
2635
2636 __ subl(length, tmp);
2637 __ addl(src_pos, tmp);
2638 __ addl(dst_pos, tmp);
2639 }
2640
2641 __ jmp(*stub->entry());
2642
2643 __ bind(cont);
2644 __ pop(dst);
2645 __ pop(src);
2646 }
2647 }
2648
2649 #ifdef ASSERT
2650 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2651 // Sanity check the known type with the incoming class. For the
2652 // primitive case the types must match exactly with src.klass and
2653 // dst.klass each exactly matching the default type. For the
2654 // object array case, if no type check is needed then either the
2655 // dst type is exactly the expected type and the src type is a
2656 // subtype which we can't check or src is the same array as dst
2657 // but not necessarily exactly of type default_type.
2658 Label known_ok, halt;
2659 __ mov_metadata(tmp, default_type->constant_encoding());
2660 if (UseCompressedClassPointers) {
2661 __ encode_klass_not_null(tmp, rscratch1);
2662 }
2663
2664 if (basic_type != T_OBJECT) {
2665 __ cmp_klass(tmp, dst, tmp2);
2666 __ jcc(Assembler::notEqual, halt);
2667 __ cmp_klass(tmp, src, tmp2);
2668 __ jcc(Assembler::equal, known_ok);
2669 } else {
2670 __ cmp_klass(tmp, dst, tmp2);
2671 __ jcc(Assembler::equal, known_ok);
2672 __ cmpptr(src, dst);
2673 __ jcc(Assembler::equal, known_ok);
2674 }
2675 __ bind(halt);
2676 __ stop("incorrect type information in arraycopy");
2677 __ bind(known_ok);
2678 }
2679 #endif
2680
2681 #ifndef PRODUCT
2682 if (PrintC1Statistics) {
2683 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2684 }
2685 #endif
2686
2687 assert_different_registers(c_rarg0, dst, dst_pos, length);
2688 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2689 assert_different_registers(c_rarg1, length);
2690 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2691 __ mov(c_rarg2, length);
2692
2693 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2694 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2695 const char *name;
2696 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2697 __ call_VM_leaf(entry, 0);
2698
2699 if (stub != nullptr) {
2700 __ bind(*stub->continuation());
2701 }
2702 }
2703
2704 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2705 assert(op->crc()->is_single_cpu(), "crc must be register");
2706 assert(op->val()->is_single_cpu(), "byte value must be register");
2707 assert(op->result_opr()->is_single_cpu(), "result must be register");
2708 Register crc = op->crc()->as_register();
2709 Register val = op->val()->as_register();
2710 Register res = op->result_opr()->as_register();
2711
2712 assert_different_registers(val, crc, res);
2713
2714 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2715 __ notl(crc); // ~crc
2716 __ update_byte_crc32(crc, val, res);
2717 __ notl(crc); // ~crc
2718 __ mov(res, crc);
2719 }
2720
2721 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2722 Register obj = op->obj_opr()->as_register(); // may not be an oop
2723 Register hdr = op->hdr_opr()->as_register();
2724 Register lock = op->lock_opr()->as_register();
2725 if (op->code() == lir_lock) {
2726 Register tmp = op->scratch_opr()->as_register();
2727 // add debug info for NullPointerException only if one is possible
2728 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2729 if (op->info() != nullptr) {
2730 add_debug_info_for_null_check(null_check_offset, op->info());
2731 }
2732 // done
2733 } else if (op->code() == lir_unlock) {
2734 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2735 } else {
2736 Unimplemented();
2737 }
2738 __ bind(*op->stub()->continuation());
2739 }
2740
2741 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2742 Register obj = op->obj()->as_pointer_register();
2743 Register result = op->result_opr()->as_pointer_register();
2744
2745 CodeEmitInfo* info = op->info();
2746 if (info != nullptr) {
2747 add_debug_info_for_null_check_here(info);
2748 }
2749
2750 __ load_klass(result, obj, rscratch1);
2751 }
2752
2753 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2754 ciMethod* method = op->profiled_method();
2755 int bci = op->profiled_bci();
2756 ciMethod* callee = op->profiled_callee();
2757 Register tmp_load_klass = rscratch1;
2758
2759 // Update counter for all call types
2760 ciMethodData* md = method->method_data_or_null();
2761 assert(md != nullptr, "Sanity");
2762 ciProfileData* data = md->bci_to_data(bci);
2763 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2764 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2765 Register mdo = op->mdo()->as_register();
2766 __ mov_metadata(mdo, md->constant_encoding());
2767 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2768 // Perform additional virtual call profiling for invokevirtual and
2769 // invokeinterface bytecodes
2770 if (op->should_profile_receiver_type()) {
2771 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2772 Register recv = op->recv()->as_register();
2773 assert_different_registers(mdo, recv);
2774 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2775 ciKlass* known_klass = op->known_holder();
2776 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2777 // We know the type that will be seen at this call site; we can
2778 // statically update the MethodData* rather than needing to do
2779 // dynamic tests on the receiver type.
2780 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2781 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2782 ciKlass* receiver = vc_data->receiver(i);
2783 if (known_klass->equals(receiver)) {
2784 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2785 __ addptr(data_addr, DataLayout::counter_increment);
2786 return;
2787 }
2788 }
2789 // Receiver type is not found in profile data.
2790 // Fall back to runtime helper to handle the rest at runtime.
2791 __ mov_metadata(recv, known_klass->constant_encoding());
2792 } else {
2793 __ load_klass(recv, recv, tmp_load_klass);
2794 }
2795 type_profile_helper(mdo, md, data, recv);
2796 } else {
2797 // Static call
2798 __ addptr(counter_addr, DataLayout::counter_increment);
2799 }
2800 }
2801
2802 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2803 Register obj = op->obj()->as_register();
2804 Register tmp = op->tmp()->as_pointer_register();
2805 Register tmp_load_klass = rscratch1;
2806 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2807 ciKlass* exact_klass = op->exact_klass();
2808 intptr_t current_klass = op->current_klass();
2809 bool not_null = op->not_null();
2810 bool no_conflict = op->no_conflict();
2811
2812 Label update, next, none;
2813
2814 bool do_null = !not_null;
2815 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2816 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2817
2818 assert(do_null || do_update, "why are we here?");
2819 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2820
2821 __ verify_oop(obj);
2822
2823 #ifdef ASSERT
2824 if (obj == tmp) {
2825 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2826 } else {
2827 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2828 }
2829 #endif
2830 if (do_null) {
2831 __ testptr(obj, obj);
2832 __ jccb(Assembler::notZero, update);
2833 if (!TypeEntries::was_null_seen(current_klass)) {
2834 __ testptr(mdo_addr, TypeEntries::null_seen);
2835 #ifndef ASSERT
2836 __ jccb(Assembler::notZero, next); // already set
2837 #else
2838 __ jcc(Assembler::notZero, next); // already set
2839 #endif
2840 // atomic update to prevent overwriting Klass* with 0
2841 __ lock();
2842 __ orptr(mdo_addr, TypeEntries::null_seen);
2843 }
2844 if (do_update) {
2845 #ifndef ASSERT
2846 __ jmpb(next);
2847 }
2848 #else
2849 __ jmp(next);
2850 }
2851 } else {
2852 __ testptr(obj, obj);
2853 __ jcc(Assembler::notZero, update);
2854 __ stop("unexpected null obj");
2855 #endif
2856 }
2857
2858 __ bind(update);
2859
2860 if (do_update) {
2861 #ifdef ASSERT
2862 if (exact_klass != nullptr) {
2863 Label ok;
2864 __ load_klass(tmp, obj, tmp_load_klass);
2865 __ push_ppx(tmp);
2866 __ mov_metadata(tmp, exact_klass->constant_encoding());
2867 __ cmpptr(tmp, Address(rsp, 0));
2868 __ jcc(Assembler::equal, ok);
2869 __ stop("exact klass and actual klass differ");
2870 __ bind(ok);
2871 __ pop_ppx(tmp);
2872 }
2873 #endif
2874 if (!no_conflict) {
2875 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2876 if (exact_klass != nullptr) {
2877 __ mov_metadata(tmp, exact_klass->constant_encoding());
2878 } else {
2879 __ load_klass(tmp, obj, tmp_load_klass);
2880 }
2881 __ mov(rscratch1, tmp); // save original value before XOR
2882 __ xorptr(tmp, mdo_addr);
2883 __ testptr(tmp, TypeEntries::type_klass_mask);
2884 // klass seen before, nothing to do. The unknown bit may have been
2885 // set already but no need to check.
2886 __ jccb(Assembler::zero, next);
2887
2888 __ testptr(tmp, TypeEntries::type_unknown);
2889 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2890
2891 if (TypeEntries::is_type_none(current_klass)) {
2892 __ testptr(mdo_addr, TypeEntries::type_mask);
2893 __ jccb(Assembler::zero, none);
2894 // There is a chance that the checks above (re-reading profiling
2895 // data from memory) fail if another thread has just set the
2896 // profiling to this obj's klass
2897 __ mov(tmp, rscratch1); // get back original value before XOR
2898 __ xorptr(tmp, mdo_addr);
2899 __ testptr(tmp, TypeEntries::type_klass_mask);
2900 __ jccb(Assembler::zero, next);
2901 }
2902 } else {
2903 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2904 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2905
2906 __ testptr(mdo_addr, TypeEntries::type_unknown);
2907 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2908 }
2909
2910 // different than before. Cannot keep accurate profile.
2911 __ orptr(mdo_addr, TypeEntries::type_unknown);
2912
2913 if (TypeEntries::is_type_none(current_klass)) {
2914 __ jmpb(next);
2915
2916 __ bind(none);
2917 // first time here. Set profile type.
2918 __ movptr(mdo_addr, tmp);
2919 #ifdef ASSERT
2920 __ andptr(tmp, TypeEntries::type_klass_mask);
2921 __ verify_klass_ptr(tmp);
2922 #endif
2923 }
2924 } else {
2925 // There's a single possible klass at this profile point
2926 assert(exact_klass != nullptr, "should be");
2927 if (TypeEntries::is_type_none(current_klass)) {
2928 __ mov_metadata(tmp, exact_klass->constant_encoding());
2929 __ xorptr(tmp, mdo_addr);
2930 __ testptr(tmp, TypeEntries::type_klass_mask);
2931 #ifdef ASSERT
2932 __ jcc(Assembler::zero, next);
2933
2934 {
2935 Label ok;
2936 __ push_ppx(tmp);
2937 __ testptr(mdo_addr, TypeEntries::type_mask);
2938 __ jcc(Assembler::zero, ok);
2939 // may have been set by another thread
2940 __ mov_metadata(tmp, exact_klass->constant_encoding());
2941 __ xorptr(tmp, mdo_addr);
2942 __ testptr(tmp, TypeEntries::type_mask);
2943 __ jcc(Assembler::zero, ok);
2944
2945 __ stop("unexpected profiling mismatch");
2946 __ bind(ok);
2947 __ pop_ppx(tmp);
2948 }
2949 #else
2950 __ jccb(Assembler::zero, next);
2951 #endif
2952 // first time here. Set profile type.
2953 __ movptr(mdo_addr, tmp);
2954 #ifdef ASSERT
2955 __ andptr(tmp, TypeEntries::type_klass_mask);
2956 __ verify_klass_ptr(tmp);
2957 #endif
2958 } else {
2959 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2960 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2961
2962 __ testptr(mdo_addr, TypeEntries::type_unknown);
2963 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2964
2965 __ orptr(mdo_addr, TypeEntries::type_unknown);
2966 }
2967 }
2968 }
2969 __ bind(next);
2970 }
2971
2972 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2973 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2974 }
2975
2976
2977 void LIR_Assembler::align_backward_branch_target() {
2978 __ align(BytesPerWord);
2979 }
2980
2981
2982 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2983 if (left->is_single_cpu()) {
2984 __ negl(left->as_register());
2985 move_regs(left->as_register(), dest->as_register());
2986
2987 } else if (left->is_double_cpu()) {
2988 Register lo = left->as_register_lo();
2989 Register dst = dest->as_register_lo();
2990 __ movptr(dst, lo);
2991 __ negptr(dst);
2992
2993 } else if (dest->is_single_xmm()) {
2994 assert(!tmp->is_valid(), "do not need temporary");
2995 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
2996 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
2997 }
2998 __ xorps(dest->as_xmm_float_reg(),
2999 ExternalAddress((address)float_signflip_pool),
3000 rscratch1);
3001 } else if (dest->is_double_xmm()) {
3002 assert(!tmp->is_valid(), "do not need temporary");
3003 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3004 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3005 }
3006 __ xorpd(dest->as_xmm_double_reg(),
3007 ExternalAddress((address)double_signflip_pool),
3008 rscratch1);
3009 } else {
3010 ShouldNotReachHere();
3011 }
3012 }
3013
3014
3015 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3016 assert(src->is_address(), "must be an address");
3017 assert(dest->is_register(), "must be a register");
3018
3019 PatchingStub* patch = nullptr;
3020 if (patch_code != lir_patch_none) {
3021 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3022 }
3023
3024 Register reg = dest->as_pointer_register();
3025 LIR_Address* addr = src->as_address_ptr();
3026 __ lea(reg, as_Address(addr));
3027
3028 if (patch != nullptr) {
3029 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3030 }
3031 }
3032
3033
3034
3035 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3036 assert(!tmp->is_valid(), "don't need temporary");
3037 __ call(RuntimeAddress(dest));
3038 if (info != nullptr) {
3039 add_call_info_here(info);
3040 }
3041 __ post_call_nop();
3042 }
3043
3044
3045 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3046 assert(type == T_LONG, "only for volatile long fields");
3047
3048 if (info != nullptr) {
3049 add_debug_info_for_null_check_here(info);
3050 }
3051
3052 if (src->is_double_xmm()) {
3053 if (dest->is_double_cpu()) {
3054 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3055 } else if (dest->is_double_stack()) {
3056 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3057 } else if (dest->is_address()) {
3058 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3059 } else {
3060 ShouldNotReachHere();
3061 }
3062
3063 } else if (dest->is_double_xmm()) {
3064 if (src->is_double_stack()) {
3065 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3066 } else if (src->is_address()) {
3067 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3068 } else {
3069 ShouldNotReachHere();
3070 }
3071
3072 } else {
3073 ShouldNotReachHere();
3074 }
3075 }
3076
3077 #ifdef ASSERT
3078 // emit run-time assertion
3079 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3080 assert(op->code() == lir_assert, "must be");
3081
3082 if (op->in_opr1()->is_valid()) {
3083 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3084 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3085 } else {
3086 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3087 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3088 }
3089
3090 Label ok;
3091 if (op->condition() != lir_cond_always) {
3092 Assembler::Condition acond = Assembler::zero;
3093 switch (op->condition()) {
3094 case lir_cond_equal: acond = Assembler::equal; break;
3095 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3096 case lir_cond_less: acond = Assembler::less; break;
3097 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3098 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3099 case lir_cond_greater: acond = Assembler::greater; break;
3100 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3101 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3102 default: ShouldNotReachHere();
3103 }
3104 __ jcc(acond, ok);
3105 }
3106 if (op->halt()) {
3107 const char* str = __ code_string(op->msg());
3108 __ stop(str);
3109 } else {
3110 breakpoint();
3111 }
3112 __ bind(ok);
3113 }
3114 #endif
3115
3116 void LIR_Assembler::membar() {
3117 // QQQ sparc TSO uses this,
3118 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3119 }
3120
3121 void LIR_Assembler::membar_acquire() {
3122 // No x86 machines currently require load fences
3123 }
3124
3125 void LIR_Assembler::membar_release() {
3126 // No x86 machines currently require store fences
3127 }
3128
3129 void LIR_Assembler::membar_loadload() {
3130 // no-op
3131 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3132 }
3133
3134 void LIR_Assembler::membar_storestore() {
3135 // no-op
3136 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3137 }
3138
3139 void LIR_Assembler::membar_loadstore() {
3140 // no-op
3141 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3142 }
3143
3144 void LIR_Assembler::membar_storeload() {
3145 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3146 }
3147
3148 void LIR_Assembler::on_spin_wait() {
3149 __ pause ();
3150 }
3151
3152 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3153 assert(result_reg->is_register(), "check");
3154 __ mov(result_reg->as_register(), r15_thread);
3155 }
3156
3157
3158 void LIR_Assembler::peephole(LIR_List*) {
3159 // do nothing for now
3160 }
3161
3162 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3163 assert(data == dest, "xchg/xadd uses only 2 operands");
3164
3165 if (data->type() == T_INT) {
3166 if (code == lir_xadd) {
3167 __ lock();
3168 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3169 } else {
3170 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3171 }
3172 } else if (data->is_oop()) {
3173 assert (code == lir_xchg, "xadd for oops");
3174 Register obj = data->as_register();
3175 if (UseCompressedOops) {
3176 __ encode_heap_oop(obj);
3177 __ xchgl(obj, as_Address(src->as_address_ptr()));
3178 __ decode_heap_oop(obj);
3179 } else {
3180 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3181 }
3182 } else if (data->type() == T_LONG) {
3183 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3184 if (code == lir_xadd) {
3185 __ lock();
3186 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3187 } else {
3188 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3189 }
3190 } else {
3191 ShouldNotReachHere();
3192 }
3193 }
3194
3195 #undef __