1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "ci/ciUtilities.hpp"
36 #include "code/aotCodeCache.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456 InternalAddress here(__ pc());
457
458 __ pushptr(here.addr(), rscratch1);
459 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
460 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
461 __ end_a_stub();
462
463 return offset;
464 }
465
466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
467 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
468 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
469 assert(result->fpu() == 0, "result must already be on TOS");
470 }
471
472 // Pop the stack before the safepoint code
473 __ remove_frame(initial_frame_size_in_bytes());
474
475 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
476 __ reserved_stack_check();
477 }
478
479 // Note: we do not need to round double result; float result has the right precision
480 // the poll sets the condition code, but no data registers
481
482 code_stub->set_safepoint_offset(__ offset());
483 __ relocate(relocInfo::poll_return_type);
484 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
485 __ ret(0);
486 }
487
488
489 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
490 guarantee(info != nullptr, "Shouldn't be null");
491 int offset = __ offset();
492 const Register poll_addr = rscratch1;
493 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
494 add_debug_info_for_branch(info);
495 __ relocate(relocInfo::poll_type);
496 address pre_pc = __ pc();
497 __ testl(rax, Address(poll_addr, 0));
498 address post_pc = __ pc();
499 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
500 return offset;
501 }
502
503
504 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
505 if (from_reg != to_reg) __ mov(to_reg, from_reg);
506 }
507
508 void LIR_Assembler::swap_reg(Register a, Register b) {
509 __ xchgptr(a, b);
510 }
511
512
513 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
514 assert(src->is_constant(), "should not call otherwise");
515 assert(dest->is_register(), "should not call otherwise");
516 LIR_Const* c = src->as_constant_ptr();
517
518 switch (c->type()) {
519 case T_INT: {
520 assert(patch_code == lir_patch_none, "no patching handled here");
521 __ movl(dest->as_register(), c->as_jint());
522 break;
523 }
524
525 case T_ADDRESS: {
526 assert(patch_code == lir_patch_none, "no patching handled here");
527 __ movptr(dest->as_register(), c->as_jint());
528 break;
529 }
530
531 case T_LONG: {
532 assert(patch_code == lir_patch_none, "no patching handled here");
533 #if INCLUDE_CDS
534 if (AOTCodeCache::is_on_for_dump()) {
535 address b = c->as_pointer();
536 if (b == (address)ThreadIdentifier::unsafe_offset()) {
537 __ lea(dest->as_register_lo(), ExternalAddress(b));
538 break;
539 }
540 if (AOTRuntimeConstants::contains(b)) {
541 __ load_aotrc_address(dest->as_register_lo(), b);
542 break;
543 }
544 }
545 #endif
546 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
547 break;
548 }
549
550 case T_OBJECT: {
551 if (patch_code != lir_patch_none) {
552 jobject2reg_with_patching(dest->as_register(), info);
553 } else {
554 __ movoop(dest->as_register(), c->as_jobject());
555 }
556 break;
557 }
558
559 case T_METADATA: {
560 if (patch_code != lir_patch_none) {
561 klass2reg_with_patching(dest->as_register(), info);
562 } else {
563 __ mov_metadata(dest->as_register(), c->as_metadata());
564 }
565 break;
566 }
567
568 case T_FLOAT: {
569 if (dest->is_single_xmm()) {
570 if (UseAVX <= 2 && c->is_zero_float()) {
571 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
572 } else {
573 __ movflt(dest->as_xmm_float_reg(),
574 InternalAddress(float_constant(c->as_jfloat())));
575 }
576 } else {
577 ShouldNotReachHere();
578 }
579 break;
580 }
581
582 case T_DOUBLE: {
583 if (dest->is_double_xmm()) {
584 if (UseAVX <= 2 && c->is_zero_double()) {
585 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
586 } else {
587 __ movdbl(dest->as_xmm_double_reg(),
588 InternalAddress(double_constant(c->as_jdouble())));
589 }
590 } else {
591 ShouldNotReachHere();
592 }
593 break;
594 }
595
596 default:
597 ShouldNotReachHere();
598 }
599 }
600
601 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
602 assert(src->is_constant(), "should not call otherwise");
603 assert(dest->is_stack(), "should not call otherwise");
604 LIR_Const* c = src->as_constant_ptr();
605
606 switch (c->type()) {
607 case T_INT: // fall through
608 case T_FLOAT:
609 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
610 break;
611
612 case T_ADDRESS:
613 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
614 break;
615
616 case T_OBJECT:
617 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
618 break;
619
620 case T_LONG: // fall through
621 case T_DOUBLE:
622 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
623 lo_word_offset_in_bytes),
624 (intptr_t)c->as_jlong_bits(),
625 rscratch1);
626 break;
627
628 default:
629 ShouldNotReachHere();
630 }
631 }
632
633 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
634 assert(src->is_constant(), "should not call otherwise");
635 assert(dest->is_address(), "should not call otherwise");
636 LIR_Const* c = src->as_constant_ptr();
637 LIR_Address* addr = dest->as_address_ptr();
638
639 int null_check_here = code_offset();
640 switch (type) {
641 case T_INT: // fall through
642 case T_FLOAT:
643 __ movl(as_Address(addr), c->as_jint_bits());
644 break;
645
646 case T_ADDRESS:
647 __ movptr(as_Address(addr), c->as_jint_bits());
648 break;
649
650 case T_OBJECT: // fall through
651 case T_ARRAY:
652 if (c->as_jobject() == nullptr) {
653 if (UseCompressedOops && !wide) {
654 __ movl(as_Address(addr), NULL_WORD);
655 } else {
656 __ xorptr(rscratch1, rscratch1);
657 null_check_here = code_offset();
658 __ movptr(as_Address(addr), rscratch1);
659 }
660 } else {
661 if (is_literal_address(addr)) {
662 ShouldNotReachHere();
663 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
664 } else {
665 __ movoop(rscratch1, c->as_jobject());
666 if (UseCompressedOops && !wide) {
667 __ encode_heap_oop(rscratch1);
668 null_check_here = code_offset();
669 __ movl(as_Address_lo(addr), rscratch1);
670 } else {
671 null_check_here = code_offset();
672 __ movptr(as_Address_lo(addr), rscratch1);
673 }
674 }
675 }
676 break;
677
678 case T_LONG: // fall through
679 case T_DOUBLE:
680 if (is_literal_address(addr)) {
681 ShouldNotReachHere();
682 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
683 } else {
684 __ movptr(r10, (intptr_t)c->as_jlong_bits());
685 null_check_here = code_offset();
686 __ movptr(as_Address_lo(addr), r10);
687 }
688 break;
689
690 case T_BOOLEAN: // fall through
691 case T_BYTE:
692 __ movb(as_Address(addr), c->as_jint() & 0xFF);
693 break;
694
695 case T_CHAR: // fall through
696 case T_SHORT:
697 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
698 break;
699
700 default:
701 ShouldNotReachHere();
702 };
703
704 if (info != nullptr) {
705 add_debug_info_for_null_check(null_check_here, info);
706 }
707 }
708
709
710 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
711 assert(src->is_register(), "should not call otherwise");
712 assert(dest->is_register(), "should not call otherwise");
713
714 // move between cpu-registers
715 if (dest->is_single_cpu()) {
716 if (src->type() == T_LONG) {
717 // Can do LONG -> OBJECT
718 move_regs(src->as_register_lo(), dest->as_register());
719 return;
720 }
721 assert(src->is_single_cpu(), "must match");
722 if (src->type() == T_OBJECT) {
723 __ verify_oop(src->as_register());
724 }
725 move_regs(src->as_register(), dest->as_register());
726
727 } else if (dest->is_double_cpu()) {
728 if (is_reference_type(src->type())) {
729 // Surprising to me but we can see move of a long to t_object
730 __ verify_oop(src->as_register());
731 move_regs(src->as_register(), dest->as_register_lo());
732 return;
733 }
734 assert(src->is_double_cpu(), "must match");
735 Register f_lo = src->as_register_lo();
736 Register f_hi = src->as_register_hi();
737 Register t_lo = dest->as_register_lo();
738 Register t_hi = dest->as_register_hi();
739 assert(f_hi == f_lo, "must be same");
740 assert(t_hi == t_lo, "must be same");
741 move_regs(f_lo, t_lo);
742
743 // move between xmm-registers
744 } else if (dest->is_single_xmm()) {
745 assert(src->is_single_xmm(), "must match");
746 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
747 } else if (dest->is_double_xmm()) {
748 assert(src->is_double_xmm(), "must match");
749 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
750
751 } else {
752 ShouldNotReachHere();
753 }
754 }
755
756 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
757 assert(src->is_register(), "should not call otherwise");
758 assert(dest->is_stack(), "should not call otherwise");
759
760 if (src->is_single_cpu()) {
761 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
762 if (is_reference_type(type)) {
763 __ verify_oop(src->as_register());
764 __ movptr (dst, src->as_register());
765 } else if (type == T_METADATA || type == T_ADDRESS) {
766 __ movptr (dst, src->as_register());
767 } else {
768 __ movl (dst, src->as_register());
769 }
770
771 } else if (src->is_double_cpu()) {
772 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
773 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
774 __ movptr (dstLO, src->as_register_lo());
775
776 } else if (src->is_single_xmm()) {
777 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
778 __ movflt(dst_addr, src->as_xmm_float_reg());
779
780 } else if (src->is_double_xmm()) {
781 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
782 __ movdbl(dst_addr, src->as_xmm_double_reg());
783
784 } else {
785 ShouldNotReachHere();
786 }
787 }
788
789
790 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
791 LIR_Address* to_addr = dest->as_address_ptr();
792 PatchingStub* patch = nullptr;
793 Register compressed_src = rscratch1;
794
795 if (is_reference_type(type)) {
796 __ verify_oop(src->as_register());
797 if (UseCompressedOops && !wide) {
798 __ movptr(compressed_src, src->as_register());
799 __ encode_heap_oop(compressed_src);
800 if (patch_code != lir_patch_none) {
801 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
802 }
803 }
804 }
805
806 if (patch_code != lir_patch_none) {
807 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
808 Address toa = as_Address(to_addr);
809 assert(toa.disp() != 0, "must have");
810 }
811
812 int null_check_here = code_offset();
813 switch (type) {
814 case T_FLOAT: {
815 assert(src->is_single_xmm(), "not a float");
816 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
817 break;
818 }
819
820 case T_DOUBLE: {
821 assert(src->is_double_xmm(), "not a double");
822 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
823 break;
824 }
825
826 case T_ARRAY: // fall through
827 case T_OBJECT: // fall through
828 if (UseCompressedOops && !wide) {
829 __ movl(as_Address(to_addr), compressed_src);
830 } else {
831 __ movptr(as_Address(to_addr), src->as_register());
832 }
833 break;
834 case T_ADDRESS:
835 __ movptr(as_Address(to_addr), src->as_register());
836 break;
837 case T_INT:
838 __ movl(as_Address(to_addr), src->as_register());
839 break;
840
841 case T_LONG: {
842 Register from_lo = src->as_register_lo();
843 Register from_hi = src->as_register_hi();
844 __ movptr(as_Address_lo(to_addr), from_lo);
845 break;
846 }
847
848 case T_BYTE: // fall through
849 case T_BOOLEAN: {
850 Register src_reg = src->as_register();
851 Address dst_addr = as_Address(to_addr);
852 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
853 __ movb(dst_addr, src_reg);
854 break;
855 }
856
857 case T_CHAR: // fall through
858 case T_SHORT:
859 __ movw(as_Address(to_addr), src->as_register());
860 break;
861
862 default:
863 ShouldNotReachHere();
864 }
865 if (info != nullptr) {
866 add_debug_info_for_null_check(null_check_here, info);
867 }
868
869 if (patch_code != lir_patch_none) {
870 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
871 }
872 }
873
874
875 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
876 assert(src->is_stack(), "should not call otherwise");
877 assert(dest->is_register(), "should not call otherwise");
878
879 if (dest->is_single_cpu()) {
880 if (is_reference_type(type)) {
881 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
882 __ verify_oop(dest->as_register());
883 } else if (type == T_METADATA || type == T_ADDRESS) {
884 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
885 } else {
886 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
887 }
888
889 } else if (dest->is_double_cpu()) {
890 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
891 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
892 __ movptr(dest->as_register_lo(), src_addr_LO);
893
894 } else if (dest->is_single_xmm()) {
895 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
896 __ movflt(dest->as_xmm_float_reg(), src_addr);
897
898 } else if (dest->is_double_xmm()) {
899 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
900 __ movdbl(dest->as_xmm_double_reg(), src_addr);
901
902 } else {
903 ShouldNotReachHere();
904 }
905 }
906
907
908 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
909 if (src->is_single_stack()) {
910 if (is_reference_type(type)) {
911 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
912 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
913 } else {
914 //no pushl on 64bits
915 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
916 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
917 }
918
919 } else if (src->is_double_stack()) {
920 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
921 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
922
923 } else {
924 ShouldNotReachHere();
925 }
926 }
927
928
929 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
930 assert(src->is_address(), "should not call otherwise");
931 assert(dest->is_register(), "should not call otherwise");
932
933 LIR_Address* addr = src->as_address_ptr();
934 Address from_addr = as_Address(addr);
935
936 if (addr->base()->type() == T_OBJECT) {
937 __ verify_oop(addr->base()->as_pointer_register());
938 }
939
940 switch (type) {
941 case T_BOOLEAN: // fall through
942 case T_BYTE: // fall through
943 case T_CHAR: // fall through
944 case T_SHORT:
945 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
946 // on pre P6 processors we may get partial register stalls
947 // so blow away the value of to_rinfo before loading a
948 // partial word into it. Do it here so that it precedes
949 // the potential patch point below.
950 __ xorptr(dest->as_register(), dest->as_register());
951 }
952 break;
953 default:
954 break;
955 }
956
957 PatchingStub* patch = nullptr;
958 if (patch_code != lir_patch_none) {
959 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
960 assert(from_addr.disp() != 0, "must have");
961 }
962 if (info != nullptr) {
963 add_debug_info_for_null_check_here(info);
964 }
965
966 switch (type) {
967 case T_FLOAT: {
968 if (dest->is_single_xmm()) {
969 __ movflt(dest->as_xmm_float_reg(), from_addr);
970 } else {
971 ShouldNotReachHere();
972 }
973 break;
974 }
975
976 case T_DOUBLE: {
977 if (dest->is_double_xmm()) {
978 __ movdbl(dest->as_xmm_double_reg(), from_addr);
979 } else {
980 ShouldNotReachHere();
981 }
982 break;
983 }
984
985 case T_OBJECT: // fall through
986 case T_ARRAY: // fall through
987 if (UseCompressedOops && !wide) {
988 __ movl(dest->as_register(), from_addr);
989 } else {
990 __ movptr(dest->as_register(), from_addr);
991 }
992 break;
993
994 case T_ADDRESS:
995 __ movptr(dest->as_register(), from_addr);
996 break;
997 case T_INT:
998 __ movl(dest->as_register(), from_addr);
999 break;
1000
1001 case T_LONG: {
1002 Register to_lo = dest->as_register_lo();
1003 Register to_hi = dest->as_register_hi();
1004 __ movptr(to_lo, as_Address_lo(addr));
1005 break;
1006 }
1007
1008 case T_BOOLEAN: // fall through
1009 case T_BYTE: {
1010 Register dest_reg = dest->as_register();
1011 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1012 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1013 __ movsbl(dest_reg, from_addr);
1014 } else {
1015 __ movb(dest_reg, from_addr);
1016 __ shll(dest_reg, 24);
1017 __ sarl(dest_reg, 24);
1018 }
1019 break;
1020 }
1021
1022 case T_CHAR: {
1023 Register dest_reg = dest->as_register();
1024 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1025 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1026 __ movzwl(dest_reg, from_addr);
1027 } else {
1028 __ movw(dest_reg, from_addr);
1029 }
1030 break;
1031 }
1032
1033 case T_SHORT: {
1034 Register dest_reg = dest->as_register();
1035 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1036 __ movswl(dest_reg, from_addr);
1037 } else {
1038 __ movw(dest_reg, from_addr);
1039 __ shll(dest_reg, 16);
1040 __ sarl(dest_reg, 16);
1041 }
1042 break;
1043 }
1044
1045 default:
1046 ShouldNotReachHere();
1047 }
1048
1049 if (patch != nullptr) {
1050 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1051 }
1052
1053 if (is_reference_type(type)) {
1054 if (UseCompressedOops && !wide) {
1055 __ decode_heap_oop(dest->as_register());
1056 }
1057
1058 __ verify_oop(dest->as_register());
1059 }
1060 }
1061
1062
1063 NEEDS_CLEANUP; // This could be static?
1064 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1065 int elem_size = type2aelembytes(type);
1066 switch (elem_size) {
1067 case 1: return Address::times_1;
1068 case 2: return Address::times_2;
1069 case 4: return Address::times_4;
1070 case 8: return Address::times_8;
1071 }
1072 ShouldNotReachHere();
1073 return Address::no_scale;
1074 }
1075
1076
1077 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1078 switch (op->code()) {
1079 case lir_idiv:
1080 case lir_irem:
1081 arithmetic_idiv(op->code(),
1082 op->in_opr1(),
1083 op->in_opr2(),
1084 op->in_opr3(),
1085 op->result_opr(),
1086 op->info());
1087 break;
1088 case lir_fmad:
1089 __ fmad(op->result_opr()->as_xmm_double_reg(),
1090 op->in_opr1()->as_xmm_double_reg(),
1091 op->in_opr2()->as_xmm_double_reg(),
1092 op->in_opr3()->as_xmm_double_reg());
1093 break;
1094 case lir_fmaf:
1095 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1096 op->in_opr1()->as_xmm_float_reg(),
1097 op->in_opr2()->as_xmm_float_reg(),
1098 op->in_opr3()->as_xmm_float_reg());
1099 break;
1100 default: ShouldNotReachHere(); break;
1101 }
1102 }
1103
1104 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1105 #ifdef ASSERT
1106 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1107 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1108 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1109 #endif
1110
1111 if (op->cond() == lir_cond_always) {
1112 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1113 __ jmp (*(op->label()));
1114 } else {
1115 Assembler::Condition acond = Assembler::zero;
1116 if (op->code() == lir_cond_float_branch) {
1117 assert(op->ublock() != nullptr, "must have unordered successor");
1118 __ jcc(Assembler::parity, *(op->ublock()->label()));
1119 switch(op->cond()) {
1120 case lir_cond_equal: acond = Assembler::equal; break;
1121 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1122 case lir_cond_less: acond = Assembler::below; break;
1123 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1124 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1125 case lir_cond_greater: acond = Assembler::above; break;
1126 default: ShouldNotReachHere();
1127 }
1128 } else {
1129 switch (op->cond()) {
1130 case lir_cond_equal: acond = Assembler::equal; break;
1131 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1132 case lir_cond_less: acond = Assembler::less; break;
1133 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1134 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1135 case lir_cond_greater: acond = Assembler::greater; break;
1136 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1137 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1138 default: ShouldNotReachHere();
1139 }
1140 }
1141 __ jcc(acond,*(op->label()));
1142 }
1143 }
1144
1145 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1146 LIR_Opr src = op->in_opr();
1147 LIR_Opr dest = op->result_opr();
1148
1149 switch (op->bytecode()) {
1150 case Bytecodes::_i2l:
1151 __ movl2ptr(dest->as_register_lo(), src->as_register());
1152 break;
1153
1154 case Bytecodes::_l2i:
1155 __ movl(dest->as_register(), src->as_register_lo());
1156 break;
1157
1158 case Bytecodes::_i2b:
1159 move_regs(src->as_register(), dest->as_register());
1160 __ sign_extend_byte(dest->as_register());
1161 break;
1162
1163 case Bytecodes::_i2c:
1164 move_regs(src->as_register(), dest->as_register());
1165 __ andl(dest->as_register(), 0xFFFF);
1166 break;
1167
1168 case Bytecodes::_i2s:
1169 move_regs(src->as_register(), dest->as_register());
1170 __ sign_extend_short(dest->as_register());
1171 break;
1172
1173 case Bytecodes::_f2d:
1174 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1175 break;
1176
1177 case Bytecodes::_d2f:
1178 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1179 break;
1180
1181 case Bytecodes::_i2f:
1182 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1183 break;
1184
1185 case Bytecodes::_i2d:
1186 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1187 break;
1188
1189 case Bytecodes::_l2f:
1190 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1191 break;
1192
1193 case Bytecodes::_l2d:
1194 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1195 break;
1196
1197 case Bytecodes::_f2i:
1198 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1199 break;
1200
1201 case Bytecodes::_d2i:
1202 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1203 break;
1204
1205 case Bytecodes::_f2l:
1206 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1207 break;
1208
1209 case Bytecodes::_d2l:
1210 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1211 break;
1212
1213 default: ShouldNotReachHere();
1214 }
1215 }
1216
1217 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1218 if (op->init_check()) {
1219 add_debug_info_for_null_check_here(op->stub()->info());
1220 // init_state needs acquire, but x86 is TSO, and so we are already good.
1221 __ cmpb(Address(op->klass()->as_register(),
1222 InstanceKlass::init_state_offset()),
1223 InstanceKlass::fully_initialized);
1224 __ jcc(Assembler::notEqual, *op->stub()->entry());
1225 }
1226 __ allocate_object(op->obj()->as_register(),
1227 op->tmp1()->as_register(),
1228 op->tmp2()->as_register(),
1229 op->header_size(),
1230 op->object_size(),
1231 op->klass()->as_register(),
1232 *op->stub()->entry());
1233 __ bind(*op->stub()->continuation());
1234 }
1235
1236 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1237 Register len = op->len()->as_register();
1238 __ movslq(len, len);
1239
1240 if (UseSlowPath ||
1241 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1242 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1243 __ jmp(*op->stub()->entry());
1244 } else {
1245 Register tmp1 = op->tmp1()->as_register();
1246 Register tmp2 = op->tmp2()->as_register();
1247 Register tmp3 = op->tmp3()->as_register();
1248 if (len == tmp1) {
1249 tmp1 = tmp3;
1250 } else if (len == tmp2) {
1251 tmp2 = tmp3;
1252 } else if (len == tmp3) {
1253 // everything is ok
1254 } else {
1255 __ mov(tmp3, len);
1256 }
1257 __ allocate_array(op->obj()->as_register(),
1258 len,
1259 tmp1,
1260 tmp2,
1261 arrayOopDesc::base_offset_in_bytes(op->type()),
1262 array_element_size(op->type()),
1263 op->klass()->as_register(),
1264 *op->stub()->entry(),
1265 op->zero_array());
1266 }
1267 __ bind(*op->stub()->continuation());
1268 }
1269
1270 void LIR_Assembler::type_profile_helper(Register mdo,
1271 ciMethodData *md, ciProfileData *data,
1272 Register recv, Label* update_done) {
1273 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1274 Label next_test;
1275 // See if the receiver is receiver[n].
1276 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1277 __ jccb(Assembler::notEqual, next_test);
1278 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1279 __ addptr(data_addr, DataLayout::counter_increment);
1280 __ jmp(*update_done);
1281 __ bind(next_test);
1282 }
1283
1284 // Didn't find receiver; find next empty slot and fill it in
1285 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1286 Label next_test;
1287 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1288 __ cmpptr(recv_addr, NULL_WORD);
1289 __ jccb(Assembler::notEqual, next_test);
1290 __ movptr(recv_addr, recv);
1291 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1292 __ jmp(*update_done);
1293 __ bind(next_test);
1294 }
1295 }
1296
1297 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1298 // we always need a stub for the failure case.
1299 CodeStub* stub = op->stub();
1300 Register obj = op->object()->as_register();
1301 Register k_RInfo = op->tmp1()->as_register();
1302 Register klass_RInfo = op->tmp2()->as_register();
1303 Register dst = op->result_opr()->as_register();
1304 ciKlass* k = op->klass();
1305 Register Rtmp1 = noreg;
1306 Register tmp_load_klass = rscratch1;
1307
1308 // check if it needs to be profiled
1309 ciMethodData* md = nullptr;
1310 ciProfileData* data = nullptr;
1311
1312 if (op->should_profile()) {
1313 ciMethod* method = op->profiled_method();
1314 assert(method != nullptr, "Should have method");
1315 int bci = op->profiled_bci();
1316 md = method->method_data_or_null();
1317 assert(md != nullptr, "Sanity");
1318 data = md->bci_to_data(bci);
1319 assert(data != nullptr, "need data for type check");
1320 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1321 }
1322 Label* success_target = success;
1323 Label* failure_target = failure;
1324
1325 if (obj == k_RInfo) {
1326 k_RInfo = dst;
1327 } else if (obj == klass_RInfo) {
1328 klass_RInfo = dst;
1329 }
1330 if (k->is_loaded() && !UseCompressedClassPointers) {
1331 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1332 } else {
1333 Rtmp1 = op->tmp3()->as_register();
1334 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1335 }
1336
1337 assert_different_registers(obj, k_RInfo, klass_RInfo);
1338
1339 __ testptr(obj, obj);
1340 if (op->should_profile()) {
1341 Label not_null;
1342 Register mdo = klass_RInfo;
1343 __ mov_metadata(mdo, md->constant_encoding());
1344 __ jccb(Assembler::notEqual, not_null);
1345 // Object is null; update MDO and exit
1346 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1347 int header_bits = BitData::null_seen_byte_constant();
1348 __ orb(data_addr, header_bits);
1349 __ jmp(*obj_is_null);
1350 __ bind(not_null);
1351
1352 Label update_done;
1353 Register recv = k_RInfo;
1354 __ load_klass(recv, obj, tmp_load_klass);
1355 type_profile_helper(mdo, md, data, recv, &update_done);
1356
1357 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1358 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1359
1360 __ bind(update_done);
1361 } else {
1362 __ jcc(Assembler::equal, *obj_is_null);
1363 }
1364
1365 if (!k->is_loaded()) {
1366 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1367 } else {
1368 __ mov_metadata(k_RInfo, k->constant_encoding());
1369 }
1370 __ verify_oop(obj);
1371
1372 if (op->fast_check()) {
1373 // get object class
1374 // not a safepoint as obj null check happens earlier
1375 if (UseCompressedClassPointers) {
1376 __ load_klass(Rtmp1, obj, tmp_load_klass);
1377 __ cmpptr(k_RInfo, Rtmp1);
1378 } else {
1379 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1380 }
1381 __ jcc(Assembler::notEqual, *failure_target);
1382 // successful cast, fall through to profile or jump
1383 } else {
1384 // get object class
1385 // not a safepoint as obj null check happens earlier
1386 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1387 if (k->is_loaded()) {
1388 // See if we get an immediate positive hit
1389 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1390 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1391 __ jcc(Assembler::notEqual, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 } else {
1394 // See if we get an immediate positive hit
1395 __ jcc(Assembler::equal, *success_target);
1396 // check for self
1397 __ cmpptr(klass_RInfo, k_RInfo);
1398 __ jcc(Assembler::equal, *success_target);
1399
1400 __ push_ppx(klass_RInfo);
1401 __ push_ppx(k_RInfo);
1402 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1403 __ pop_ppx(klass_RInfo);
1404 __ pop_ppx(klass_RInfo);
1405 // result is a boolean
1406 __ testl(klass_RInfo, klass_RInfo);
1407 __ jcc(Assembler::equal, *failure_target);
1408 // successful cast, fall through to profile or jump
1409 }
1410 } else {
1411 // perform the fast part of the checking logic
1412 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1413 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1414 __ push_ppx(klass_RInfo);
1415 __ push_ppx(k_RInfo);
1416 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1417 __ pop_ppx(klass_RInfo);
1418 __ pop_ppx(k_RInfo);
1419 // result is a boolean
1420 __ testl(k_RInfo, k_RInfo);
1421 __ jcc(Assembler::equal, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 }
1424 }
1425 __ jmp(*success);
1426 }
1427
1428
1429 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1430 Register tmp_load_klass = rscratch1;
1431 LIR_Code code = op->code();
1432 if (code == lir_store_check) {
1433 Register value = op->object()->as_register();
1434 Register array = op->array()->as_register();
1435 Register k_RInfo = op->tmp1()->as_register();
1436 Register klass_RInfo = op->tmp2()->as_register();
1437 Register Rtmp1 = op->tmp3()->as_register();
1438
1439 CodeStub* stub = op->stub();
1440
1441 // check if it needs to be profiled
1442 ciMethodData* md = nullptr;
1443 ciProfileData* data = nullptr;
1444
1445 if (op->should_profile()) {
1446 ciMethod* method = op->profiled_method();
1447 assert(method != nullptr, "Should have method");
1448 int bci = op->profiled_bci();
1449 md = method->method_data_or_null();
1450 assert(md != nullptr, "Sanity");
1451 data = md->bci_to_data(bci);
1452 assert(data != nullptr, "need data for type check");
1453 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1454 }
1455 Label done;
1456 Label* success_target = &done;
1457 Label* failure_target = stub->entry();
1458
1459 __ testptr(value, value);
1460 if (op->should_profile()) {
1461 Label not_null;
1462 Register mdo = klass_RInfo;
1463 __ mov_metadata(mdo, md->constant_encoding());
1464 __ jccb(Assembler::notEqual, not_null);
1465 // Object is null; update MDO and exit
1466 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1467 int header_bits = BitData::null_seen_byte_constant();
1468 __ orb(data_addr, header_bits);
1469 __ jmp(done);
1470 __ bind(not_null);
1471
1472 Label update_done;
1473 Register recv = k_RInfo;
1474 __ load_klass(recv, value, tmp_load_klass);
1475 type_profile_helper(mdo, md, data, recv, &update_done);
1476
1477 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1478 __ addptr(counter_addr, DataLayout::counter_increment);
1479 __ bind(update_done);
1480 } else {
1481 __ jcc(Assembler::equal, done);
1482 }
1483
1484 add_debug_info_for_null_check_here(op->info_for_exception());
1485 __ load_klass(k_RInfo, array, tmp_load_klass);
1486 __ load_klass(klass_RInfo, value, tmp_load_klass);
1487
1488 // get instance klass (it's already uncompressed)
1489 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1490 // perform the fast part of the checking logic
1491 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1492 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1493 __ push_ppx(klass_RInfo);
1494 __ push_ppx(k_RInfo);
1495 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1496 __ pop_ppx(klass_RInfo);
1497 __ pop_ppx(k_RInfo);
1498 // result is a boolean
1499 __ testl(k_RInfo, k_RInfo);
1500 __ jcc(Assembler::equal, *failure_target);
1501 // fall through to the success case
1502
1503 __ bind(done);
1504 } else
1505 if (code == lir_checkcast) {
1506 Register obj = op->object()->as_register();
1507 Register dst = op->result_opr()->as_register();
1508 Label success;
1509 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1510 __ bind(success);
1511 if (dst != obj) {
1512 __ mov(dst, obj);
1513 }
1514 } else
1515 if (code == lir_instanceof) {
1516 Register obj = op->object()->as_register();
1517 Register dst = op->result_opr()->as_register();
1518 Label success, failure, done;
1519 emit_typecheck_helper(op, &success, &failure, &failure);
1520 __ bind(failure);
1521 __ xorptr(dst, dst);
1522 __ jmpb(done);
1523 __ bind(success);
1524 __ movptr(dst, 1);
1525 __ bind(done);
1526 } else {
1527 ShouldNotReachHere();
1528 }
1529
1530 }
1531
1532
1533 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1534 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1535 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1536 Register newval = op->new_value()->as_register();
1537 Register cmpval = op->cmp_value()->as_register();
1538 assert(cmpval == rax, "wrong register");
1539 assert(newval != noreg, "new val must be register");
1540 assert(cmpval != newval, "cmp and new values must be in different registers");
1541 assert(cmpval != addr, "cmp and addr must be in different registers");
1542 assert(newval != addr, "new value and addr must be in different registers");
1543
1544 if (op->code() == lir_cas_obj) {
1545 if (UseCompressedOops) {
1546 __ encode_heap_oop(cmpval);
1547 __ mov(rscratch1, newval);
1548 __ encode_heap_oop(rscratch1);
1549 __ lock();
1550 // cmpval (rax) is implicitly used by this instruction
1551 __ cmpxchgl(rscratch1, Address(addr, 0));
1552 } else {
1553 __ lock();
1554 __ cmpxchgptr(newval, Address(addr, 0));
1555 }
1556 } else {
1557 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1558 __ lock();
1559 __ cmpxchgl(newval, Address(addr, 0));
1560 }
1561 } else if (op->code() == lir_cas_long) {
1562 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1563 Register newval = op->new_value()->as_register_lo();
1564 Register cmpval = op->cmp_value()->as_register_lo();
1565 assert(cmpval == rax, "wrong register");
1566 assert(newval != noreg, "new val must be register");
1567 assert(cmpval != newval, "cmp and new values must be in different registers");
1568 assert(cmpval != addr, "cmp and addr must be in different registers");
1569 assert(newval != addr, "new value and addr must be in different registers");
1570 __ lock();
1571 __ cmpxchgq(newval, Address(addr, 0));
1572 } else {
1573 Unimplemented();
1574 }
1575 }
1576
1577 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1578 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1579 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1580
1581 Assembler::Condition acond, ncond;
1582 switch (condition) {
1583 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1584 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1585 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1586 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1587 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1588 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1589 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1590 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1591 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1592 ShouldNotReachHere();
1593 }
1594
1595 if (opr1->is_cpu_register()) {
1596 reg2reg(opr1, result);
1597 } else if (opr1->is_stack()) {
1598 stack2reg(opr1, result, result->type());
1599 } else if (opr1->is_constant()) {
1600 const2reg(opr1, result, lir_patch_none, nullptr);
1601 } else {
1602 ShouldNotReachHere();
1603 }
1604
1605 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1606 // optimized version that does not require a branch
1607 if (opr2->is_single_cpu()) {
1608 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1609 __ cmov(ncond, result->as_register(), opr2->as_register());
1610 } else if (opr2->is_double_cpu()) {
1611 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1612 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1613 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1614 } else if (opr2->is_single_stack()) {
1615 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1616 } else if (opr2->is_double_stack()) {
1617 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1618 } else {
1619 ShouldNotReachHere();
1620 }
1621
1622 } else {
1623 Label skip;
1624 __ jccb(acond, skip);
1625 if (opr2->is_cpu_register()) {
1626 reg2reg(opr2, result);
1627 } else if (opr2->is_stack()) {
1628 stack2reg(opr2, result, result->type());
1629 } else if (opr2->is_constant()) {
1630 const2reg(opr2, result, lir_patch_none, nullptr);
1631 } else {
1632 ShouldNotReachHere();
1633 }
1634 __ bind(skip);
1635 }
1636 }
1637
1638
1639 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1640 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1641
1642 if (left->is_single_cpu()) {
1643 assert(left == dest, "left and dest must be equal");
1644 Register lreg = left->as_register();
1645
1646 if (right->is_single_cpu()) {
1647 // cpu register - cpu register
1648 Register rreg = right->as_register();
1649 switch (code) {
1650 case lir_add: __ addl (lreg, rreg); break;
1651 case lir_sub: __ subl (lreg, rreg); break;
1652 case lir_mul: __ imull(lreg, rreg); break;
1653 default: ShouldNotReachHere();
1654 }
1655
1656 } else if (right->is_stack()) {
1657 // cpu register - stack
1658 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1659 switch (code) {
1660 case lir_add: __ addl(lreg, raddr); break;
1661 case lir_sub: __ subl(lreg, raddr); break;
1662 default: ShouldNotReachHere();
1663 }
1664
1665 } else if (right->is_constant()) {
1666 // cpu register - constant
1667 jint c = right->as_constant_ptr()->as_jint();
1668 switch (code) {
1669 case lir_add: {
1670 __ incrementl(lreg, c);
1671 break;
1672 }
1673 case lir_sub: {
1674 __ decrementl(lreg, c);
1675 break;
1676 }
1677 default: ShouldNotReachHere();
1678 }
1679
1680 } else {
1681 ShouldNotReachHere();
1682 }
1683
1684 } else if (left->is_double_cpu()) {
1685 assert(left == dest, "left and dest must be equal");
1686 Register lreg_lo = left->as_register_lo();
1687 Register lreg_hi = left->as_register_hi();
1688
1689 if (right->is_double_cpu()) {
1690 // cpu register - cpu register
1691 Register rreg_lo = right->as_register_lo();
1692 Register rreg_hi = right->as_register_hi();
1693 assert_different_registers(lreg_lo, rreg_lo);
1694 switch (code) {
1695 case lir_add:
1696 __ addptr(lreg_lo, rreg_lo);
1697 break;
1698 case lir_sub:
1699 __ subptr(lreg_lo, rreg_lo);
1700 break;
1701 case lir_mul:
1702 __ imulq(lreg_lo, rreg_lo);
1703 break;
1704 default:
1705 ShouldNotReachHere();
1706 }
1707
1708 } else if (right->is_constant()) {
1709 // cpu register - constant
1710 jlong c = right->as_constant_ptr()->as_jlong_bits();
1711 __ movptr(r10, (intptr_t) c);
1712 switch (code) {
1713 case lir_add:
1714 __ addptr(lreg_lo, r10);
1715 break;
1716 case lir_sub:
1717 __ subptr(lreg_lo, r10);
1718 break;
1719 default:
1720 ShouldNotReachHere();
1721 }
1722
1723 } else {
1724 ShouldNotReachHere();
1725 }
1726
1727 } else if (left->is_single_xmm()) {
1728 assert(left == dest, "left and dest must be equal");
1729 XMMRegister lreg = left->as_xmm_float_reg();
1730
1731 if (right->is_single_xmm()) {
1732 XMMRegister rreg = right->as_xmm_float_reg();
1733 switch (code) {
1734 case lir_add: __ addss(lreg, rreg); break;
1735 case lir_sub: __ subss(lreg, rreg); break;
1736 case lir_mul: __ mulss(lreg, rreg); break;
1737 case lir_div: __ divss(lreg, rreg); break;
1738 default: ShouldNotReachHere();
1739 }
1740 } else {
1741 Address raddr;
1742 if (right->is_single_stack()) {
1743 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1744 } else if (right->is_constant()) {
1745 // hack for now
1746 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1747 } else {
1748 ShouldNotReachHere();
1749 }
1750 switch (code) {
1751 case lir_add: __ addss(lreg, raddr); break;
1752 case lir_sub: __ subss(lreg, raddr); break;
1753 case lir_mul: __ mulss(lreg, raddr); break;
1754 case lir_div: __ divss(lreg, raddr); break;
1755 default: ShouldNotReachHere();
1756 }
1757 }
1758
1759 } else if (left->is_double_xmm()) {
1760 assert(left == dest, "left and dest must be equal");
1761
1762 XMMRegister lreg = left->as_xmm_double_reg();
1763 if (right->is_double_xmm()) {
1764 XMMRegister rreg = right->as_xmm_double_reg();
1765 switch (code) {
1766 case lir_add: __ addsd(lreg, rreg); break;
1767 case lir_sub: __ subsd(lreg, rreg); break;
1768 case lir_mul: __ mulsd(lreg, rreg); break;
1769 case lir_div: __ divsd(lreg, rreg); break;
1770 default: ShouldNotReachHere();
1771 }
1772 } else {
1773 Address raddr;
1774 if (right->is_double_stack()) {
1775 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1776 } else if (right->is_constant()) {
1777 // hack for now
1778 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1779 } else {
1780 ShouldNotReachHere();
1781 }
1782 switch (code) {
1783 case lir_add: __ addsd(lreg, raddr); break;
1784 case lir_sub: __ subsd(lreg, raddr); break;
1785 case lir_mul: __ mulsd(lreg, raddr); break;
1786 case lir_div: __ divsd(lreg, raddr); break;
1787 default: ShouldNotReachHere();
1788 }
1789 }
1790
1791 } else if (left->is_single_stack() || left->is_address()) {
1792 assert(left == dest, "left and dest must be equal");
1793
1794 Address laddr;
1795 if (left->is_single_stack()) {
1796 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1797 } else if (left->is_address()) {
1798 laddr = as_Address(left->as_address_ptr());
1799 } else {
1800 ShouldNotReachHere();
1801 }
1802
1803 if (right->is_single_cpu()) {
1804 Register rreg = right->as_register();
1805 switch (code) {
1806 case lir_add: __ addl(laddr, rreg); break;
1807 case lir_sub: __ subl(laddr, rreg); break;
1808 default: ShouldNotReachHere();
1809 }
1810 } else if (right->is_constant()) {
1811 jint c = right->as_constant_ptr()->as_jint();
1812 switch (code) {
1813 case lir_add: {
1814 __ incrementl(laddr, c);
1815 break;
1816 }
1817 case lir_sub: {
1818 __ decrementl(laddr, c);
1819 break;
1820 }
1821 default: ShouldNotReachHere();
1822 }
1823 } else {
1824 ShouldNotReachHere();
1825 }
1826
1827 } else {
1828 ShouldNotReachHere();
1829 }
1830 }
1831
1832
1833 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1834 if (value->is_double_xmm()) {
1835 switch(code) {
1836 case lir_abs :
1837 {
1838 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1839 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1840 }
1841 assert(!tmp->is_valid(), "do not need temporary");
1842 __ andpd(dest->as_xmm_double_reg(),
1843 ExternalAddress(LIR_Assembler::double_signmask_pool),
1844 rscratch1);
1845 }
1846 break;
1847
1848 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1849 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1850 default : ShouldNotReachHere();
1851 }
1852
1853 } else if (code == lir_f2hf) {
1854 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1855 } else if (code == lir_hf2f) {
1856 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1857 } else {
1858 Unimplemented();
1859 }
1860 }
1861
1862 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1863 // assert(left->destroys_register(), "check");
1864 if (left->is_single_cpu()) {
1865 Register reg = left->as_register();
1866 if (right->is_constant()) {
1867 int val = right->as_constant_ptr()->as_jint();
1868 switch (code) {
1869 case lir_logic_and: __ andl (reg, val); break;
1870 case lir_logic_or: __ orl (reg, val); break;
1871 case lir_logic_xor: __ xorl (reg, val); break;
1872 default: ShouldNotReachHere();
1873 }
1874 } else if (right->is_stack()) {
1875 // added support for stack operands
1876 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1877 switch (code) {
1878 case lir_logic_and: __ andl (reg, raddr); break;
1879 case lir_logic_or: __ orl (reg, raddr); break;
1880 case lir_logic_xor: __ xorl (reg, raddr); break;
1881 default: ShouldNotReachHere();
1882 }
1883 } else {
1884 Register rright = right->as_register();
1885 switch (code) {
1886 case lir_logic_and: __ andptr (reg, rright); break;
1887 case lir_logic_or : __ orptr (reg, rright); break;
1888 case lir_logic_xor: __ xorptr (reg, rright); break;
1889 default: ShouldNotReachHere();
1890 }
1891 }
1892 move_regs(reg, dst->as_register());
1893 } else {
1894 Register l_lo = left->as_register_lo();
1895 Register l_hi = left->as_register_hi();
1896 if (right->is_constant()) {
1897 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1898 switch (code) {
1899 case lir_logic_and:
1900 __ andq(l_lo, rscratch1);
1901 break;
1902 case lir_logic_or:
1903 __ orq(l_lo, rscratch1);
1904 break;
1905 case lir_logic_xor:
1906 __ xorq(l_lo, rscratch1);
1907 break;
1908 default: ShouldNotReachHere();
1909 }
1910 } else {
1911 Register r_lo;
1912 if (is_reference_type(right->type())) {
1913 r_lo = right->as_register();
1914 } else {
1915 r_lo = right->as_register_lo();
1916 }
1917 switch (code) {
1918 case lir_logic_and:
1919 __ andptr(l_lo, r_lo);
1920 break;
1921 case lir_logic_or:
1922 __ orptr(l_lo, r_lo);
1923 break;
1924 case lir_logic_xor:
1925 __ xorptr(l_lo, r_lo);
1926 break;
1927 default: ShouldNotReachHere();
1928 }
1929 }
1930
1931 Register dst_lo = dst->as_register_lo();
1932 Register dst_hi = dst->as_register_hi();
1933
1934 move_regs(l_lo, dst_lo);
1935 }
1936 }
1937
1938
1939 // we assume that rax, and rdx can be overwritten
1940 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1941
1942 assert(left->is_single_cpu(), "left must be register");
1943 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1944 assert(result->is_single_cpu(), "result must be register");
1945
1946 // assert(left->destroys_register(), "check");
1947 // assert(right->destroys_register(), "check");
1948
1949 Register lreg = left->as_register();
1950 Register dreg = result->as_register();
1951
1952 if (right->is_constant()) {
1953 jint divisor = right->as_constant_ptr()->as_jint();
1954 assert(divisor > 0 && is_power_of_2(divisor), "must be");
1955 if (code == lir_idiv) {
1956 assert(lreg == rax, "must be rax,");
1957 assert(temp->as_register() == rdx, "tmp register must be rdx");
1958 __ cdql(); // sign extend into rdx:rax
1959 if (divisor == 2) {
1960 __ subl(lreg, rdx);
1961 } else {
1962 __ andl(rdx, divisor - 1);
1963 __ addl(lreg, rdx);
1964 }
1965 __ sarl(lreg, log2i_exact(divisor));
1966 move_regs(lreg, dreg);
1967 } else if (code == lir_irem) {
1968 Label done;
1969 __ mov(dreg, lreg);
1970 __ andl(dreg, 0x80000000 | (divisor - 1));
1971 __ jcc(Assembler::positive, done);
1972 __ decrement(dreg);
1973 __ orl(dreg, ~(divisor - 1));
1974 __ increment(dreg);
1975 __ bind(done);
1976 } else {
1977 ShouldNotReachHere();
1978 }
1979 } else {
1980 Register rreg = right->as_register();
1981 assert(lreg == rax, "left register must be rax,");
1982 assert(rreg != rdx, "right register must not be rdx");
1983 assert(temp->as_register() == rdx, "tmp register must be rdx");
1984
1985 move_regs(lreg, rax);
1986
1987 int idivl_offset = __ corrected_idivl(rreg);
1988 if (ImplicitDiv0Checks) {
1989 add_debug_info_for_div0(idivl_offset, info);
1990 }
1991 if (code == lir_irem) {
1992 move_regs(rdx, dreg); // result is in rdx
1993 } else {
1994 move_regs(rax, dreg);
1995 }
1996 }
1997 }
1998
1999
2000 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2001 if (opr1->is_single_cpu()) {
2002 Register reg1 = opr1->as_register();
2003 if (opr2->is_single_cpu()) {
2004 // cpu register - cpu register
2005 if (is_reference_type(opr1->type())) {
2006 __ cmpoop(reg1, opr2->as_register());
2007 } else {
2008 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2009 __ cmpl(reg1, opr2->as_register());
2010 }
2011 } else if (opr2->is_stack()) {
2012 // cpu register - stack
2013 if (is_reference_type(opr1->type())) {
2014 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2015 } else {
2016 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2017 }
2018 } else if (opr2->is_constant()) {
2019 // cpu register - constant
2020 LIR_Const* c = opr2->as_constant_ptr();
2021 if (c->type() == T_INT) {
2022 jint i = c->as_jint();
2023 if (i == 0) {
2024 __ testl(reg1, reg1);
2025 } else {
2026 __ cmpl(reg1, i);
2027 }
2028 } else if (c->type() == T_METADATA) {
2029 // All we need for now is a comparison with null for equality.
2030 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2031 Metadata* m = c->as_metadata();
2032 if (m == nullptr) {
2033 __ testptr(reg1, reg1);
2034 } else {
2035 ShouldNotReachHere();
2036 }
2037 } else if (is_reference_type(c->type())) {
2038 // In 64bit oops are single register
2039 jobject o = c->as_jobject();
2040 if (o == nullptr) {
2041 __ testptr(reg1, reg1);
2042 } else {
2043 __ cmpoop(reg1, o, rscratch1);
2044 }
2045 } else {
2046 fatal("unexpected type: %s", basictype_to_str(c->type()));
2047 }
2048 // cpu register - address
2049 } else if (opr2->is_address()) {
2050 if (op->info() != nullptr) {
2051 add_debug_info_for_null_check_here(op->info());
2052 }
2053 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2054 } else {
2055 ShouldNotReachHere();
2056 }
2057
2058 } else if(opr1->is_double_cpu()) {
2059 Register xlo = opr1->as_register_lo();
2060 Register xhi = opr1->as_register_hi();
2061 if (opr2->is_double_cpu()) {
2062 __ cmpptr(xlo, opr2->as_register_lo());
2063 } else if (opr2->is_constant()) {
2064 // cpu register - constant 0
2065 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2066 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2067 } else {
2068 ShouldNotReachHere();
2069 }
2070
2071 } else if (opr1->is_single_xmm()) {
2072 XMMRegister reg1 = opr1->as_xmm_float_reg();
2073 if (opr2->is_single_xmm()) {
2074 // xmm register - xmm register
2075 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2076 } else if (opr2->is_stack()) {
2077 // xmm register - stack
2078 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2079 } else if (opr2->is_constant()) {
2080 // xmm register - constant
2081 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2082 } else if (opr2->is_address()) {
2083 // xmm register - address
2084 if (op->info() != nullptr) {
2085 add_debug_info_for_null_check_here(op->info());
2086 }
2087 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2088 } else {
2089 ShouldNotReachHere();
2090 }
2091
2092 } else if (opr1->is_double_xmm()) {
2093 XMMRegister reg1 = opr1->as_xmm_double_reg();
2094 if (opr2->is_double_xmm()) {
2095 // xmm register - xmm register
2096 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2097 } else if (opr2->is_stack()) {
2098 // xmm register - stack
2099 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2100 } else if (opr2->is_constant()) {
2101 // xmm register - constant
2102 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2103 } else if (opr2->is_address()) {
2104 // xmm register - address
2105 if (op->info() != nullptr) {
2106 add_debug_info_for_null_check_here(op->info());
2107 }
2108 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2109 } else {
2110 ShouldNotReachHere();
2111 }
2112
2113 } else if (opr1->is_address() && opr2->is_constant()) {
2114 LIR_Const* c = opr2->as_constant_ptr();
2115 if (is_reference_type(c->type())) {
2116 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2117 __ movoop(rscratch1, c->as_jobject());
2118 }
2119 if (op->info() != nullptr) {
2120 add_debug_info_for_null_check_here(op->info());
2121 }
2122 // special case: address - constant
2123 LIR_Address* addr = opr1->as_address_ptr();
2124 if (c->type() == T_INT) {
2125 __ cmpl(as_Address(addr), c->as_jint());
2126 } else if (is_reference_type(c->type())) {
2127 // %%% Make this explode if addr isn't reachable until we figure out a
2128 // better strategy by giving noreg as the temp for as_Address
2129 __ cmpoop(rscratch1, as_Address(addr, noreg));
2130 } else {
2131 ShouldNotReachHere();
2132 }
2133
2134 } else {
2135 ShouldNotReachHere();
2136 }
2137 }
2138
2139 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2140 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2141 if (left->is_single_xmm()) {
2142 assert(right->is_single_xmm(), "must match");
2143 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2144 } else if (left->is_double_xmm()) {
2145 assert(right->is_double_xmm(), "must match");
2146 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2147
2148 } else {
2149 ShouldNotReachHere();
2150 }
2151 } else {
2152 assert(code == lir_cmp_l2i, "check");
2153 Label done;
2154 Register dest = dst->as_register();
2155 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2156 __ movl(dest, -1);
2157 __ jccb(Assembler::less, done);
2158 __ setb(Assembler::notZero, dest);
2159 __ movzbl(dest, dest);
2160 __ bind(done);
2161 }
2162 }
2163
2164
2165 void LIR_Assembler::align_call(LIR_Code code) {
2166 // make sure that the displacement word of the call ends up word aligned
2167 int offset = __ offset();
2168 switch (code) {
2169 case lir_static_call:
2170 case lir_optvirtual_call:
2171 case lir_dynamic_call:
2172 offset += NativeCall::displacement_offset;
2173 break;
2174 case lir_icvirtual_call:
2175 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2176 break;
2177 default: ShouldNotReachHere();
2178 }
2179 __ align(BytesPerWord, offset);
2180 }
2181
2182
2183 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2184 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2185 "must be aligned");
2186 __ call(AddressLiteral(op->addr(), rtype));
2187 add_call_info(code_offset(), op->info());
2188 __ post_call_nop();
2189 }
2190
2191
2192 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2193 __ ic_call(op->addr());
2194 add_call_info(code_offset(), op->info());
2195 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2196 "must be aligned");
2197 __ post_call_nop();
2198 }
2199
2200
2201 void LIR_Assembler::emit_static_call_stub() {
2202 address call_pc = __ pc();
2203 address stub = __ start_a_stub(call_stub_size());
2204 if (stub == nullptr) {
2205 bailout("static call stub overflow");
2206 return;
2207 }
2208
2209 int start = __ offset();
2210
2211 // make sure that the displacement word of the call ends up word aligned
2212 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2213 __ relocate(static_stub_Relocation::spec(call_pc));
2214 __ mov_metadata(rbx, (Metadata*)nullptr);
2215 // must be set to -1 at code generation time
2216 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2217 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2218 __ jump(RuntimeAddress(__ pc()));
2219
2220 assert(__ offset() - start <= call_stub_size(), "stub too big");
2221 __ end_a_stub();
2222 }
2223
2224
2225 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2226 assert(exceptionOop->as_register() == rax, "must match");
2227 assert(exceptionPC->as_register() == rdx, "must match");
2228
2229 // exception object is not added to oop map by LinearScan
2230 // (LinearScan assumes that no oops are in fixed registers)
2231 info->add_register_oop(exceptionOop);
2232 StubId unwind_id;
2233
2234 // get current pc information
2235 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2236 int pc_for_athrow_offset = __ offset();
2237 InternalAddress pc_for_athrow(__ pc());
2238 __ lea(exceptionPC->as_register(), pc_for_athrow);
2239 add_call_info(pc_for_athrow_offset, info); // for exception handler
2240
2241 __ verify_not_null_oop(rax);
2242 // search an exception handler (rax: exception oop, rdx: throwing pc)
2243 if (compilation()->has_fpu_code()) {
2244 unwind_id = StubId::c1_handle_exception_id;
2245 } else {
2246 unwind_id = StubId::c1_handle_exception_nofpu_id;
2247 }
2248 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2249
2250 // enough room for two byte trap
2251 __ nop();
2252 }
2253
2254
2255 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2256 assert(exceptionOop->as_register() == rax, "must match");
2257
2258 __ jmp(_unwind_handler_entry);
2259 }
2260
2261
2262 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2263
2264 // optimized version for linear scan:
2265 // * count must be already in ECX (guaranteed by LinearScan)
2266 // * left and dest must be equal
2267 // * tmp must be unused
2268 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2269 assert(left == dest, "left and dest must be equal");
2270 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2271
2272 if (left->is_single_cpu()) {
2273 Register value = left->as_register();
2274 assert(value != SHIFT_count, "left cannot be ECX");
2275
2276 switch (code) {
2277 case lir_shl: __ shll(value); break;
2278 case lir_shr: __ sarl(value); break;
2279 case lir_ushr: __ shrl(value); break;
2280 default: ShouldNotReachHere();
2281 }
2282 } else if (left->is_double_cpu()) {
2283 Register lo = left->as_register_lo();
2284 Register hi = left->as_register_hi();
2285 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2286 switch (code) {
2287 case lir_shl: __ shlptr(lo); break;
2288 case lir_shr: __ sarptr(lo); break;
2289 case lir_ushr: __ shrptr(lo); break;
2290 default: ShouldNotReachHere();
2291 }
2292 } else {
2293 ShouldNotReachHere();
2294 }
2295 }
2296
2297
2298 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2299 if (dest->is_single_cpu()) {
2300 // first move left into dest so that left is not destroyed by the shift
2301 Register value = dest->as_register();
2302 count = count & 0x1F; // Java spec
2303
2304 move_regs(left->as_register(), value);
2305 switch (code) {
2306 case lir_shl: __ shll(value, count); break;
2307 case lir_shr: __ sarl(value, count); break;
2308 case lir_ushr: __ shrl(value, count); break;
2309 default: ShouldNotReachHere();
2310 }
2311 } else if (dest->is_double_cpu()) {
2312 // first move left into dest so that left is not destroyed by the shift
2313 Register value = dest->as_register_lo();
2314 count = count & 0x1F; // Java spec
2315
2316 move_regs(left->as_register_lo(), value);
2317 switch (code) {
2318 case lir_shl: __ shlptr(value, count); break;
2319 case lir_shr: __ sarptr(value, count); break;
2320 case lir_ushr: __ shrptr(value, count); break;
2321 default: ShouldNotReachHere();
2322 }
2323 } else {
2324 ShouldNotReachHere();
2325 }
2326 }
2327
2328
2329 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2330 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2338 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2342 }
2343
2344
2345 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2346 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2347 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2348 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2349 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2350 }
2351
2352
2353 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2354 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2355 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2356 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2357 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2358 }
2359
2360
2361 // This code replaces a call to arraycopy; no exception may
2362 // be thrown in this code, they must be thrown in the System.arraycopy
2363 // activation frame; we could save some checks if this would not be the case
2364 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2365 ciArrayKlass* default_type = op->expected_type();
2366 Register src = op->src()->as_register();
2367 Register dst = op->dst()->as_register();
2368 Register src_pos = op->src_pos()->as_register();
2369 Register dst_pos = op->dst_pos()->as_register();
2370 Register length = op->length()->as_register();
2371 Register tmp = op->tmp()->as_register();
2372 Register tmp_load_klass = rscratch1;
2373 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2374
2375 CodeStub* stub = op->stub();
2376 int flags = op->flags();
2377 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2378 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2379
2380 // if we don't know anything, just go through the generic arraycopy
2381 if (default_type == nullptr) {
2382 // save outgoing arguments on stack in case call to System.arraycopy is needed
2383 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2384 // for interpreter calling conventions. Now we have to do it in new style conventions.
2385 // For the moment until C1 gets the new register allocator I just force all the
2386 // args to the right place (except the register args) and then on the back side
2387 // reload the register args properly if we go slow path. Yuck
2388
2389 // These are proper for the calling convention
2390 store_parameter(length, 2);
2391 store_parameter(dst_pos, 1);
2392 store_parameter(dst, 0);
2393
2394 // these are just temporary placements until we need to reload
2395 store_parameter(src_pos, 3);
2396 store_parameter(src, 4);
2397
2398 address copyfunc_addr = StubRoutines::generic_arraycopy();
2399 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2400
2401 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2402 // The arguments are in java calling convention so we can trivially shift them to C
2403 // convention
2404 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2405 __ mov(c_rarg0, j_rarg0);
2406 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2407 __ mov(c_rarg1, j_rarg1);
2408 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2409 __ mov(c_rarg2, j_rarg2);
2410 assert_different_registers(c_rarg3, j_rarg4);
2411 __ mov(c_rarg3, j_rarg3);
2412 #ifdef _WIN64
2413 // Allocate abi space for args but be sure to keep stack aligned
2414 __ subptr(rsp, 6*wordSize);
2415 store_parameter(j_rarg4, 4);
2416 #ifndef PRODUCT
2417 if (PrintC1Statistics) {
2418 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2419 }
2420 #endif
2421 __ call(RuntimeAddress(copyfunc_addr));
2422 __ addptr(rsp, 6*wordSize);
2423 #else
2424 __ mov(c_rarg4, j_rarg4);
2425 #ifndef PRODUCT
2426 if (PrintC1Statistics) {
2427 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2428 }
2429 #endif
2430 __ call(RuntimeAddress(copyfunc_addr));
2431 #endif // _WIN64
2432
2433 __ testl(rax, rax);
2434 __ jcc(Assembler::equal, *stub->continuation());
2435
2436 __ mov(tmp, rax);
2437 __ xorl(tmp, -1);
2438
2439 // Reload values from the stack so they are where the stub
2440 // expects them.
2441 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2442 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2443 __ movptr (length, Address(rsp, 2*BytesPerWord));
2444 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2445 __ movptr (src, Address(rsp, 4*BytesPerWord));
2446
2447 __ subl(length, tmp);
2448 __ addl(src_pos, tmp);
2449 __ addl(dst_pos, tmp);
2450 __ jmp(*stub->entry());
2451
2452 __ bind(*stub->continuation());
2453 return;
2454 }
2455
2456 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2457
2458 int elem_size = type2aelembytes(basic_type);
2459 Address::ScaleFactor scale;
2460
2461 switch (elem_size) {
2462 case 1 :
2463 scale = Address::times_1;
2464 break;
2465 case 2 :
2466 scale = Address::times_2;
2467 break;
2468 case 4 :
2469 scale = Address::times_4;
2470 break;
2471 case 8 :
2472 scale = Address::times_8;
2473 break;
2474 default:
2475 scale = Address::no_scale;
2476 ShouldNotReachHere();
2477 }
2478
2479 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2480 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2481
2482 // length and pos's are all sign extended at this point on 64bit
2483
2484 // test for null
2485 if (flags & LIR_OpArrayCopy::src_null_check) {
2486 __ testptr(src, src);
2487 __ jcc(Assembler::zero, *stub->entry());
2488 }
2489 if (flags & LIR_OpArrayCopy::dst_null_check) {
2490 __ testptr(dst, dst);
2491 __ jcc(Assembler::zero, *stub->entry());
2492 }
2493
2494 // If the compiler was not able to prove that exact type of the source or the destination
2495 // of the arraycopy is an array type, check at runtime if the source or the destination is
2496 // an instance type.
2497 if (flags & LIR_OpArrayCopy::type_check) {
2498 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2499 __ load_klass(tmp, dst, tmp_load_klass);
2500 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2501 __ jcc(Assembler::greaterEqual, *stub->entry());
2502 }
2503
2504 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2505 __ load_klass(tmp, src, tmp_load_klass);
2506 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2507 __ jcc(Assembler::greaterEqual, *stub->entry());
2508 }
2509 }
2510
2511 // check if negative
2512 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2513 __ testl(src_pos, src_pos);
2514 __ jcc(Assembler::less, *stub->entry());
2515 }
2516 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2517 __ testl(dst_pos, dst_pos);
2518 __ jcc(Assembler::less, *stub->entry());
2519 }
2520
2521 if (flags & LIR_OpArrayCopy::src_range_check) {
2522 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2523 __ cmpl(tmp, src_length_addr);
2524 __ jcc(Assembler::above, *stub->entry());
2525 }
2526 if (flags & LIR_OpArrayCopy::dst_range_check) {
2527 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2528 __ cmpl(tmp, dst_length_addr);
2529 __ jcc(Assembler::above, *stub->entry());
2530 }
2531
2532 if (flags & LIR_OpArrayCopy::length_positive_check) {
2533 __ testl(length, length);
2534 __ jcc(Assembler::less, *stub->entry());
2535 }
2536
2537 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2538 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2539
2540 if (flags & LIR_OpArrayCopy::type_check) {
2541 // We don't know the array types are compatible
2542 if (basic_type != T_OBJECT) {
2543 // Simple test for basic type arrays
2544 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2545 __ jcc(Assembler::notEqual, *stub->entry());
2546 } else {
2547 // For object arrays, if src is a sub class of dst then we can
2548 // safely do the copy.
2549 Label cont, slow;
2550
2551 __ push_ppx(src);
2552 __ push_ppx(dst);
2553
2554 __ load_klass(src, src, tmp_load_klass);
2555 __ load_klass(dst, dst, tmp_load_klass);
2556
2557 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2558
2559 __ push_ppx(src);
2560 __ push_ppx(dst);
2561 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2562 __ pop_ppx(dst);
2563 __ pop_ppx(src);
2564
2565 __ testl(src, src);
2566 __ jcc(Assembler::notEqual, cont);
2567
2568 __ bind(slow);
2569 __ pop_ppx(dst);
2570 __ pop_ppx(src);
2571
2572 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2573 if (copyfunc_addr != nullptr) { // use stub if available
2574 // src is not a sub class of dst so we have to do a
2575 // per-element check.
2576
2577 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2578 if ((flags & mask) != mask) {
2579 // Check that at least both of them object arrays.
2580 assert(flags & mask, "one of the two should be known to be an object array");
2581
2582 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2583 __ load_klass(tmp, src, tmp_load_klass);
2584 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2585 __ load_klass(tmp, dst, tmp_load_klass);
2586 }
2587 int lh_offset = in_bytes(Klass::layout_helper_offset());
2588 Address klass_lh_addr(tmp, lh_offset);
2589 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2590 __ cmpl(klass_lh_addr, objArray_lh);
2591 __ jcc(Assembler::notEqual, *stub->entry());
2592 }
2593
2594 // Spill because stubs can use any register they like and it's
2595 // easier to restore just those that we care about.
2596 store_parameter(dst, 0);
2597 store_parameter(dst_pos, 1);
2598 store_parameter(length, 2);
2599 store_parameter(src_pos, 3);
2600 store_parameter(src, 4);
2601
2602 __ movl2ptr(length, length); //higher 32bits must be null
2603
2604 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2605 assert_different_registers(c_rarg0, dst, dst_pos, length);
2606 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2607 assert_different_registers(c_rarg1, dst, length);
2608
2609 __ mov(c_rarg2, length);
2610 assert_different_registers(c_rarg2, dst);
2611
2612 #ifdef _WIN64
2613 // Allocate abi space for args but be sure to keep stack aligned
2614 __ subptr(rsp, 6*wordSize);
2615 __ load_klass(c_rarg3, dst, tmp_load_klass);
2616 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2617 store_parameter(c_rarg3, 4);
2618 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2619 __ call(RuntimeAddress(copyfunc_addr));
2620 __ addptr(rsp, 6*wordSize);
2621 #else
2622 __ load_klass(c_rarg4, dst, tmp_load_klass);
2623 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2624 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2625 __ call(RuntimeAddress(copyfunc_addr));
2626 #endif
2627
2628 #ifndef PRODUCT
2629 if (PrintC1Statistics) {
2630 Label failed;
2631 __ testl(rax, rax);
2632 __ jcc(Assembler::notZero, failed);
2633 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2634 __ bind(failed);
2635 }
2636 #endif
2637
2638 __ testl(rax, rax);
2639 __ jcc(Assembler::zero, *stub->continuation());
2640
2641 #ifndef PRODUCT
2642 if (PrintC1Statistics) {
2643 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2644 }
2645 #endif
2646
2647 __ mov(tmp, rax);
2648
2649 __ xorl(tmp, -1);
2650
2651 // Restore previously spilled arguments
2652 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2653 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2654 __ movptr (length, Address(rsp, 2*BytesPerWord));
2655 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2656 __ movptr (src, Address(rsp, 4*BytesPerWord));
2657
2658
2659 __ subl(length, tmp);
2660 __ addl(src_pos, tmp);
2661 __ addl(dst_pos, tmp);
2662 }
2663
2664 __ jmp(*stub->entry());
2665
2666 __ bind(cont);
2667 __ pop(dst);
2668 __ pop(src);
2669 }
2670 }
2671
2672 #ifdef ASSERT
2673 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2674 // Sanity check the known type with the incoming class. For the
2675 // primitive case the types must match exactly with src.klass and
2676 // dst.klass each exactly matching the default type. For the
2677 // object array case, if no type check is needed then either the
2678 // dst type is exactly the expected type and the src type is a
2679 // subtype which we can't check or src is the same array as dst
2680 // but not necessarily exactly of type default_type.
2681 Label known_ok, halt;
2682 __ mov_metadata(tmp, default_type->constant_encoding());
2683 if (UseCompressedClassPointers) {
2684 __ encode_klass_not_null(tmp, rscratch1);
2685 }
2686
2687 if (basic_type != T_OBJECT) {
2688 __ cmp_klass(tmp, dst, tmp2);
2689 __ jcc(Assembler::notEqual, halt);
2690 __ cmp_klass(tmp, src, tmp2);
2691 __ jcc(Assembler::equal, known_ok);
2692 } else {
2693 __ cmp_klass(tmp, dst, tmp2);
2694 __ jcc(Assembler::equal, known_ok);
2695 __ cmpptr(src, dst);
2696 __ jcc(Assembler::equal, known_ok);
2697 }
2698 __ bind(halt);
2699 __ stop("incorrect type information in arraycopy");
2700 __ bind(known_ok);
2701 }
2702 #endif
2703
2704 #ifndef PRODUCT
2705 if (PrintC1Statistics) {
2706 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2707 }
2708 #endif
2709
2710 assert_different_registers(c_rarg0, dst, dst_pos, length);
2711 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2712 assert_different_registers(c_rarg1, length);
2713 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2714 __ mov(c_rarg2, length);
2715
2716 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2717 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2718 const char *name;
2719 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2720 __ call_VM_leaf(entry, 0);
2721
2722 if (stub != nullptr) {
2723 __ bind(*stub->continuation());
2724 }
2725 }
2726
2727 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2728 assert(op->crc()->is_single_cpu(), "crc must be register");
2729 assert(op->val()->is_single_cpu(), "byte value must be register");
2730 assert(op->result_opr()->is_single_cpu(), "result must be register");
2731 Register crc = op->crc()->as_register();
2732 Register val = op->val()->as_register();
2733 Register res = op->result_opr()->as_register();
2734
2735 assert_different_registers(val, crc, res);
2736
2737 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2738 __ notl(crc); // ~crc
2739 __ update_byte_crc32(crc, val, res);
2740 __ notl(crc); // ~crc
2741 __ mov(res, crc);
2742 }
2743
2744 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2745 Register obj = op->obj_opr()->as_register(); // may not be an oop
2746 Register hdr = op->hdr_opr()->as_register();
2747 Register lock = op->lock_opr()->as_register();
2748 if (op->code() == lir_lock) {
2749 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2750 Register tmp = op->scratch_opr()->as_register();
2751 // add debug info for NullPointerException only if one is possible
2752 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2753 if (op->info() != nullptr) {
2754 add_debug_info_for_null_check(null_check_offset, op->info());
2755 }
2756 // done
2757 } else if (op->code() == lir_unlock) {
2758 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2759 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2760 } else {
2761 Unimplemented();
2762 }
2763 __ bind(*op->stub()->continuation());
2764 }
2765
2766 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2767 Register obj = op->obj()->as_pointer_register();
2768 Register result = op->result_opr()->as_pointer_register();
2769
2770 CodeEmitInfo* info = op->info();
2771 if (info != nullptr) {
2772 add_debug_info_for_null_check_here(info);
2773 }
2774
2775 __ load_klass(result, obj, rscratch1);
2776 }
2777
2778 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2779 ciMethod* method = op->profiled_method();
2780 int bci = op->profiled_bci();
2781 ciMethod* callee = op->profiled_callee();
2782 Register tmp_load_klass = rscratch1;
2783
2784 // Update counter for all call types
2785 ciMethodData* md = method->method_data_or_null();
2786 assert(md != nullptr, "Sanity");
2787 ciProfileData* data = md->bci_to_data(bci);
2788 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2789 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2790 Register mdo = op->mdo()->as_register();
2791 __ mov_metadata(mdo, md->constant_encoding());
2792 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2793 // Perform additional virtual call profiling for invokevirtual and
2794 // invokeinterface bytecodes
2795 if (op->should_profile_receiver_type()) {
2796 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2797 Register recv = op->recv()->as_register();
2798 assert_different_registers(mdo, recv);
2799 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2800 ciKlass* known_klass = op->known_holder();
2801 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2802 // We know the type that will be seen at this call site; we can
2803 // statically update the MethodData* rather than needing to do
2804 // dynamic tests on the receiver type
2805
2806 // NOTE: we should probably put a lock around this search to
2807 // avoid collisions by concurrent compilations
2808 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2809 uint i;
2810 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2811 ciKlass* receiver = vc_data->receiver(i);
2812 if (known_klass->equals(receiver)) {
2813 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2814 __ addptr(data_addr, DataLayout::counter_increment);
2815 return;
2816 }
2817 }
2818
2819 // Receiver type not found in profile data; select an empty slot
2820
2821 // Note that this is less efficient than it should be because it
2822 // always does a write to the receiver part of the
2823 // VirtualCallData rather than just the first time
2824 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2825 ciKlass* receiver = vc_data->receiver(i);
2826 if (receiver == nullptr) {
2827 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2828 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
2829 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2830 __ addptr(data_addr, DataLayout::counter_increment);
2831 return;
2832 }
2833 }
2834 } else {
2835 __ load_klass(recv, recv, tmp_load_klass);
2836 Label update_done;
2837 type_profile_helper(mdo, md, data, recv, &update_done);
2838 // Receiver did not match any saved receiver and there is no empty row for it.
2839 // Increment total counter to indicate polymorphic case.
2840 __ addptr(counter_addr, DataLayout::counter_increment);
2841
2842 __ bind(update_done);
2843 }
2844 } else {
2845 // Static call
2846 __ addptr(counter_addr, DataLayout::counter_increment);
2847 }
2848 }
2849
2850 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2851 Register obj = op->obj()->as_register();
2852 Register tmp = op->tmp()->as_pointer_register();
2853 Register tmp_load_klass = rscratch1;
2854 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2855 ciKlass* exact_klass = op->exact_klass();
2856 intptr_t current_klass = op->current_klass();
2857 bool not_null = op->not_null();
2858 bool no_conflict = op->no_conflict();
2859
2860 Label update, next, none;
2861
2862 bool do_null = !not_null;
2863 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2864 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2865
2866 assert(do_null || do_update, "why are we here?");
2867 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2868
2869 __ verify_oop(obj);
2870
2871 #ifdef ASSERT
2872 if (obj == tmp) {
2873 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2874 } else {
2875 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2876 }
2877 #endif
2878 if (do_null) {
2879 __ testptr(obj, obj);
2880 __ jccb(Assembler::notZero, update);
2881 if (!TypeEntries::was_null_seen(current_klass)) {
2882 __ testptr(mdo_addr, TypeEntries::null_seen);
2883 #ifndef ASSERT
2884 __ jccb(Assembler::notZero, next); // already set
2885 #else
2886 __ jcc(Assembler::notZero, next); // already set
2887 #endif
2888 // atomic update to prevent overwriting Klass* with 0
2889 __ lock();
2890 __ orptr(mdo_addr, TypeEntries::null_seen);
2891 }
2892 if (do_update) {
2893 #ifndef ASSERT
2894 __ jmpb(next);
2895 }
2896 #else
2897 __ jmp(next);
2898 }
2899 } else {
2900 __ testptr(obj, obj);
2901 __ jcc(Assembler::notZero, update);
2902 __ stop("unexpected null obj");
2903 #endif
2904 }
2905
2906 __ bind(update);
2907
2908 if (do_update) {
2909 #ifdef ASSERT
2910 if (exact_klass != nullptr) {
2911 Label ok;
2912 __ load_klass(tmp, obj, tmp_load_klass);
2913 __ push_ppx(tmp);
2914 __ mov_metadata(tmp, exact_klass->constant_encoding());
2915 __ cmpptr(tmp, Address(rsp, 0));
2916 __ jcc(Assembler::equal, ok);
2917 __ stop("exact klass and actual klass differ");
2918 __ bind(ok);
2919 __ pop_ppx(tmp);
2920 }
2921 #endif
2922 if (!no_conflict) {
2923 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2924 if (exact_klass != nullptr) {
2925 __ mov_metadata(tmp, exact_klass->constant_encoding());
2926 } else {
2927 __ load_klass(tmp, obj, tmp_load_klass);
2928 }
2929 __ mov(rscratch1, tmp); // save original value before XOR
2930 __ xorptr(tmp, mdo_addr);
2931 __ testptr(tmp, TypeEntries::type_klass_mask);
2932 // klass seen before, nothing to do. The unknown bit may have been
2933 // set already but no need to check.
2934 __ jccb(Assembler::zero, next);
2935
2936 __ testptr(tmp, TypeEntries::type_unknown);
2937 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2938
2939 if (TypeEntries::is_type_none(current_klass)) {
2940 __ testptr(mdo_addr, TypeEntries::type_mask);
2941 __ jccb(Assembler::zero, none);
2942 // There is a chance that the checks above (re-reading profiling
2943 // data from memory) fail if another thread has just set the
2944 // profiling to this obj's klass
2945 __ mov(tmp, rscratch1); // get back original value before XOR
2946 __ xorptr(tmp, mdo_addr);
2947 __ testptr(tmp, TypeEntries::type_klass_mask);
2948 __ jccb(Assembler::zero, next);
2949 }
2950 } else {
2951 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2952 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2953
2954 __ testptr(mdo_addr, TypeEntries::type_unknown);
2955 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2956 }
2957
2958 // different than before. Cannot keep accurate profile.
2959 __ orptr(mdo_addr, TypeEntries::type_unknown);
2960
2961 if (TypeEntries::is_type_none(current_klass)) {
2962 __ jmpb(next);
2963
2964 __ bind(none);
2965 // first time here. Set profile type.
2966 __ movptr(mdo_addr, tmp);
2967 #ifdef ASSERT
2968 __ andptr(tmp, TypeEntries::type_klass_mask);
2969 __ verify_klass_ptr(tmp);
2970 #endif
2971 }
2972 } else {
2973 // There's a single possible klass at this profile point
2974 assert(exact_klass != nullptr, "should be");
2975 if (TypeEntries::is_type_none(current_klass)) {
2976 __ mov_metadata(tmp, exact_klass->constant_encoding());
2977 __ xorptr(tmp, mdo_addr);
2978 __ testptr(tmp, TypeEntries::type_klass_mask);
2979 #ifdef ASSERT
2980 __ jcc(Assembler::zero, next);
2981
2982 {
2983 Label ok;
2984 __ push_ppx(tmp);
2985 __ testptr(mdo_addr, TypeEntries::type_mask);
2986 __ jcc(Assembler::zero, ok);
2987 // may have been set by another thread
2988 __ mov_metadata(tmp, exact_klass->constant_encoding());
2989 __ xorptr(tmp, mdo_addr);
2990 __ testptr(tmp, TypeEntries::type_mask);
2991 __ jcc(Assembler::zero, ok);
2992
2993 __ stop("unexpected profiling mismatch");
2994 __ bind(ok);
2995 __ pop_ppx(tmp);
2996 }
2997 #else
2998 __ jccb(Assembler::zero, next);
2999 #endif
3000 // first time here. Set profile type.
3001 __ movptr(mdo_addr, tmp);
3002 #ifdef ASSERT
3003 __ andptr(tmp, TypeEntries::type_klass_mask);
3004 __ verify_klass_ptr(tmp);
3005 #endif
3006 } else {
3007 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3008 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3009
3010 __ testptr(mdo_addr, TypeEntries::type_unknown);
3011 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3012
3013 __ orptr(mdo_addr, TypeEntries::type_unknown);
3014 }
3015 }
3016 }
3017 __ bind(next);
3018 }
3019
3020 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3021 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3022 }
3023
3024
3025 void LIR_Assembler::align_backward_branch_target() {
3026 __ align(BytesPerWord);
3027 }
3028
3029
3030 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3031 if (left->is_single_cpu()) {
3032 __ negl(left->as_register());
3033 move_regs(left->as_register(), dest->as_register());
3034
3035 } else if (left->is_double_cpu()) {
3036 Register lo = left->as_register_lo();
3037 Register dst = dest->as_register_lo();
3038 __ movptr(dst, lo);
3039 __ negptr(dst);
3040
3041 } else if (dest->is_single_xmm()) {
3042 assert(!tmp->is_valid(), "do not need temporary");
3043 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3044 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3045 }
3046 __ xorps(dest->as_xmm_float_reg(),
3047 ExternalAddress((address)float_signflip_pool),
3048 rscratch1);
3049 } else if (dest->is_double_xmm()) {
3050 assert(!tmp->is_valid(), "do not need temporary");
3051 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3052 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3053 }
3054 __ xorpd(dest->as_xmm_double_reg(),
3055 ExternalAddress((address)double_signflip_pool),
3056 rscratch1);
3057 } else {
3058 ShouldNotReachHere();
3059 }
3060 }
3061
3062
3063 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3064 assert(src->is_address(), "must be an address");
3065 assert(dest->is_register(), "must be a register");
3066
3067 PatchingStub* patch = nullptr;
3068 if (patch_code != lir_patch_none) {
3069 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3070 }
3071
3072 Register reg = dest->as_pointer_register();
3073 LIR_Address* addr = src->as_address_ptr();
3074 __ lea(reg, as_Address(addr));
3075
3076 if (patch != nullptr) {
3077 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3078 }
3079 }
3080
3081
3082
3083 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3084 assert(!tmp->is_valid(), "don't need temporary");
3085 __ call(RuntimeAddress(dest));
3086 if (info != nullptr) {
3087 add_call_info_here(info);
3088 }
3089 __ post_call_nop();
3090 }
3091
3092
3093 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3094 assert(type == T_LONG, "only for volatile long fields");
3095
3096 if (info != nullptr) {
3097 add_debug_info_for_null_check_here(info);
3098 }
3099
3100 if (src->is_double_xmm()) {
3101 if (dest->is_double_cpu()) {
3102 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3103 } else if (dest->is_double_stack()) {
3104 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3105 } else if (dest->is_address()) {
3106 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3107 } else {
3108 ShouldNotReachHere();
3109 }
3110
3111 } else if (dest->is_double_xmm()) {
3112 if (src->is_double_stack()) {
3113 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3114 } else if (src->is_address()) {
3115 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3116 } else {
3117 ShouldNotReachHere();
3118 }
3119
3120 } else {
3121 ShouldNotReachHere();
3122 }
3123 }
3124
3125 #ifdef ASSERT
3126 // emit run-time assertion
3127 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3128 assert(op->code() == lir_assert, "must be");
3129
3130 if (op->in_opr1()->is_valid()) {
3131 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3132 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3133 } else {
3134 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3135 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3136 }
3137
3138 Label ok;
3139 if (op->condition() != lir_cond_always) {
3140 Assembler::Condition acond = Assembler::zero;
3141 switch (op->condition()) {
3142 case lir_cond_equal: acond = Assembler::equal; break;
3143 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3144 case lir_cond_less: acond = Assembler::less; break;
3145 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3146 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3147 case lir_cond_greater: acond = Assembler::greater; break;
3148 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3149 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3150 default: ShouldNotReachHere();
3151 }
3152 __ jcc(acond, ok);
3153 }
3154 if (op->halt()) {
3155 const char* str = __ code_string(op->msg());
3156 __ stop(str);
3157 } else {
3158 breakpoint();
3159 }
3160 __ bind(ok);
3161 }
3162 #endif
3163
3164 void LIR_Assembler::membar() {
3165 // QQQ sparc TSO uses this,
3166 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3167 }
3168
3169 void LIR_Assembler::membar_acquire() {
3170 // No x86 machines currently require load fences
3171 }
3172
3173 void LIR_Assembler::membar_release() {
3174 // No x86 machines currently require store fences
3175 }
3176
3177 void LIR_Assembler::membar_loadload() {
3178 // no-op
3179 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3180 }
3181
3182 void LIR_Assembler::membar_storestore() {
3183 // no-op
3184 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3185 }
3186
3187 void LIR_Assembler::membar_loadstore() {
3188 // no-op
3189 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3190 }
3191
3192 void LIR_Assembler::membar_storeload() {
3193 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3194 }
3195
3196 void LIR_Assembler::on_spin_wait() {
3197 __ pause ();
3198 }
3199
3200 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3201 assert(result_reg->is_register(), "check");
3202 __ mov(result_reg->as_register(), r15_thread);
3203 }
3204
3205
3206 void LIR_Assembler::peephole(LIR_List*) {
3207 // do nothing for now
3208 }
3209
3210 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3211 assert(data == dest, "xchg/xadd uses only 2 operands");
3212
3213 if (data->type() == T_INT) {
3214 if (code == lir_xadd) {
3215 __ lock();
3216 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3217 } else {
3218 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3219 }
3220 } else if (data->is_oop()) {
3221 assert (code == lir_xchg, "xadd for oops");
3222 Register obj = data->as_register();
3223 if (UseCompressedOops) {
3224 __ encode_heap_oop(obj);
3225 __ xchgl(obj, as_Address(src->as_address_ptr()));
3226 __ decode_heap_oop(obj);
3227 } else {
3228 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3229 }
3230 } else if (data->type() == T_LONG) {
3231 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3232 if (code == lir_xadd) {
3233 __ lock();
3234 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3235 } else {
3236 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3237 }
3238 } else {
3239 ShouldNotReachHere();
3240 }
3241 }
3242
3243 #undef __