1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "ci/ciUtilities.hpp"
36 #include "code/aotCodeCache.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456 InternalAddress here(__ pc());
457
458 __ pushptr(here.addr(), rscratch1);
459 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
460 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
461 __ end_a_stub();
462
463 return offset;
464 }
465
466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
467 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
468 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
469 assert(result->fpu() == 0, "result must already be on TOS");
470 }
471
472 // Pop the stack before the safepoint code
473 __ remove_frame(initial_frame_size_in_bytes());
474
475 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
476 __ reserved_stack_check();
477 }
478
479 // Note: we do not need to round double result; float result has the right precision
480 // the poll sets the condition code, but no data registers
481
482 code_stub->set_safepoint_offset(__ offset());
483 __ relocate(relocInfo::poll_return_type);
484 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
485 __ ret(0);
486 }
487
488
489 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
490 guarantee(info != nullptr, "Shouldn't be null");
491 int offset = __ offset();
492 const Register poll_addr = rscratch1;
493 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
494 add_debug_info_for_branch(info);
495 __ relocate(relocInfo::poll_type);
496 address pre_pc = __ pc();
497 __ testl(rax, Address(poll_addr, 0));
498 address post_pc = __ pc();
499 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
500 return offset;
501 }
502
503
504 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
505 if (from_reg != to_reg) __ mov(to_reg, from_reg);
506 }
507
508 void LIR_Assembler::swap_reg(Register a, Register b) {
509 __ xchgptr(a, b);
510 }
511
512
513 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
514 assert(src->is_constant(), "should not call otherwise");
515 assert(dest->is_register(), "should not call otherwise");
516 LIR_Const* c = src->as_constant_ptr();
517
518 switch (c->type()) {
519 case T_INT: {
520 assert(patch_code == lir_patch_none, "no patching handled here");
521 __ movl(dest->as_register(), c->as_jint());
522 break;
523 }
524
525 case T_ADDRESS: {
526 assert(patch_code == lir_patch_none, "no patching handled here");
527 __ movptr(dest->as_register(), c->as_jint());
528 break;
529 }
530
531 case T_LONG: {
532 assert(patch_code == lir_patch_none, "no patching handled here");
533 if (AOTCodeCache::is_on_for_dump()) {
534 // AOTCodeCache needs relocation info for card table base
535 address b = c->as_pointer();
536 if (is_card_table_address(b)) {
537 __ lea(dest->as_register_lo(), ExternalAddress(b));
538 break;
539 }
540 if (b == (address)ThreadIdentifier::unsafe_offset()) {
541 __ lea(dest->as_register_lo(), ExternalAddress(b));
542 break;
543 }
544 #if INCLUDE_CDS
545 if (AOTRuntimeConstants::contains(b)) {
546 __ load_aotrc_address(dest->as_register_lo(), b);
547 break;
548 }
549 #endif
550 }
551 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
552 break;
553 }
554
555 case T_OBJECT: {
556 if (patch_code != lir_patch_none) {
557 jobject2reg_with_patching(dest->as_register(), info);
558 } else {
559 __ movoop(dest->as_register(), c->as_jobject());
560 }
561 break;
562 }
563
564 case T_METADATA: {
565 if (patch_code != lir_patch_none) {
566 klass2reg_with_patching(dest->as_register(), info);
567 } else {
568 __ mov_metadata(dest->as_register(), c->as_metadata());
569 }
570 break;
571 }
572
573 case T_FLOAT: {
574 if (dest->is_single_xmm()) {
575 if (UseAVX <= 2 && c->is_zero_float()) {
576 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
577 } else {
578 __ movflt(dest->as_xmm_float_reg(),
579 InternalAddress(float_constant(c->as_jfloat())));
580 }
581 } else {
582 ShouldNotReachHere();
583 }
584 break;
585 }
586
587 case T_DOUBLE: {
588 if (dest->is_double_xmm()) {
589 if (UseAVX <= 2 && c->is_zero_double()) {
590 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
591 } else {
592 __ movdbl(dest->as_xmm_double_reg(),
593 InternalAddress(double_constant(c->as_jdouble())));
594 }
595 } else {
596 ShouldNotReachHere();
597 }
598 break;
599 }
600
601 default:
602 ShouldNotReachHere();
603 }
604 }
605
606 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
607 assert(src->is_constant(), "should not call otherwise");
608 assert(dest->is_stack(), "should not call otherwise");
609 LIR_Const* c = src->as_constant_ptr();
610
611 switch (c->type()) {
612 case T_INT: // fall through
613 case T_FLOAT:
614 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
615 break;
616
617 case T_ADDRESS:
618 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
619 break;
620
621 case T_OBJECT:
622 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
623 break;
624
625 case T_LONG: // fall through
626 case T_DOUBLE:
627 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
628 lo_word_offset_in_bytes),
629 (intptr_t)c->as_jlong_bits(),
630 rscratch1);
631 break;
632
633 default:
634 ShouldNotReachHere();
635 }
636 }
637
638 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
639 assert(src->is_constant(), "should not call otherwise");
640 assert(dest->is_address(), "should not call otherwise");
641 LIR_Const* c = src->as_constant_ptr();
642 LIR_Address* addr = dest->as_address_ptr();
643
644 int null_check_here = code_offset();
645 switch (type) {
646 case T_INT: // fall through
647 case T_FLOAT:
648 __ movl(as_Address(addr), c->as_jint_bits());
649 break;
650
651 case T_ADDRESS:
652 __ movptr(as_Address(addr), c->as_jint_bits());
653 break;
654
655 case T_OBJECT: // fall through
656 case T_ARRAY:
657 if (c->as_jobject() == nullptr) {
658 if (UseCompressedOops && !wide) {
659 __ movl(as_Address(addr), NULL_WORD);
660 } else {
661 __ xorptr(rscratch1, rscratch1);
662 null_check_here = code_offset();
663 __ movptr(as_Address(addr), rscratch1);
664 }
665 } else {
666 if (is_literal_address(addr)) {
667 ShouldNotReachHere();
668 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
669 } else {
670 __ movoop(rscratch1, c->as_jobject());
671 if (UseCompressedOops && !wide) {
672 __ encode_heap_oop(rscratch1);
673 null_check_here = code_offset();
674 __ movl(as_Address_lo(addr), rscratch1);
675 } else {
676 null_check_here = code_offset();
677 __ movptr(as_Address_lo(addr), rscratch1);
678 }
679 }
680 }
681 break;
682
683 case T_LONG: // fall through
684 case T_DOUBLE:
685 if (is_literal_address(addr)) {
686 ShouldNotReachHere();
687 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
688 } else {
689 __ movptr(r10, (intptr_t)c->as_jlong_bits());
690 null_check_here = code_offset();
691 __ movptr(as_Address_lo(addr), r10);
692 }
693 break;
694
695 case T_BOOLEAN: // fall through
696 case T_BYTE:
697 __ movb(as_Address(addr), c->as_jint() & 0xFF);
698 break;
699
700 case T_CHAR: // fall through
701 case T_SHORT:
702 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
703 break;
704
705 default:
706 ShouldNotReachHere();
707 };
708
709 if (info != nullptr) {
710 add_debug_info_for_null_check(null_check_here, info);
711 }
712 }
713
714
715 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
716 assert(src->is_register(), "should not call otherwise");
717 assert(dest->is_register(), "should not call otherwise");
718
719 // move between cpu-registers
720 if (dest->is_single_cpu()) {
721 if (src->type() == T_LONG) {
722 // Can do LONG -> OBJECT
723 move_regs(src->as_register_lo(), dest->as_register());
724 return;
725 }
726 assert(src->is_single_cpu(), "must match");
727 if (src->type() == T_OBJECT) {
728 __ verify_oop(src->as_register());
729 }
730 move_regs(src->as_register(), dest->as_register());
731
732 } else if (dest->is_double_cpu()) {
733 if (is_reference_type(src->type())) {
734 // Surprising to me but we can see move of a long to t_object
735 __ verify_oop(src->as_register());
736 move_regs(src->as_register(), dest->as_register_lo());
737 return;
738 }
739 assert(src->is_double_cpu(), "must match");
740 Register f_lo = src->as_register_lo();
741 Register f_hi = src->as_register_hi();
742 Register t_lo = dest->as_register_lo();
743 Register t_hi = dest->as_register_hi();
744 assert(f_hi == f_lo, "must be same");
745 assert(t_hi == t_lo, "must be same");
746 move_regs(f_lo, t_lo);
747
748 // move between xmm-registers
749 } else if (dest->is_single_xmm()) {
750 assert(src->is_single_xmm(), "must match");
751 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
752 } else if (dest->is_double_xmm()) {
753 assert(src->is_double_xmm(), "must match");
754 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
755
756 } else {
757 ShouldNotReachHere();
758 }
759 }
760
761 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
762 assert(src->is_register(), "should not call otherwise");
763 assert(dest->is_stack(), "should not call otherwise");
764
765 if (src->is_single_cpu()) {
766 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
767 if (is_reference_type(type)) {
768 __ verify_oop(src->as_register());
769 __ movptr (dst, src->as_register());
770 } else if (type == T_METADATA || type == T_ADDRESS) {
771 __ movptr (dst, src->as_register());
772 } else {
773 __ movl (dst, src->as_register());
774 }
775
776 } else if (src->is_double_cpu()) {
777 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
778 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
779 __ movptr (dstLO, src->as_register_lo());
780
781 } else if (src->is_single_xmm()) {
782 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
783 __ movflt(dst_addr, src->as_xmm_float_reg());
784
785 } else if (src->is_double_xmm()) {
786 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
787 __ movdbl(dst_addr, src->as_xmm_double_reg());
788
789 } else {
790 ShouldNotReachHere();
791 }
792 }
793
794
795 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
796 LIR_Address* to_addr = dest->as_address_ptr();
797 PatchingStub* patch = nullptr;
798 Register compressed_src = rscratch1;
799
800 if (is_reference_type(type)) {
801 __ verify_oop(src->as_register());
802 if (UseCompressedOops && !wide) {
803 __ movptr(compressed_src, src->as_register());
804 __ encode_heap_oop(compressed_src);
805 if (patch_code != lir_patch_none) {
806 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
807 }
808 }
809 }
810
811 if (patch_code != lir_patch_none) {
812 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
813 Address toa = as_Address(to_addr);
814 assert(toa.disp() != 0, "must have");
815 }
816
817 int null_check_here = code_offset();
818 switch (type) {
819 case T_FLOAT: {
820 assert(src->is_single_xmm(), "not a float");
821 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
822 break;
823 }
824
825 case T_DOUBLE: {
826 assert(src->is_double_xmm(), "not a double");
827 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
828 break;
829 }
830
831 case T_ARRAY: // fall through
832 case T_OBJECT: // fall through
833 if (UseCompressedOops && !wide) {
834 __ movl(as_Address(to_addr), compressed_src);
835 } else {
836 __ movptr(as_Address(to_addr), src->as_register());
837 }
838 break;
839 case T_ADDRESS:
840 __ movptr(as_Address(to_addr), src->as_register());
841 break;
842 case T_INT:
843 __ movl(as_Address(to_addr), src->as_register());
844 break;
845
846 case T_LONG: {
847 Register from_lo = src->as_register_lo();
848 Register from_hi = src->as_register_hi();
849 __ movptr(as_Address_lo(to_addr), from_lo);
850 break;
851 }
852
853 case T_BYTE: // fall through
854 case T_BOOLEAN: {
855 Register src_reg = src->as_register();
856 Address dst_addr = as_Address(to_addr);
857 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
858 __ movb(dst_addr, src_reg);
859 break;
860 }
861
862 case T_CHAR: // fall through
863 case T_SHORT:
864 __ movw(as_Address(to_addr), src->as_register());
865 break;
866
867 default:
868 ShouldNotReachHere();
869 }
870 if (info != nullptr) {
871 add_debug_info_for_null_check(null_check_here, info);
872 }
873
874 if (patch_code != lir_patch_none) {
875 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
876 }
877 }
878
879
880 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
881 assert(src->is_stack(), "should not call otherwise");
882 assert(dest->is_register(), "should not call otherwise");
883
884 if (dest->is_single_cpu()) {
885 if (is_reference_type(type)) {
886 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
887 __ verify_oop(dest->as_register());
888 } else if (type == T_METADATA || type == T_ADDRESS) {
889 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
890 } else {
891 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
892 }
893
894 } else if (dest->is_double_cpu()) {
895 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
896 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
897 __ movptr(dest->as_register_lo(), src_addr_LO);
898
899 } else if (dest->is_single_xmm()) {
900 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
901 __ movflt(dest->as_xmm_float_reg(), src_addr);
902
903 } else if (dest->is_double_xmm()) {
904 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
905 __ movdbl(dest->as_xmm_double_reg(), src_addr);
906
907 } else {
908 ShouldNotReachHere();
909 }
910 }
911
912
913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
914 if (src->is_single_stack()) {
915 if (is_reference_type(type)) {
916 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
917 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
918 } else {
919 //no pushl on 64bits
920 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
921 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
922 }
923
924 } else if (src->is_double_stack()) {
925 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
926 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
927
928 } else {
929 ShouldNotReachHere();
930 }
931 }
932
933
934 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
935 assert(src->is_address(), "should not call otherwise");
936 assert(dest->is_register(), "should not call otherwise");
937
938 LIR_Address* addr = src->as_address_ptr();
939 Address from_addr = as_Address(addr);
940
941 if (addr->base()->type() == T_OBJECT) {
942 __ verify_oop(addr->base()->as_pointer_register());
943 }
944
945 switch (type) {
946 case T_BOOLEAN: // fall through
947 case T_BYTE: // fall through
948 case T_CHAR: // fall through
949 case T_SHORT:
950 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
951 // on pre P6 processors we may get partial register stalls
952 // so blow away the value of to_rinfo before loading a
953 // partial word into it. Do it here so that it precedes
954 // the potential patch point below.
955 __ xorptr(dest->as_register(), dest->as_register());
956 }
957 break;
958 default:
959 break;
960 }
961
962 PatchingStub* patch = nullptr;
963 if (patch_code != lir_patch_none) {
964 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
965 assert(from_addr.disp() != 0, "must have");
966 }
967 if (info != nullptr) {
968 add_debug_info_for_null_check_here(info);
969 }
970
971 switch (type) {
972 case T_FLOAT: {
973 if (dest->is_single_xmm()) {
974 __ movflt(dest->as_xmm_float_reg(), from_addr);
975 } else {
976 ShouldNotReachHere();
977 }
978 break;
979 }
980
981 case T_DOUBLE: {
982 if (dest->is_double_xmm()) {
983 __ movdbl(dest->as_xmm_double_reg(), from_addr);
984 } else {
985 ShouldNotReachHere();
986 }
987 break;
988 }
989
990 case T_OBJECT: // fall through
991 case T_ARRAY: // fall through
992 if (UseCompressedOops && !wide) {
993 __ movl(dest->as_register(), from_addr);
994 } else {
995 __ movptr(dest->as_register(), from_addr);
996 }
997 break;
998
999 case T_ADDRESS:
1000 __ movptr(dest->as_register(), from_addr);
1001 break;
1002 case T_INT:
1003 __ movl(dest->as_register(), from_addr);
1004 break;
1005
1006 case T_LONG: {
1007 Register to_lo = dest->as_register_lo();
1008 Register to_hi = dest->as_register_hi();
1009 __ movptr(to_lo, as_Address_lo(addr));
1010 break;
1011 }
1012
1013 case T_BOOLEAN: // fall through
1014 case T_BYTE: {
1015 Register dest_reg = dest->as_register();
1016 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1017 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1018 __ movsbl(dest_reg, from_addr);
1019 } else {
1020 __ movb(dest_reg, from_addr);
1021 __ shll(dest_reg, 24);
1022 __ sarl(dest_reg, 24);
1023 }
1024 break;
1025 }
1026
1027 case T_CHAR: {
1028 Register dest_reg = dest->as_register();
1029 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1030 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1031 __ movzwl(dest_reg, from_addr);
1032 } else {
1033 __ movw(dest_reg, from_addr);
1034 }
1035 break;
1036 }
1037
1038 case T_SHORT: {
1039 Register dest_reg = dest->as_register();
1040 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1041 __ movswl(dest_reg, from_addr);
1042 } else {
1043 __ movw(dest_reg, from_addr);
1044 __ shll(dest_reg, 16);
1045 __ sarl(dest_reg, 16);
1046 }
1047 break;
1048 }
1049
1050 default:
1051 ShouldNotReachHere();
1052 }
1053
1054 if (patch != nullptr) {
1055 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1056 }
1057
1058 if (is_reference_type(type)) {
1059 if (UseCompressedOops && !wide) {
1060 __ decode_heap_oop(dest->as_register());
1061 }
1062
1063 __ verify_oop(dest->as_register());
1064 }
1065 }
1066
1067
1068 NEEDS_CLEANUP; // This could be static?
1069 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1070 int elem_size = type2aelembytes(type);
1071 switch (elem_size) {
1072 case 1: return Address::times_1;
1073 case 2: return Address::times_2;
1074 case 4: return Address::times_4;
1075 case 8: return Address::times_8;
1076 }
1077 ShouldNotReachHere();
1078 return Address::no_scale;
1079 }
1080
1081
1082 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1083 switch (op->code()) {
1084 case lir_idiv:
1085 case lir_irem:
1086 arithmetic_idiv(op->code(),
1087 op->in_opr1(),
1088 op->in_opr2(),
1089 op->in_opr3(),
1090 op->result_opr(),
1091 op->info());
1092 break;
1093 case lir_fmad:
1094 __ fmad(op->result_opr()->as_xmm_double_reg(),
1095 op->in_opr1()->as_xmm_double_reg(),
1096 op->in_opr2()->as_xmm_double_reg(),
1097 op->in_opr3()->as_xmm_double_reg());
1098 break;
1099 case lir_fmaf:
1100 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1101 op->in_opr1()->as_xmm_float_reg(),
1102 op->in_opr2()->as_xmm_float_reg(),
1103 op->in_opr3()->as_xmm_float_reg());
1104 break;
1105 default: ShouldNotReachHere(); break;
1106 }
1107 }
1108
1109 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1110 #ifdef ASSERT
1111 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1112 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1113 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1114 #endif
1115
1116 if (op->cond() == lir_cond_always) {
1117 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1118 __ jmp (*(op->label()));
1119 } else {
1120 Assembler::Condition acond = Assembler::zero;
1121 if (op->code() == lir_cond_float_branch) {
1122 assert(op->ublock() != nullptr, "must have unordered successor");
1123 __ jcc(Assembler::parity, *(op->ublock()->label()));
1124 switch(op->cond()) {
1125 case lir_cond_equal: acond = Assembler::equal; break;
1126 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1127 case lir_cond_less: acond = Assembler::below; break;
1128 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1129 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1130 case lir_cond_greater: acond = Assembler::above; break;
1131 default: ShouldNotReachHere();
1132 }
1133 } else {
1134 switch (op->cond()) {
1135 case lir_cond_equal: acond = Assembler::equal; break;
1136 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1137 case lir_cond_less: acond = Assembler::less; break;
1138 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1139 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1140 case lir_cond_greater: acond = Assembler::greater; break;
1141 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1142 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1143 default: ShouldNotReachHere();
1144 }
1145 }
1146 __ jcc(acond,*(op->label()));
1147 }
1148 }
1149
1150 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1151 LIR_Opr src = op->in_opr();
1152 LIR_Opr dest = op->result_opr();
1153
1154 switch (op->bytecode()) {
1155 case Bytecodes::_i2l:
1156 __ movl2ptr(dest->as_register_lo(), src->as_register());
1157 break;
1158
1159 case Bytecodes::_l2i:
1160 __ movl(dest->as_register(), src->as_register_lo());
1161 break;
1162
1163 case Bytecodes::_i2b:
1164 move_regs(src->as_register(), dest->as_register());
1165 __ sign_extend_byte(dest->as_register());
1166 break;
1167
1168 case Bytecodes::_i2c:
1169 move_regs(src->as_register(), dest->as_register());
1170 __ andl(dest->as_register(), 0xFFFF);
1171 break;
1172
1173 case Bytecodes::_i2s:
1174 move_regs(src->as_register(), dest->as_register());
1175 __ sign_extend_short(dest->as_register());
1176 break;
1177
1178 case Bytecodes::_f2d:
1179 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1180 break;
1181
1182 case Bytecodes::_d2f:
1183 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1184 break;
1185
1186 case Bytecodes::_i2f:
1187 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1188 break;
1189
1190 case Bytecodes::_i2d:
1191 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1192 break;
1193
1194 case Bytecodes::_l2f:
1195 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1196 break;
1197
1198 case Bytecodes::_l2d:
1199 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1200 break;
1201
1202 case Bytecodes::_f2i:
1203 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1204 break;
1205
1206 case Bytecodes::_d2i:
1207 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1208 break;
1209
1210 case Bytecodes::_f2l:
1211 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1212 break;
1213
1214 case Bytecodes::_d2l:
1215 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1216 break;
1217
1218 default: ShouldNotReachHere();
1219 }
1220 }
1221
1222 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1223 if (op->init_check()) {
1224 add_debug_info_for_null_check_here(op->stub()->info());
1225 // init_state needs acquire, but x86 is TSO, and so we are already good.
1226 __ cmpb(Address(op->klass()->as_register(),
1227 InstanceKlass::init_state_offset()),
1228 InstanceKlass::fully_initialized);
1229 __ jcc(Assembler::notEqual, *op->stub()->entry());
1230 }
1231 __ allocate_object(op->obj()->as_register(),
1232 op->tmp1()->as_register(),
1233 op->tmp2()->as_register(),
1234 op->header_size(),
1235 op->object_size(),
1236 op->klass()->as_register(),
1237 *op->stub()->entry());
1238 __ bind(*op->stub()->continuation());
1239 }
1240
1241 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1242 Register len = op->len()->as_register();
1243 __ movslq(len, len);
1244
1245 if (UseSlowPath ||
1246 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1247 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1248 __ jmp(*op->stub()->entry());
1249 } else {
1250 Register tmp1 = op->tmp1()->as_register();
1251 Register tmp2 = op->tmp2()->as_register();
1252 Register tmp3 = op->tmp3()->as_register();
1253 if (len == tmp1) {
1254 tmp1 = tmp3;
1255 } else if (len == tmp2) {
1256 tmp2 = tmp3;
1257 } else if (len == tmp3) {
1258 // everything is ok
1259 } else {
1260 __ mov(tmp3, len);
1261 }
1262 __ allocate_array(op->obj()->as_register(),
1263 len,
1264 tmp1,
1265 tmp2,
1266 arrayOopDesc::base_offset_in_bytes(op->type()),
1267 array_element_size(op->type()),
1268 op->klass()->as_register(),
1269 *op->stub()->entry(),
1270 op->zero_array());
1271 }
1272 __ bind(*op->stub()->continuation());
1273 }
1274
1275 void LIR_Assembler::type_profile_helper(Register mdo,
1276 ciMethodData *md, ciProfileData *data,
1277 Register recv, Label* update_done) {
1278 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1279 Label next_test;
1280 // See if the receiver is receiver[n].
1281 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1282 __ jccb(Assembler::notEqual, next_test);
1283 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1284 __ addptr(data_addr, DataLayout::counter_increment);
1285 __ jmp(*update_done);
1286 __ bind(next_test);
1287 }
1288
1289 // Didn't find receiver; find next empty slot and fill it in
1290 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1291 Label next_test;
1292 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1293 __ cmpptr(recv_addr, NULL_WORD);
1294 __ jccb(Assembler::notEqual, next_test);
1295 __ movptr(recv_addr, recv);
1296 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1297 __ jmp(*update_done);
1298 __ bind(next_test);
1299 }
1300 }
1301
1302 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1303 // we always need a stub for the failure case.
1304 CodeStub* stub = op->stub();
1305 Register obj = op->object()->as_register();
1306 Register k_RInfo = op->tmp1()->as_register();
1307 Register klass_RInfo = op->tmp2()->as_register();
1308 Register dst = op->result_opr()->as_register();
1309 ciKlass* k = op->klass();
1310 Register Rtmp1 = noreg;
1311 Register tmp_load_klass = rscratch1;
1312
1313 // check if it needs to be profiled
1314 ciMethodData* md = nullptr;
1315 ciProfileData* data = nullptr;
1316
1317 if (op->should_profile()) {
1318 ciMethod* method = op->profiled_method();
1319 assert(method != nullptr, "Should have method");
1320 int bci = op->profiled_bci();
1321 md = method->method_data_or_null();
1322 assert(md != nullptr, "Sanity");
1323 data = md->bci_to_data(bci);
1324 assert(data != nullptr, "need data for type check");
1325 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1326 }
1327 Label* success_target = success;
1328 Label* failure_target = failure;
1329
1330 if (obj == k_RInfo) {
1331 k_RInfo = dst;
1332 } else if (obj == klass_RInfo) {
1333 klass_RInfo = dst;
1334 }
1335 if (k->is_loaded() && !UseCompressedClassPointers) {
1336 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1337 } else {
1338 Rtmp1 = op->tmp3()->as_register();
1339 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1340 }
1341
1342 assert_different_registers(obj, k_RInfo, klass_RInfo);
1343
1344 __ testptr(obj, obj);
1345 if (op->should_profile()) {
1346 Label not_null;
1347 Register mdo = klass_RInfo;
1348 __ mov_metadata(mdo, md->constant_encoding());
1349 __ jccb(Assembler::notEqual, not_null);
1350 // Object is null; update MDO and exit
1351 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1352 int header_bits = BitData::null_seen_byte_constant();
1353 __ orb(data_addr, header_bits);
1354 __ jmp(*obj_is_null);
1355 __ bind(not_null);
1356
1357 Label update_done;
1358 Register recv = k_RInfo;
1359 __ load_klass(recv, obj, tmp_load_klass);
1360 type_profile_helper(mdo, md, data, recv, &update_done);
1361
1362 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1363 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1364
1365 __ bind(update_done);
1366 } else {
1367 __ jcc(Assembler::equal, *obj_is_null);
1368 }
1369
1370 if (!k->is_loaded()) {
1371 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1372 } else {
1373 __ mov_metadata(k_RInfo, k->constant_encoding());
1374 }
1375 __ verify_oop(obj);
1376
1377 if (op->fast_check()) {
1378 // get object class
1379 // not a safepoint as obj null check happens earlier
1380 if (UseCompressedClassPointers) {
1381 __ load_klass(Rtmp1, obj, tmp_load_klass);
1382 __ cmpptr(k_RInfo, Rtmp1);
1383 } else {
1384 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1385 }
1386 __ jcc(Assembler::notEqual, *failure_target);
1387 // successful cast, fall through to profile or jump
1388 } else {
1389 // get object class
1390 // not a safepoint as obj null check happens earlier
1391 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1392 if (k->is_loaded()) {
1393 // See if we get an immediate positive hit
1394 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1395 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1396 __ jcc(Assembler::notEqual, *failure_target);
1397 // successful cast, fall through to profile or jump
1398 } else {
1399 // See if we get an immediate positive hit
1400 __ jcc(Assembler::equal, *success_target);
1401 // check for self
1402 __ cmpptr(klass_RInfo, k_RInfo);
1403 __ jcc(Assembler::equal, *success_target);
1404
1405 __ push_ppx(klass_RInfo);
1406 __ push_ppx(k_RInfo);
1407 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1408 __ pop_ppx(klass_RInfo);
1409 __ pop_ppx(klass_RInfo);
1410 // result is a boolean
1411 __ testl(klass_RInfo, klass_RInfo);
1412 __ jcc(Assembler::equal, *failure_target);
1413 // successful cast, fall through to profile or jump
1414 }
1415 } else {
1416 // perform the fast part of the checking logic
1417 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1418 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1419 __ push_ppx(klass_RInfo);
1420 __ push_ppx(k_RInfo);
1421 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1422 __ pop_ppx(klass_RInfo);
1423 __ pop_ppx(k_RInfo);
1424 // result is a boolean
1425 __ testl(k_RInfo, k_RInfo);
1426 __ jcc(Assembler::equal, *failure_target);
1427 // successful cast, fall through to profile or jump
1428 }
1429 }
1430 __ jmp(*success);
1431 }
1432
1433
1434 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1435 Register tmp_load_klass = rscratch1;
1436 LIR_Code code = op->code();
1437 if (code == lir_store_check) {
1438 Register value = op->object()->as_register();
1439 Register array = op->array()->as_register();
1440 Register k_RInfo = op->tmp1()->as_register();
1441 Register klass_RInfo = op->tmp2()->as_register();
1442 Register Rtmp1 = op->tmp3()->as_register();
1443
1444 CodeStub* stub = op->stub();
1445
1446 // check if it needs to be profiled
1447 ciMethodData* md = nullptr;
1448 ciProfileData* data = nullptr;
1449
1450 if (op->should_profile()) {
1451 ciMethod* method = op->profiled_method();
1452 assert(method != nullptr, "Should have method");
1453 int bci = op->profiled_bci();
1454 md = method->method_data_or_null();
1455 assert(md != nullptr, "Sanity");
1456 data = md->bci_to_data(bci);
1457 assert(data != nullptr, "need data for type check");
1458 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1459 }
1460 Label done;
1461 Label* success_target = &done;
1462 Label* failure_target = stub->entry();
1463
1464 __ testptr(value, value);
1465 if (op->should_profile()) {
1466 Label not_null;
1467 Register mdo = klass_RInfo;
1468 __ mov_metadata(mdo, md->constant_encoding());
1469 __ jccb(Assembler::notEqual, not_null);
1470 // Object is null; update MDO and exit
1471 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1472 int header_bits = BitData::null_seen_byte_constant();
1473 __ orb(data_addr, header_bits);
1474 __ jmp(done);
1475 __ bind(not_null);
1476
1477 Label update_done;
1478 Register recv = k_RInfo;
1479 __ load_klass(recv, value, tmp_load_klass);
1480 type_profile_helper(mdo, md, data, recv, &update_done);
1481
1482 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1483 __ addptr(counter_addr, DataLayout::counter_increment);
1484 __ bind(update_done);
1485 } else {
1486 __ jcc(Assembler::equal, done);
1487 }
1488
1489 add_debug_info_for_null_check_here(op->info_for_exception());
1490 __ load_klass(k_RInfo, array, tmp_load_klass);
1491 __ load_klass(klass_RInfo, value, tmp_load_klass);
1492
1493 // get instance klass (it's already uncompressed)
1494 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1495 // perform the fast part of the checking logic
1496 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1497 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1498 __ push_ppx(klass_RInfo);
1499 __ push_ppx(k_RInfo);
1500 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1501 __ pop_ppx(klass_RInfo);
1502 __ pop_ppx(k_RInfo);
1503 // result is a boolean
1504 __ testl(k_RInfo, k_RInfo);
1505 __ jcc(Assembler::equal, *failure_target);
1506 // fall through to the success case
1507
1508 __ bind(done);
1509 } else
1510 if (code == lir_checkcast) {
1511 Register obj = op->object()->as_register();
1512 Register dst = op->result_opr()->as_register();
1513 Label success;
1514 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1515 __ bind(success);
1516 if (dst != obj) {
1517 __ mov(dst, obj);
1518 }
1519 } else
1520 if (code == lir_instanceof) {
1521 Register obj = op->object()->as_register();
1522 Register dst = op->result_opr()->as_register();
1523 Label success, failure, done;
1524 emit_typecheck_helper(op, &success, &failure, &failure);
1525 __ bind(failure);
1526 __ xorptr(dst, dst);
1527 __ jmpb(done);
1528 __ bind(success);
1529 __ movptr(dst, 1);
1530 __ bind(done);
1531 } else {
1532 ShouldNotReachHere();
1533 }
1534
1535 }
1536
1537
1538 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1539 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1540 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1541 Register newval = op->new_value()->as_register();
1542 Register cmpval = op->cmp_value()->as_register();
1543 assert(cmpval == rax, "wrong register");
1544 assert(newval != noreg, "new val must be register");
1545 assert(cmpval != newval, "cmp and new values must be in different registers");
1546 assert(cmpval != addr, "cmp and addr must be in different registers");
1547 assert(newval != addr, "new value and addr must be in different registers");
1548
1549 if (op->code() == lir_cas_obj) {
1550 if (UseCompressedOops) {
1551 __ encode_heap_oop(cmpval);
1552 __ mov(rscratch1, newval);
1553 __ encode_heap_oop(rscratch1);
1554 __ lock();
1555 // cmpval (rax) is implicitly used by this instruction
1556 __ cmpxchgl(rscratch1, Address(addr, 0));
1557 } else {
1558 __ lock();
1559 __ cmpxchgptr(newval, Address(addr, 0));
1560 }
1561 } else {
1562 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1563 __ lock();
1564 __ cmpxchgl(newval, Address(addr, 0));
1565 }
1566 } else if (op->code() == lir_cas_long) {
1567 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1568 Register newval = op->new_value()->as_register_lo();
1569 Register cmpval = op->cmp_value()->as_register_lo();
1570 assert(cmpval == rax, "wrong register");
1571 assert(newval != noreg, "new val must be register");
1572 assert(cmpval != newval, "cmp and new values must be in different registers");
1573 assert(cmpval != addr, "cmp and addr must be in different registers");
1574 assert(newval != addr, "new value and addr must be in different registers");
1575 __ lock();
1576 __ cmpxchgq(newval, Address(addr, 0));
1577 } else {
1578 Unimplemented();
1579 }
1580 }
1581
1582 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1583 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1584 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1585
1586 Assembler::Condition acond, ncond;
1587 switch (condition) {
1588 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1589 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1590 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1591 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1592 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1593 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1594 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1595 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1596 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1597 ShouldNotReachHere();
1598 }
1599
1600 if (opr1->is_cpu_register()) {
1601 reg2reg(opr1, result);
1602 } else if (opr1->is_stack()) {
1603 stack2reg(opr1, result, result->type());
1604 } else if (opr1->is_constant()) {
1605 const2reg(opr1, result, lir_patch_none, nullptr);
1606 } else {
1607 ShouldNotReachHere();
1608 }
1609
1610 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1611 // optimized version that does not require a branch
1612 if (opr2->is_single_cpu()) {
1613 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1614 __ cmov(ncond, result->as_register(), opr2->as_register());
1615 } else if (opr2->is_double_cpu()) {
1616 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1617 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1618 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1619 } else if (opr2->is_single_stack()) {
1620 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1621 } else if (opr2->is_double_stack()) {
1622 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1623 } else {
1624 ShouldNotReachHere();
1625 }
1626
1627 } else {
1628 Label skip;
1629 __ jccb(acond, skip);
1630 if (opr2->is_cpu_register()) {
1631 reg2reg(opr2, result);
1632 } else if (opr2->is_stack()) {
1633 stack2reg(opr2, result, result->type());
1634 } else if (opr2->is_constant()) {
1635 const2reg(opr2, result, lir_patch_none, nullptr);
1636 } else {
1637 ShouldNotReachHere();
1638 }
1639 __ bind(skip);
1640 }
1641 }
1642
1643
1644 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1645 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1646
1647 if (left->is_single_cpu()) {
1648 assert(left == dest, "left and dest must be equal");
1649 Register lreg = left->as_register();
1650
1651 if (right->is_single_cpu()) {
1652 // cpu register - cpu register
1653 Register rreg = right->as_register();
1654 switch (code) {
1655 case lir_add: __ addl (lreg, rreg); break;
1656 case lir_sub: __ subl (lreg, rreg); break;
1657 case lir_mul: __ imull(lreg, rreg); break;
1658 default: ShouldNotReachHere();
1659 }
1660
1661 } else if (right->is_stack()) {
1662 // cpu register - stack
1663 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1664 switch (code) {
1665 case lir_add: __ addl(lreg, raddr); break;
1666 case lir_sub: __ subl(lreg, raddr); break;
1667 default: ShouldNotReachHere();
1668 }
1669
1670 } else if (right->is_constant()) {
1671 // cpu register - constant
1672 jint c = right->as_constant_ptr()->as_jint();
1673 switch (code) {
1674 case lir_add: {
1675 __ incrementl(lreg, c);
1676 break;
1677 }
1678 case lir_sub: {
1679 __ decrementl(lreg, c);
1680 break;
1681 }
1682 default: ShouldNotReachHere();
1683 }
1684
1685 } else {
1686 ShouldNotReachHere();
1687 }
1688
1689 } else if (left->is_double_cpu()) {
1690 assert(left == dest, "left and dest must be equal");
1691 Register lreg_lo = left->as_register_lo();
1692 Register lreg_hi = left->as_register_hi();
1693
1694 if (right->is_double_cpu()) {
1695 // cpu register - cpu register
1696 Register rreg_lo = right->as_register_lo();
1697 Register rreg_hi = right->as_register_hi();
1698 assert_different_registers(lreg_lo, rreg_lo);
1699 switch (code) {
1700 case lir_add:
1701 __ addptr(lreg_lo, rreg_lo);
1702 break;
1703 case lir_sub:
1704 __ subptr(lreg_lo, rreg_lo);
1705 break;
1706 case lir_mul:
1707 __ imulq(lreg_lo, rreg_lo);
1708 break;
1709 default:
1710 ShouldNotReachHere();
1711 }
1712
1713 } else if (right->is_constant()) {
1714 // cpu register - constant
1715 jlong c = right->as_constant_ptr()->as_jlong_bits();
1716 __ movptr(r10, (intptr_t) c);
1717 switch (code) {
1718 case lir_add:
1719 __ addptr(lreg_lo, r10);
1720 break;
1721 case lir_sub:
1722 __ subptr(lreg_lo, r10);
1723 break;
1724 default:
1725 ShouldNotReachHere();
1726 }
1727
1728 } else {
1729 ShouldNotReachHere();
1730 }
1731
1732 } else if (left->is_single_xmm()) {
1733 assert(left == dest, "left and dest must be equal");
1734 XMMRegister lreg = left->as_xmm_float_reg();
1735
1736 if (right->is_single_xmm()) {
1737 XMMRegister rreg = right->as_xmm_float_reg();
1738 switch (code) {
1739 case lir_add: __ addss(lreg, rreg); break;
1740 case lir_sub: __ subss(lreg, rreg); break;
1741 case lir_mul: __ mulss(lreg, rreg); break;
1742 case lir_div: __ divss(lreg, rreg); break;
1743 default: ShouldNotReachHere();
1744 }
1745 } else {
1746 Address raddr;
1747 if (right->is_single_stack()) {
1748 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1749 } else if (right->is_constant()) {
1750 // hack for now
1751 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1752 } else {
1753 ShouldNotReachHere();
1754 }
1755 switch (code) {
1756 case lir_add: __ addss(lreg, raddr); break;
1757 case lir_sub: __ subss(lreg, raddr); break;
1758 case lir_mul: __ mulss(lreg, raddr); break;
1759 case lir_div: __ divss(lreg, raddr); break;
1760 default: ShouldNotReachHere();
1761 }
1762 }
1763
1764 } else if (left->is_double_xmm()) {
1765 assert(left == dest, "left and dest must be equal");
1766
1767 XMMRegister lreg = left->as_xmm_double_reg();
1768 if (right->is_double_xmm()) {
1769 XMMRegister rreg = right->as_xmm_double_reg();
1770 switch (code) {
1771 case lir_add: __ addsd(lreg, rreg); break;
1772 case lir_sub: __ subsd(lreg, rreg); break;
1773 case lir_mul: __ mulsd(lreg, rreg); break;
1774 case lir_div: __ divsd(lreg, rreg); break;
1775 default: ShouldNotReachHere();
1776 }
1777 } else {
1778 Address raddr;
1779 if (right->is_double_stack()) {
1780 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1781 } else if (right->is_constant()) {
1782 // hack for now
1783 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1784 } else {
1785 ShouldNotReachHere();
1786 }
1787 switch (code) {
1788 case lir_add: __ addsd(lreg, raddr); break;
1789 case lir_sub: __ subsd(lreg, raddr); break;
1790 case lir_mul: __ mulsd(lreg, raddr); break;
1791 case lir_div: __ divsd(lreg, raddr); break;
1792 default: ShouldNotReachHere();
1793 }
1794 }
1795
1796 } else if (left->is_single_stack() || left->is_address()) {
1797 assert(left == dest, "left and dest must be equal");
1798
1799 Address laddr;
1800 if (left->is_single_stack()) {
1801 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1802 } else if (left->is_address()) {
1803 laddr = as_Address(left->as_address_ptr());
1804 } else {
1805 ShouldNotReachHere();
1806 }
1807
1808 if (right->is_single_cpu()) {
1809 Register rreg = right->as_register();
1810 switch (code) {
1811 case lir_add: __ addl(laddr, rreg); break;
1812 case lir_sub: __ subl(laddr, rreg); break;
1813 default: ShouldNotReachHere();
1814 }
1815 } else if (right->is_constant()) {
1816 jint c = right->as_constant_ptr()->as_jint();
1817 switch (code) {
1818 case lir_add: {
1819 __ incrementl(laddr, c);
1820 break;
1821 }
1822 case lir_sub: {
1823 __ decrementl(laddr, c);
1824 break;
1825 }
1826 default: ShouldNotReachHere();
1827 }
1828 } else {
1829 ShouldNotReachHere();
1830 }
1831
1832 } else {
1833 ShouldNotReachHere();
1834 }
1835 }
1836
1837
1838 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1839 if (value->is_double_xmm()) {
1840 switch(code) {
1841 case lir_abs :
1842 {
1843 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1844 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1845 }
1846 assert(!tmp->is_valid(), "do not need temporary");
1847 __ andpd(dest->as_xmm_double_reg(),
1848 ExternalAddress(LIR_Assembler::double_signmask_pool),
1849 rscratch1);
1850 }
1851 break;
1852
1853 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1854 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1855 default : ShouldNotReachHere();
1856 }
1857
1858 } else if (code == lir_f2hf) {
1859 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1860 } else if (code == lir_hf2f) {
1861 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1862 } else {
1863 Unimplemented();
1864 }
1865 }
1866
1867 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1868 // assert(left->destroys_register(), "check");
1869 if (left->is_single_cpu()) {
1870 Register reg = left->as_register();
1871 if (right->is_constant()) {
1872 int val = right->as_constant_ptr()->as_jint();
1873 switch (code) {
1874 case lir_logic_and: __ andl (reg, val); break;
1875 case lir_logic_or: __ orl (reg, val); break;
1876 case lir_logic_xor: __ xorl (reg, val); break;
1877 default: ShouldNotReachHere();
1878 }
1879 } else if (right->is_stack()) {
1880 // added support for stack operands
1881 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1882 switch (code) {
1883 case lir_logic_and: __ andl (reg, raddr); break;
1884 case lir_logic_or: __ orl (reg, raddr); break;
1885 case lir_logic_xor: __ xorl (reg, raddr); break;
1886 default: ShouldNotReachHere();
1887 }
1888 } else {
1889 Register rright = right->as_register();
1890 switch (code) {
1891 case lir_logic_and: __ andptr (reg, rright); break;
1892 case lir_logic_or : __ orptr (reg, rright); break;
1893 case lir_logic_xor: __ xorptr (reg, rright); break;
1894 default: ShouldNotReachHere();
1895 }
1896 }
1897 move_regs(reg, dst->as_register());
1898 } else {
1899 Register l_lo = left->as_register_lo();
1900 Register l_hi = left->as_register_hi();
1901 if (right->is_constant()) {
1902 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1903 switch (code) {
1904 case lir_logic_and:
1905 __ andq(l_lo, rscratch1);
1906 break;
1907 case lir_logic_or:
1908 __ orq(l_lo, rscratch1);
1909 break;
1910 case lir_logic_xor:
1911 __ xorq(l_lo, rscratch1);
1912 break;
1913 default: ShouldNotReachHere();
1914 }
1915 } else {
1916 Register r_lo;
1917 if (is_reference_type(right->type())) {
1918 r_lo = right->as_register();
1919 } else {
1920 r_lo = right->as_register_lo();
1921 }
1922 switch (code) {
1923 case lir_logic_and:
1924 __ andptr(l_lo, r_lo);
1925 break;
1926 case lir_logic_or:
1927 __ orptr(l_lo, r_lo);
1928 break;
1929 case lir_logic_xor:
1930 __ xorptr(l_lo, r_lo);
1931 break;
1932 default: ShouldNotReachHere();
1933 }
1934 }
1935
1936 Register dst_lo = dst->as_register_lo();
1937 Register dst_hi = dst->as_register_hi();
1938
1939 move_regs(l_lo, dst_lo);
1940 }
1941 }
1942
1943
1944 // we assume that rax, and rdx can be overwritten
1945 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1946
1947 assert(left->is_single_cpu(), "left must be register");
1948 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1949 assert(result->is_single_cpu(), "result must be register");
1950
1951 // assert(left->destroys_register(), "check");
1952 // assert(right->destroys_register(), "check");
1953
1954 Register lreg = left->as_register();
1955 Register dreg = result->as_register();
1956
1957 if (right->is_constant()) {
1958 jint divisor = right->as_constant_ptr()->as_jint();
1959 assert(divisor > 0 && is_power_of_2(divisor), "must be");
1960 if (code == lir_idiv) {
1961 assert(lreg == rax, "must be rax,");
1962 assert(temp->as_register() == rdx, "tmp register must be rdx");
1963 __ cdql(); // sign extend into rdx:rax
1964 if (divisor == 2) {
1965 __ subl(lreg, rdx);
1966 } else {
1967 __ andl(rdx, divisor - 1);
1968 __ addl(lreg, rdx);
1969 }
1970 __ sarl(lreg, log2i_exact(divisor));
1971 move_regs(lreg, dreg);
1972 } else if (code == lir_irem) {
1973 Label done;
1974 __ mov(dreg, lreg);
1975 __ andl(dreg, 0x80000000 | (divisor - 1));
1976 __ jcc(Assembler::positive, done);
1977 __ decrement(dreg);
1978 __ orl(dreg, ~(divisor - 1));
1979 __ increment(dreg);
1980 __ bind(done);
1981 } else {
1982 ShouldNotReachHere();
1983 }
1984 } else {
1985 Register rreg = right->as_register();
1986 assert(lreg == rax, "left register must be rax,");
1987 assert(rreg != rdx, "right register must not be rdx");
1988 assert(temp->as_register() == rdx, "tmp register must be rdx");
1989
1990 move_regs(lreg, rax);
1991
1992 int idivl_offset = __ corrected_idivl(rreg);
1993 if (ImplicitDiv0Checks) {
1994 add_debug_info_for_div0(idivl_offset, info);
1995 }
1996 if (code == lir_irem) {
1997 move_regs(rdx, dreg); // result is in rdx
1998 } else {
1999 move_regs(rax, dreg);
2000 }
2001 }
2002 }
2003
2004
2005 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2006 if (opr1->is_single_cpu()) {
2007 Register reg1 = opr1->as_register();
2008 if (opr2->is_single_cpu()) {
2009 // cpu register - cpu register
2010 if (is_reference_type(opr1->type())) {
2011 __ cmpoop(reg1, opr2->as_register());
2012 } else {
2013 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2014 __ cmpl(reg1, opr2->as_register());
2015 }
2016 } else if (opr2->is_stack()) {
2017 // cpu register - stack
2018 if (is_reference_type(opr1->type())) {
2019 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2020 } else {
2021 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2022 }
2023 } else if (opr2->is_constant()) {
2024 // cpu register - constant
2025 LIR_Const* c = opr2->as_constant_ptr();
2026 if (c->type() == T_INT) {
2027 jint i = c->as_jint();
2028 if (i == 0) {
2029 __ testl(reg1, reg1);
2030 } else {
2031 __ cmpl(reg1, i);
2032 }
2033 } else if (c->type() == T_METADATA) {
2034 // All we need for now is a comparison with null for equality.
2035 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2036 Metadata* m = c->as_metadata();
2037 if (m == nullptr) {
2038 __ testptr(reg1, reg1);
2039 } else {
2040 ShouldNotReachHere();
2041 }
2042 } else if (is_reference_type(c->type())) {
2043 // In 64bit oops are single register
2044 jobject o = c->as_jobject();
2045 if (o == nullptr) {
2046 __ testptr(reg1, reg1);
2047 } else {
2048 __ cmpoop(reg1, o, rscratch1);
2049 }
2050 } else {
2051 fatal("unexpected type: %s", basictype_to_str(c->type()));
2052 }
2053 // cpu register - address
2054 } else if (opr2->is_address()) {
2055 if (op->info() != nullptr) {
2056 add_debug_info_for_null_check_here(op->info());
2057 }
2058 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2059 } else {
2060 ShouldNotReachHere();
2061 }
2062
2063 } else if(opr1->is_double_cpu()) {
2064 Register xlo = opr1->as_register_lo();
2065 Register xhi = opr1->as_register_hi();
2066 if (opr2->is_double_cpu()) {
2067 __ cmpptr(xlo, opr2->as_register_lo());
2068 } else if (opr2->is_constant()) {
2069 // cpu register - constant 0
2070 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2071 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2072 } else {
2073 ShouldNotReachHere();
2074 }
2075
2076 } else if (opr1->is_single_xmm()) {
2077 XMMRegister reg1 = opr1->as_xmm_float_reg();
2078 if (opr2->is_single_xmm()) {
2079 // xmm register - xmm register
2080 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2081 } else if (opr2->is_stack()) {
2082 // xmm register - stack
2083 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2084 } else if (opr2->is_constant()) {
2085 // xmm register - constant
2086 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2087 } else if (opr2->is_address()) {
2088 // xmm register - address
2089 if (op->info() != nullptr) {
2090 add_debug_info_for_null_check_here(op->info());
2091 }
2092 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2093 } else {
2094 ShouldNotReachHere();
2095 }
2096
2097 } else if (opr1->is_double_xmm()) {
2098 XMMRegister reg1 = opr1->as_xmm_double_reg();
2099 if (opr2->is_double_xmm()) {
2100 // xmm register - xmm register
2101 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2102 } else if (opr2->is_stack()) {
2103 // xmm register - stack
2104 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2105 } else if (opr2->is_constant()) {
2106 // xmm register - constant
2107 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2108 } else if (opr2->is_address()) {
2109 // xmm register - address
2110 if (op->info() != nullptr) {
2111 add_debug_info_for_null_check_here(op->info());
2112 }
2113 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2114 } else {
2115 ShouldNotReachHere();
2116 }
2117
2118 } else if (opr1->is_address() && opr2->is_constant()) {
2119 LIR_Const* c = opr2->as_constant_ptr();
2120 if (is_reference_type(c->type())) {
2121 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2122 __ movoop(rscratch1, c->as_jobject());
2123 }
2124 if (op->info() != nullptr) {
2125 add_debug_info_for_null_check_here(op->info());
2126 }
2127 // special case: address - constant
2128 LIR_Address* addr = opr1->as_address_ptr();
2129 if (c->type() == T_INT) {
2130 __ cmpl(as_Address(addr), c->as_jint());
2131 } else if (is_reference_type(c->type())) {
2132 // %%% Make this explode if addr isn't reachable until we figure out a
2133 // better strategy by giving noreg as the temp for as_Address
2134 __ cmpoop(rscratch1, as_Address(addr, noreg));
2135 } else {
2136 ShouldNotReachHere();
2137 }
2138
2139 } else {
2140 ShouldNotReachHere();
2141 }
2142 }
2143
2144 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2145 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2146 if (left->is_single_xmm()) {
2147 assert(right->is_single_xmm(), "must match");
2148 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2149 } else if (left->is_double_xmm()) {
2150 assert(right->is_double_xmm(), "must match");
2151 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2152
2153 } else {
2154 ShouldNotReachHere();
2155 }
2156 } else {
2157 assert(code == lir_cmp_l2i, "check");
2158 Label done;
2159 Register dest = dst->as_register();
2160 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2161 __ movl(dest, -1);
2162 __ jccb(Assembler::less, done);
2163 __ setb(Assembler::notZero, dest);
2164 __ movzbl(dest, dest);
2165 __ bind(done);
2166 }
2167 }
2168
2169
2170 void LIR_Assembler::align_call(LIR_Code code) {
2171 // make sure that the displacement word of the call ends up word aligned
2172 int offset = __ offset();
2173 switch (code) {
2174 case lir_static_call:
2175 case lir_optvirtual_call:
2176 case lir_dynamic_call:
2177 offset += NativeCall::displacement_offset;
2178 break;
2179 case lir_icvirtual_call:
2180 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2181 break;
2182 default: ShouldNotReachHere();
2183 }
2184 __ align(BytesPerWord, offset);
2185 }
2186
2187
2188 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2189 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2190 "must be aligned");
2191 __ call(AddressLiteral(op->addr(), rtype));
2192 add_call_info(code_offset(), op->info());
2193 __ post_call_nop();
2194 }
2195
2196
2197 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2198 __ ic_call(op->addr());
2199 add_call_info(code_offset(), op->info());
2200 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2201 "must be aligned");
2202 __ post_call_nop();
2203 }
2204
2205
2206 void LIR_Assembler::emit_static_call_stub() {
2207 address call_pc = __ pc();
2208 address stub = __ start_a_stub(call_stub_size());
2209 if (stub == nullptr) {
2210 bailout("static call stub overflow");
2211 return;
2212 }
2213
2214 int start = __ offset();
2215
2216 // make sure that the displacement word of the call ends up word aligned
2217 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2218 __ relocate(static_stub_Relocation::spec(call_pc));
2219 __ mov_metadata(rbx, (Metadata*)nullptr);
2220 // must be set to -1 at code generation time
2221 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2222 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2223 __ jump(RuntimeAddress(__ pc()));
2224
2225 assert(__ offset() - start <= call_stub_size(), "stub too big");
2226 __ end_a_stub();
2227 }
2228
2229
2230 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2231 assert(exceptionOop->as_register() == rax, "must match");
2232 assert(exceptionPC->as_register() == rdx, "must match");
2233
2234 // exception object is not added to oop map by LinearScan
2235 // (LinearScan assumes that no oops are in fixed registers)
2236 info->add_register_oop(exceptionOop);
2237 StubId unwind_id;
2238
2239 // get current pc information
2240 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2241 int pc_for_athrow_offset = __ offset();
2242 InternalAddress pc_for_athrow(__ pc());
2243 __ lea(exceptionPC->as_register(), pc_for_athrow);
2244 add_call_info(pc_for_athrow_offset, info); // for exception handler
2245
2246 __ verify_not_null_oop(rax);
2247 // search an exception handler (rax: exception oop, rdx: throwing pc)
2248 if (compilation()->has_fpu_code()) {
2249 unwind_id = StubId::c1_handle_exception_id;
2250 } else {
2251 unwind_id = StubId::c1_handle_exception_nofpu_id;
2252 }
2253 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2254
2255 // enough room for two byte trap
2256 __ nop();
2257 }
2258
2259
2260 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2261 assert(exceptionOop->as_register() == rax, "must match");
2262
2263 __ jmp(_unwind_handler_entry);
2264 }
2265
2266
2267 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2268
2269 // optimized version for linear scan:
2270 // * count must be already in ECX (guaranteed by LinearScan)
2271 // * left and dest must be equal
2272 // * tmp must be unused
2273 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2274 assert(left == dest, "left and dest must be equal");
2275 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2276
2277 if (left->is_single_cpu()) {
2278 Register value = left->as_register();
2279 assert(value != SHIFT_count, "left cannot be ECX");
2280
2281 switch (code) {
2282 case lir_shl: __ shll(value); break;
2283 case lir_shr: __ sarl(value); break;
2284 case lir_ushr: __ shrl(value); break;
2285 default: ShouldNotReachHere();
2286 }
2287 } else if (left->is_double_cpu()) {
2288 Register lo = left->as_register_lo();
2289 Register hi = left->as_register_hi();
2290 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2291 switch (code) {
2292 case lir_shl: __ shlptr(lo); break;
2293 case lir_shr: __ sarptr(lo); break;
2294 case lir_ushr: __ shrptr(lo); break;
2295 default: ShouldNotReachHere();
2296 }
2297 } else {
2298 ShouldNotReachHere();
2299 }
2300 }
2301
2302
2303 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2304 if (dest->is_single_cpu()) {
2305 // first move left into dest so that left is not destroyed by the shift
2306 Register value = dest->as_register();
2307 count = count & 0x1F; // Java spec
2308
2309 move_regs(left->as_register(), value);
2310 switch (code) {
2311 case lir_shl: __ shll(value, count); break;
2312 case lir_shr: __ sarl(value, count); break;
2313 case lir_ushr: __ shrl(value, count); break;
2314 default: ShouldNotReachHere();
2315 }
2316 } else if (dest->is_double_cpu()) {
2317 // first move left into dest so that left is not destroyed by the shift
2318 Register value = dest->as_register_lo();
2319 count = count & 0x1F; // Java spec
2320
2321 move_regs(left->as_register_lo(), value);
2322 switch (code) {
2323 case lir_shl: __ shlptr(value, count); break;
2324 case lir_shr: __ sarptr(value, count); break;
2325 case lir_ushr: __ shrptr(value, count); break;
2326 default: ShouldNotReachHere();
2327 }
2328 } else {
2329 ShouldNotReachHere();
2330 }
2331 }
2332
2333
2334 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2335 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2336 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2337 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2338 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2339 }
2340
2341
2342 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2343 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2344 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2345 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2346 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2347 }
2348
2349
2350 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2351 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2352 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2353 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2354 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2355 }
2356
2357
2358 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2359 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2360 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2361 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2362 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2363 }
2364
2365
2366 // This code replaces a call to arraycopy; no exception may
2367 // be thrown in this code, they must be thrown in the System.arraycopy
2368 // activation frame; we could save some checks if this would not be the case
2369 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2370 ciArrayKlass* default_type = op->expected_type();
2371 Register src = op->src()->as_register();
2372 Register dst = op->dst()->as_register();
2373 Register src_pos = op->src_pos()->as_register();
2374 Register dst_pos = op->dst_pos()->as_register();
2375 Register length = op->length()->as_register();
2376 Register tmp = op->tmp()->as_register();
2377 Register tmp_load_klass = rscratch1;
2378 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2379
2380 CodeStub* stub = op->stub();
2381 int flags = op->flags();
2382 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2383 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2384
2385 // if we don't know anything, just go through the generic arraycopy
2386 if (default_type == nullptr) {
2387 // save outgoing arguments on stack in case call to System.arraycopy is needed
2388 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2389 // for interpreter calling conventions. Now we have to do it in new style conventions.
2390 // For the moment until C1 gets the new register allocator I just force all the
2391 // args to the right place (except the register args) and then on the back side
2392 // reload the register args properly if we go slow path. Yuck
2393
2394 // These are proper for the calling convention
2395 store_parameter(length, 2);
2396 store_parameter(dst_pos, 1);
2397 store_parameter(dst, 0);
2398
2399 // these are just temporary placements until we need to reload
2400 store_parameter(src_pos, 3);
2401 store_parameter(src, 4);
2402
2403 address copyfunc_addr = StubRoutines::generic_arraycopy();
2404 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2405
2406 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2407 // The arguments are in java calling convention so we can trivially shift them to C
2408 // convention
2409 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2410 __ mov(c_rarg0, j_rarg0);
2411 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2412 __ mov(c_rarg1, j_rarg1);
2413 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2414 __ mov(c_rarg2, j_rarg2);
2415 assert_different_registers(c_rarg3, j_rarg4);
2416 __ mov(c_rarg3, j_rarg3);
2417 #ifdef _WIN64
2418 // Allocate abi space for args but be sure to keep stack aligned
2419 __ subptr(rsp, 6*wordSize);
2420 store_parameter(j_rarg4, 4);
2421 #ifndef PRODUCT
2422 if (PrintC1Statistics) {
2423 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2424 }
2425 #endif
2426 __ call(RuntimeAddress(copyfunc_addr));
2427 __ addptr(rsp, 6*wordSize);
2428 #else
2429 __ mov(c_rarg4, j_rarg4);
2430 #ifndef PRODUCT
2431 if (PrintC1Statistics) {
2432 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2433 }
2434 #endif
2435 __ call(RuntimeAddress(copyfunc_addr));
2436 #endif // _WIN64
2437
2438 __ testl(rax, rax);
2439 __ jcc(Assembler::equal, *stub->continuation());
2440
2441 __ mov(tmp, rax);
2442 __ xorl(tmp, -1);
2443
2444 // Reload values from the stack so they are where the stub
2445 // expects them.
2446 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2447 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2448 __ movptr (length, Address(rsp, 2*BytesPerWord));
2449 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2450 __ movptr (src, Address(rsp, 4*BytesPerWord));
2451
2452 __ subl(length, tmp);
2453 __ addl(src_pos, tmp);
2454 __ addl(dst_pos, tmp);
2455 __ jmp(*stub->entry());
2456
2457 __ bind(*stub->continuation());
2458 return;
2459 }
2460
2461 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2462
2463 int elem_size = type2aelembytes(basic_type);
2464 Address::ScaleFactor scale;
2465
2466 switch (elem_size) {
2467 case 1 :
2468 scale = Address::times_1;
2469 break;
2470 case 2 :
2471 scale = Address::times_2;
2472 break;
2473 case 4 :
2474 scale = Address::times_4;
2475 break;
2476 case 8 :
2477 scale = Address::times_8;
2478 break;
2479 default:
2480 scale = Address::no_scale;
2481 ShouldNotReachHere();
2482 }
2483
2484 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2485 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2486
2487 // length and pos's are all sign extended at this point on 64bit
2488
2489 // test for null
2490 if (flags & LIR_OpArrayCopy::src_null_check) {
2491 __ testptr(src, src);
2492 __ jcc(Assembler::zero, *stub->entry());
2493 }
2494 if (flags & LIR_OpArrayCopy::dst_null_check) {
2495 __ testptr(dst, dst);
2496 __ jcc(Assembler::zero, *stub->entry());
2497 }
2498
2499 // If the compiler was not able to prove that exact type of the source or the destination
2500 // of the arraycopy is an array type, check at runtime if the source or the destination is
2501 // an instance type.
2502 if (flags & LIR_OpArrayCopy::type_check) {
2503 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2504 __ load_klass(tmp, dst, tmp_load_klass);
2505 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2506 __ jcc(Assembler::greaterEqual, *stub->entry());
2507 }
2508
2509 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2510 __ load_klass(tmp, src, tmp_load_klass);
2511 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2512 __ jcc(Assembler::greaterEqual, *stub->entry());
2513 }
2514 }
2515
2516 // check if negative
2517 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2518 __ testl(src_pos, src_pos);
2519 __ jcc(Assembler::less, *stub->entry());
2520 }
2521 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2522 __ testl(dst_pos, dst_pos);
2523 __ jcc(Assembler::less, *stub->entry());
2524 }
2525
2526 if (flags & LIR_OpArrayCopy::src_range_check) {
2527 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2528 __ cmpl(tmp, src_length_addr);
2529 __ jcc(Assembler::above, *stub->entry());
2530 }
2531 if (flags & LIR_OpArrayCopy::dst_range_check) {
2532 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2533 __ cmpl(tmp, dst_length_addr);
2534 __ jcc(Assembler::above, *stub->entry());
2535 }
2536
2537 if (flags & LIR_OpArrayCopy::length_positive_check) {
2538 __ testl(length, length);
2539 __ jcc(Assembler::less, *stub->entry());
2540 }
2541
2542 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2543 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2544
2545 if (flags & LIR_OpArrayCopy::type_check) {
2546 // We don't know the array types are compatible
2547 if (basic_type != T_OBJECT) {
2548 // Simple test for basic type arrays
2549 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2550 __ jcc(Assembler::notEqual, *stub->entry());
2551 } else {
2552 // For object arrays, if src is a sub class of dst then we can
2553 // safely do the copy.
2554 Label cont, slow;
2555
2556 __ push_ppx(src);
2557 __ push_ppx(dst);
2558
2559 __ load_klass(src, src, tmp_load_klass);
2560 __ load_klass(dst, dst, tmp_load_klass);
2561
2562 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2563
2564 __ push_ppx(src);
2565 __ push_ppx(dst);
2566 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2567 __ pop_ppx(dst);
2568 __ pop_ppx(src);
2569
2570 __ testl(src, src);
2571 __ jcc(Assembler::notEqual, cont);
2572
2573 __ bind(slow);
2574 __ pop_ppx(dst);
2575 __ pop_ppx(src);
2576
2577 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2578 if (copyfunc_addr != nullptr) { // use stub if available
2579 // src is not a sub class of dst so we have to do a
2580 // per-element check.
2581
2582 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2583 if ((flags & mask) != mask) {
2584 // Check that at least both of them object arrays.
2585 assert(flags & mask, "one of the two should be known to be an object array");
2586
2587 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2588 __ load_klass(tmp, src, tmp_load_klass);
2589 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2590 __ load_klass(tmp, dst, tmp_load_klass);
2591 }
2592 int lh_offset = in_bytes(Klass::layout_helper_offset());
2593 Address klass_lh_addr(tmp, lh_offset);
2594 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2595 __ cmpl(klass_lh_addr, objArray_lh);
2596 __ jcc(Assembler::notEqual, *stub->entry());
2597 }
2598
2599 // Spill because stubs can use any register they like and it's
2600 // easier to restore just those that we care about.
2601 store_parameter(dst, 0);
2602 store_parameter(dst_pos, 1);
2603 store_parameter(length, 2);
2604 store_parameter(src_pos, 3);
2605 store_parameter(src, 4);
2606
2607 __ movl2ptr(length, length); //higher 32bits must be null
2608
2609 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2610 assert_different_registers(c_rarg0, dst, dst_pos, length);
2611 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2612 assert_different_registers(c_rarg1, dst, length);
2613
2614 __ mov(c_rarg2, length);
2615 assert_different_registers(c_rarg2, dst);
2616
2617 #ifdef _WIN64
2618 // Allocate abi space for args but be sure to keep stack aligned
2619 __ subptr(rsp, 6*wordSize);
2620 __ load_klass(c_rarg3, dst, tmp_load_klass);
2621 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2622 store_parameter(c_rarg3, 4);
2623 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2624 __ call(RuntimeAddress(copyfunc_addr));
2625 __ addptr(rsp, 6*wordSize);
2626 #else
2627 __ load_klass(c_rarg4, dst, tmp_load_klass);
2628 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2629 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2630 __ call(RuntimeAddress(copyfunc_addr));
2631 #endif
2632
2633 #ifndef PRODUCT
2634 if (PrintC1Statistics) {
2635 Label failed;
2636 __ testl(rax, rax);
2637 __ jcc(Assembler::notZero, failed);
2638 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2639 __ bind(failed);
2640 }
2641 #endif
2642
2643 __ testl(rax, rax);
2644 __ jcc(Assembler::zero, *stub->continuation());
2645
2646 #ifndef PRODUCT
2647 if (PrintC1Statistics) {
2648 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2649 }
2650 #endif
2651
2652 __ mov(tmp, rax);
2653
2654 __ xorl(tmp, -1);
2655
2656 // Restore previously spilled arguments
2657 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2658 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2659 __ movptr (length, Address(rsp, 2*BytesPerWord));
2660 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2661 __ movptr (src, Address(rsp, 4*BytesPerWord));
2662
2663
2664 __ subl(length, tmp);
2665 __ addl(src_pos, tmp);
2666 __ addl(dst_pos, tmp);
2667 }
2668
2669 __ jmp(*stub->entry());
2670
2671 __ bind(cont);
2672 __ pop(dst);
2673 __ pop(src);
2674 }
2675 }
2676
2677 #ifdef ASSERT
2678 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2679 // Sanity check the known type with the incoming class. For the
2680 // primitive case the types must match exactly with src.klass and
2681 // dst.klass each exactly matching the default type. For the
2682 // object array case, if no type check is needed then either the
2683 // dst type is exactly the expected type and the src type is a
2684 // subtype which we can't check or src is the same array as dst
2685 // but not necessarily exactly of type default_type.
2686 Label known_ok, halt;
2687 __ mov_metadata(tmp, default_type->constant_encoding());
2688 if (UseCompressedClassPointers) {
2689 __ encode_klass_not_null(tmp, rscratch1);
2690 }
2691
2692 if (basic_type != T_OBJECT) {
2693 __ cmp_klass(tmp, dst, tmp2);
2694 __ jcc(Assembler::notEqual, halt);
2695 __ cmp_klass(tmp, src, tmp2);
2696 __ jcc(Assembler::equal, known_ok);
2697 } else {
2698 __ cmp_klass(tmp, dst, tmp2);
2699 __ jcc(Assembler::equal, known_ok);
2700 __ cmpptr(src, dst);
2701 __ jcc(Assembler::equal, known_ok);
2702 }
2703 __ bind(halt);
2704 __ stop("incorrect type information in arraycopy");
2705 __ bind(known_ok);
2706 }
2707 #endif
2708
2709 #ifndef PRODUCT
2710 if (PrintC1Statistics) {
2711 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2712 }
2713 #endif
2714
2715 assert_different_registers(c_rarg0, dst, dst_pos, length);
2716 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2717 assert_different_registers(c_rarg1, length);
2718 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2719 __ mov(c_rarg2, length);
2720
2721 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2722 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2723 const char *name;
2724 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2725 __ call_VM_leaf(entry, 0);
2726
2727 if (stub != nullptr) {
2728 __ bind(*stub->continuation());
2729 }
2730 }
2731
2732 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2733 assert(op->crc()->is_single_cpu(), "crc must be register");
2734 assert(op->val()->is_single_cpu(), "byte value must be register");
2735 assert(op->result_opr()->is_single_cpu(), "result must be register");
2736 Register crc = op->crc()->as_register();
2737 Register val = op->val()->as_register();
2738 Register res = op->result_opr()->as_register();
2739
2740 assert_different_registers(val, crc, res);
2741
2742 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2743 __ notl(crc); // ~crc
2744 __ update_byte_crc32(crc, val, res);
2745 __ notl(crc); // ~crc
2746 __ mov(res, crc);
2747 }
2748
2749 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2750 Register obj = op->obj_opr()->as_register(); // may not be an oop
2751 Register hdr = op->hdr_opr()->as_register();
2752 Register lock = op->lock_opr()->as_register();
2753 if (op->code() == lir_lock) {
2754 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2755 Register tmp = op->scratch_opr()->as_register();
2756 // add debug info for NullPointerException only if one is possible
2757 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2758 if (op->info() != nullptr) {
2759 add_debug_info_for_null_check(null_check_offset, op->info());
2760 }
2761 // done
2762 } else if (op->code() == lir_unlock) {
2763 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2764 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2765 } else {
2766 Unimplemented();
2767 }
2768 __ bind(*op->stub()->continuation());
2769 }
2770
2771 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2772 Register obj = op->obj()->as_pointer_register();
2773 Register result = op->result_opr()->as_pointer_register();
2774
2775 CodeEmitInfo* info = op->info();
2776 if (info != nullptr) {
2777 add_debug_info_for_null_check_here(info);
2778 }
2779
2780 __ load_klass(result, obj, rscratch1);
2781 }
2782
2783 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2784 ciMethod* method = op->profiled_method();
2785 int bci = op->profiled_bci();
2786 ciMethod* callee = op->profiled_callee();
2787 Register tmp_load_klass = rscratch1;
2788
2789 // Update counter for all call types
2790 ciMethodData* md = method->method_data_or_null();
2791 assert(md != nullptr, "Sanity");
2792 ciProfileData* data = md->bci_to_data(bci);
2793 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2794 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2795 Register mdo = op->mdo()->as_register();
2796 __ mov_metadata(mdo, md->constant_encoding());
2797 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2798 // Perform additional virtual call profiling for invokevirtual and
2799 // invokeinterface bytecodes
2800 if (op->should_profile_receiver_type()) {
2801 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2802 Register recv = op->recv()->as_register();
2803 assert_different_registers(mdo, recv);
2804 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2805 ciKlass* known_klass = op->known_holder();
2806 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2807 // We know the type that will be seen at this call site; we can
2808 // statically update the MethodData* rather than needing to do
2809 // dynamic tests on the receiver type
2810
2811 // NOTE: we should probably put a lock around this search to
2812 // avoid collisions by concurrent compilations
2813 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2814 uint i;
2815 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2816 ciKlass* receiver = vc_data->receiver(i);
2817 if (known_klass->equals(receiver)) {
2818 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2819 __ addptr(data_addr, DataLayout::counter_increment);
2820 return;
2821 }
2822 }
2823
2824 // Receiver type not found in profile data; select an empty slot
2825
2826 // Note that this is less efficient than it should be because it
2827 // always does a write to the receiver part of the
2828 // VirtualCallData rather than just the first time
2829 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2830 ciKlass* receiver = vc_data->receiver(i);
2831 if (receiver == nullptr) {
2832 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2833 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
2834 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2835 __ addptr(data_addr, DataLayout::counter_increment);
2836 return;
2837 }
2838 }
2839 } else {
2840 __ load_klass(recv, recv, tmp_load_klass);
2841 Label update_done;
2842 type_profile_helper(mdo, md, data, recv, &update_done);
2843 // Receiver did not match any saved receiver and there is no empty row for it.
2844 // Increment total counter to indicate polymorphic case.
2845 __ addptr(counter_addr, DataLayout::counter_increment);
2846
2847 __ bind(update_done);
2848 }
2849 } else {
2850 // Static call
2851 __ addptr(counter_addr, DataLayout::counter_increment);
2852 }
2853 }
2854
2855 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2856 Register obj = op->obj()->as_register();
2857 Register tmp = op->tmp()->as_pointer_register();
2858 Register tmp_load_klass = rscratch1;
2859 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2860 ciKlass* exact_klass = op->exact_klass();
2861 intptr_t current_klass = op->current_klass();
2862 bool not_null = op->not_null();
2863 bool no_conflict = op->no_conflict();
2864
2865 Label update, next, none;
2866
2867 bool do_null = !not_null;
2868 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2869 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2870
2871 assert(do_null || do_update, "why are we here?");
2872 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2873
2874 __ verify_oop(obj);
2875
2876 #ifdef ASSERT
2877 if (obj == tmp) {
2878 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2879 } else {
2880 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2881 }
2882 #endif
2883 if (do_null) {
2884 __ testptr(obj, obj);
2885 __ jccb(Assembler::notZero, update);
2886 if (!TypeEntries::was_null_seen(current_klass)) {
2887 __ testptr(mdo_addr, TypeEntries::null_seen);
2888 #ifndef ASSERT
2889 __ jccb(Assembler::notZero, next); // already set
2890 #else
2891 __ jcc(Assembler::notZero, next); // already set
2892 #endif
2893 // atomic update to prevent overwriting Klass* with 0
2894 __ lock();
2895 __ orptr(mdo_addr, TypeEntries::null_seen);
2896 }
2897 if (do_update) {
2898 #ifndef ASSERT
2899 __ jmpb(next);
2900 }
2901 #else
2902 __ jmp(next);
2903 }
2904 } else {
2905 __ testptr(obj, obj);
2906 __ jcc(Assembler::notZero, update);
2907 __ stop("unexpected null obj");
2908 #endif
2909 }
2910
2911 __ bind(update);
2912
2913 if (do_update) {
2914 #ifdef ASSERT
2915 if (exact_klass != nullptr) {
2916 Label ok;
2917 __ load_klass(tmp, obj, tmp_load_klass);
2918 __ push_ppx(tmp);
2919 __ mov_metadata(tmp, exact_klass->constant_encoding());
2920 __ cmpptr(tmp, Address(rsp, 0));
2921 __ jcc(Assembler::equal, ok);
2922 __ stop("exact klass and actual klass differ");
2923 __ bind(ok);
2924 __ pop_ppx(tmp);
2925 }
2926 #endif
2927 if (!no_conflict) {
2928 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2929 if (exact_klass != nullptr) {
2930 __ mov_metadata(tmp, exact_klass->constant_encoding());
2931 } else {
2932 __ load_klass(tmp, obj, tmp_load_klass);
2933 }
2934 __ mov(rscratch1, tmp); // save original value before XOR
2935 __ xorptr(tmp, mdo_addr);
2936 __ testptr(tmp, TypeEntries::type_klass_mask);
2937 // klass seen before, nothing to do. The unknown bit may have been
2938 // set already but no need to check.
2939 __ jccb(Assembler::zero, next);
2940
2941 __ testptr(tmp, TypeEntries::type_unknown);
2942 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2943
2944 if (TypeEntries::is_type_none(current_klass)) {
2945 __ testptr(mdo_addr, TypeEntries::type_mask);
2946 __ jccb(Assembler::zero, none);
2947 // There is a chance that the checks above (re-reading profiling
2948 // data from memory) fail if another thread has just set the
2949 // profiling to this obj's klass
2950 __ mov(tmp, rscratch1); // get back original value before XOR
2951 __ xorptr(tmp, mdo_addr);
2952 __ testptr(tmp, TypeEntries::type_klass_mask);
2953 __ jccb(Assembler::zero, next);
2954 }
2955 } else {
2956 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2957 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2958
2959 __ testptr(mdo_addr, TypeEntries::type_unknown);
2960 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2961 }
2962
2963 // different than before. Cannot keep accurate profile.
2964 __ orptr(mdo_addr, TypeEntries::type_unknown);
2965
2966 if (TypeEntries::is_type_none(current_klass)) {
2967 __ jmpb(next);
2968
2969 __ bind(none);
2970 // first time here. Set profile type.
2971 __ movptr(mdo_addr, tmp);
2972 #ifdef ASSERT
2973 __ andptr(tmp, TypeEntries::type_klass_mask);
2974 __ verify_klass_ptr(tmp);
2975 #endif
2976 }
2977 } else {
2978 // There's a single possible klass at this profile point
2979 assert(exact_klass != nullptr, "should be");
2980 if (TypeEntries::is_type_none(current_klass)) {
2981 __ mov_metadata(tmp, exact_klass->constant_encoding());
2982 __ xorptr(tmp, mdo_addr);
2983 __ testptr(tmp, TypeEntries::type_klass_mask);
2984 #ifdef ASSERT
2985 __ jcc(Assembler::zero, next);
2986
2987 {
2988 Label ok;
2989 __ push_ppx(tmp);
2990 __ testptr(mdo_addr, TypeEntries::type_mask);
2991 __ jcc(Assembler::zero, ok);
2992 // may have been set by another thread
2993 __ mov_metadata(tmp, exact_klass->constant_encoding());
2994 __ xorptr(tmp, mdo_addr);
2995 __ testptr(tmp, TypeEntries::type_mask);
2996 __ jcc(Assembler::zero, ok);
2997
2998 __ stop("unexpected profiling mismatch");
2999 __ bind(ok);
3000 __ pop_ppx(tmp);
3001 }
3002 #else
3003 __ jccb(Assembler::zero, next);
3004 #endif
3005 // first time here. Set profile type.
3006 __ movptr(mdo_addr, tmp);
3007 #ifdef ASSERT
3008 __ andptr(tmp, TypeEntries::type_klass_mask);
3009 __ verify_klass_ptr(tmp);
3010 #endif
3011 } else {
3012 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3013 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3014
3015 __ testptr(mdo_addr, TypeEntries::type_unknown);
3016 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3017
3018 __ orptr(mdo_addr, TypeEntries::type_unknown);
3019 }
3020 }
3021 }
3022 __ bind(next);
3023 }
3024
3025 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3026 Unimplemented();
3027 }
3028
3029
3030 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3031 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3032 }
3033
3034
3035 void LIR_Assembler::align_backward_branch_target() {
3036 __ align(BytesPerWord);
3037 }
3038
3039
3040 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3041 if (left->is_single_cpu()) {
3042 __ negl(left->as_register());
3043 move_regs(left->as_register(), dest->as_register());
3044
3045 } else if (left->is_double_cpu()) {
3046 Register lo = left->as_register_lo();
3047 Register dst = dest->as_register_lo();
3048 __ movptr(dst, lo);
3049 __ negptr(dst);
3050
3051 } else if (dest->is_single_xmm()) {
3052 assert(!tmp->is_valid(), "do not need temporary");
3053 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3054 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3055 }
3056 __ xorps(dest->as_xmm_float_reg(),
3057 ExternalAddress((address)float_signflip_pool),
3058 rscratch1);
3059 } else if (dest->is_double_xmm()) {
3060 assert(!tmp->is_valid(), "do not need temporary");
3061 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3062 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3063 }
3064 __ xorpd(dest->as_xmm_double_reg(),
3065 ExternalAddress((address)double_signflip_pool),
3066 rscratch1);
3067 } else {
3068 ShouldNotReachHere();
3069 }
3070 }
3071
3072
3073 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3074 assert(src->is_address(), "must be an address");
3075 assert(dest->is_register(), "must be a register");
3076
3077 PatchingStub* patch = nullptr;
3078 if (patch_code != lir_patch_none) {
3079 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3080 }
3081
3082 Register reg = dest->as_pointer_register();
3083 LIR_Address* addr = src->as_address_ptr();
3084 __ lea(reg, as_Address(addr));
3085
3086 if (patch != nullptr) {
3087 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3088 }
3089 }
3090
3091
3092
3093 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3094 assert(!tmp->is_valid(), "don't need temporary");
3095 __ call(RuntimeAddress(dest));
3096 if (info != nullptr) {
3097 add_call_info_here(info);
3098 }
3099 __ post_call_nop();
3100 }
3101
3102
3103 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3104 assert(type == T_LONG, "only for volatile long fields");
3105
3106 if (info != nullptr) {
3107 add_debug_info_for_null_check_here(info);
3108 }
3109
3110 if (src->is_double_xmm()) {
3111 if (dest->is_double_cpu()) {
3112 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3113 } else if (dest->is_double_stack()) {
3114 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3115 } else if (dest->is_address()) {
3116 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3117 } else {
3118 ShouldNotReachHere();
3119 }
3120
3121 } else if (dest->is_double_xmm()) {
3122 if (src->is_double_stack()) {
3123 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3124 } else if (src->is_address()) {
3125 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3126 } else {
3127 ShouldNotReachHere();
3128 }
3129
3130 } else {
3131 ShouldNotReachHere();
3132 }
3133 }
3134
3135 #ifdef ASSERT
3136 // emit run-time assertion
3137 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3138 assert(op->code() == lir_assert, "must be");
3139
3140 if (op->in_opr1()->is_valid()) {
3141 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3142 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3143 } else {
3144 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3145 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3146 }
3147
3148 Label ok;
3149 if (op->condition() != lir_cond_always) {
3150 Assembler::Condition acond = Assembler::zero;
3151 switch (op->condition()) {
3152 case lir_cond_equal: acond = Assembler::equal; break;
3153 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3154 case lir_cond_less: acond = Assembler::less; break;
3155 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3156 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3157 case lir_cond_greater: acond = Assembler::greater; break;
3158 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3159 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3160 default: ShouldNotReachHere();
3161 }
3162 __ jcc(acond, ok);
3163 }
3164 if (op->halt()) {
3165 const char* str = __ code_string(op->msg());
3166 __ stop(str);
3167 } else {
3168 breakpoint();
3169 }
3170 __ bind(ok);
3171 }
3172 #endif
3173
3174 void LIR_Assembler::membar() {
3175 // QQQ sparc TSO uses this,
3176 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3177 }
3178
3179 void LIR_Assembler::membar_acquire() {
3180 // No x86 machines currently require load fences
3181 }
3182
3183 void LIR_Assembler::membar_release() {
3184 // No x86 machines currently require store fences
3185 }
3186
3187 void LIR_Assembler::membar_loadload() {
3188 // no-op
3189 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3190 }
3191
3192 void LIR_Assembler::membar_storestore() {
3193 // no-op
3194 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3195 }
3196
3197 void LIR_Assembler::membar_loadstore() {
3198 // no-op
3199 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3200 }
3201
3202 void LIR_Assembler::membar_storeload() {
3203 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3204 }
3205
3206 void LIR_Assembler::on_spin_wait() {
3207 __ pause ();
3208 }
3209
3210 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3211 assert(result_reg->is_register(), "check");
3212 __ mov(result_reg->as_register(), r15_thread);
3213 }
3214
3215
3216 void LIR_Assembler::peephole(LIR_List*) {
3217 // do nothing for now
3218 }
3219
3220 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3221 assert(data == dest, "xchg/xadd uses only 2 operands");
3222
3223 if (data->type() == T_INT) {
3224 if (code == lir_xadd) {
3225 __ lock();
3226 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3227 } else {
3228 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3229 }
3230 } else if (data->is_oop()) {
3231 assert (code == lir_xchg, "xadd for oops");
3232 Register obj = data->as_register();
3233 if (UseCompressedOops) {
3234 __ encode_heap_oop(obj);
3235 __ xchgl(obj, as_Address(src->as_address_ptr()));
3236 __ decode_heap_oop(obj);
3237 } else {
3238 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3239 }
3240 } else if (data->type() == T_LONG) {
3241 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3242 if (code == lir_xadd) {
3243 __ lock();
3244 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3245 } else {
3246 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3247 }
3248 } else {
3249 ShouldNotReachHere();
3250 }
3251 }
3252
3253 #undef __