1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
59 operand[1] = hi;
60 return operand;
61 }
62
63 // Buffer for 128-bits masks used by SSE instructions.
64 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
65
66 // Static initialization during VM startup.
67 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
68 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
69 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
70 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
71
72
73 NEEDS_CLEANUP // remove this definitions ?
74 const Register SYNC_header = rax; // synchronization header
75 const Register SHIFT_count = rcx; // where count for shift operations must be
76
77 #define __ _masm->
78
79
80 static void select_different_registers(Register preserve,
81 Register extra,
82 Register &tmp1,
83 Register &tmp2) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp2 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2);
92 }
93
94
95
96 static void select_different_registers(Register preserve,
97 Register extra,
98 Register &tmp1,
99 Register &tmp2,
100 Register &tmp3) {
101 if (tmp1 == preserve) {
102 assert_different_registers(tmp1, tmp2, tmp3, extra);
103 tmp1 = extra;
104 } else if (tmp2 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp2 = extra;
107 } else if (tmp3 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp3 = extra;
110 }
111 assert_different_registers(preserve, tmp1, tmp2, tmp3);
112 }
113
114
115
116 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
117 if (opr->is_constant()) {
118 LIR_Const* constant = opr->as_constant_ptr();
119 switch (constant->type()) {
120 case T_INT: {
121 return true;
122 }
123
124 default:
125 return false;
126 }
127 }
128 return false;
129 }
130
131
132 LIR_Opr LIR_Assembler::receiverOpr() {
133 return FrameMap::receiver_opr;
134 }
135
136 LIR_Opr LIR_Assembler::osrBufferPointer() {
137 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
138 }
139
140 //--------------fpu register translations-----------------------
141
142
143 address LIR_Assembler::float_constant(float f) {
144 address const_addr = __ float_constant(f);
145 if (const_addr == nullptr) {
146 bailout("const section overflow");
147 return __ code()->consts()->start();
148 } else {
149 return const_addr;
150 }
151 }
152
153
154 address LIR_Assembler::double_constant(double d) {
155 address const_addr = __ double_constant(d);
156 if (const_addr == nullptr) {
157 bailout("const section overflow");
158 return __ code()->consts()->start();
159 } else {
160 return const_addr;
161 }
162 }
163
164 void LIR_Assembler::breakpoint() {
165 __ int3();
166 }
167
168 void LIR_Assembler::push(LIR_Opr opr) {
169 if (opr->is_single_cpu()) {
170 __ push_reg(opr->as_register());
171 } else if (opr->is_double_cpu()) {
172 __ push_reg(opr->as_register_lo());
173 } else if (opr->is_stack()) {
174 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
175 } else if (opr->is_constant()) {
176 LIR_Const* const_opr = opr->as_constant_ptr();
177 if (const_opr->type() == T_OBJECT) {
178 __ push_oop(const_opr->as_jobject(), rscratch1);
179 } else if (const_opr->type() == T_INT) {
180 __ push_jint(const_opr->as_jint());
181 } else {
182 ShouldNotReachHere();
183 }
184
185 } else {
186 ShouldNotReachHere();
187 }
188 }
189
190 void LIR_Assembler::pop(LIR_Opr opr) {
191 if (opr->is_single_cpu()) {
192 __ pop_reg(opr->as_register());
193 } else {
194 ShouldNotReachHere();
195 }
196 }
197
198 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
199 return addr->base()->is_illegal() && addr->index()->is_illegal();
200 }
201
202 //-------------------------------------------
203
204 Address LIR_Assembler::as_Address(LIR_Address* addr) {
205 return as_Address(addr, rscratch1);
206 }
207
208 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
209 if (addr->base()->is_illegal()) {
210 assert(addr->index()->is_illegal(), "must be illegal too");
211 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
212 if (! __ reachable(laddr)) {
213 __ movptr(tmp, laddr.addr());
214 Address res(tmp, 0);
215 return res;
216 } else {
217 return __ as_Address(laddr);
218 }
219 }
220
221 Register base = addr->base()->as_pointer_register();
222
223 if (addr->index()->is_illegal()) {
224 return Address( base, addr->disp());
225 } else if (addr->index()->is_cpu_register()) {
226 Register index = addr->index()->as_pointer_register();
227 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
228 } else if (addr->index()->is_constant()) {
229 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
230 assert(Assembler::is_simm32(addr_offset), "must be");
231
232 return Address(base, addr_offset);
233 } else {
234 Unimplemented();
235 return Address();
236 }
237 }
238
239
240 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
241 Address base = as_Address(addr);
242 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
243 }
244
245
246 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
247 return as_Address(addr);
248 }
249
250
251 void LIR_Assembler::osr_entry() {
252 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
253 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
254 ValueStack* entry_state = osr_entry->state();
255 int number_of_locks = entry_state->locks_size();
256
257 // we jump here if osr happens with the interpreter
258 // state set up to continue at the beginning of the
259 // loop that triggered osr - in particular, we have
260 // the following registers setup:
261 //
262 // rcx: osr buffer
263 //
264
265 // build frame
266 ciMethod* m = compilation()->method();
267 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
268
269 // OSR buffer is
270 //
271 // locals[nlocals-1..0]
272 // monitors[0..number_of_locks]
273 //
274 // locals is a direct copy of the interpreter frame so in the osr buffer
275 // so first slot in the local array is the last local from the interpreter
276 // and last slot is local[0] (receiver) from the interpreter
277 //
278 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
279 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
280 // in the interpreter frame (the method lock if a sync method)
281
282 // Initialize monitors in the compiled activation.
283 // rcx: pointer to osr buffer
284 //
285 // All other registers are dead at this point and the locals will be
286 // copied into place by code emitted in the IR.
287
288 Register OSR_buf = osrBufferPointer()->as_pointer_register();
289 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
290 int monitor_offset = BytesPerWord * method()->max_locals() +
291 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
292 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
293 // the OSR buffer using 2 word entries: first the lock and then
294 // the oop.
295 for (int i = 0; i < number_of_locks; i++) {
296 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
297 #ifdef ASSERT
298 // verify the interpreter's monitor has a non-null object
299 {
300 Label L;
301 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
302 __ jcc(Assembler::notZero, L);
303 __ stop("locked object is null");
304 __ bind(L);
305 }
306 #endif
307 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
308 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
309 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
310 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
311 }
312 }
313 }
314
315
316 // inline cache check; done before the frame is built.
317 int LIR_Assembler::check_icache() {
318 return __ ic_check(CodeEntryAlignment);
319 }
320
321 void LIR_Assembler::clinit_barrier(ciMethod* method) {
322 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
323 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
324
325 Label L_skip_barrier;
326 Register klass = rscratch1;
327
328 __ mov_metadata(klass, method->holder()->constant_encoding());
329 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
330
331 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
332
333 __ bind(L_skip_barrier);
334 }
335
336 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
337 jobject o = nullptr;
338 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
339 __ movoop(reg, o);
340 patching_epilog(patch, lir_patch_normal, reg, info);
341 }
342
343 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
344 Metadata* o = nullptr;
345 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
346 __ mov_metadata(reg, o);
347 patching_epilog(patch, lir_patch_normal, reg, info);
348 }
349
350 // This specifies the rsp decrement needed to build the frame
351 int LIR_Assembler::initial_frame_size_in_bytes() const {
352 // if rounding, must let FrameMap know!
353
354 // The frame_map records size in slots (32bit word)
355
356 // subtract two words to account for return address and link
357 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
358 }
359
360
361 int LIR_Assembler::emit_exception_handler() {
362 // generate code for exception handler
363 address handler_base = __ start_a_stub(exception_handler_size());
364 if (handler_base == nullptr) {
365 // not enough space left for the handler
366 bailout("exception handler overflow");
367 return -1;
368 }
369
370 int offset = code_offset();
371
372 // the exception oop and pc are in rax, and rdx
373 // no other registers need to be preserved, so invalidate them
374 __ invalidate_registers(false, true, true, false, true, true);
375
376 // check that there is really an exception
377 __ verify_not_null_oop(rax);
378
379 // search an exception handler (rax: exception oop, rdx: throwing pc)
380 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
381 __ should_not_reach_here();
382 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
383 __ end_a_stub();
384
385 return offset;
386 }
387
388
389 // Emit the code to remove the frame from the stack in the exception
390 // unwind path.
391 int LIR_Assembler::emit_unwind_handler() {
392 #ifndef PRODUCT
393 if (CommentedAssembly) {
394 _masm->block_comment("Unwind handler");
395 }
396 #endif
397
398 int offset = code_offset();
399
400 // Fetch the exception from TLS and clear out exception related thread state
401 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
402 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
403 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
404
405 __ bind(_unwind_handler_entry);
406 __ verify_not_null_oop(rax);
407 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
408 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
409 }
410
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
452 int offset = code_offset();
453 InternalAddress here(__ pc());
454
455 __ pushptr(here.addr(), rscratch1);
456 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
458 __ end_a_stub();
459
460 return offset;
461 }
462
463 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
464 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
465 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
466 assert(result->fpu() == 0, "result must already be on TOS");
467 }
468
469 // Pop the stack before the safepoint code
470 __ remove_frame(initial_frame_size_in_bytes());
471
472 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
473 __ reserved_stack_check();
474 }
475
476 // Note: we do not need to round double result; float result has the right precision
477 // the poll sets the condition code, but no data registers
478
479 code_stub->set_safepoint_offset(__ offset());
480 __ relocate(relocInfo::poll_return_type);
481 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
482 __ ret(0);
483 }
484
485
486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
487 guarantee(info != nullptr, "Shouldn't be null");
488 int offset = __ offset();
489 const Register poll_addr = rscratch1;
490 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
491 add_debug_info_for_branch(info);
492 __ relocate(relocInfo::poll_type);
493 address pre_pc = __ pc();
494 __ testl(rax, Address(poll_addr, 0));
495 address post_pc = __ pc();
496 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
497 return offset;
498 }
499
500
501 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
502 if (from_reg != to_reg) __ mov(to_reg, from_reg);
503 }
504
505 void LIR_Assembler::swap_reg(Register a, Register b) {
506 __ xchgptr(a, b);
507 }
508
509
510 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
511 assert(src->is_constant(), "should not call otherwise");
512 assert(dest->is_register(), "should not call otherwise");
513 LIR_Const* c = src->as_constant_ptr();
514
515 switch (c->type()) {
516 case T_INT: {
517 assert(patch_code == lir_patch_none, "no patching handled here");
518 __ movl(dest->as_register(), c->as_jint());
519 break;
520 }
521
522 case T_ADDRESS: {
523 assert(patch_code == lir_patch_none, "no patching handled here");
524 __ movptr(dest->as_register(), c->as_jint());
525 break;
526 }
527
528 case T_LONG: {
529 assert(patch_code == lir_patch_none, "no patching handled here");
530 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
531 break;
532 }
533
534 case T_OBJECT: {
535 if (patch_code != lir_patch_none) {
536 jobject2reg_with_patching(dest->as_register(), info);
537 } else {
538 __ movoop(dest->as_register(), c->as_jobject());
539 }
540 break;
541 }
542
543 case T_METADATA: {
544 if (patch_code != lir_patch_none) {
545 klass2reg_with_patching(dest->as_register(), info);
546 } else {
547 __ mov_metadata(dest->as_register(), c->as_metadata());
548 }
549 break;
550 }
551
552 case T_FLOAT: {
553 if (dest->is_single_xmm()) {
554 if (UseAVX <= 2 && c->is_zero_float()) {
555 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
556 } else {
557 __ movflt(dest->as_xmm_float_reg(),
558 InternalAddress(float_constant(c->as_jfloat())));
559 }
560 } else {
561 ShouldNotReachHere();
562 }
563 break;
564 }
565
566 case T_DOUBLE: {
567 if (dest->is_double_xmm()) {
568 if (UseAVX <= 2 && c->is_zero_double()) {
569 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
570 } else {
571 __ movdbl(dest->as_xmm_double_reg(),
572 InternalAddress(double_constant(c->as_jdouble())));
573 }
574 } else {
575 ShouldNotReachHere();
576 }
577 break;
578 }
579
580 default:
581 ShouldNotReachHere();
582 }
583 }
584
585 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
586 assert(src->is_constant(), "should not call otherwise");
587 assert(dest->is_stack(), "should not call otherwise");
588 LIR_Const* c = src->as_constant_ptr();
589
590 switch (c->type()) {
591 case T_INT: // fall through
592 case T_FLOAT:
593 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
594 break;
595
596 case T_ADDRESS:
597 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
598 break;
599
600 case T_OBJECT:
601 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
602 break;
603
604 case T_LONG: // fall through
605 case T_DOUBLE:
606 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
607 lo_word_offset_in_bytes),
608 (intptr_t)c->as_jlong_bits(),
609 rscratch1);
610 break;
611
612 default:
613 ShouldNotReachHere();
614 }
615 }
616
617 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
618 assert(src->is_constant(), "should not call otherwise");
619 assert(dest->is_address(), "should not call otherwise");
620 LIR_Const* c = src->as_constant_ptr();
621 LIR_Address* addr = dest->as_address_ptr();
622
623 int null_check_here = code_offset();
624 switch (type) {
625 case T_INT: // fall through
626 case T_FLOAT:
627 __ movl(as_Address(addr), c->as_jint_bits());
628 break;
629
630 case T_ADDRESS:
631 __ movptr(as_Address(addr), c->as_jint_bits());
632 break;
633
634 case T_OBJECT: // fall through
635 case T_ARRAY:
636 if (c->as_jobject() == nullptr) {
637 if (UseCompressedOops && !wide) {
638 __ movl(as_Address(addr), NULL_WORD);
639 } else {
640 __ xorptr(rscratch1, rscratch1);
641 null_check_here = code_offset();
642 __ movptr(as_Address(addr), rscratch1);
643 }
644 } else {
645 if (is_literal_address(addr)) {
646 ShouldNotReachHere();
647 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
648 } else {
649 __ movoop(rscratch1, c->as_jobject());
650 if (UseCompressedOops && !wide) {
651 __ encode_heap_oop(rscratch1);
652 null_check_here = code_offset();
653 __ movl(as_Address_lo(addr), rscratch1);
654 } else {
655 null_check_here = code_offset();
656 __ movptr(as_Address_lo(addr), rscratch1);
657 }
658 }
659 }
660 break;
661
662 case T_LONG: // fall through
663 case T_DOUBLE:
664 if (is_literal_address(addr)) {
665 ShouldNotReachHere();
666 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
667 } else {
668 __ movptr(r10, (intptr_t)c->as_jlong_bits());
669 null_check_here = code_offset();
670 __ movptr(as_Address_lo(addr), r10);
671 }
672 break;
673
674 case T_BOOLEAN: // fall through
675 case T_BYTE:
676 __ movb(as_Address(addr), c->as_jint() & 0xFF);
677 break;
678
679 case T_CHAR: // fall through
680 case T_SHORT:
681 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
682 break;
683
684 default:
685 ShouldNotReachHere();
686 };
687
688 if (info != nullptr) {
689 add_debug_info_for_null_check(null_check_here, info);
690 }
691 }
692
693
694 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
695 assert(src->is_register(), "should not call otherwise");
696 assert(dest->is_register(), "should not call otherwise");
697
698 // move between cpu-registers
699 if (dest->is_single_cpu()) {
700 if (src->type() == T_LONG) {
701 // Can do LONG -> OBJECT
702 move_regs(src->as_register_lo(), dest->as_register());
703 return;
704 }
705 assert(src->is_single_cpu(), "must match");
706 if (src->type() == T_OBJECT) {
707 __ verify_oop(src->as_register());
708 }
709 move_regs(src->as_register(), dest->as_register());
710
711 } else if (dest->is_double_cpu()) {
712 if (is_reference_type(src->type())) {
713 // Surprising to me but we can see move of a long to t_object
714 __ verify_oop(src->as_register());
715 move_regs(src->as_register(), dest->as_register_lo());
716 return;
717 }
718 assert(src->is_double_cpu(), "must match");
719 Register f_lo = src->as_register_lo();
720 Register f_hi = src->as_register_hi();
721 Register t_lo = dest->as_register_lo();
722 Register t_hi = dest->as_register_hi();
723 assert(f_hi == f_lo, "must be same");
724 assert(t_hi == t_lo, "must be same");
725 move_regs(f_lo, t_lo);
726
727 // move between xmm-registers
728 } else if (dest->is_single_xmm()) {
729 assert(src->is_single_xmm(), "must match");
730 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
731 } else if (dest->is_double_xmm()) {
732 assert(src->is_double_xmm(), "must match");
733 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
734
735 } else {
736 ShouldNotReachHere();
737 }
738 }
739
740 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
741 assert(src->is_register(), "should not call otherwise");
742 assert(dest->is_stack(), "should not call otherwise");
743
744 if (src->is_single_cpu()) {
745 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
746 if (is_reference_type(type)) {
747 __ verify_oop(src->as_register());
748 __ movptr (dst, src->as_register());
749 } else if (type == T_METADATA || type == T_ADDRESS) {
750 __ movptr (dst, src->as_register());
751 } else {
752 __ movl (dst, src->as_register());
753 }
754
755 } else if (src->is_double_cpu()) {
756 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
757 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
758 __ movptr (dstLO, src->as_register_lo());
759
760 } else if (src->is_single_xmm()) {
761 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
762 __ movflt(dst_addr, src->as_xmm_float_reg());
763
764 } else if (src->is_double_xmm()) {
765 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
766 __ movdbl(dst_addr, src->as_xmm_double_reg());
767
768 } else {
769 ShouldNotReachHere();
770 }
771 }
772
773
774 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
775 LIR_Address* to_addr = dest->as_address_ptr();
776 PatchingStub* patch = nullptr;
777 Register compressed_src = rscratch1;
778
779 if (is_reference_type(type)) {
780 __ verify_oop(src->as_register());
781 if (UseCompressedOops && !wide) {
782 __ movptr(compressed_src, src->as_register());
783 __ encode_heap_oop(compressed_src);
784 if (patch_code != lir_patch_none) {
785 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
786 }
787 }
788 }
789
790 if (patch_code != lir_patch_none) {
791 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
792 Address toa = as_Address(to_addr);
793 assert(toa.disp() != 0, "must have");
794 }
795
796 int null_check_here = code_offset();
797 switch (type) {
798 case T_FLOAT: {
799 assert(src->is_single_xmm(), "not a float");
800 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
801 break;
802 }
803
804 case T_DOUBLE: {
805 assert(src->is_double_xmm(), "not a double");
806 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
807 break;
808 }
809
810 case T_ARRAY: // fall through
811 case T_OBJECT: // fall through
812 if (UseCompressedOops && !wide) {
813 __ movl(as_Address(to_addr), compressed_src);
814 } else {
815 __ movptr(as_Address(to_addr), src->as_register());
816 }
817 break;
818 case T_ADDRESS:
819 __ movptr(as_Address(to_addr), src->as_register());
820 break;
821 case T_INT:
822 __ movl(as_Address(to_addr), src->as_register());
823 break;
824
825 case T_LONG: {
826 Register from_lo = src->as_register_lo();
827 Register from_hi = src->as_register_hi();
828 __ movptr(as_Address_lo(to_addr), from_lo);
829 break;
830 }
831
832 case T_BYTE: // fall through
833 case T_BOOLEAN: {
834 Register src_reg = src->as_register();
835 Address dst_addr = as_Address(to_addr);
836 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
837 __ movb(dst_addr, src_reg);
838 break;
839 }
840
841 case T_CHAR: // fall through
842 case T_SHORT:
843 __ movw(as_Address(to_addr), src->as_register());
844 break;
845
846 default:
847 ShouldNotReachHere();
848 }
849 if (info != nullptr) {
850 add_debug_info_for_null_check(null_check_here, info);
851 }
852
853 if (patch_code != lir_patch_none) {
854 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
855 }
856 }
857
858
859 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
860 assert(src->is_stack(), "should not call otherwise");
861 assert(dest->is_register(), "should not call otherwise");
862
863 if (dest->is_single_cpu()) {
864 if (is_reference_type(type)) {
865 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
866 __ verify_oop(dest->as_register());
867 } else if (type == T_METADATA || type == T_ADDRESS) {
868 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
869 } else {
870 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
871 }
872
873 } else if (dest->is_double_cpu()) {
874 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
875 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
876 __ movptr(dest->as_register_lo(), src_addr_LO);
877
878 } else if (dest->is_single_xmm()) {
879 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
880 __ movflt(dest->as_xmm_float_reg(), src_addr);
881
882 } else if (dest->is_double_xmm()) {
883 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
884 __ movdbl(dest->as_xmm_double_reg(), src_addr);
885
886 } else {
887 ShouldNotReachHere();
888 }
889 }
890
891
892 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 if (src->is_single_stack()) {
894 if (is_reference_type(type)) {
895 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
896 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
897 } else {
898 //no pushl on 64bits
899 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
900 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
901 }
902
903 } else if (src->is_double_stack()) {
904 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
905 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
906
907 } else {
908 ShouldNotReachHere();
909 }
910 }
911
912
913 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
914 assert(src->is_address(), "should not call otherwise");
915 assert(dest->is_register(), "should not call otherwise");
916
917 LIR_Address* addr = src->as_address_ptr();
918 Address from_addr = as_Address(addr);
919
920 if (addr->base()->type() == T_OBJECT) {
921 __ verify_oop(addr->base()->as_pointer_register());
922 }
923
924 switch (type) {
925 case T_BOOLEAN: // fall through
926 case T_BYTE: // fall through
927 case T_CHAR: // fall through
928 case T_SHORT:
929 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
930 // on pre P6 processors we may get partial register stalls
931 // so blow away the value of to_rinfo before loading a
932 // partial word into it. Do it here so that it precedes
933 // the potential patch point below.
934 __ xorptr(dest->as_register(), dest->as_register());
935 }
936 break;
937 default:
938 break;
939 }
940
941 PatchingStub* patch = nullptr;
942 if (patch_code != lir_patch_none) {
943 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
944 assert(from_addr.disp() != 0, "must have");
945 }
946 if (info != nullptr) {
947 add_debug_info_for_null_check_here(info);
948 }
949
950 switch (type) {
951 case T_FLOAT: {
952 if (dest->is_single_xmm()) {
953 __ movflt(dest->as_xmm_float_reg(), from_addr);
954 } else {
955 ShouldNotReachHere();
956 }
957 break;
958 }
959
960 case T_DOUBLE: {
961 if (dest->is_double_xmm()) {
962 __ movdbl(dest->as_xmm_double_reg(), from_addr);
963 } else {
964 ShouldNotReachHere();
965 }
966 break;
967 }
968
969 case T_OBJECT: // fall through
970 case T_ARRAY: // fall through
971 if (UseCompressedOops && !wide) {
972 __ movl(dest->as_register(), from_addr);
973 } else {
974 __ movptr(dest->as_register(), from_addr);
975 }
976 break;
977
978 case T_ADDRESS:
979 __ movptr(dest->as_register(), from_addr);
980 break;
981 case T_INT:
982 __ movl(dest->as_register(), from_addr);
983 break;
984
985 case T_LONG: {
986 Register to_lo = dest->as_register_lo();
987 Register to_hi = dest->as_register_hi();
988 __ movptr(to_lo, as_Address_lo(addr));
989 break;
990 }
991
992 case T_BOOLEAN: // fall through
993 case T_BYTE: {
994 Register dest_reg = dest->as_register();
995 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
996 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
997 __ movsbl(dest_reg, from_addr);
998 } else {
999 __ movb(dest_reg, from_addr);
1000 __ shll(dest_reg, 24);
1001 __ sarl(dest_reg, 24);
1002 }
1003 break;
1004 }
1005
1006 case T_CHAR: {
1007 Register dest_reg = dest->as_register();
1008 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1009 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1010 __ movzwl(dest_reg, from_addr);
1011 } else {
1012 __ movw(dest_reg, from_addr);
1013 }
1014 break;
1015 }
1016
1017 case T_SHORT: {
1018 Register dest_reg = dest->as_register();
1019 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1020 __ movswl(dest_reg, from_addr);
1021 } else {
1022 __ movw(dest_reg, from_addr);
1023 __ shll(dest_reg, 16);
1024 __ sarl(dest_reg, 16);
1025 }
1026 break;
1027 }
1028
1029 default:
1030 ShouldNotReachHere();
1031 }
1032
1033 if (patch != nullptr) {
1034 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1035 }
1036
1037 if (is_reference_type(type)) {
1038 if (UseCompressedOops && !wide) {
1039 __ decode_heap_oop(dest->as_register());
1040 }
1041
1042 __ verify_oop(dest->as_register());
1043 }
1044 }
1045
1046
1047 NEEDS_CLEANUP; // This could be static?
1048 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1049 int elem_size = type2aelembytes(type);
1050 switch (elem_size) {
1051 case 1: return Address::times_1;
1052 case 2: return Address::times_2;
1053 case 4: return Address::times_4;
1054 case 8: return Address::times_8;
1055 }
1056 ShouldNotReachHere();
1057 return Address::no_scale;
1058 }
1059
1060
1061 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1062 switch (op->code()) {
1063 case lir_idiv:
1064 case lir_irem:
1065 arithmetic_idiv(op->code(),
1066 op->in_opr1(),
1067 op->in_opr2(),
1068 op->in_opr3(),
1069 op->result_opr(),
1070 op->info());
1071 break;
1072 case lir_fmad:
1073 __ fmad(op->result_opr()->as_xmm_double_reg(),
1074 op->in_opr1()->as_xmm_double_reg(),
1075 op->in_opr2()->as_xmm_double_reg(),
1076 op->in_opr3()->as_xmm_double_reg());
1077 break;
1078 case lir_fmaf:
1079 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1080 op->in_opr1()->as_xmm_float_reg(),
1081 op->in_opr2()->as_xmm_float_reg(),
1082 op->in_opr3()->as_xmm_float_reg());
1083 break;
1084 default: ShouldNotReachHere(); break;
1085 }
1086 }
1087
1088 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1089 #ifdef ASSERT
1090 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1091 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1092 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1093 #endif
1094
1095 if (op->cond() == lir_cond_always) {
1096 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1097 __ jmp (*(op->label()));
1098 } else {
1099 Assembler::Condition acond = Assembler::zero;
1100 if (op->code() == lir_cond_float_branch) {
1101 assert(op->ublock() != nullptr, "must have unordered successor");
1102 __ jcc(Assembler::parity, *(op->ublock()->label()));
1103 switch(op->cond()) {
1104 case lir_cond_equal: acond = Assembler::equal; break;
1105 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1106 case lir_cond_less: acond = Assembler::below; break;
1107 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1108 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1109 case lir_cond_greater: acond = Assembler::above; break;
1110 default: ShouldNotReachHere();
1111 }
1112 } else {
1113 switch (op->cond()) {
1114 case lir_cond_equal: acond = Assembler::equal; break;
1115 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1116 case lir_cond_less: acond = Assembler::less; break;
1117 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1118 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1119 case lir_cond_greater: acond = Assembler::greater; break;
1120 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1121 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1122 default: ShouldNotReachHere();
1123 }
1124 }
1125 __ jcc(acond,*(op->label()));
1126 }
1127 }
1128
1129 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1130 LIR_Opr src = op->in_opr();
1131 LIR_Opr dest = op->result_opr();
1132
1133 switch (op->bytecode()) {
1134 case Bytecodes::_i2l:
1135 __ movl2ptr(dest->as_register_lo(), src->as_register());
1136 break;
1137
1138 case Bytecodes::_l2i:
1139 __ movl(dest->as_register(), src->as_register_lo());
1140 break;
1141
1142 case Bytecodes::_i2b:
1143 move_regs(src->as_register(), dest->as_register());
1144 __ sign_extend_byte(dest->as_register());
1145 break;
1146
1147 case Bytecodes::_i2c:
1148 move_regs(src->as_register(), dest->as_register());
1149 __ andl(dest->as_register(), 0xFFFF);
1150 break;
1151
1152 case Bytecodes::_i2s:
1153 move_regs(src->as_register(), dest->as_register());
1154 __ sign_extend_short(dest->as_register());
1155 break;
1156
1157 case Bytecodes::_f2d:
1158 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1159 break;
1160
1161 case Bytecodes::_d2f:
1162 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1163 break;
1164
1165 case Bytecodes::_i2f:
1166 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1167 break;
1168
1169 case Bytecodes::_i2d:
1170 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1171 break;
1172
1173 case Bytecodes::_l2f:
1174 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1175 break;
1176
1177 case Bytecodes::_l2d:
1178 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1179 break;
1180
1181 case Bytecodes::_f2i:
1182 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1183 break;
1184
1185 case Bytecodes::_d2i:
1186 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1187 break;
1188
1189 case Bytecodes::_f2l:
1190 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1191 break;
1192
1193 case Bytecodes::_d2l:
1194 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1195 break;
1196
1197 default: ShouldNotReachHere();
1198 }
1199 }
1200
1201 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1202 if (op->init_check()) {
1203 add_debug_info_for_null_check_here(op->stub()->info());
1204 // init_state needs acquire, but x86 is TSO, and so we are already good.
1205 __ cmpb(Address(op->klass()->as_register(),
1206 InstanceKlass::init_state_offset()),
1207 InstanceKlass::fully_initialized);
1208 __ jcc(Assembler::notEqual, *op->stub()->entry());
1209 }
1210 __ allocate_object(op->obj()->as_register(),
1211 op->tmp1()->as_register(),
1212 op->tmp2()->as_register(),
1213 op->header_size(),
1214 op->object_size(),
1215 op->klass()->as_register(),
1216 *op->stub()->entry());
1217 __ bind(*op->stub()->continuation());
1218 }
1219
1220 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1221 Register len = op->len()->as_register();
1222 __ movslq(len, len);
1223
1224 if (UseSlowPath ||
1225 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1226 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1227 __ jmp(*op->stub()->entry());
1228 } else {
1229 Register tmp1 = op->tmp1()->as_register();
1230 Register tmp2 = op->tmp2()->as_register();
1231 Register tmp3 = op->tmp3()->as_register();
1232 if (len == tmp1) {
1233 tmp1 = tmp3;
1234 } else if (len == tmp2) {
1235 tmp2 = tmp3;
1236 } else if (len == tmp3) {
1237 // everything is ok
1238 } else {
1239 __ mov(tmp3, len);
1240 }
1241 __ allocate_array(op->obj()->as_register(),
1242 len,
1243 tmp1,
1244 tmp2,
1245 arrayOopDesc::base_offset_in_bytes(op->type()),
1246 array_element_size(op->type()),
1247 op->klass()->as_register(),
1248 *op->stub()->entry(),
1249 op->zero_array());
1250 }
1251 __ bind(*op->stub()->continuation());
1252 }
1253
1254 void LIR_Assembler::type_profile_helper(Register mdo,
1255 ciMethodData *md, ciProfileData *data,
1256 Register recv, Label* update_done) {
1257 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1258 Label next_test;
1259 // See if the receiver is receiver[n].
1260 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1261 __ jccb(Assembler::notEqual, next_test);
1262 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1263 __ addptr(data_addr, DataLayout::counter_increment);
1264 __ jmp(*update_done);
1265 __ bind(next_test);
1266 }
1267
1268 // Didn't find receiver; find next empty slot and fill it in
1269 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1270 Label next_test;
1271 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1272 __ cmpptr(recv_addr, NULL_WORD);
1273 __ jccb(Assembler::notEqual, next_test);
1274 __ movptr(recv_addr, recv);
1275 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1276 __ jmp(*update_done);
1277 __ bind(next_test);
1278 }
1279 }
1280
1281 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1282 // we always need a stub for the failure case.
1283 CodeStub* stub = op->stub();
1284 Register obj = op->object()->as_register();
1285 Register k_RInfo = op->tmp1()->as_register();
1286 Register klass_RInfo = op->tmp2()->as_register();
1287 Register dst = op->result_opr()->as_register();
1288 ciKlass* k = op->klass();
1289 Register Rtmp1 = noreg;
1290 Register tmp_load_klass = rscratch1;
1291
1292 // check if it needs to be profiled
1293 ciMethodData* md = nullptr;
1294 ciProfileData* data = nullptr;
1295
1296 if (op->should_profile()) {
1297 ciMethod* method = op->profiled_method();
1298 assert(method != nullptr, "Should have method");
1299 int bci = op->profiled_bci();
1300 md = method->method_data_or_null();
1301 assert(md != nullptr, "Sanity");
1302 data = md->bci_to_data(bci);
1303 assert(data != nullptr, "need data for type check");
1304 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305 }
1306 Label* success_target = success;
1307 Label* failure_target = failure;
1308
1309 if (obj == k_RInfo) {
1310 k_RInfo = dst;
1311 } else if (obj == klass_RInfo) {
1312 klass_RInfo = dst;
1313 }
1314 if (k->is_loaded() && !UseCompressedClassPointers) {
1315 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1316 } else {
1317 Rtmp1 = op->tmp3()->as_register();
1318 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1319 }
1320
1321 assert_different_registers(obj, k_RInfo, klass_RInfo);
1322
1323 __ testptr(obj, obj);
1324 if (op->should_profile()) {
1325 Label not_null;
1326 Register mdo = klass_RInfo;
1327 __ mov_metadata(mdo, md->constant_encoding());
1328 __ jccb(Assembler::notEqual, not_null);
1329 // Object is null; update MDO and exit
1330 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1331 int header_bits = BitData::null_seen_byte_constant();
1332 __ orb(data_addr, header_bits);
1333 __ jmp(*obj_is_null);
1334 __ bind(not_null);
1335
1336 Label update_done;
1337 Register recv = k_RInfo;
1338 __ load_klass(recv, obj, tmp_load_klass);
1339 type_profile_helper(mdo, md, data, recv, &update_done);
1340
1341 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1342 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1343
1344 __ bind(update_done);
1345 } else {
1346 __ jcc(Assembler::equal, *obj_is_null);
1347 }
1348
1349 if (!k->is_loaded()) {
1350 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1351 } else {
1352 __ mov_metadata(k_RInfo, k->constant_encoding());
1353 }
1354 __ verify_oop(obj);
1355
1356 if (op->fast_check()) {
1357 // get object class
1358 // not a safepoint as obj null check happens earlier
1359 if (UseCompressedClassPointers) {
1360 __ load_klass(Rtmp1, obj, tmp_load_klass);
1361 __ cmpptr(k_RInfo, Rtmp1);
1362 } else {
1363 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1364 }
1365 __ jcc(Assembler::notEqual, *failure_target);
1366 // successful cast, fall through to profile or jump
1367 } else {
1368 // get object class
1369 // not a safepoint as obj null check happens earlier
1370 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1371 if (k->is_loaded()) {
1372 // See if we get an immediate positive hit
1373 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1374 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1375 __ jcc(Assembler::notEqual, *failure_target);
1376 // successful cast, fall through to profile or jump
1377 } else {
1378 // See if we get an immediate positive hit
1379 __ jcc(Assembler::equal, *success_target);
1380 // check for self
1381 __ cmpptr(klass_RInfo, k_RInfo);
1382 __ jcc(Assembler::equal, *success_target);
1383
1384 __ push_ppx(klass_RInfo);
1385 __ push_ppx(k_RInfo);
1386 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1387 __ pop_ppx(klass_RInfo);
1388 __ pop_ppx(klass_RInfo);
1389 // result is a boolean
1390 __ testl(klass_RInfo, klass_RInfo);
1391 __ jcc(Assembler::equal, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 }
1394 } else {
1395 // perform the fast part of the checking logic
1396 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1397 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1398 __ push_ppx(klass_RInfo);
1399 __ push_ppx(k_RInfo);
1400 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1401 __ pop_ppx(klass_RInfo);
1402 __ pop_ppx(k_RInfo);
1403 // result is a boolean
1404 __ testl(k_RInfo, k_RInfo);
1405 __ jcc(Assembler::equal, *failure_target);
1406 // successful cast, fall through to profile or jump
1407 }
1408 }
1409 __ jmp(*success);
1410 }
1411
1412
1413 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1414 Register tmp_load_klass = rscratch1;
1415 LIR_Code code = op->code();
1416 if (code == lir_store_check) {
1417 Register value = op->object()->as_register();
1418 Register array = op->array()->as_register();
1419 Register k_RInfo = op->tmp1()->as_register();
1420 Register klass_RInfo = op->tmp2()->as_register();
1421 Register Rtmp1 = op->tmp3()->as_register();
1422
1423 CodeStub* stub = op->stub();
1424
1425 // check if it needs to be profiled
1426 ciMethodData* md = nullptr;
1427 ciProfileData* data = nullptr;
1428
1429 if (op->should_profile()) {
1430 ciMethod* method = op->profiled_method();
1431 assert(method != nullptr, "Should have method");
1432 int bci = op->profiled_bci();
1433 md = method->method_data_or_null();
1434 assert(md != nullptr, "Sanity");
1435 data = md->bci_to_data(bci);
1436 assert(data != nullptr, "need data for type check");
1437 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1438 }
1439 Label done;
1440 Label* success_target = &done;
1441 Label* failure_target = stub->entry();
1442
1443 __ testptr(value, value);
1444 if (op->should_profile()) {
1445 Label not_null;
1446 Register mdo = klass_RInfo;
1447 __ mov_metadata(mdo, md->constant_encoding());
1448 __ jccb(Assembler::notEqual, not_null);
1449 // Object is null; update MDO and exit
1450 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1451 int header_bits = BitData::null_seen_byte_constant();
1452 __ orb(data_addr, header_bits);
1453 __ jmp(done);
1454 __ bind(not_null);
1455
1456 Label update_done;
1457 Register recv = k_RInfo;
1458 __ load_klass(recv, value, tmp_load_klass);
1459 type_profile_helper(mdo, md, data, recv, &update_done);
1460
1461 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1462 __ addptr(counter_addr, DataLayout::counter_increment);
1463 __ bind(update_done);
1464 } else {
1465 __ jcc(Assembler::equal, done);
1466 }
1467
1468 add_debug_info_for_null_check_here(op->info_for_exception());
1469 __ load_klass(k_RInfo, array, tmp_load_klass);
1470 __ load_klass(klass_RInfo, value, tmp_load_klass);
1471
1472 // get instance klass (it's already uncompressed)
1473 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1474 // perform the fast part of the checking logic
1475 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1476 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1477 __ push_ppx(klass_RInfo);
1478 __ push_ppx(k_RInfo);
1479 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1480 __ pop_ppx(klass_RInfo);
1481 __ pop_ppx(k_RInfo);
1482 // result is a boolean
1483 __ testl(k_RInfo, k_RInfo);
1484 __ jcc(Assembler::equal, *failure_target);
1485 // fall through to the success case
1486
1487 __ bind(done);
1488 } else
1489 if (code == lir_checkcast) {
1490 Register obj = op->object()->as_register();
1491 Register dst = op->result_opr()->as_register();
1492 Label success;
1493 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1494 __ bind(success);
1495 if (dst != obj) {
1496 __ mov(dst, obj);
1497 }
1498 } else
1499 if (code == lir_instanceof) {
1500 Register obj = op->object()->as_register();
1501 Register dst = op->result_opr()->as_register();
1502 Label success, failure, done;
1503 emit_typecheck_helper(op, &success, &failure, &failure);
1504 __ bind(failure);
1505 __ xorptr(dst, dst);
1506 __ jmpb(done);
1507 __ bind(success);
1508 __ movptr(dst, 1);
1509 __ bind(done);
1510 } else {
1511 ShouldNotReachHere();
1512 }
1513
1514 }
1515
1516
1517 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1518 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1519 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1520 Register newval = op->new_value()->as_register();
1521 Register cmpval = op->cmp_value()->as_register();
1522 assert(cmpval == rax, "wrong register");
1523 assert(newval != noreg, "new val must be register");
1524 assert(cmpval != newval, "cmp and new values must be in different registers");
1525 assert(cmpval != addr, "cmp and addr must be in different registers");
1526 assert(newval != addr, "new value and addr must be in different registers");
1527
1528 if (op->code() == lir_cas_obj) {
1529 if (UseCompressedOops) {
1530 __ encode_heap_oop(cmpval);
1531 __ mov(rscratch1, newval);
1532 __ encode_heap_oop(rscratch1);
1533 __ lock();
1534 // cmpval (rax) is implicitly used by this instruction
1535 __ cmpxchgl(rscratch1, Address(addr, 0));
1536 } else {
1537 __ lock();
1538 __ cmpxchgptr(newval, Address(addr, 0));
1539 }
1540 } else {
1541 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1542 __ lock();
1543 __ cmpxchgl(newval, Address(addr, 0));
1544 }
1545 } else if (op->code() == lir_cas_long) {
1546 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1547 Register newval = op->new_value()->as_register_lo();
1548 Register cmpval = op->cmp_value()->as_register_lo();
1549 assert(cmpval == rax, "wrong register");
1550 assert(newval != noreg, "new val must be register");
1551 assert(cmpval != newval, "cmp and new values must be in different registers");
1552 assert(cmpval != addr, "cmp and addr must be in different registers");
1553 assert(newval != addr, "new value and addr must be in different registers");
1554 __ lock();
1555 __ cmpxchgq(newval, Address(addr, 0));
1556 } else {
1557 Unimplemented();
1558 }
1559 }
1560
1561 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1562 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1563 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1564
1565 Assembler::Condition acond, ncond;
1566 switch (condition) {
1567 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1568 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1569 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1570 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1571 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1572 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1573 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1574 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1575 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1576 ShouldNotReachHere();
1577 }
1578
1579 if (opr1->is_cpu_register()) {
1580 reg2reg(opr1, result);
1581 } else if (opr1->is_stack()) {
1582 stack2reg(opr1, result, result->type());
1583 } else if (opr1->is_constant()) {
1584 const2reg(opr1, result, lir_patch_none, nullptr);
1585 } else {
1586 ShouldNotReachHere();
1587 }
1588
1589 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1590 // optimized version that does not require a branch
1591 if (opr2->is_single_cpu()) {
1592 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1593 __ cmov(ncond, result->as_register(), opr2->as_register());
1594 } else if (opr2->is_double_cpu()) {
1595 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1596 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1597 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1598 } else if (opr2->is_single_stack()) {
1599 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1600 } else if (opr2->is_double_stack()) {
1601 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1602 } else {
1603 ShouldNotReachHere();
1604 }
1605
1606 } else {
1607 Label skip;
1608 __ jccb(acond, skip);
1609 if (opr2->is_cpu_register()) {
1610 reg2reg(opr2, result);
1611 } else if (opr2->is_stack()) {
1612 stack2reg(opr2, result, result->type());
1613 } else if (opr2->is_constant()) {
1614 const2reg(opr2, result, lir_patch_none, nullptr);
1615 } else {
1616 ShouldNotReachHere();
1617 }
1618 __ bind(skip);
1619 }
1620 }
1621
1622
1623 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1624 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1625
1626 if (left->is_single_cpu()) {
1627 assert(left == dest, "left and dest must be equal");
1628 Register lreg = left->as_register();
1629
1630 if (right->is_single_cpu()) {
1631 // cpu register - cpu register
1632 Register rreg = right->as_register();
1633 switch (code) {
1634 case lir_add: __ addl (lreg, rreg); break;
1635 case lir_sub: __ subl (lreg, rreg); break;
1636 case lir_mul: __ imull(lreg, rreg); break;
1637 default: ShouldNotReachHere();
1638 }
1639
1640 } else if (right->is_stack()) {
1641 // cpu register - stack
1642 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1643 switch (code) {
1644 case lir_add: __ addl(lreg, raddr); break;
1645 case lir_sub: __ subl(lreg, raddr); break;
1646 default: ShouldNotReachHere();
1647 }
1648
1649 } else if (right->is_constant()) {
1650 // cpu register - constant
1651 jint c = right->as_constant_ptr()->as_jint();
1652 switch (code) {
1653 case lir_add: {
1654 __ incrementl(lreg, c);
1655 break;
1656 }
1657 case lir_sub: {
1658 __ decrementl(lreg, c);
1659 break;
1660 }
1661 default: ShouldNotReachHere();
1662 }
1663
1664 } else {
1665 ShouldNotReachHere();
1666 }
1667
1668 } else if (left->is_double_cpu()) {
1669 assert(left == dest, "left and dest must be equal");
1670 Register lreg_lo = left->as_register_lo();
1671 Register lreg_hi = left->as_register_hi();
1672
1673 if (right->is_double_cpu()) {
1674 // cpu register - cpu register
1675 Register rreg_lo = right->as_register_lo();
1676 Register rreg_hi = right->as_register_hi();
1677 assert_different_registers(lreg_lo, rreg_lo);
1678 switch (code) {
1679 case lir_add:
1680 __ addptr(lreg_lo, rreg_lo);
1681 break;
1682 case lir_sub:
1683 __ subptr(lreg_lo, rreg_lo);
1684 break;
1685 case lir_mul:
1686 __ imulq(lreg_lo, rreg_lo);
1687 break;
1688 default:
1689 ShouldNotReachHere();
1690 }
1691
1692 } else if (right->is_constant()) {
1693 // cpu register - constant
1694 jlong c = right->as_constant_ptr()->as_jlong_bits();
1695 __ movptr(r10, (intptr_t) c);
1696 switch (code) {
1697 case lir_add:
1698 __ addptr(lreg_lo, r10);
1699 break;
1700 case lir_sub:
1701 __ subptr(lreg_lo, r10);
1702 break;
1703 default:
1704 ShouldNotReachHere();
1705 }
1706
1707 } else {
1708 ShouldNotReachHere();
1709 }
1710
1711 } else if (left->is_single_xmm()) {
1712 assert(left == dest, "left and dest must be equal");
1713 XMMRegister lreg = left->as_xmm_float_reg();
1714
1715 if (right->is_single_xmm()) {
1716 XMMRegister rreg = right->as_xmm_float_reg();
1717 switch (code) {
1718 case lir_add: __ addss(lreg, rreg); break;
1719 case lir_sub: __ subss(lreg, rreg); break;
1720 case lir_mul: __ mulss(lreg, rreg); break;
1721 case lir_div: __ divss(lreg, rreg); break;
1722 default: ShouldNotReachHere();
1723 }
1724 } else {
1725 Address raddr;
1726 if (right->is_single_stack()) {
1727 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1728 } else if (right->is_constant()) {
1729 // hack for now
1730 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1731 } else {
1732 ShouldNotReachHere();
1733 }
1734 switch (code) {
1735 case lir_add: __ addss(lreg, raddr); break;
1736 case lir_sub: __ subss(lreg, raddr); break;
1737 case lir_mul: __ mulss(lreg, raddr); break;
1738 case lir_div: __ divss(lreg, raddr); break;
1739 default: ShouldNotReachHere();
1740 }
1741 }
1742
1743 } else if (left->is_double_xmm()) {
1744 assert(left == dest, "left and dest must be equal");
1745
1746 XMMRegister lreg = left->as_xmm_double_reg();
1747 if (right->is_double_xmm()) {
1748 XMMRegister rreg = right->as_xmm_double_reg();
1749 switch (code) {
1750 case lir_add: __ addsd(lreg, rreg); break;
1751 case lir_sub: __ subsd(lreg, rreg); break;
1752 case lir_mul: __ mulsd(lreg, rreg); break;
1753 case lir_div: __ divsd(lreg, rreg); break;
1754 default: ShouldNotReachHere();
1755 }
1756 } else {
1757 Address raddr;
1758 if (right->is_double_stack()) {
1759 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1760 } else if (right->is_constant()) {
1761 // hack for now
1762 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1763 } else {
1764 ShouldNotReachHere();
1765 }
1766 switch (code) {
1767 case lir_add: __ addsd(lreg, raddr); break;
1768 case lir_sub: __ subsd(lreg, raddr); break;
1769 case lir_mul: __ mulsd(lreg, raddr); break;
1770 case lir_div: __ divsd(lreg, raddr); break;
1771 default: ShouldNotReachHere();
1772 }
1773 }
1774
1775 } else if (left->is_single_stack() || left->is_address()) {
1776 assert(left == dest, "left and dest must be equal");
1777
1778 Address laddr;
1779 if (left->is_single_stack()) {
1780 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1781 } else if (left->is_address()) {
1782 laddr = as_Address(left->as_address_ptr());
1783 } else {
1784 ShouldNotReachHere();
1785 }
1786
1787 if (right->is_single_cpu()) {
1788 Register rreg = right->as_register();
1789 switch (code) {
1790 case lir_add: __ addl(laddr, rreg); break;
1791 case lir_sub: __ subl(laddr, rreg); break;
1792 default: ShouldNotReachHere();
1793 }
1794 } else if (right->is_constant()) {
1795 jint c = right->as_constant_ptr()->as_jint();
1796 switch (code) {
1797 case lir_add: {
1798 __ incrementl(laddr, c);
1799 break;
1800 }
1801 case lir_sub: {
1802 __ decrementl(laddr, c);
1803 break;
1804 }
1805 default: ShouldNotReachHere();
1806 }
1807 } else {
1808 ShouldNotReachHere();
1809 }
1810
1811 } else {
1812 ShouldNotReachHere();
1813 }
1814 }
1815
1816
1817 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1818 if (value->is_double_xmm()) {
1819 switch(code) {
1820 case lir_abs :
1821 {
1822 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1823 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1824 }
1825 assert(!tmp->is_valid(), "do not need temporary");
1826 __ andpd(dest->as_xmm_double_reg(),
1827 ExternalAddress((address)double_signmask_pool),
1828 rscratch1);
1829 }
1830 break;
1831
1832 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1833 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1834 default : ShouldNotReachHere();
1835 }
1836
1837 } else if (code == lir_f2hf) {
1838 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1839 } else if (code == lir_hf2f) {
1840 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1841 } else {
1842 Unimplemented();
1843 }
1844 }
1845
1846 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1847 // assert(left->destroys_register(), "check");
1848 if (left->is_single_cpu()) {
1849 Register reg = left->as_register();
1850 if (right->is_constant()) {
1851 int val = right->as_constant_ptr()->as_jint();
1852 switch (code) {
1853 case lir_logic_and: __ andl (reg, val); break;
1854 case lir_logic_or: __ orl (reg, val); break;
1855 case lir_logic_xor: __ xorl (reg, val); break;
1856 default: ShouldNotReachHere();
1857 }
1858 } else if (right->is_stack()) {
1859 // added support for stack operands
1860 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1861 switch (code) {
1862 case lir_logic_and: __ andl (reg, raddr); break;
1863 case lir_logic_or: __ orl (reg, raddr); break;
1864 case lir_logic_xor: __ xorl (reg, raddr); break;
1865 default: ShouldNotReachHere();
1866 }
1867 } else {
1868 Register rright = right->as_register();
1869 switch (code) {
1870 case lir_logic_and: __ andptr (reg, rright); break;
1871 case lir_logic_or : __ orptr (reg, rright); break;
1872 case lir_logic_xor: __ xorptr (reg, rright); break;
1873 default: ShouldNotReachHere();
1874 }
1875 }
1876 move_regs(reg, dst->as_register());
1877 } else {
1878 Register l_lo = left->as_register_lo();
1879 Register l_hi = left->as_register_hi();
1880 if (right->is_constant()) {
1881 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1882 switch (code) {
1883 case lir_logic_and:
1884 __ andq(l_lo, rscratch1);
1885 break;
1886 case lir_logic_or:
1887 __ orq(l_lo, rscratch1);
1888 break;
1889 case lir_logic_xor:
1890 __ xorq(l_lo, rscratch1);
1891 break;
1892 default: ShouldNotReachHere();
1893 }
1894 } else {
1895 Register r_lo;
1896 if (is_reference_type(right->type())) {
1897 r_lo = right->as_register();
1898 } else {
1899 r_lo = right->as_register_lo();
1900 }
1901 switch (code) {
1902 case lir_logic_and:
1903 __ andptr(l_lo, r_lo);
1904 break;
1905 case lir_logic_or:
1906 __ orptr(l_lo, r_lo);
1907 break;
1908 case lir_logic_xor:
1909 __ xorptr(l_lo, r_lo);
1910 break;
1911 default: ShouldNotReachHere();
1912 }
1913 }
1914
1915 Register dst_lo = dst->as_register_lo();
1916 Register dst_hi = dst->as_register_hi();
1917
1918 move_regs(l_lo, dst_lo);
1919 }
1920 }
1921
1922
1923 // we assume that rax, and rdx can be overwritten
1924 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1925
1926 assert(left->is_single_cpu(), "left must be register");
1927 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1928 assert(result->is_single_cpu(), "result must be register");
1929
1930 // assert(left->destroys_register(), "check");
1931 // assert(right->destroys_register(), "check");
1932
1933 Register lreg = left->as_register();
1934 Register dreg = result->as_register();
1935
1936 if (right->is_constant()) {
1937 jint divisor = right->as_constant_ptr()->as_jint();
1938 assert(divisor > 0 && is_power_of_2(divisor), "must be");
1939 if (code == lir_idiv) {
1940 assert(lreg == rax, "must be rax,");
1941 assert(temp->as_register() == rdx, "tmp register must be rdx");
1942 __ cdql(); // sign extend into rdx:rax
1943 if (divisor == 2) {
1944 __ subl(lreg, rdx);
1945 } else {
1946 __ andl(rdx, divisor - 1);
1947 __ addl(lreg, rdx);
1948 }
1949 __ sarl(lreg, log2i_exact(divisor));
1950 move_regs(lreg, dreg);
1951 } else if (code == lir_irem) {
1952 Label done;
1953 __ mov(dreg, lreg);
1954 __ andl(dreg, 0x80000000 | (divisor - 1));
1955 __ jcc(Assembler::positive, done);
1956 __ decrement(dreg);
1957 __ orl(dreg, ~(divisor - 1));
1958 __ increment(dreg);
1959 __ bind(done);
1960 } else {
1961 ShouldNotReachHere();
1962 }
1963 } else {
1964 Register rreg = right->as_register();
1965 assert(lreg == rax, "left register must be rax,");
1966 assert(rreg != rdx, "right register must not be rdx");
1967 assert(temp->as_register() == rdx, "tmp register must be rdx");
1968
1969 move_regs(lreg, rax);
1970
1971 int idivl_offset = __ corrected_idivl(rreg);
1972 if (ImplicitDiv0Checks) {
1973 add_debug_info_for_div0(idivl_offset, info);
1974 }
1975 if (code == lir_irem) {
1976 move_regs(rdx, dreg); // result is in rdx
1977 } else {
1978 move_regs(rax, dreg);
1979 }
1980 }
1981 }
1982
1983
1984 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1985 if (opr1->is_single_cpu()) {
1986 Register reg1 = opr1->as_register();
1987 if (opr2->is_single_cpu()) {
1988 // cpu register - cpu register
1989 if (is_reference_type(opr1->type())) {
1990 __ cmpoop(reg1, opr2->as_register());
1991 } else {
1992 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1993 __ cmpl(reg1, opr2->as_register());
1994 }
1995 } else if (opr2->is_stack()) {
1996 // cpu register - stack
1997 if (is_reference_type(opr1->type())) {
1998 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1999 } else {
2000 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2001 }
2002 } else if (opr2->is_constant()) {
2003 // cpu register - constant
2004 LIR_Const* c = opr2->as_constant_ptr();
2005 if (c->type() == T_INT) {
2006 jint i = c->as_jint();
2007 if (i == 0) {
2008 __ testl(reg1, reg1);
2009 } else {
2010 __ cmpl(reg1, i);
2011 }
2012 } else if (c->type() == T_METADATA) {
2013 // All we need for now is a comparison with null for equality.
2014 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2015 Metadata* m = c->as_metadata();
2016 if (m == nullptr) {
2017 __ testptr(reg1, reg1);
2018 } else {
2019 ShouldNotReachHere();
2020 }
2021 } else if (is_reference_type(c->type())) {
2022 // In 64bit oops are single register
2023 jobject o = c->as_jobject();
2024 if (o == nullptr) {
2025 __ testptr(reg1, reg1);
2026 } else {
2027 __ cmpoop(reg1, o, rscratch1);
2028 }
2029 } else {
2030 fatal("unexpected type: %s", basictype_to_str(c->type()));
2031 }
2032 // cpu register - address
2033 } else if (opr2->is_address()) {
2034 if (op->info() != nullptr) {
2035 add_debug_info_for_null_check_here(op->info());
2036 }
2037 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2038 } else {
2039 ShouldNotReachHere();
2040 }
2041
2042 } else if(opr1->is_double_cpu()) {
2043 Register xlo = opr1->as_register_lo();
2044 Register xhi = opr1->as_register_hi();
2045 if (opr2->is_double_cpu()) {
2046 __ cmpptr(xlo, opr2->as_register_lo());
2047 } else if (opr2->is_constant()) {
2048 // cpu register - constant 0
2049 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2050 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2051 } else {
2052 ShouldNotReachHere();
2053 }
2054
2055 } else if (opr1->is_single_xmm()) {
2056 XMMRegister reg1 = opr1->as_xmm_float_reg();
2057 if (opr2->is_single_xmm()) {
2058 // xmm register - xmm register
2059 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2060 } else if (opr2->is_stack()) {
2061 // xmm register - stack
2062 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2063 } else if (opr2->is_constant()) {
2064 // xmm register - constant
2065 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2066 } else if (opr2->is_address()) {
2067 // xmm register - address
2068 if (op->info() != nullptr) {
2069 add_debug_info_for_null_check_here(op->info());
2070 }
2071 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2072 } else {
2073 ShouldNotReachHere();
2074 }
2075
2076 } else if (opr1->is_double_xmm()) {
2077 XMMRegister reg1 = opr1->as_xmm_double_reg();
2078 if (opr2->is_double_xmm()) {
2079 // xmm register - xmm register
2080 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2081 } else if (opr2->is_stack()) {
2082 // xmm register - stack
2083 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2084 } else if (opr2->is_constant()) {
2085 // xmm register - constant
2086 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2087 } else if (opr2->is_address()) {
2088 // xmm register - address
2089 if (op->info() != nullptr) {
2090 add_debug_info_for_null_check_here(op->info());
2091 }
2092 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2093 } else {
2094 ShouldNotReachHere();
2095 }
2096
2097 } else if (opr1->is_address() && opr2->is_constant()) {
2098 LIR_Const* c = opr2->as_constant_ptr();
2099 if (is_reference_type(c->type())) {
2100 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2101 __ movoop(rscratch1, c->as_jobject());
2102 }
2103 if (op->info() != nullptr) {
2104 add_debug_info_for_null_check_here(op->info());
2105 }
2106 // special case: address - constant
2107 LIR_Address* addr = opr1->as_address_ptr();
2108 if (c->type() == T_INT) {
2109 __ cmpl(as_Address(addr), c->as_jint());
2110 } else if (is_reference_type(c->type())) {
2111 // %%% Make this explode if addr isn't reachable until we figure out a
2112 // better strategy by giving noreg as the temp for as_Address
2113 __ cmpoop(rscratch1, as_Address(addr, noreg));
2114 } else {
2115 ShouldNotReachHere();
2116 }
2117
2118 } else {
2119 ShouldNotReachHere();
2120 }
2121 }
2122
2123 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2124 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2125 if (left->is_single_xmm()) {
2126 assert(right->is_single_xmm(), "must match");
2127 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2128 } else if (left->is_double_xmm()) {
2129 assert(right->is_double_xmm(), "must match");
2130 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2131
2132 } else {
2133 ShouldNotReachHere();
2134 }
2135 } else {
2136 assert(code == lir_cmp_l2i, "check");
2137 Label done;
2138 Register dest = dst->as_register();
2139 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2140 __ movl(dest, -1);
2141 __ jccb(Assembler::less, done);
2142 __ setb(Assembler::notZero, dest);
2143 __ movzbl(dest, dest);
2144 __ bind(done);
2145 }
2146 }
2147
2148
2149 void LIR_Assembler::align_call(LIR_Code code) {
2150 // make sure that the displacement word of the call ends up word aligned
2151 int offset = __ offset();
2152 switch (code) {
2153 case lir_static_call:
2154 case lir_optvirtual_call:
2155 case lir_dynamic_call:
2156 offset += NativeCall::displacement_offset;
2157 break;
2158 case lir_icvirtual_call:
2159 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2160 break;
2161 default: ShouldNotReachHere();
2162 }
2163 __ align(BytesPerWord, offset);
2164 }
2165
2166
2167 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2168 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2169 "must be aligned");
2170 __ call(AddressLiteral(op->addr(), rtype));
2171 add_call_info(code_offset(), op->info());
2172 __ post_call_nop();
2173 }
2174
2175
2176 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2177 __ ic_call(op->addr());
2178 add_call_info(code_offset(), op->info());
2179 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2180 "must be aligned");
2181 __ post_call_nop();
2182 }
2183
2184
2185 void LIR_Assembler::emit_static_call_stub() {
2186 address call_pc = __ pc();
2187 address stub = __ start_a_stub(call_stub_size());
2188 if (stub == nullptr) {
2189 bailout("static call stub overflow");
2190 return;
2191 }
2192
2193 int start = __ offset();
2194
2195 // make sure that the displacement word of the call ends up word aligned
2196 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2197 __ relocate(static_stub_Relocation::spec(call_pc));
2198 __ mov_metadata(rbx, (Metadata*)nullptr);
2199 // must be set to -1 at code generation time
2200 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2201 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2202 __ jump(RuntimeAddress(__ pc()));
2203
2204 assert(__ offset() - start <= call_stub_size(), "stub too big");
2205 __ end_a_stub();
2206 }
2207
2208
2209 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2210 assert(exceptionOop->as_register() == rax, "must match");
2211 assert(exceptionPC->as_register() == rdx, "must match");
2212
2213 // exception object is not added to oop map by LinearScan
2214 // (LinearScan assumes that no oops are in fixed registers)
2215 info->add_register_oop(exceptionOop);
2216 StubId unwind_id;
2217
2218 // get current pc information
2219 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2220 int pc_for_athrow_offset = __ offset();
2221 InternalAddress pc_for_athrow(__ pc());
2222 __ lea(exceptionPC->as_register(), pc_for_athrow);
2223 add_call_info(pc_for_athrow_offset, info); // for exception handler
2224
2225 __ verify_not_null_oop(rax);
2226 // search an exception handler (rax: exception oop, rdx: throwing pc)
2227 if (compilation()->has_fpu_code()) {
2228 unwind_id = StubId::c1_handle_exception_id;
2229 } else {
2230 unwind_id = StubId::c1_handle_exception_nofpu_id;
2231 }
2232 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2233
2234 // enough room for two byte trap
2235 __ nop();
2236 }
2237
2238
2239 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2240 assert(exceptionOop->as_register() == rax, "must match");
2241
2242 __ jmp(_unwind_handler_entry);
2243 }
2244
2245
2246 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2247
2248 // optimized version for linear scan:
2249 // * count must be already in ECX (guaranteed by LinearScan)
2250 // * left and dest must be equal
2251 // * tmp must be unused
2252 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2253 assert(left == dest, "left and dest must be equal");
2254 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2255
2256 if (left->is_single_cpu()) {
2257 Register value = left->as_register();
2258 assert(value != SHIFT_count, "left cannot be ECX");
2259
2260 switch (code) {
2261 case lir_shl: __ shll(value); break;
2262 case lir_shr: __ sarl(value); break;
2263 case lir_ushr: __ shrl(value); break;
2264 default: ShouldNotReachHere();
2265 }
2266 } else if (left->is_double_cpu()) {
2267 Register lo = left->as_register_lo();
2268 Register hi = left->as_register_hi();
2269 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2270 switch (code) {
2271 case lir_shl: __ shlptr(lo); break;
2272 case lir_shr: __ sarptr(lo); break;
2273 case lir_ushr: __ shrptr(lo); break;
2274 default: ShouldNotReachHere();
2275 }
2276 } else {
2277 ShouldNotReachHere();
2278 }
2279 }
2280
2281
2282 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2283 if (dest->is_single_cpu()) {
2284 // first move left into dest so that left is not destroyed by the shift
2285 Register value = dest->as_register();
2286 count = count & 0x1F; // Java spec
2287
2288 move_regs(left->as_register(), value);
2289 switch (code) {
2290 case lir_shl: __ shll(value, count); break;
2291 case lir_shr: __ sarl(value, count); break;
2292 case lir_ushr: __ shrl(value, count); break;
2293 default: ShouldNotReachHere();
2294 }
2295 } else if (dest->is_double_cpu()) {
2296 // first move left into dest so that left is not destroyed by the shift
2297 Register value = dest->as_register_lo();
2298 count = count & 0x1F; // Java spec
2299
2300 move_regs(left->as_register_lo(), value);
2301 switch (code) {
2302 case lir_shl: __ shlptr(value, count); break;
2303 case lir_shr: __ sarptr(value, count); break;
2304 case lir_ushr: __ shrptr(value, count); break;
2305 default: ShouldNotReachHere();
2306 }
2307 } else {
2308 ShouldNotReachHere();
2309 }
2310 }
2311
2312
2313 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2314 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2315 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2316 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2317 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2318 }
2319
2320
2321 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2322 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2323 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2324 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2325 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2326 }
2327
2328
2329 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2330 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2338 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2342 }
2343
2344
2345 // This code replaces a call to arraycopy; no exception may
2346 // be thrown in this code, they must be thrown in the System.arraycopy
2347 // activation frame; we could save some checks if this would not be the case
2348 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2349 ciArrayKlass* default_type = op->expected_type();
2350 Register src = op->src()->as_register();
2351 Register dst = op->dst()->as_register();
2352 Register src_pos = op->src_pos()->as_register();
2353 Register dst_pos = op->dst_pos()->as_register();
2354 Register length = op->length()->as_register();
2355 Register tmp = op->tmp()->as_register();
2356 Register tmp_load_klass = rscratch1;
2357 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2358
2359 CodeStub* stub = op->stub();
2360 int flags = op->flags();
2361 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2362 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2363
2364 // if we don't know anything, just go through the generic arraycopy
2365 if (default_type == nullptr) {
2366 // save outgoing arguments on stack in case call to System.arraycopy is needed
2367 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2368 // for interpreter calling conventions. Now we have to do it in new style conventions.
2369 // For the moment until C1 gets the new register allocator I just force all the
2370 // args to the right place (except the register args) and then on the back side
2371 // reload the register args properly if we go slow path. Yuck
2372
2373 // These are proper for the calling convention
2374 store_parameter(length, 2);
2375 store_parameter(dst_pos, 1);
2376 store_parameter(dst, 0);
2377
2378 // these are just temporary placements until we need to reload
2379 store_parameter(src_pos, 3);
2380 store_parameter(src, 4);
2381
2382 address copyfunc_addr = StubRoutines::generic_arraycopy();
2383 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2384
2385 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2386 // The arguments are in java calling convention so we can trivially shift them to C
2387 // convention
2388 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2389 __ mov(c_rarg0, j_rarg0);
2390 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2391 __ mov(c_rarg1, j_rarg1);
2392 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2393 __ mov(c_rarg2, j_rarg2);
2394 assert_different_registers(c_rarg3, j_rarg4);
2395 __ mov(c_rarg3, j_rarg3);
2396 #ifdef _WIN64
2397 // Allocate abi space for args but be sure to keep stack aligned
2398 __ subptr(rsp, 6*wordSize);
2399 store_parameter(j_rarg4, 4);
2400 #ifndef PRODUCT
2401 if (PrintC1Statistics) {
2402 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2403 }
2404 #endif
2405 __ call(RuntimeAddress(copyfunc_addr));
2406 __ addptr(rsp, 6*wordSize);
2407 #else
2408 __ mov(c_rarg4, j_rarg4);
2409 #ifndef PRODUCT
2410 if (PrintC1Statistics) {
2411 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2412 }
2413 #endif
2414 __ call(RuntimeAddress(copyfunc_addr));
2415 #endif // _WIN64
2416
2417 __ testl(rax, rax);
2418 __ jcc(Assembler::equal, *stub->continuation());
2419
2420 __ mov(tmp, rax);
2421 __ xorl(tmp, -1);
2422
2423 // Reload values from the stack so they are where the stub
2424 // expects them.
2425 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2426 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2427 __ movptr (length, Address(rsp, 2*BytesPerWord));
2428 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2429 __ movptr (src, Address(rsp, 4*BytesPerWord));
2430
2431 __ subl(length, tmp);
2432 __ addl(src_pos, tmp);
2433 __ addl(dst_pos, tmp);
2434 __ jmp(*stub->entry());
2435
2436 __ bind(*stub->continuation());
2437 return;
2438 }
2439
2440 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2441
2442 int elem_size = type2aelembytes(basic_type);
2443 Address::ScaleFactor scale;
2444
2445 switch (elem_size) {
2446 case 1 :
2447 scale = Address::times_1;
2448 break;
2449 case 2 :
2450 scale = Address::times_2;
2451 break;
2452 case 4 :
2453 scale = Address::times_4;
2454 break;
2455 case 8 :
2456 scale = Address::times_8;
2457 break;
2458 default:
2459 scale = Address::no_scale;
2460 ShouldNotReachHere();
2461 }
2462
2463 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2464 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2465
2466 // length and pos's are all sign extended at this point on 64bit
2467
2468 // test for null
2469 if (flags & LIR_OpArrayCopy::src_null_check) {
2470 __ testptr(src, src);
2471 __ jcc(Assembler::zero, *stub->entry());
2472 }
2473 if (flags & LIR_OpArrayCopy::dst_null_check) {
2474 __ testptr(dst, dst);
2475 __ jcc(Assembler::zero, *stub->entry());
2476 }
2477
2478 // If the compiler was not able to prove that exact type of the source or the destination
2479 // of the arraycopy is an array type, check at runtime if the source or the destination is
2480 // an instance type.
2481 if (flags & LIR_OpArrayCopy::type_check) {
2482 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2483 __ load_klass(tmp, dst, tmp_load_klass);
2484 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2485 __ jcc(Assembler::greaterEqual, *stub->entry());
2486 }
2487
2488 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2489 __ load_klass(tmp, src, tmp_load_klass);
2490 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2491 __ jcc(Assembler::greaterEqual, *stub->entry());
2492 }
2493 }
2494
2495 // check if negative
2496 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2497 __ testl(src_pos, src_pos);
2498 __ jcc(Assembler::less, *stub->entry());
2499 }
2500 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2501 __ testl(dst_pos, dst_pos);
2502 __ jcc(Assembler::less, *stub->entry());
2503 }
2504
2505 if (flags & LIR_OpArrayCopy::src_range_check) {
2506 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2507 __ cmpl(tmp, src_length_addr);
2508 __ jcc(Assembler::above, *stub->entry());
2509 }
2510 if (flags & LIR_OpArrayCopy::dst_range_check) {
2511 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2512 __ cmpl(tmp, dst_length_addr);
2513 __ jcc(Assembler::above, *stub->entry());
2514 }
2515
2516 if (flags & LIR_OpArrayCopy::length_positive_check) {
2517 __ testl(length, length);
2518 __ jcc(Assembler::less, *stub->entry());
2519 }
2520
2521 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2522 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2523
2524 if (flags & LIR_OpArrayCopy::type_check) {
2525 // We don't know the array types are compatible
2526 if (basic_type != T_OBJECT) {
2527 // Simple test for basic type arrays
2528 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2529 __ jcc(Assembler::notEqual, *stub->entry());
2530 } else {
2531 // For object arrays, if src is a sub class of dst then we can
2532 // safely do the copy.
2533 Label cont, slow;
2534
2535 __ push_ppx(src);
2536 __ push_ppx(dst);
2537
2538 __ load_klass(src, src, tmp_load_klass);
2539 __ load_klass(dst, dst, tmp_load_klass);
2540
2541 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2542
2543 __ push_ppx(src);
2544 __ push_ppx(dst);
2545 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2546 __ pop_ppx(dst);
2547 __ pop_ppx(src);
2548
2549 __ testl(src, src);
2550 __ jcc(Assembler::notEqual, cont);
2551
2552 __ bind(slow);
2553 __ pop_ppx(dst);
2554 __ pop_ppx(src);
2555
2556 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2557 if (copyfunc_addr != nullptr) { // use stub if available
2558 // src is not a sub class of dst so we have to do a
2559 // per-element check.
2560
2561 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2562 if ((flags & mask) != mask) {
2563 // Check that at least both of them object arrays.
2564 assert(flags & mask, "one of the two should be known to be an object array");
2565
2566 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2567 __ load_klass(tmp, src, tmp_load_klass);
2568 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2569 __ load_klass(tmp, dst, tmp_load_klass);
2570 }
2571 int lh_offset = in_bytes(Klass::layout_helper_offset());
2572 Address klass_lh_addr(tmp, lh_offset);
2573 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2574 __ cmpl(klass_lh_addr, objArray_lh);
2575 __ jcc(Assembler::notEqual, *stub->entry());
2576 }
2577
2578 // Spill because stubs can use any register they like and it's
2579 // easier to restore just those that we care about.
2580 store_parameter(dst, 0);
2581 store_parameter(dst_pos, 1);
2582 store_parameter(length, 2);
2583 store_parameter(src_pos, 3);
2584 store_parameter(src, 4);
2585
2586 __ movl2ptr(length, length); //higher 32bits must be null
2587
2588 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2589 assert_different_registers(c_rarg0, dst, dst_pos, length);
2590 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2591 assert_different_registers(c_rarg1, dst, length);
2592
2593 __ mov(c_rarg2, length);
2594 assert_different_registers(c_rarg2, dst);
2595
2596 #ifdef _WIN64
2597 // Allocate abi space for args but be sure to keep stack aligned
2598 __ subptr(rsp, 6*wordSize);
2599 __ load_klass(c_rarg3, dst, tmp_load_klass);
2600 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2601 store_parameter(c_rarg3, 4);
2602 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2603 __ call(RuntimeAddress(copyfunc_addr));
2604 __ addptr(rsp, 6*wordSize);
2605 #else
2606 __ load_klass(c_rarg4, dst, tmp_load_klass);
2607 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2608 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2609 __ call(RuntimeAddress(copyfunc_addr));
2610 #endif
2611
2612 #ifndef PRODUCT
2613 if (PrintC1Statistics) {
2614 Label failed;
2615 __ testl(rax, rax);
2616 __ jcc(Assembler::notZero, failed);
2617 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2618 __ bind(failed);
2619 }
2620 #endif
2621
2622 __ testl(rax, rax);
2623 __ jcc(Assembler::zero, *stub->continuation());
2624
2625 #ifndef PRODUCT
2626 if (PrintC1Statistics) {
2627 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2628 }
2629 #endif
2630
2631 __ mov(tmp, rax);
2632
2633 __ xorl(tmp, -1);
2634
2635 // Restore previously spilled arguments
2636 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2637 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2638 __ movptr (length, Address(rsp, 2*BytesPerWord));
2639 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2640 __ movptr (src, Address(rsp, 4*BytesPerWord));
2641
2642
2643 __ subl(length, tmp);
2644 __ addl(src_pos, tmp);
2645 __ addl(dst_pos, tmp);
2646 }
2647
2648 __ jmp(*stub->entry());
2649
2650 __ bind(cont);
2651 __ pop(dst);
2652 __ pop(src);
2653 }
2654 }
2655
2656 #ifdef ASSERT
2657 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2658 // Sanity check the known type with the incoming class. For the
2659 // primitive case the types must match exactly with src.klass and
2660 // dst.klass each exactly matching the default type. For the
2661 // object array case, if no type check is needed then either the
2662 // dst type is exactly the expected type and the src type is a
2663 // subtype which we can't check or src is the same array as dst
2664 // but not necessarily exactly of type default_type.
2665 Label known_ok, halt;
2666 __ mov_metadata(tmp, default_type->constant_encoding());
2667 if (UseCompressedClassPointers) {
2668 __ encode_klass_not_null(tmp, rscratch1);
2669 }
2670
2671 if (basic_type != T_OBJECT) {
2672 __ cmp_klass(tmp, dst, tmp2);
2673 __ jcc(Assembler::notEqual, halt);
2674 __ cmp_klass(tmp, src, tmp2);
2675 __ jcc(Assembler::equal, known_ok);
2676 } else {
2677 __ cmp_klass(tmp, dst, tmp2);
2678 __ jcc(Assembler::equal, known_ok);
2679 __ cmpptr(src, dst);
2680 __ jcc(Assembler::equal, known_ok);
2681 }
2682 __ bind(halt);
2683 __ stop("incorrect type information in arraycopy");
2684 __ bind(known_ok);
2685 }
2686 #endif
2687
2688 #ifndef PRODUCT
2689 if (PrintC1Statistics) {
2690 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2691 }
2692 #endif
2693
2694 assert_different_registers(c_rarg0, dst, dst_pos, length);
2695 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2696 assert_different_registers(c_rarg1, length);
2697 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2698 __ mov(c_rarg2, length);
2699
2700 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2701 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2702 const char *name;
2703 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2704 __ call_VM_leaf(entry, 0);
2705
2706 if (stub != nullptr) {
2707 __ bind(*stub->continuation());
2708 }
2709 }
2710
2711 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2712 assert(op->crc()->is_single_cpu(), "crc must be register");
2713 assert(op->val()->is_single_cpu(), "byte value must be register");
2714 assert(op->result_opr()->is_single_cpu(), "result must be register");
2715 Register crc = op->crc()->as_register();
2716 Register val = op->val()->as_register();
2717 Register res = op->result_opr()->as_register();
2718
2719 assert_different_registers(val, crc, res);
2720
2721 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2722 __ notl(crc); // ~crc
2723 __ update_byte_crc32(crc, val, res);
2724 __ notl(crc); // ~crc
2725 __ mov(res, crc);
2726 }
2727
2728 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2729 Register obj = op->obj_opr()->as_register(); // may not be an oop
2730 Register hdr = op->hdr_opr()->as_register();
2731 Register lock = op->lock_opr()->as_register();
2732 if (op->code() == lir_lock) {
2733 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2734 Register tmp = op->scratch_opr()->as_register();
2735 // add debug info for NullPointerException only if one is possible
2736 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2737 if (op->info() != nullptr) {
2738 add_debug_info_for_null_check(null_check_offset, op->info());
2739 }
2740 // done
2741 } else if (op->code() == lir_unlock) {
2742 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2743 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2744 } else {
2745 Unimplemented();
2746 }
2747 __ bind(*op->stub()->continuation());
2748 }
2749
2750 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2751 Register obj = op->obj()->as_pointer_register();
2752 Register result = op->result_opr()->as_pointer_register();
2753
2754 CodeEmitInfo* info = op->info();
2755 if (info != nullptr) {
2756 add_debug_info_for_null_check_here(info);
2757 }
2758
2759 __ load_klass(result, obj, rscratch1);
2760 }
2761
2762 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2763 ciMethod* method = op->profiled_method();
2764 int bci = op->profiled_bci();
2765 ciMethod* callee = op->profiled_callee();
2766 Register tmp_load_klass = rscratch1;
2767
2768 // Update counter for all call types
2769 ciMethodData* md = method->method_data_or_null();
2770 assert(md != nullptr, "Sanity");
2771 ciProfileData* data = md->bci_to_data(bci);
2772 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2773 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2774 Register mdo = op->mdo()->as_register();
2775 __ mov_metadata(mdo, md->constant_encoding());
2776 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2777 // Perform additional virtual call profiling for invokevirtual and
2778 // invokeinterface bytecodes
2779 if (op->should_profile_receiver_type()) {
2780 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2781 Register recv = op->recv()->as_register();
2782 assert_different_registers(mdo, recv);
2783 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2784 ciKlass* known_klass = op->known_holder();
2785 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2786 // We know the type that will be seen at this call site; we can
2787 // statically update the MethodData* rather than needing to do
2788 // dynamic tests on the receiver type
2789
2790 // NOTE: we should probably put a lock around this search to
2791 // avoid collisions by concurrent compilations
2792 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2793 uint i;
2794 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2795 ciKlass* receiver = vc_data->receiver(i);
2796 if (known_klass->equals(receiver)) {
2797 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2798 __ addptr(data_addr, DataLayout::counter_increment);
2799 return;
2800 }
2801 }
2802
2803 // Receiver type not found in profile data; select an empty slot
2804
2805 // Note that this is less efficient than it should be because it
2806 // always does a write to the receiver part of the
2807 // VirtualCallData rather than just the first time
2808 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2809 ciKlass* receiver = vc_data->receiver(i);
2810 if (receiver == nullptr) {
2811 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2812 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
2813 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2814 __ addptr(data_addr, DataLayout::counter_increment);
2815 return;
2816 }
2817 }
2818 } else {
2819 __ load_klass(recv, recv, tmp_load_klass);
2820 Label update_done;
2821 type_profile_helper(mdo, md, data, recv, &update_done);
2822 // Receiver did not match any saved receiver and there is no empty row for it.
2823 // Increment total counter to indicate polymorphic case.
2824 __ addptr(counter_addr, DataLayout::counter_increment);
2825
2826 __ bind(update_done);
2827 }
2828 } else {
2829 // Static call
2830 __ addptr(counter_addr, DataLayout::counter_increment);
2831 }
2832 }
2833
2834 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2835 Register obj = op->obj()->as_register();
2836 Register tmp = op->tmp()->as_pointer_register();
2837 Register tmp_load_klass = rscratch1;
2838 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2839 ciKlass* exact_klass = op->exact_klass();
2840 intptr_t current_klass = op->current_klass();
2841 bool not_null = op->not_null();
2842 bool no_conflict = op->no_conflict();
2843
2844 Label update, next, none;
2845
2846 bool do_null = !not_null;
2847 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2848 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2849
2850 assert(do_null || do_update, "why are we here?");
2851 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2852
2853 __ verify_oop(obj);
2854
2855 #ifdef ASSERT
2856 if (obj == tmp) {
2857 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2858 } else {
2859 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2860 }
2861 #endif
2862 if (do_null) {
2863 __ testptr(obj, obj);
2864 __ jccb(Assembler::notZero, update);
2865 if (!TypeEntries::was_null_seen(current_klass)) {
2866 __ testptr(mdo_addr, TypeEntries::null_seen);
2867 #ifndef ASSERT
2868 __ jccb(Assembler::notZero, next); // already set
2869 #else
2870 __ jcc(Assembler::notZero, next); // already set
2871 #endif
2872 // atomic update to prevent overwriting Klass* with 0
2873 __ lock();
2874 __ orptr(mdo_addr, TypeEntries::null_seen);
2875 }
2876 if (do_update) {
2877 #ifndef ASSERT
2878 __ jmpb(next);
2879 }
2880 #else
2881 __ jmp(next);
2882 }
2883 } else {
2884 __ testptr(obj, obj);
2885 __ jcc(Assembler::notZero, update);
2886 __ stop("unexpected null obj");
2887 #endif
2888 }
2889
2890 __ bind(update);
2891
2892 if (do_update) {
2893 #ifdef ASSERT
2894 if (exact_klass != nullptr) {
2895 Label ok;
2896 __ load_klass(tmp, obj, tmp_load_klass);
2897 __ push_ppx(tmp);
2898 __ mov_metadata(tmp, exact_klass->constant_encoding());
2899 __ cmpptr(tmp, Address(rsp, 0));
2900 __ jcc(Assembler::equal, ok);
2901 __ stop("exact klass and actual klass differ");
2902 __ bind(ok);
2903 __ pop_ppx(tmp);
2904 }
2905 #endif
2906 if (!no_conflict) {
2907 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2908 if (exact_klass != nullptr) {
2909 __ mov_metadata(tmp, exact_klass->constant_encoding());
2910 } else {
2911 __ load_klass(tmp, obj, tmp_load_klass);
2912 }
2913 __ mov(rscratch1, tmp); // save original value before XOR
2914 __ xorptr(tmp, mdo_addr);
2915 __ testptr(tmp, TypeEntries::type_klass_mask);
2916 // klass seen before, nothing to do. The unknown bit may have been
2917 // set already but no need to check.
2918 __ jccb(Assembler::zero, next);
2919
2920 __ testptr(tmp, TypeEntries::type_unknown);
2921 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2922
2923 if (TypeEntries::is_type_none(current_klass)) {
2924 __ testptr(mdo_addr, TypeEntries::type_mask);
2925 __ jccb(Assembler::zero, none);
2926 // There is a chance that the checks above (re-reading profiling
2927 // data from memory) fail if another thread has just set the
2928 // profiling to this obj's klass
2929 __ mov(tmp, rscratch1); // get back original value before XOR
2930 __ xorptr(tmp, mdo_addr);
2931 __ testptr(tmp, TypeEntries::type_klass_mask);
2932 __ jccb(Assembler::zero, next);
2933 }
2934 } else {
2935 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2936 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2937
2938 __ testptr(mdo_addr, TypeEntries::type_unknown);
2939 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2940 }
2941
2942 // different than before. Cannot keep accurate profile.
2943 __ orptr(mdo_addr, TypeEntries::type_unknown);
2944
2945 if (TypeEntries::is_type_none(current_klass)) {
2946 __ jmpb(next);
2947
2948 __ bind(none);
2949 // first time here. Set profile type.
2950 __ movptr(mdo_addr, tmp);
2951 #ifdef ASSERT
2952 __ andptr(tmp, TypeEntries::type_klass_mask);
2953 __ verify_klass_ptr(tmp);
2954 #endif
2955 }
2956 } else {
2957 // There's a single possible klass at this profile point
2958 assert(exact_klass != nullptr, "should be");
2959 if (TypeEntries::is_type_none(current_klass)) {
2960 __ mov_metadata(tmp, exact_klass->constant_encoding());
2961 __ xorptr(tmp, mdo_addr);
2962 __ testptr(tmp, TypeEntries::type_klass_mask);
2963 #ifdef ASSERT
2964 __ jcc(Assembler::zero, next);
2965
2966 {
2967 Label ok;
2968 __ push_ppx(tmp);
2969 __ testptr(mdo_addr, TypeEntries::type_mask);
2970 __ jcc(Assembler::zero, ok);
2971 // may have been set by another thread
2972 __ mov_metadata(tmp, exact_klass->constant_encoding());
2973 __ xorptr(tmp, mdo_addr);
2974 __ testptr(tmp, TypeEntries::type_mask);
2975 __ jcc(Assembler::zero, ok);
2976
2977 __ stop("unexpected profiling mismatch");
2978 __ bind(ok);
2979 __ pop_ppx(tmp);
2980 }
2981 #else
2982 __ jccb(Assembler::zero, next);
2983 #endif
2984 // first time here. Set profile type.
2985 __ movptr(mdo_addr, tmp);
2986 #ifdef ASSERT
2987 __ andptr(tmp, TypeEntries::type_klass_mask);
2988 __ verify_klass_ptr(tmp);
2989 #endif
2990 } else {
2991 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2992 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2993
2994 __ testptr(mdo_addr, TypeEntries::type_unknown);
2995 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2996
2997 __ orptr(mdo_addr, TypeEntries::type_unknown);
2998 }
2999 }
3000 }
3001 __ bind(next);
3002 }
3003
3004 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3005 Unimplemented();
3006 }
3007
3008
3009 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3010 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3011 }
3012
3013
3014 void LIR_Assembler::align_backward_branch_target() {
3015 __ align(BytesPerWord);
3016 }
3017
3018
3019 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3020 if (left->is_single_cpu()) {
3021 __ negl(left->as_register());
3022 move_regs(left->as_register(), dest->as_register());
3023
3024 } else if (left->is_double_cpu()) {
3025 Register lo = left->as_register_lo();
3026 Register dst = dest->as_register_lo();
3027 __ movptr(dst, lo);
3028 __ negptr(dst);
3029
3030 } else if (dest->is_single_xmm()) {
3031 assert(!tmp->is_valid(), "do not need temporary");
3032 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3033 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3034 }
3035 __ xorps(dest->as_xmm_float_reg(),
3036 ExternalAddress((address)float_signflip_pool),
3037 rscratch1);
3038 } else if (dest->is_double_xmm()) {
3039 assert(!tmp->is_valid(), "do not need temporary");
3040 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3041 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3042 }
3043 __ xorpd(dest->as_xmm_double_reg(),
3044 ExternalAddress((address)double_signflip_pool),
3045 rscratch1);
3046 } else {
3047 ShouldNotReachHere();
3048 }
3049 }
3050
3051
3052 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3053 assert(src->is_address(), "must be an address");
3054 assert(dest->is_register(), "must be a register");
3055
3056 PatchingStub* patch = nullptr;
3057 if (patch_code != lir_patch_none) {
3058 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3059 }
3060
3061 Register reg = dest->as_pointer_register();
3062 LIR_Address* addr = src->as_address_ptr();
3063 __ lea(reg, as_Address(addr));
3064
3065 if (patch != nullptr) {
3066 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3067 }
3068 }
3069
3070
3071
3072 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3073 assert(!tmp->is_valid(), "don't need temporary");
3074 __ call(RuntimeAddress(dest));
3075 if (info != nullptr) {
3076 add_call_info_here(info);
3077 }
3078 __ post_call_nop();
3079 }
3080
3081
3082 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3083 assert(type == T_LONG, "only for volatile long fields");
3084
3085 if (info != nullptr) {
3086 add_debug_info_for_null_check_here(info);
3087 }
3088
3089 if (src->is_double_xmm()) {
3090 if (dest->is_double_cpu()) {
3091 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3092 } else if (dest->is_double_stack()) {
3093 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3094 } else if (dest->is_address()) {
3095 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3096 } else {
3097 ShouldNotReachHere();
3098 }
3099
3100 } else if (dest->is_double_xmm()) {
3101 if (src->is_double_stack()) {
3102 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3103 } else if (src->is_address()) {
3104 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3105 } else {
3106 ShouldNotReachHere();
3107 }
3108
3109 } else {
3110 ShouldNotReachHere();
3111 }
3112 }
3113
3114 #ifdef ASSERT
3115 // emit run-time assertion
3116 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3117 assert(op->code() == lir_assert, "must be");
3118
3119 if (op->in_opr1()->is_valid()) {
3120 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3121 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3122 } else {
3123 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3124 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3125 }
3126
3127 Label ok;
3128 if (op->condition() != lir_cond_always) {
3129 Assembler::Condition acond = Assembler::zero;
3130 switch (op->condition()) {
3131 case lir_cond_equal: acond = Assembler::equal; break;
3132 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3133 case lir_cond_less: acond = Assembler::less; break;
3134 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3135 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3136 case lir_cond_greater: acond = Assembler::greater; break;
3137 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3138 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3139 default: ShouldNotReachHere();
3140 }
3141 __ jcc(acond, ok);
3142 }
3143 if (op->halt()) {
3144 const char* str = __ code_string(op->msg());
3145 __ stop(str);
3146 } else {
3147 breakpoint();
3148 }
3149 __ bind(ok);
3150 }
3151 #endif
3152
3153 void LIR_Assembler::membar() {
3154 // QQQ sparc TSO uses this,
3155 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3156 }
3157
3158 void LIR_Assembler::membar_acquire() {
3159 // No x86 machines currently require load fences
3160 }
3161
3162 void LIR_Assembler::membar_release() {
3163 // No x86 machines currently require store fences
3164 }
3165
3166 void LIR_Assembler::membar_loadload() {
3167 // no-op
3168 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3169 }
3170
3171 void LIR_Assembler::membar_storestore() {
3172 // no-op
3173 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3174 }
3175
3176 void LIR_Assembler::membar_loadstore() {
3177 // no-op
3178 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3179 }
3180
3181 void LIR_Assembler::membar_storeload() {
3182 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3183 }
3184
3185 void LIR_Assembler::on_spin_wait() {
3186 __ pause ();
3187 }
3188
3189 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3190 assert(result_reg->is_register(), "check");
3191 __ mov(result_reg->as_register(), r15_thread);
3192 }
3193
3194
3195 void LIR_Assembler::peephole(LIR_List*) {
3196 // do nothing for now
3197 }
3198
3199 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3200 assert(data == dest, "xchg/xadd uses only 2 operands");
3201
3202 if (data->type() == T_INT) {
3203 if (code == lir_xadd) {
3204 __ lock();
3205 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3206 } else {
3207 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3208 }
3209 } else if (data->is_oop()) {
3210 assert (code == lir_xchg, "xadd for oops");
3211 Register obj = data->as_register();
3212 if (UseCompressedOops) {
3213 __ encode_heap_oop(obj);
3214 __ xchgl(obj, as_Address(src->as_address_ptr()));
3215 __ decode_heap_oop(obj);
3216 } else {
3217 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3218 }
3219 } else if (data->type() == T_LONG) {
3220 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3221 if (code == lir_xadd) {
3222 __ lock();
3223 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3224 } else {
3225 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3226 }
3227 } else {
3228 ShouldNotReachHere();
3229 }
3230 }
3231
3232 #undef __