1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "ci/ciUtilities.hpp"
36 #include "code/aotCodeCache.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
419 if (LockingMode == LM_MONITOR) {
420 __ jmp(*stub->entry());
421 } else {
422 __ unlock_object(rdi, rsi, rax, *stub->entry());
423 }
424 __ bind(*stub->continuation());
425 }
426
427 if (compilation()->env()->dtrace_method_probes()) {
428 __ mov(rdi, r15_thread);
429 __ mov_metadata(rsi, method()->constant_encoding());
430 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
431 }
432
433 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
434 __ mov(rax, rbx); // Restore the exception
435 }
436
437 // remove the activation and dispatch to the unwind handler
438 __ remove_frame(initial_frame_size_in_bytes());
439 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
440
441 // Emit the slow path assembly
442 if (stub != nullptr) {
443 stub->emit_code(this);
444 }
445
446 return offset;
447 }
448
449
450 int LIR_Assembler::emit_deopt_handler() {
451 // generate code for exception handler
452 address handler_base = __ start_a_stub(deopt_handler_size());
453 if (handler_base == nullptr) {
454 // not enough space left for the handler
455 bailout("deopt handler overflow");
456 return -1;
457 }
458
459 int offset = code_offset();
460 InternalAddress here(__ pc());
461
462 __ pushptr(here.addr(), rscratch1);
463 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
464 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
465 __ end_a_stub();
466
467 return offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475
476 // Pop the stack before the safepoint code
477 __ remove_frame(initial_frame_size_in_bytes());
478
479 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
480 __ reserved_stack_check();
481 }
482
483 // Note: we do not need to round double result; float result has the right precision
484 // the poll sets the condition code, but no data registers
485
486 code_stub->set_safepoint_offset(__ offset());
487 __ relocate(relocInfo::poll_return_type);
488 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
489 __ ret(0);
490 }
491
492
493 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
494 guarantee(info != nullptr, "Shouldn't be null");
495 int offset = __ offset();
496 const Register poll_addr = rscratch1;
497 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
498 add_debug_info_for_branch(info);
499 __ relocate(relocInfo::poll_type);
500 address pre_pc = __ pc();
501 __ testl(rax, Address(poll_addr, 0));
502 address post_pc = __ pc();
503 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
504 return offset;
505 }
506
507
508 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
509 if (from_reg != to_reg) __ mov(to_reg, from_reg);
510 }
511
512 void LIR_Assembler::swap_reg(Register a, Register b) {
513 __ xchgptr(a, b);
514 }
515
516
517 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
518 assert(src->is_constant(), "should not call otherwise");
519 assert(dest->is_register(), "should not call otherwise");
520 LIR_Const* c = src->as_constant_ptr();
521
522 switch (c->type()) {
523 case T_INT: {
524 assert(patch_code == lir_patch_none, "no patching handled here");
525 __ movl(dest->as_register(), c->as_jint());
526 break;
527 }
528
529 case T_ADDRESS: {
530 assert(patch_code == lir_patch_none, "no patching handled here");
531 __ movptr(dest->as_register(), c->as_jint());
532 break;
533 }
534
535 case T_LONG: {
536 assert(patch_code == lir_patch_none, "no patching handled here");
537 if (AOTCodeCache::is_on_for_dump()) {
538 // AOTCodeCache needs relocation info for card table base
539 address b = c->as_pointer();
540 if (is_card_table_address(b)) {
541 __ lea(dest->as_register_lo(), ExternalAddress(b));
542 break;
543 }
544 if (b == (address)ThreadIdentifier::unsafe_offset()) {
545 __ lea(dest->as_register_lo(), ExternalAddress(b));
546 break;
547 }
548 #if INCLUDE_CDS
549 if (AOTRuntimeConstants::contains(b)) {
550 __ load_aotrc_address(dest->as_register_lo(), b);
551 break;
552 }
553 #endif
554 }
555 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
556 break;
557 }
558
559 case T_OBJECT: {
560 if (patch_code != lir_patch_none) {
561 jobject2reg_with_patching(dest->as_register(), info);
562 } else {
563 __ movoop(dest->as_register(), c->as_jobject());
564 }
565 break;
566 }
567
568 case T_METADATA: {
569 if (patch_code != lir_patch_none) {
570 klass2reg_with_patching(dest->as_register(), info);
571 } else {
572 __ mov_metadata(dest->as_register(), c->as_metadata());
573 }
574 break;
575 }
576
577 case T_FLOAT: {
578 if (dest->is_single_xmm()) {
579 if (UseAVX <= 2 && c->is_zero_float()) {
580 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
581 } else {
582 __ movflt(dest->as_xmm_float_reg(),
583 InternalAddress(float_constant(c->as_jfloat())));
584 }
585 } else {
586 ShouldNotReachHere();
587 }
588 break;
589 }
590
591 case T_DOUBLE: {
592 if (dest->is_double_xmm()) {
593 if (UseAVX <= 2 && c->is_zero_double()) {
594 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
595 } else {
596 __ movdbl(dest->as_xmm_double_reg(),
597 InternalAddress(double_constant(c->as_jdouble())));
598 }
599 } else {
600 ShouldNotReachHere();
601 }
602 break;
603 }
604
605 default:
606 ShouldNotReachHere();
607 }
608 }
609
610 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
611 assert(src->is_constant(), "should not call otherwise");
612 assert(dest->is_stack(), "should not call otherwise");
613 LIR_Const* c = src->as_constant_ptr();
614
615 switch (c->type()) {
616 case T_INT: // fall through
617 case T_FLOAT:
618 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
619 break;
620
621 case T_ADDRESS:
622 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
623 break;
624
625 case T_OBJECT:
626 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
627 break;
628
629 case T_LONG: // fall through
630 case T_DOUBLE:
631 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
632 lo_word_offset_in_bytes),
633 (intptr_t)c->as_jlong_bits(),
634 rscratch1);
635 break;
636
637 default:
638 ShouldNotReachHere();
639 }
640 }
641
642 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
643 assert(src->is_constant(), "should not call otherwise");
644 assert(dest->is_address(), "should not call otherwise");
645 LIR_Const* c = src->as_constant_ptr();
646 LIR_Address* addr = dest->as_address_ptr();
647
648 int null_check_here = code_offset();
649 switch (type) {
650 case T_INT: // fall through
651 case T_FLOAT:
652 __ movl(as_Address(addr), c->as_jint_bits());
653 break;
654
655 case T_ADDRESS:
656 __ movptr(as_Address(addr), c->as_jint_bits());
657 break;
658
659 case T_OBJECT: // fall through
660 case T_ARRAY:
661 if (c->as_jobject() == nullptr) {
662 if (UseCompressedOops && !wide) {
663 __ movl(as_Address(addr), NULL_WORD);
664 } else {
665 __ xorptr(rscratch1, rscratch1);
666 null_check_here = code_offset();
667 __ movptr(as_Address(addr), rscratch1);
668 }
669 } else {
670 if (is_literal_address(addr)) {
671 ShouldNotReachHere();
672 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
673 } else {
674 __ movoop(rscratch1, c->as_jobject());
675 if (UseCompressedOops && !wide) {
676 __ encode_heap_oop(rscratch1);
677 null_check_here = code_offset();
678 __ movl(as_Address_lo(addr), rscratch1);
679 } else {
680 null_check_here = code_offset();
681 __ movptr(as_Address_lo(addr), rscratch1);
682 }
683 }
684 }
685 break;
686
687 case T_LONG: // fall through
688 case T_DOUBLE:
689 if (is_literal_address(addr)) {
690 ShouldNotReachHere();
691 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
692 } else {
693 __ movptr(r10, (intptr_t)c->as_jlong_bits());
694 null_check_here = code_offset();
695 __ movptr(as_Address_lo(addr), r10);
696 }
697 break;
698
699 case T_BOOLEAN: // fall through
700 case T_BYTE:
701 __ movb(as_Address(addr), c->as_jint() & 0xFF);
702 break;
703
704 case T_CHAR: // fall through
705 case T_SHORT:
706 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
707 break;
708
709 default:
710 ShouldNotReachHere();
711 };
712
713 if (info != nullptr) {
714 add_debug_info_for_null_check(null_check_here, info);
715 }
716 }
717
718
719 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
720 assert(src->is_register(), "should not call otherwise");
721 assert(dest->is_register(), "should not call otherwise");
722
723 // move between cpu-registers
724 if (dest->is_single_cpu()) {
725 if (src->type() == T_LONG) {
726 // Can do LONG -> OBJECT
727 move_regs(src->as_register_lo(), dest->as_register());
728 return;
729 }
730 assert(src->is_single_cpu(), "must match");
731 if (src->type() == T_OBJECT) {
732 __ verify_oop(src->as_register());
733 }
734 move_regs(src->as_register(), dest->as_register());
735
736 } else if (dest->is_double_cpu()) {
737 if (is_reference_type(src->type())) {
738 // Surprising to me but we can see move of a long to t_object
739 __ verify_oop(src->as_register());
740 move_regs(src->as_register(), dest->as_register_lo());
741 return;
742 }
743 assert(src->is_double_cpu(), "must match");
744 Register f_lo = src->as_register_lo();
745 Register f_hi = src->as_register_hi();
746 Register t_lo = dest->as_register_lo();
747 Register t_hi = dest->as_register_hi();
748 assert(f_hi == f_lo, "must be same");
749 assert(t_hi == t_lo, "must be same");
750 move_regs(f_lo, t_lo);
751
752 // move between xmm-registers
753 } else if (dest->is_single_xmm()) {
754 assert(src->is_single_xmm(), "must match");
755 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
756 } else if (dest->is_double_xmm()) {
757 assert(src->is_double_xmm(), "must match");
758 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
759
760 } else {
761 ShouldNotReachHere();
762 }
763 }
764
765 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
766 assert(src->is_register(), "should not call otherwise");
767 assert(dest->is_stack(), "should not call otherwise");
768
769 if (src->is_single_cpu()) {
770 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
771 if (is_reference_type(type)) {
772 __ verify_oop(src->as_register());
773 __ movptr (dst, src->as_register());
774 } else if (type == T_METADATA || type == T_ADDRESS) {
775 __ movptr (dst, src->as_register());
776 } else {
777 __ movl (dst, src->as_register());
778 }
779
780 } else if (src->is_double_cpu()) {
781 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
782 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
783 __ movptr (dstLO, src->as_register_lo());
784
785 } else if (src->is_single_xmm()) {
786 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
787 __ movflt(dst_addr, src->as_xmm_float_reg());
788
789 } else if (src->is_double_xmm()) {
790 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
791 __ movdbl(dst_addr, src->as_xmm_double_reg());
792
793 } else {
794 ShouldNotReachHere();
795 }
796 }
797
798
799 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
800 LIR_Address* to_addr = dest->as_address_ptr();
801 PatchingStub* patch = nullptr;
802 Register compressed_src = rscratch1;
803
804 if (is_reference_type(type)) {
805 __ verify_oop(src->as_register());
806 if (UseCompressedOops && !wide) {
807 __ movptr(compressed_src, src->as_register());
808 __ encode_heap_oop(compressed_src);
809 if (patch_code != lir_patch_none) {
810 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
811 }
812 }
813 }
814
815 if (patch_code != lir_patch_none) {
816 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
817 Address toa = as_Address(to_addr);
818 assert(toa.disp() != 0, "must have");
819 }
820
821 int null_check_here = code_offset();
822 switch (type) {
823 case T_FLOAT: {
824 assert(src->is_single_xmm(), "not a float");
825 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
826 break;
827 }
828
829 case T_DOUBLE: {
830 assert(src->is_double_xmm(), "not a double");
831 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
832 break;
833 }
834
835 case T_ARRAY: // fall through
836 case T_OBJECT: // fall through
837 if (UseCompressedOops && !wide) {
838 __ movl(as_Address(to_addr), compressed_src);
839 } else {
840 __ movptr(as_Address(to_addr), src->as_register());
841 }
842 break;
843 case T_ADDRESS:
844 __ movptr(as_Address(to_addr), src->as_register());
845 break;
846 case T_INT:
847 __ movl(as_Address(to_addr), src->as_register());
848 break;
849
850 case T_LONG: {
851 Register from_lo = src->as_register_lo();
852 Register from_hi = src->as_register_hi();
853 __ movptr(as_Address_lo(to_addr), from_lo);
854 break;
855 }
856
857 case T_BYTE: // fall through
858 case T_BOOLEAN: {
859 Register src_reg = src->as_register();
860 Address dst_addr = as_Address(to_addr);
861 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
862 __ movb(dst_addr, src_reg);
863 break;
864 }
865
866 case T_CHAR: // fall through
867 case T_SHORT:
868 __ movw(as_Address(to_addr), src->as_register());
869 break;
870
871 default:
872 ShouldNotReachHere();
873 }
874 if (info != nullptr) {
875 add_debug_info_for_null_check(null_check_here, info);
876 }
877
878 if (patch_code != lir_patch_none) {
879 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
880 }
881 }
882
883
884 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
885 assert(src->is_stack(), "should not call otherwise");
886 assert(dest->is_register(), "should not call otherwise");
887
888 if (dest->is_single_cpu()) {
889 if (is_reference_type(type)) {
890 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
891 __ verify_oop(dest->as_register());
892 } else if (type == T_METADATA || type == T_ADDRESS) {
893 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
894 } else {
895 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
896 }
897
898 } else if (dest->is_double_cpu()) {
899 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
900 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
901 __ movptr(dest->as_register_lo(), src_addr_LO);
902
903 } else if (dest->is_single_xmm()) {
904 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
905 __ movflt(dest->as_xmm_float_reg(), src_addr);
906
907 } else if (dest->is_double_xmm()) {
908 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
909 __ movdbl(dest->as_xmm_double_reg(), src_addr);
910
911 } else {
912 ShouldNotReachHere();
913 }
914 }
915
916
917 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
918 if (src->is_single_stack()) {
919 if (is_reference_type(type)) {
920 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
921 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
922 } else {
923 //no pushl on 64bits
924 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
925 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
926 }
927
928 } else if (src->is_double_stack()) {
929 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
930 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
931
932 } else {
933 ShouldNotReachHere();
934 }
935 }
936
937
938 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
939 assert(src->is_address(), "should not call otherwise");
940 assert(dest->is_register(), "should not call otherwise");
941
942 LIR_Address* addr = src->as_address_ptr();
943 Address from_addr = as_Address(addr);
944
945 if (addr->base()->type() == T_OBJECT) {
946 __ verify_oop(addr->base()->as_pointer_register());
947 }
948
949 switch (type) {
950 case T_BOOLEAN: // fall through
951 case T_BYTE: // fall through
952 case T_CHAR: // fall through
953 case T_SHORT:
954 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
955 // on pre P6 processors we may get partial register stalls
956 // so blow away the value of to_rinfo before loading a
957 // partial word into it. Do it here so that it precedes
958 // the potential patch point below.
959 __ xorptr(dest->as_register(), dest->as_register());
960 }
961 break;
962 default:
963 break;
964 }
965
966 PatchingStub* patch = nullptr;
967 if (patch_code != lir_patch_none) {
968 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
969 assert(from_addr.disp() != 0, "must have");
970 }
971 if (info != nullptr) {
972 add_debug_info_for_null_check_here(info);
973 }
974
975 switch (type) {
976 case T_FLOAT: {
977 if (dest->is_single_xmm()) {
978 __ movflt(dest->as_xmm_float_reg(), from_addr);
979 } else {
980 ShouldNotReachHere();
981 }
982 break;
983 }
984
985 case T_DOUBLE: {
986 if (dest->is_double_xmm()) {
987 __ movdbl(dest->as_xmm_double_reg(), from_addr);
988 } else {
989 ShouldNotReachHere();
990 }
991 break;
992 }
993
994 case T_OBJECT: // fall through
995 case T_ARRAY: // fall through
996 if (UseCompressedOops && !wide) {
997 __ movl(dest->as_register(), from_addr);
998 } else {
999 __ movptr(dest->as_register(), from_addr);
1000 }
1001 break;
1002
1003 case T_ADDRESS:
1004 __ movptr(dest->as_register(), from_addr);
1005 break;
1006 case T_INT:
1007 __ movl(dest->as_register(), from_addr);
1008 break;
1009
1010 case T_LONG: {
1011 Register to_lo = dest->as_register_lo();
1012 Register to_hi = dest->as_register_hi();
1013 __ movptr(to_lo, as_Address_lo(addr));
1014 break;
1015 }
1016
1017 case T_BOOLEAN: // fall through
1018 case T_BYTE: {
1019 Register dest_reg = dest->as_register();
1020 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1021 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1022 __ movsbl(dest_reg, from_addr);
1023 } else {
1024 __ movb(dest_reg, from_addr);
1025 __ shll(dest_reg, 24);
1026 __ sarl(dest_reg, 24);
1027 }
1028 break;
1029 }
1030
1031 case T_CHAR: {
1032 Register dest_reg = dest->as_register();
1033 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1034 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1035 __ movzwl(dest_reg, from_addr);
1036 } else {
1037 __ movw(dest_reg, from_addr);
1038 }
1039 break;
1040 }
1041
1042 case T_SHORT: {
1043 Register dest_reg = dest->as_register();
1044 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1045 __ movswl(dest_reg, from_addr);
1046 } else {
1047 __ movw(dest_reg, from_addr);
1048 __ shll(dest_reg, 16);
1049 __ sarl(dest_reg, 16);
1050 }
1051 break;
1052 }
1053
1054 default:
1055 ShouldNotReachHere();
1056 }
1057
1058 if (patch != nullptr) {
1059 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1060 }
1061
1062 if (is_reference_type(type)) {
1063 if (UseCompressedOops && !wide) {
1064 __ decode_heap_oop(dest->as_register());
1065 }
1066
1067 __ verify_oop(dest->as_register());
1068 }
1069 }
1070
1071
1072 NEEDS_CLEANUP; // This could be static?
1073 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1074 int elem_size = type2aelembytes(type);
1075 switch (elem_size) {
1076 case 1: return Address::times_1;
1077 case 2: return Address::times_2;
1078 case 4: return Address::times_4;
1079 case 8: return Address::times_8;
1080 }
1081 ShouldNotReachHere();
1082 return Address::no_scale;
1083 }
1084
1085
1086 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1087 switch (op->code()) {
1088 case lir_idiv:
1089 case lir_irem:
1090 arithmetic_idiv(op->code(),
1091 op->in_opr1(),
1092 op->in_opr2(),
1093 op->in_opr3(),
1094 op->result_opr(),
1095 op->info());
1096 break;
1097 case lir_fmad:
1098 __ fmad(op->result_opr()->as_xmm_double_reg(),
1099 op->in_opr1()->as_xmm_double_reg(),
1100 op->in_opr2()->as_xmm_double_reg(),
1101 op->in_opr3()->as_xmm_double_reg());
1102 break;
1103 case lir_fmaf:
1104 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1105 op->in_opr1()->as_xmm_float_reg(),
1106 op->in_opr2()->as_xmm_float_reg(),
1107 op->in_opr3()->as_xmm_float_reg());
1108 break;
1109 default: ShouldNotReachHere(); break;
1110 }
1111 }
1112
1113 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1114 #ifdef ASSERT
1115 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1116 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1117 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1118 #endif
1119
1120 if (op->cond() == lir_cond_always) {
1121 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1122 __ jmp (*(op->label()));
1123 } else {
1124 Assembler::Condition acond = Assembler::zero;
1125 if (op->code() == lir_cond_float_branch) {
1126 assert(op->ublock() != nullptr, "must have unordered successor");
1127 __ jcc(Assembler::parity, *(op->ublock()->label()));
1128 switch(op->cond()) {
1129 case lir_cond_equal: acond = Assembler::equal; break;
1130 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1131 case lir_cond_less: acond = Assembler::below; break;
1132 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1133 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1134 case lir_cond_greater: acond = Assembler::above; break;
1135 default: ShouldNotReachHere();
1136 }
1137 } else {
1138 switch (op->cond()) {
1139 case lir_cond_equal: acond = Assembler::equal; break;
1140 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1141 case lir_cond_less: acond = Assembler::less; break;
1142 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1143 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1144 case lir_cond_greater: acond = Assembler::greater; break;
1145 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1146 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1147 default: ShouldNotReachHere();
1148 }
1149 }
1150 __ jcc(acond,*(op->label()));
1151 }
1152 }
1153
1154 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1155 LIR_Opr src = op->in_opr();
1156 LIR_Opr dest = op->result_opr();
1157
1158 switch (op->bytecode()) {
1159 case Bytecodes::_i2l:
1160 __ movl2ptr(dest->as_register_lo(), src->as_register());
1161 break;
1162
1163 case Bytecodes::_l2i:
1164 __ movl(dest->as_register(), src->as_register_lo());
1165 break;
1166
1167 case Bytecodes::_i2b:
1168 move_regs(src->as_register(), dest->as_register());
1169 __ sign_extend_byte(dest->as_register());
1170 break;
1171
1172 case Bytecodes::_i2c:
1173 move_regs(src->as_register(), dest->as_register());
1174 __ andl(dest->as_register(), 0xFFFF);
1175 break;
1176
1177 case Bytecodes::_i2s:
1178 move_regs(src->as_register(), dest->as_register());
1179 __ sign_extend_short(dest->as_register());
1180 break;
1181
1182 case Bytecodes::_f2d:
1183 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1184 break;
1185
1186 case Bytecodes::_d2f:
1187 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1188 break;
1189
1190 case Bytecodes::_i2f:
1191 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1192 break;
1193
1194 case Bytecodes::_i2d:
1195 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1196 break;
1197
1198 case Bytecodes::_l2f:
1199 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1200 break;
1201
1202 case Bytecodes::_l2d:
1203 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1204 break;
1205
1206 case Bytecodes::_f2i:
1207 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1208 break;
1209
1210 case Bytecodes::_d2i:
1211 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1212 break;
1213
1214 case Bytecodes::_f2l:
1215 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1216 break;
1217
1218 case Bytecodes::_d2l:
1219 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1220 break;
1221
1222 default: ShouldNotReachHere();
1223 }
1224 }
1225
1226 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1227 if (op->init_check()) {
1228 add_debug_info_for_null_check_here(op->stub()->info());
1229 // init_state needs acquire, but x86 is TSO, and so we are already good.
1230 __ cmpb(Address(op->klass()->as_register(),
1231 InstanceKlass::init_state_offset()),
1232 InstanceKlass::fully_initialized);
1233 __ jcc(Assembler::notEqual, *op->stub()->entry());
1234 }
1235 __ allocate_object(op->obj()->as_register(),
1236 op->tmp1()->as_register(),
1237 op->tmp2()->as_register(),
1238 op->header_size(),
1239 op->object_size(),
1240 op->klass()->as_register(),
1241 *op->stub()->entry());
1242 __ bind(*op->stub()->continuation());
1243 }
1244
1245 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1246 Register len = op->len()->as_register();
1247 __ movslq(len, len);
1248
1249 if (UseSlowPath ||
1250 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1251 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1252 __ jmp(*op->stub()->entry());
1253 } else {
1254 Register tmp1 = op->tmp1()->as_register();
1255 Register tmp2 = op->tmp2()->as_register();
1256 Register tmp3 = op->tmp3()->as_register();
1257 if (len == tmp1) {
1258 tmp1 = tmp3;
1259 } else if (len == tmp2) {
1260 tmp2 = tmp3;
1261 } else if (len == tmp3) {
1262 // everything is ok
1263 } else {
1264 __ mov(tmp3, len);
1265 }
1266 __ allocate_array(op->obj()->as_register(),
1267 len,
1268 tmp1,
1269 tmp2,
1270 arrayOopDesc::base_offset_in_bytes(op->type()),
1271 array_element_size(op->type()),
1272 op->klass()->as_register(),
1273 *op->stub()->entry(),
1274 op->zero_array());
1275 }
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::type_profile_helper(Register mdo,
1280 ciMethodData *md, ciProfileData *data,
1281 Register recv, Label* update_done) {
1282 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1283 Label next_test;
1284 // See if the receiver is receiver[n].
1285 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1286 __ jccb(Assembler::notEqual, next_test);
1287 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1288 __ addptr(data_addr, DataLayout::counter_increment);
1289 __ jmp(*update_done);
1290 __ bind(next_test);
1291 }
1292
1293 // Didn't find receiver; find next empty slot and fill it in
1294 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1295 Label next_test;
1296 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1297 __ cmpptr(recv_addr, NULL_WORD);
1298 __ jccb(Assembler::notEqual, next_test);
1299 __ movptr(recv_addr, recv);
1300 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1301 __ jmp(*update_done);
1302 __ bind(next_test);
1303 }
1304 }
1305
1306 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1307 // we always need a stub for the failure case.
1308 CodeStub* stub = op->stub();
1309 Register obj = op->object()->as_register();
1310 Register k_RInfo = op->tmp1()->as_register();
1311 Register klass_RInfo = op->tmp2()->as_register();
1312 Register dst = op->result_opr()->as_register();
1313 ciKlass* k = op->klass();
1314 Register Rtmp1 = noreg;
1315 Register tmp_load_klass = rscratch1;
1316
1317 // check if it needs to be profiled
1318 ciMethodData* md = nullptr;
1319 ciProfileData* data = nullptr;
1320
1321 if (op->should_profile()) {
1322 ciMethod* method = op->profiled_method();
1323 assert(method != nullptr, "Should have method");
1324 int bci = op->profiled_bci();
1325 md = method->method_data_or_null();
1326 assert(md != nullptr, "Sanity");
1327 data = md->bci_to_data(bci);
1328 assert(data != nullptr, "need data for type check");
1329 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1330 }
1331 Label* success_target = success;
1332 Label* failure_target = failure;
1333
1334 if (obj == k_RInfo) {
1335 k_RInfo = dst;
1336 } else if (obj == klass_RInfo) {
1337 klass_RInfo = dst;
1338 }
1339 if (k->is_loaded() && !UseCompressedClassPointers) {
1340 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1341 } else {
1342 Rtmp1 = op->tmp3()->as_register();
1343 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1344 }
1345
1346 assert_different_registers(obj, k_RInfo, klass_RInfo);
1347
1348 __ testptr(obj, obj);
1349 if (op->should_profile()) {
1350 Label not_null;
1351 Register mdo = klass_RInfo;
1352 __ mov_metadata(mdo, md->constant_encoding());
1353 __ jccb(Assembler::notEqual, not_null);
1354 // Object is null; update MDO and exit
1355 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1356 int header_bits = BitData::null_seen_byte_constant();
1357 __ orb(data_addr, header_bits);
1358 __ jmp(*obj_is_null);
1359 __ bind(not_null);
1360
1361 Label update_done;
1362 Register recv = k_RInfo;
1363 __ load_klass(recv, obj, tmp_load_klass);
1364 type_profile_helper(mdo, md, data, recv, &update_done);
1365
1366 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1367 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1368
1369 __ bind(update_done);
1370 } else {
1371 __ jcc(Assembler::equal, *obj_is_null);
1372 }
1373
1374 if (!k->is_loaded()) {
1375 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1376 } else {
1377 __ mov_metadata(k_RInfo, k->constant_encoding());
1378 }
1379 __ verify_oop(obj);
1380
1381 if (op->fast_check()) {
1382 // get object class
1383 // not a safepoint as obj null check happens earlier
1384 if (UseCompressedClassPointers) {
1385 __ load_klass(Rtmp1, obj, tmp_load_klass);
1386 __ cmpptr(k_RInfo, Rtmp1);
1387 } else {
1388 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1389 }
1390 __ jcc(Assembler::notEqual, *failure_target);
1391 // successful cast, fall through to profile or jump
1392 } else {
1393 // get object class
1394 // not a safepoint as obj null check happens earlier
1395 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1396 if (k->is_loaded()) {
1397 // See if we get an immediate positive hit
1398 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1399 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1400 __ jcc(Assembler::notEqual, *failure_target);
1401 // successful cast, fall through to profile or jump
1402 } else {
1403 // See if we get an immediate positive hit
1404 __ jcc(Assembler::equal, *success_target);
1405 // check for self
1406 __ cmpptr(klass_RInfo, k_RInfo);
1407 __ jcc(Assembler::equal, *success_target);
1408
1409 __ push(klass_RInfo);
1410 __ push(k_RInfo);
1411 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1412 __ pop(klass_RInfo);
1413 __ pop(klass_RInfo);
1414 // result is a boolean
1415 __ testl(klass_RInfo, klass_RInfo);
1416 __ jcc(Assembler::equal, *failure_target);
1417 // successful cast, fall through to profile or jump
1418 }
1419 } else {
1420 // perform the fast part of the checking logic
1421 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1422 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1423 __ push(klass_RInfo);
1424 __ push(k_RInfo);
1425 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1426 __ pop(klass_RInfo);
1427 __ pop(k_RInfo);
1428 // result is a boolean
1429 __ testl(k_RInfo, k_RInfo);
1430 __ jcc(Assembler::equal, *failure_target);
1431 // successful cast, fall through to profile or jump
1432 }
1433 }
1434 __ jmp(*success);
1435 }
1436
1437
1438 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1439 Register tmp_load_klass = rscratch1;
1440 LIR_Code code = op->code();
1441 if (code == lir_store_check) {
1442 Register value = op->object()->as_register();
1443 Register array = op->array()->as_register();
1444 Register k_RInfo = op->tmp1()->as_register();
1445 Register klass_RInfo = op->tmp2()->as_register();
1446 Register Rtmp1 = op->tmp3()->as_register();
1447
1448 CodeStub* stub = op->stub();
1449
1450 // check if it needs to be profiled
1451 ciMethodData* md = nullptr;
1452 ciProfileData* data = nullptr;
1453
1454 if (op->should_profile()) {
1455 ciMethod* method = op->profiled_method();
1456 assert(method != nullptr, "Should have method");
1457 int bci = op->profiled_bci();
1458 md = method->method_data_or_null();
1459 assert(md != nullptr, "Sanity");
1460 data = md->bci_to_data(bci);
1461 assert(data != nullptr, "need data for type check");
1462 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1463 }
1464 Label done;
1465 Label* success_target = &done;
1466 Label* failure_target = stub->entry();
1467
1468 __ testptr(value, value);
1469 if (op->should_profile()) {
1470 Label not_null;
1471 Register mdo = klass_RInfo;
1472 __ mov_metadata(mdo, md->constant_encoding());
1473 __ jccb(Assembler::notEqual, not_null);
1474 // Object is null; update MDO and exit
1475 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1476 int header_bits = BitData::null_seen_byte_constant();
1477 __ orb(data_addr, header_bits);
1478 __ jmp(done);
1479 __ bind(not_null);
1480
1481 Label update_done;
1482 Register recv = k_RInfo;
1483 __ load_klass(recv, value, tmp_load_klass);
1484 type_profile_helper(mdo, md, data, recv, &update_done);
1485
1486 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1487 __ addptr(counter_addr, DataLayout::counter_increment);
1488 __ bind(update_done);
1489 } else {
1490 __ jcc(Assembler::equal, done);
1491 }
1492
1493 add_debug_info_for_null_check_here(op->info_for_exception());
1494 __ load_klass(k_RInfo, array, tmp_load_klass);
1495 __ load_klass(klass_RInfo, value, tmp_load_klass);
1496
1497 // get instance klass (it's already uncompressed)
1498 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1499 // perform the fast part of the checking logic
1500 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1501 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1502 __ push(klass_RInfo);
1503 __ push(k_RInfo);
1504 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1505 __ pop(klass_RInfo);
1506 __ pop(k_RInfo);
1507 // result is a boolean
1508 __ testl(k_RInfo, k_RInfo);
1509 __ jcc(Assembler::equal, *failure_target);
1510 // fall through to the success case
1511
1512 __ bind(done);
1513 } else
1514 if (code == lir_checkcast) {
1515 Register obj = op->object()->as_register();
1516 Register dst = op->result_opr()->as_register();
1517 Label success;
1518 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1519 __ bind(success);
1520 if (dst != obj) {
1521 __ mov(dst, obj);
1522 }
1523 } else
1524 if (code == lir_instanceof) {
1525 Register obj = op->object()->as_register();
1526 Register dst = op->result_opr()->as_register();
1527 Label success, failure, done;
1528 emit_typecheck_helper(op, &success, &failure, &failure);
1529 __ bind(failure);
1530 __ xorptr(dst, dst);
1531 __ jmpb(done);
1532 __ bind(success);
1533 __ movptr(dst, 1);
1534 __ bind(done);
1535 } else {
1536 ShouldNotReachHere();
1537 }
1538
1539 }
1540
1541
1542 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1543 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1544 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1545 Register newval = op->new_value()->as_register();
1546 Register cmpval = op->cmp_value()->as_register();
1547 assert(cmpval == rax, "wrong register");
1548 assert(newval != noreg, "new val must be register");
1549 assert(cmpval != newval, "cmp and new values must be in different registers");
1550 assert(cmpval != addr, "cmp and addr must be in different registers");
1551 assert(newval != addr, "new value and addr must be in different registers");
1552
1553 if (op->code() == lir_cas_obj) {
1554 if (UseCompressedOops) {
1555 __ encode_heap_oop(cmpval);
1556 __ mov(rscratch1, newval);
1557 __ encode_heap_oop(rscratch1);
1558 __ lock();
1559 // cmpval (rax) is implicitly used by this instruction
1560 __ cmpxchgl(rscratch1, Address(addr, 0));
1561 } else {
1562 __ lock();
1563 __ cmpxchgptr(newval, Address(addr, 0));
1564 }
1565 } else {
1566 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1567 __ lock();
1568 __ cmpxchgl(newval, Address(addr, 0));
1569 }
1570 } else if (op->code() == lir_cas_long) {
1571 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1572 Register newval = op->new_value()->as_register_lo();
1573 Register cmpval = op->cmp_value()->as_register_lo();
1574 assert(cmpval == rax, "wrong register");
1575 assert(newval != noreg, "new val must be register");
1576 assert(cmpval != newval, "cmp and new values must be in different registers");
1577 assert(cmpval != addr, "cmp and addr must be in different registers");
1578 assert(newval != addr, "new value and addr must be in different registers");
1579 __ lock();
1580 __ cmpxchgq(newval, Address(addr, 0));
1581 } else {
1582 Unimplemented();
1583 }
1584 }
1585
1586 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1587 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1588 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1589
1590 Assembler::Condition acond, ncond;
1591 switch (condition) {
1592 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1593 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1594 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1595 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1596 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1597 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1598 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1599 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1600 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1601 ShouldNotReachHere();
1602 }
1603
1604 if (opr1->is_cpu_register()) {
1605 reg2reg(opr1, result);
1606 } else if (opr1->is_stack()) {
1607 stack2reg(opr1, result, result->type());
1608 } else if (opr1->is_constant()) {
1609 const2reg(opr1, result, lir_patch_none, nullptr);
1610 } else {
1611 ShouldNotReachHere();
1612 }
1613
1614 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1615 // optimized version that does not require a branch
1616 if (opr2->is_single_cpu()) {
1617 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1618 __ cmov(ncond, result->as_register(), opr2->as_register());
1619 } else if (opr2->is_double_cpu()) {
1620 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1621 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1622 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1623 } else if (opr2->is_single_stack()) {
1624 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1625 } else if (opr2->is_double_stack()) {
1626 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1627 } else {
1628 ShouldNotReachHere();
1629 }
1630
1631 } else {
1632 Label skip;
1633 __ jccb(acond, skip);
1634 if (opr2->is_cpu_register()) {
1635 reg2reg(opr2, result);
1636 } else if (opr2->is_stack()) {
1637 stack2reg(opr2, result, result->type());
1638 } else if (opr2->is_constant()) {
1639 const2reg(opr2, result, lir_patch_none, nullptr);
1640 } else {
1641 ShouldNotReachHere();
1642 }
1643 __ bind(skip);
1644 }
1645 }
1646
1647
1648 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1649 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1650
1651 if (left->is_single_cpu()) {
1652 assert(left == dest, "left and dest must be equal");
1653 Register lreg = left->as_register();
1654
1655 if (right->is_single_cpu()) {
1656 // cpu register - cpu register
1657 Register rreg = right->as_register();
1658 switch (code) {
1659 case lir_add: __ addl (lreg, rreg); break;
1660 case lir_sub: __ subl (lreg, rreg); break;
1661 case lir_mul: __ imull(lreg, rreg); break;
1662 default: ShouldNotReachHere();
1663 }
1664
1665 } else if (right->is_stack()) {
1666 // cpu register - stack
1667 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1668 switch (code) {
1669 case lir_add: __ addl(lreg, raddr); break;
1670 case lir_sub: __ subl(lreg, raddr); break;
1671 default: ShouldNotReachHere();
1672 }
1673
1674 } else if (right->is_constant()) {
1675 // cpu register - constant
1676 jint c = right->as_constant_ptr()->as_jint();
1677 switch (code) {
1678 case lir_add: {
1679 __ incrementl(lreg, c);
1680 break;
1681 }
1682 case lir_sub: {
1683 __ decrementl(lreg, c);
1684 break;
1685 }
1686 default: ShouldNotReachHere();
1687 }
1688
1689 } else {
1690 ShouldNotReachHere();
1691 }
1692
1693 } else if (left->is_double_cpu()) {
1694 assert(left == dest, "left and dest must be equal");
1695 Register lreg_lo = left->as_register_lo();
1696 Register lreg_hi = left->as_register_hi();
1697
1698 if (right->is_double_cpu()) {
1699 // cpu register - cpu register
1700 Register rreg_lo = right->as_register_lo();
1701 Register rreg_hi = right->as_register_hi();
1702 assert_different_registers(lreg_lo, rreg_lo);
1703 switch (code) {
1704 case lir_add:
1705 __ addptr(lreg_lo, rreg_lo);
1706 break;
1707 case lir_sub:
1708 __ subptr(lreg_lo, rreg_lo);
1709 break;
1710 case lir_mul:
1711 __ imulq(lreg_lo, rreg_lo);
1712 break;
1713 default:
1714 ShouldNotReachHere();
1715 }
1716
1717 } else if (right->is_constant()) {
1718 // cpu register - constant
1719 jlong c = right->as_constant_ptr()->as_jlong_bits();
1720 __ movptr(r10, (intptr_t) c);
1721 switch (code) {
1722 case lir_add:
1723 __ addptr(lreg_lo, r10);
1724 break;
1725 case lir_sub:
1726 __ subptr(lreg_lo, r10);
1727 break;
1728 default:
1729 ShouldNotReachHere();
1730 }
1731
1732 } else {
1733 ShouldNotReachHere();
1734 }
1735
1736 } else if (left->is_single_xmm()) {
1737 assert(left == dest, "left and dest must be equal");
1738 XMMRegister lreg = left->as_xmm_float_reg();
1739
1740 if (right->is_single_xmm()) {
1741 XMMRegister rreg = right->as_xmm_float_reg();
1742 switch (code) {
1743 case lir_add: __ addss(lreg, rreg); break;
1744 case lir_sub: __ subss(lreg, rreg); break;
1745 case lir_mul: __ mulss(lreg, rreg); break;
1746 case lir_div: __ divss(lreg, rreg); break;
1747 default: ShouldNotReachHere();
1748 }
1749 } else {
1750 Address raddr;
1751 if (right->is_single_stack()) {
1752 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1753 } else if (right->is_constant()) {
1754 // hack for now
1755 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1756 } else {
1757 ShouldNotReachHere();
1758 }
1759 switch (code) {
1760 case lir_add: __ addss(lreg, raddr); break;
1761 case lir_sub: __ subss(lreg, raddr); break;
1762 case lir_mul: __ mulss(lreg, raddr); break;
1763 case lir_div: __ divss(lreg, raddr); break;
1764 default: ShouldNotReachHere();
1765 }
1766 }
1767
1768 } else if (left->is_double_xmm()) {
1769 assert(left == dest, "left and dest must be equal");
1770
1771 XMMRegister lreg = left->as_xmm_double_reg();
1772 if (right->is_double_xmm()) {
1773 XMMRegister rreg = right->as_xmm_double_reg();
1774 switch (code) {
1775 case lir_add: __ addsd(lreg, rreg); break;
1776 case lir_sub: __ subsd(lreg, rreg); break;
1777 case lir_mul: __ mulsd(lreg, rreg); break;
1778 case lir_div: __ divsd(lreg, rreg); break;
1779 default: ShouldNotReachHere();
1780 }
1781 } else {
1782 Address raddr;
1783 if (right->is_double_stack()) {
1784 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1785 } else if (right->is_constant()) {
1786 // hack for now
1787 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1788 } else {
1789 ShouldNotReachHere();
1790 }
1791 switch (code) {
1792 case lir_add: __ addsd(lreg, raddr); break;
1793 case lir_sub: __ subsd(lreg, raddr); break;
1794 case lir_mul: __ mulsd(lreg, raddr); break;
1795 case lir_div: __ divsd(lreg, raddr); break;
1796 default: ShouldNotReachHere();
1797 }
1798 }
1799
1800 } else if (left->is_single_stack() || left->is_address()) {
1801 assert(left == dest, "left and dest must be equal");
1802
1803 Address laddr;
1804 if (left->is_single_stack()) {
1805 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1806 } else if (left->is_address()) {
1807 laddr = as_Address(left->as_address_ptr());
1808 } else {
1809 ShouldNotReachHere();
1810 }
1811
1812 if (right->is_single_cpu()) {
1813 Register rreg = right->as_register();
1814 switch (code) {
1815 case lir_add: __ addl(laddr, rreg); break;
1816 case lir_sub: __ subl(laddr, rreg); break;
1817 default: ShouldNotReachHere();
1818 }
1819 } else if (right->is_constant()) {
1820 jint c = right->as_constant_ptr()->as_jint();
1821 switch (code) {
1822 case lir_add: {
1823 __ incrementl(laddr, c);
1824 break;
1825 }
1826 case lir_sub: {
1827 __ decrementl(laddr, c);
1828 break;
1829 }
1830 default: ShouldNotReachHere();
1831 }
1832 } else {
1833 ShouldNotReachHere();
1834 }
1835
1836 } else {
1837 ShouldNotReachHere();
1838 }
1839 }
1840
1841
1842 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1843 if (value->is_double_xmm()) {
1844 switch(code) {
1845 case lir_abs :
1846 {
1847 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1848 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1849 }
1850 assert(!tmp->is_valid(), "do not need temporary");
1851 __ andpd(dest->as_xmm_double_reg(),
1852 ExternalAddress(LIR_Assembler::double_signmask_pool),
1853 rscratch1);
1854 }
1855 break;
1856
1857 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1858 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1859 default : ShouldNotReachHere();
1860 }
1861
1862 } else if (code == lir_f2hf) {
1863 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1864 } else if (code == lir_hf2f) {
1865 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1866 } else {
1867 Unimplemented();
1868 }
1869 }
1870
1871 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1872 // assert(left->destroys_register(), "check");
1873 if (left->is_single_cpu()) {
1874 Register reg = left->as_register();
1875 if (right->is_constant()) {
1876 int val = right->as_constant_ptr()->as_jint();
1877 switch (code) {
1878 case lir_logic_and: __ andl (reg, val); break;
1879 case lir_logic_or: __ orl (reg, val); break;
1880 case lir_logic_xor: __ xorl (reg, val); break;
1881 default: ShouldNotReachHere();
1882 }
1883 } else if (right->is_stack()) {
1884 // added support for stack operands
1885 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1886 switch (code) {
1887 case lir_logic_and: __ andl (reg, raddr); break;
1888 case lir_logic_or: __ orl (reg, raddr); break;
1889 case lir_logic_xor: __ xorl (reg, raddr); break;
1890 default: ShouldNotReachHere();
1891 }
1892 } else {
1893 Register rright = right->as_register();
1894 switch (code) {
1895 case lir_logic_and: __ andptr (reg, rright); break;
1896 case lir_logic_or : __ orptr (reg, rright); break;
1897 case lir_logic_xor: __ xorptr (reg, rright); break;
1898 default: ShouldNotReachHere();
1899 }
1900 }
1901 move_regs(reg, dst->as_register());
1902 } else {
1903 Register l_lo = left->as_register_lo();
1904 Register l_hi = left->as_register_hi();
1905 if (right->is_constant()) {
1906 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1907 switch (code) {
1908 case lir_logic_and:
1909 __ andq(l_lo, rscratch1);
1910 break;
1911 case lir_logic_or:
1912 __ orq(l_lo, rscratch1);
1913 break;
1914 case lir_logic_xor:
1915 __ xorq(l_lo, rscratch1);
1916 break;
1917 default: ShouldNotReachHere();
1918 }
1919 } else {
1920 Register r_lo;
1921 if (is_reference_type(right->type())) {
1922 r_lo = right->as_register();
1923 } else {
1924 r_lo = right->as_register_lo();
1925 }
1926 switch (code) {
1927 case lir_logic_and:
1928 __ andptr(l_lo, r_lo);
1929 break;
1930 case lir_logic_or:
1931 __ orptr(l_lo, r_lo);
1932 break;
1933 case lir_logic_xor:
1934 __ xorptr(l_lo, r_lo);
1935 break;
1936 default: ShouldNotReachHere();
1937 }
1938 }
1939
1940 Register dst_lo = dst->as_register_lo();
1941 Register dst_hi = dst->as_register_hi();
1942
1943 move_regs(l_lo, dst_lo);
1944 }
1945 }
1946
1947
1948 // we assume that rax, and rdx can be overwritten
1949 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1950
1951 assert(left->is_single_cpu(), "left must be register");
1952 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1953 assert(result->is_single_cpu(), "result must be register");
1954
1955 // assert(left->destroys_register(), "check");
1956 // assert(right->destroys_register(), "check");
1957
1958 Register lreg = left->as_register();
1959 Register dreg = result->as_register();
1960
1961 if (right->is_constant()) {
1962 jint divisor = right->as_constant_ptr()->as_jint();
1963 assert(divisor > 0 && is_power_of_2(divisor), "must be");
1964 if (code == lir_idiv) {
1965 assert(lreg == rax, "must be rax,");
1966 assert(temp->as_register() == rdx, "tmp register must be rdx");
1967 __ cdql(); // sign extend into rdx:rax
1968 if (divisor == 2) {
1969 __ subl(lreg, rdx);
1970 } else {
1971 __ andl(rdx, divisor - 1);
1972 __ addl(lreg, rdx);
1973 }
1974 __ sarl(lreg, log2i_exact(divisor));
1975 move_regs(lreg, dreg);
1976 } else if (code == lir_irem) {
1977 Label done;
1978 __ mov(dreg, lreg);
1979 __ andl(dreg, 0x80000000 | (divisor - 1));
1980 __ jcc(Assembler::positive, done);
1981 __ decrement(dreg);
1982 __ orl(dreg, ~(divisor - 1));
1983 __ increment(dreg);
1984 __ bind(done);
1985 } else {
1986 ShouldNotReachHere();
1987 }
1988 } else {
1989 Register rreg = right->as_register();
1990 assert(lreg == rax, "left register must be rax,");
1991 assert(rreg != rdx, "right register must not be rdx");
1992 assert(temp->as_register() == rdx, "tmp register must be rdx");
1993
1994 move_regs(lreg, rax);
1995
1996 int idivl_offset = __ corrected_idivl(rreg);
1997 if (ImplicitDiv0Checks) {
1998 add_debug_info_for_div0(idivl_offset, info);
1999 }
2000 if (code == lir_irem) {
2001 move_regs(rdx, dreg); // result is in rdx
2002 } else {
2003 move_regs(rax, dreg);
2004 }
2005 }
2006 }
2007
2008
2009 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2010 if (opr1->is_single_cpu()) {
2011 Register reg1 = opr1->as_register();
2012 if (opr2->is_single_cpu()) {
2013 // cpu register - cpu register
2014 if (is_reference_type(opr1->type())) {
2015 __ cmpoop(reg1, opr2->as_register());
2016 } else {
2017 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2018 __ cmpl(reg1, opr2->as_register());
2019 }
2020 } else if (opr2->is_stack()) {
2021 // cpu register - stack
2022 if (is_reference_type(opr1->type())) {
2023 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2024 } else {
2025 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2026 }
2027 } else if (opr2->is_constant()) {
2028 // cpu register - constant
2029 LIR_Const* c = opr2->as_constant_ptr();
2030 if (c->type() == T_INT) {
2031 jint i = c->as_jint();
2032 if (i == 0) {
2033 __ testl(reg1, reg1);
2034 } else {
2035 __ cmpl(reg1, i);
2036 }
2037 } else if (c->type() == T_METADATA) {
2038 // All we need for now is a comparison with null for equality.
2039 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2040 Metadata* m = c->as_metadata();
2041 if (m == nullptr) {
2042 __ testptr(reg1, reg1);
2043 } else {
2044 ShouldNotReachHere();
2045 }
2046 } else if (is_reference_type(c->type())) {
2047 // In 64bit oops are single register
2048 jobject o = c->as_jobject();
2049 if (o == nullptr) {
2050 __ testptr(reg1, reg1);
2051 } else {
2052 __ cmpoop(reg1, o, rscratch1);
2053 }
2054 } else {
2055 fatal("unexpected type: %s", basictype_to_str(c->type()));
2056 }
2057 // cpu register - address
2058 } else if (opr2->is_address()) {
2059 if (op->info() != nullptr) {
2060 add_debug_info_for_null_check_here(op->info());
2061 }
2062 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2063 } else {
2064 ShouldNotReachHere();
2065 }
2066
2067 } else if(opr1->is_double_cpu()) {
2068 Register xlo = opr1->as_register_lo();
2069 Register xhi = opr1->as_register_hi();
2070 if (opr2->is_double_cpu()) {
2071 __ cmpptr(xlo, opr2->as_register_lo());
2072 } else if (opr2->is_constant()) {
2073 // cpu register - constant 0
2074 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2075 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2076 } else {
2077 ShouldNotReachHere();
2078 }
2079
2080 } else if (opr1->is_single_xmm()) {
2081 XMMRegister reg1 = opr1->as_xmm_float_reg();
2082 if (opr2->is_single_xmm()) {
2083 // xmm register - xmm register
2084 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2085 } else if (opr2->is_stack()) {
2086 // xmm register - stack
2087 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2088 } else if (opr2->is_constant()) {
2089 // xmm register - constant
2090 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2091 } else if (opr2->is_address()) {
2092 // xmm register - address
2093 if (op->info() != nullptr) {
2094 add_debug_info_for_null_check_here(op->info());
2095 }
2096 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2097 } else {
2098 ShouldNotReachHere();
2099 }
2100
2101 } else if (opr1->is_double_xmm()) {
2102 XMMRegister reg1 = opr1->as_xmm_double_reg();
2103 if (opr2->is_double_xmm()) {
2104 // xmm register - xmm register
2105 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2106 } else if (opr2->is_stack()) {
2107 // xmm register - stack
2108 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2109 } else if (opr2->is_constant()) {
2110 // xmm register - constant
2111 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2112 } else if (opr2->is_address()) {
2113 // xmm register - address
2114 if (op->info() != nullptr) {
2115 add_debug_info_for_null_check_here(op->info());
2116 }
2117 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2118 } else {
2119 ShouldNotReachHere();
2120 }
2121
2122 } else if (opr1->is_address() && opr2->is_constant()) {
2123 LIR_Const* c = opr2->as_constant_ptr();
2124 if (is_reference_type(c->type())) {
2125 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2126 __ movoop(rscratch1, c->as_jobject());
2127 }
2128 if (op->info() != nullptr) {
2129 add_debug_info_for_null_check_here(op->info());
2130 }
2131 // special case: address - constant
2132 LIR_Address* addr = opr1->as_address_ptr();
2133 if (c->type() == T_INT) {
2134 __ cmpl(as_Address(addr), c->as_jint());
2135 } else if (is_reference_type(c->type())) {
2136 // %%% Make this explode if addr isn't reachable until we figure out a
2137 // better strategy by giving noreg as the temp for as_Address
2138 __ cmpoop(rscratch1, as_Address(addr, noreg));
2139 } else {
2140 ShouldNotReachHere();
2141 }
2142
2143 } else {
2144 ShouldNotReachHere();
2145 }
2146 }
2147
2148 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2149 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2150 if (left->is_single_xmm()) {
2151 assert(right->is_single_xmm(), "must match");
2152 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2153 } else if (left->is_double_xmm()) {
2154 assert(right->is_double_xmm(), "must match");
2155 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2156
2157 } else {
2158 ShouldNotReachHere();
2159 }
2160 } else {
2161 assert(code == lir_cmp_l2i, "check");
2162 Label done;
2163 Register dest = dst->as_register();
2164 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2165 __ movl(dest, -1);
2166 __ jccb(Assembler::less, done);
2167 __ setb(Assembler::notZero, dest);
2168 __ movzbl(dest, dest);
2169 __ bind(done);
2170 }
2171 }
2172
2173
2174 void LIR_Assembler::align_call(LIR_Code code) {
2175 // make sure that the displacement word of the call ends up word aligned
2176 int offset = __ offset();
2177 switch (code) {
2178 case lir_static_call:
2179 case lir_optvirtual_call:
2180 case lir_dynamic_call:
2181 offset += NativeCall::displacement_offset;
2182 break;
2183 case lir_icvirtual_call:
2184 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2185 break;
2186 default: ShouldNotReachHere();
2187 }
2188 __ align(BytesPerWord, offset);
2189 }
2190
2191
2192 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2193 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2194 "must be aligned");
2195 __ call(AddressLiteral(op->addr(), rtype));
2196 add_call_info(code_offset(), op->info());
2197 __ post_call_nop();
2198 }
2199
2200
2201 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2202 __ ic_call(op->addr());
2203 add_call_info(code_offset(), op->info());
2204 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2205 "must be aligned");
2206 __ post_call_nop();
2207 }
2208
2209
2210 void LIR_Assembler::emit_static_call_stub() {
2211 address call_pc = __ pc();
2212 address stub = __ start_a_stub(call_stub_size());
2213 if (stub == nullptr) {
2214 bailout("static call stub overflow");
2215 return;
2216 }
2217
2218 int start = __ offset();
2219
2220 // make sure that the displacement word of the call ends up word aligned
2221 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2222 __ relocate(static_stub_Relocation::spec(call_pc));
2223 __ mov_metadata(rbx, (Metadata*)nullptr);
2224 // must be set to -1 at code generation time
2225 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2226 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2227 __ jump(RuntimeAddress(__ pc()));
2228
2229 assert(__ offset() - start <= call_stub_size(), "stub too big");
2230 __ end_a_stub();
2231 }
2232
2233
2234 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2235 assert(exceptionOop->as_register() == rax, "must match");
2236 assert(exceptionPC->as_register() == rdx, "must match");
2237
2238 // exception object is not added to oop map by LinearScan
2239 // (LinearScan assumes that no oops are in fixed registers)
2240 info->add_register_oop(exceptionOop);
2241 C1StubId unwind_id;
2242
2243 // get current pc information
2244 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2245 int pc_for_athrow_offset = __ offset();
2246 InternalAddress pc_for_athrow(__ pc());
2247 __ lea(exceptionPC->as_register(), pc_for_athrow);
2248 add_call_info(pc_for_athrow_offset, info); // for exception handler
2249
2250 __ verify_not_null_oop(rax);
2251 // search an exception handler (rax: exception oop, rdx: throwing pc)
2252 if (compilation()->has_fpu_code()) {
2253 unwind_id = C1StubId::handle_exception_id;
2254 } else {
2255 unwind_id = C1StubId::handle_exception_nofpu_id;
2256 }
2257 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2258
2259 // enough room for two byte trap
2260 __ nop();
2261 }
2262
2263
2264 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2265 assert(exceptionOop->as_register() == rax, "must match");
2266
2267 __ jmp(_unwind_handler_entry);
2268 }
2269
2270
2271 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2272
2273 // optimized version for linear scan:
2274 // * count must be already in ECX (guaranteed by LinearScan)
2275 // * left and dest must be equal
2276 // * tmp must be unused
2277 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2278 assert(left == dest, "left and dest must be equal");
2279 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2280
2281 if (left->is_single_cpu()) {
2282 Register value = left->as_register();
2283 assert(value != SHIFT_count, "left cannot be ECX");
2284
2285 switch (code) {
2286 case lir_shl: __ shll(value); break;
2287 case lir_shr: __ sarl(value); break;
2288 case lir_ushr: __ shrl(value); break;
2289 default: ShouldNotReachHere();
2290 }
2291 } else if (left->is_double_cpu()) {
2292 Register lo = left->as_register_lo();
2293 Register hi = left->as_register_hi();
2294 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2295 switch (code) {
2296 case lir_shl: __ shlptr(lo); break;
2297 case lir_shr: __ sarptr(lo); break;
2298 case lir_ushr: __ shrptr(lo); break;
2299 default: ShouldNotReachHere();
2300 }
2301 } else {
2302 ShouldNotReachHere();
2303 }
2304 }
2305
2306
2307 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2308 if (dest->is_single_cpu()) {
2309 // first move left into dest so that left is not destroyed by the shift
2310 Register value = dest->as_register();
2311 count = count & 0x1F; // Java spec
2312
2313 move_regs(left->as_register(), value);
2314 switch (code) {
2315 case lir_shl: __ shll(value, count); break;
2316 case lir_shr: __ sarl(value, count); break;
2317 case lir_ushr: __ shrl(value, count); break;
2318 default: ShouldNotReachHere();
2319 }
2320 } else if (dest->is_double_cpu()) {
2321 // first move left into dest so that left is not destroyed by the shift
2322 Register value = dest->as_register_lo();
2323 count = count & 0x1F; // Java spec
2324
2325 move_regs(left->as_register_lo(), value);
2326 switch (code) {
2327 case lir_shl: __ shlptr(value, count); break;
2328 case lir_shr: __ sarptr(value, count); break;
2329 case lir_ushr: __ shrptr(value, count); break;
2330 default: ShouldNotReachHere();
2331 }
2332 } else {
2333 ShouldNotReachHere();
2334 }
2335 }
2336
2337
2338 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2339 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2340 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2341 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2342 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2343 }
2344
2345
2346 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2347 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2348 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2349 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2350 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2351 }
2352
2353
2354 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2355 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2356 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2357 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2358 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2359 }
2360
2361
2362 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2363 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2364 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2365 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2366 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2367 }
2368
2369
2370 // This code replaces a call to arraycopy; no exception may
2371 // be thrown in this code, they must be thrown in the System.arraycopy
2372 // activation frame; we could save some checks if this would not be the case
2373 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2374 ciArrayKlass* default_type = op->expected_type();
2375 Register src = op->src()->as_register();
2376 Register dst = op->dst()->as_register();
2377 Register src_pos = op->src_pos()->as_register();
2378 Register dst_pos = op->dst_pos()->as_register();
2379 Register length = op->length()->as_register();
2380 Register tmp = op->tmp()->as_register();
2381 Register tmp_load_klass = rscratch1;
2382 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2383
2384 CodeStub* stub = op->stub();
2385 int flags = op->flags();
2386 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2387 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2388
2389 // if we don't know anything, just go through the generic arraycopy
2390 if (default_type == nullptr) {
2391 // save outgoing arguments on stack in case call to System.arraycopy is needed
2392 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2393 // for interpreter calling conventions. Now we have to do it in new style conventions.
2394 // For the moment until C1 gets the new register allocator I just force all the
2395 // args to the right place (except the register args) and then on the back side
2396 // reload the register args properly if we go slow path. Yuck
2397
2398 // These are proper for the calling convention
2399 store_parameter(length, 2);
2400 store_parameter(dst_pos, 1);
2401 store_parameter(dst, 0);
2402
2403 // these are just temporary placements until we need to reload
2404 store_parameter(src_pos, 3);
2405 store_parameter(src, 4);
2406
2407 address copyfunc_addr = StubRoutines::generic_arraycopy();
2408 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2409
2410 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2411 // The arguments are in java calling convention so we can trivially shift them to C
2412 // convention
2413 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2414 __ mov(c_rarg0, j_rarg0);
2415 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2416 __ mov(c_rarg1, j_rarg1);
2417 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2418 __ mov(c_rarg2, j_rarg2);
2419 assert_different_registers(c_rarg3, j_rarg4);
2420 __ mov(c_rarg3, j_rarg3);
2421 #ifdef _WIN64
2422 // Allocate abi space for args but be sure to keep stack aligned
2423 __ subptr(rsp, 6*wordSize);
2424 store_parameter(j_rarg4, 4);
2425 #ifndef PRODUCT
2426 if (PrintC1Statistics) {
2427 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2428 }
2429 #endif
2430 __ call(RuntimeAddress(copyfunc_addr));
2431 __ addptr(rsp, 6*wordSize);
2432 #else
2433 __ mov(c_rarg4, j_rarg4);
2434 #ifndef PRODUCT
2435 if (PrintC1Statistics) {
2436 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2437 }
2438 #endif
2439 __ call(RuntimeAddress(copyfunc_addr));
2440 #endif // _WIN64
2441
2442 __ testl(rax, rax);
2443 __ jcc(Assembler::equal, *stub->continuation());
2444
2445 __ mov(tmp, rax);
2446 __ xorl(tmp, -1);
2447
2448 // Reload values from the stack so they are where the stub
2449 // expects them.
2450 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2451 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2452 __ movptr (length, Address(rsp, 2*BytesPerWord));
2453 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2454 __ movptr (src, Address(rsp, 4*BytesPerWord));
2455
2456 __ subl(length, tmp);
2457 __ addl(src_pos, tmp);
2458 __ addl(dst_pos, tmp);
2459 __ jmp(*stub->entry());
2460
2461 __ bind(*stub->continuation());
2462 return;
2463 }
2464
2465 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2466
2467 int elem_size = type2aelembytes(basic_type);
2468 Address::ScaleFactor scale;
2469
2470 switch (elem_size) {
2471 case 1 :
2472 scale = Address::times_1;
2473 break;
2474 case 2 :
2475 scale = Address::times_2;
2476 break;
2477 case 4 :
2478 scale = Address::times_4;
2479 break;
2480 case 8 :
2481 scale = Address::times_8;
2482 break;
2483 default:
2484 scale = Address::no_scale;
2485 ShouldNotReachHere();
2486 }
2487
2488 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2489 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2490
2491 // length and pos's are all sign extended at this point on 64bit
2492
2493 // test for null
2494 if (flags & LIR_OpArrayCopy::src_null_check) {
2495 __ testptr(src, src);
2496 __ jcc(Assembler::zero, *stub->entry());
2497 }
2498 if (flags & LIR_OpArrayCopy::dst_null_check) {
2499 __ testptr(dst, dst);
2500 __ jcc(Assembler::zero, *stub->entry());
2501 }
2502
2503 // If the compiler was not able to prove that exact type of the source or the destination
2504 // of the arraycopy is an array type, check at runtime if the source or the destination is
2505 // an instance type.
2506 if (flags & LIR_OpArrayCopy::type_check) {
2507 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2508 __ load_klass(tmp, dst, tmp_load_klass);
2509 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2510 __ jcc(Assembler::greaterEqual, *stub->entry());
2511 }
2512
2513 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2514 __ load_klass(tmp, src, tmp_load_klass);
2515 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2516 __ jcc(Assembler::greaterEqual, *stub->entry());
2517 }
2518 }
2519
2520 // check if negative
2521 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2522 __ testl(src_pos, src_pos);
2523 __ jcc(Assembler::less, *stub->entry());
2524 }
2525 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2526 __ testl(dst_pos, dst_pos);
2527 __ jcc(Assembler::less, *stub->entry());
2528 }
2529
2530 if (flags & LIR_OpArrayCopy::src_range_check) {
2531 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2532 __ cmpl(tmp, src_length_addr);
2533 __ jcc(Assembler::above, *stub->entry());
2534 }
2535 if (flags & LIR_OpArrayCopy::dst_range_check) {
2536 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2537 __ cmpl(tmp, dst_length_addr);
2538 __ jcc(Assembler::above, *stub->entry());
2539 }
2540
2541 if (flags & LIR_OpArrayCopy::length_positive_check) {
2542 __ testl(length, length);
2543 __ jcc(Assembler::less, *stub->entry());
2544 }
2545
2546 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2547 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2548
2549 if (flags & LIR_OpArrayCopy::type_check) {
2550 // We don't know the array types are compatible
2551 if (basic_type != T_OBJECT) {
2552 // Simple test for basic type arrays
2553 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2554 __ jcc(Assembler::notEqual, *stub->entry());
2555 } else {
2556 // For object arrays, if src is a sub class of dst then we can
2557 // safely do the copy.
2558 Label cont, slow;
2559
2560 __ push(src);
2561 __ push(dst);
2562
2563 __ load_klass(src, src, tmp_load_klass);
2564 __ load_klass(dst, dst, tmp_load_klass);
2565
2566 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2567
2568 __ push(src);
2569 __ push(dst);
2570 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
2571 __ pop(dst);
2572 __ pop(src);
2573
2574 __ testl(src, src);
2575 __ jcc(Assembler::notEqual, cont);
2576
2577 __ bind(slow);
2578 __ pop(dst);
2579 __ pop(src);
2580
2581 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2582 if (copyfunc_addr != nullptr) { // use stub if available
2583 // src is not a sub class of dst so we have to do a
2584 // per-element check.
2585
2586 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2587 if ((flags & mask) != mask) {
2588 // Check that at least both of them object arrays.
2589 assert(flags & mask, "one of the two should be known to be an object array");
2590
2591 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2592 __ load_klass(tmp, src, tmp_load_klass);
2593 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2594 __ load_klass(tmp, dst, tmp_load_klass);
2595 }
2596 int lh_offset = in_bytes(Klass::layout_helper_offset());
2597 Address klass_lh_addr(tmp, lh_offset);
2598 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2599 __ cmpl(klass_lh_addr, objArray_lh);
2600 __ jcc(Assembler::notEqual, *stub->entry());
2601 }
2602
2603 // Spill because stubs can use any register they like and it's
2604 // easier to restore just those that we care about.
2605 store_parameter(dst, 0);
2606 store_parameter(dst_pos, 1);
2607 store_parameter(length, 2);
2608 store_parameter(src_pos, 3);
2609 store_parameter(src, 4);
2610
2611 __ movl2ptr(length, length); //higher 32bits must be null
2612
2613 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2614 assert_different_registers(c_rarg0, dst, dst_pos, length);
2615 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2616 assert_different_registers(c_rarg1, dst, length);
2617
2618 __ mov(c_rarg2, length);
2619 assert_different_registers(c_rarg2, dst);
2620
2621 #ifdef _WIN64
2622 // Allocate abi space for args but be sure to keep stack aligned
2623 __ subptr(rsp, 6*wordSize);
2624 __ load_klass(c_rarg3, dst, tmp_load_klass);
2625 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2626 store_parameter(c_rarg3, 4);
2627 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2628 __ call(RuntimeAddress(copyfunc_addr));
2629 __ addptr(rsp, 6*wordSize);
2630 #else
2631 __ load_klass(c_rarg4, dst, tmp_load_klass);
2632 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2633 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2634 __ call(RuntimeAddress(copyfunc_addr));
2635 #endif
2636
2637 #ifndef PRODUCT
2638 if (PrintC1Statistics) {
2639 Label failed;
2640 __ testl(rax, rax);
2641 __ jcc(Assembler::notZero, failed);
2642 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2643 __ bind(failed);
2644 }
2645 #endif
2646
2647 __ testl(rax, rax);
2648 __ jcc(Assembler::zero, *stub->continuation());
2649
2650 #ifndef PRODUCT
2651 if (PrintC1Statistics) {
2652 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2653 }
2654 #endif
2655
2656 __ mov(tmp, rax);
2657
2658 __ xorl(tmp, -1);
2659
2660 // Restore previously spilled arguments
2661 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2662 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2663 __ movptr (length, Address(rsp, 2*BytesPerWord));
2664 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2665 __ movptr (src, Address(rsp, 4*BytesPerWord));
2666
2667
2668 __ subl(length, tmp);
2669 __ addl(src_pos, tmp);
2670 __ addl(dst_pos, tmp);
2671 }
2672
2673 __ jmp(*stub->entry());
2674
2675 __ bind(cont);
2676 __ pop(dst);
2677 __ pop(src);
2678 }
2679 }
2680
2681 #ifdef ASSERT
2682 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2683 // Sanity check the known type with the incoming class. For the
2684 // primitive case the types must match exactly with src.klass and
2685 // dst.klass each exactly matching the default type. For the
2686 // object array case, if no type check is needed then either the
2687 // dst type is exactly the expected type and the src type is a
2688 // subtype which we can't check or src is the same array as dst
2689 // but not necessarily exactly of type default_type.
2690 Label known_ok, halt;
2691 __ mov_metadata(tmp, default_type->constant_encoding());
2692 if (UseCompressedClassPointers) {
2693 __ encode_klass_not_null(tmp, rscratch1);
2694 }
2695
2696 if (basic_type != T_OBJECT) {
2697 __ cmp_klass(tmp, dst, tmp2);
2698 __ jcc(Assembler::notEqual, halt);
2699 __ cmp_klass(tmp, src, tmp2);
2700 __ jcc(Assembler::equal, known_ok);
2701 } else {
2702 __ cmp_klass(tmp, dst, tmp2);
2703 __ jcc(Assembler::equal, known_ok);
2704 __ cmpptr(src, dst);
2705 __ jcc(Assembler::equal, known_ok);
2706 }
2707 __ bind(halt);
2708 __ stop("incorrect type information in arraycopy");
2709 __ bind(known_ok);
2710 }
2711 #endif
2712
2713 #ifndef PRODUCT
2714 if (PrintC1Statistics) {
2715 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2716 }
2717 #endif
2718
2719 assert_different_registers(c_rarg0, dst, dst_pos, length);
2720 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2721 assert_different_registers(c_rarg1, length);
2722 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2723 __ mov(c_rarg2, length);
2724
2725 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2726 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2727 const char *name;
2728 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2729 __ call_VM_leaf(entry, 0);
2730
2731 if (stub != nullptr) {
2732 __ bind(*stub->continuation());
2733 }
2734 }
2735
2736 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2737 assert(op->crc()->is_single_cpu(), "crc must be register");
2738 assert(op->val()->is_single_cpu(), "byte value must be register");
2739 assert(op->result_opr()->is_single_cpu(), "result must be register");
2740 Register crc = op->crc()->as_register();
2741 Register val = op->val()->as_register();
2742 Register res = op->result_opr()->as_register();
2743
2744 assert_different_registers(val, crc, res);
2745
2746 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2747 __ notl(crc); // ~crc
2748 __ update_byte_crc32(crc, val, res);
2749 __ notl(crc); // ~crc
2750 __ mov(res, crc);
2751 }
2752
2753 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2754 Register obj = op->obj_opr()->as_register(); // may not be an oop
2755 Register hdr = op->hdr_opr()->as_register();
2756 Register lock = op->lock_opr()->as_register();
2757 if (LockingMode == LM_MONITOR) {
2758 if (op->info() != nullptr) {
2759 add_debug_info_for_null_check_here(op->info());
2760 __ null_check(obj);
2761 }
2762 __ jmp(*op->stub()->entry());
2763 } else if (op->code() == lir_lock) {
2764 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2765 Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
2766 // add debug info for NullPointerException only if one is possible
2767 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2768 if (op->info() != nullptr) {
2769 add_debug_info_for_null_check(null_check_offset, op->info());
2770 }
2771 // done
2772 } else if (op->code() == lir_unlock) {
2773 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2774 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2775 } else {
2776 Unimplemented();
2777 }
2778 __ bind(*op->stub()->continuation());
2779 }
2780
2781 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2782 Register obj = op->obj()->as_pointer_register();
2783 Register result = op->result_opr()->as_pointer_register();
2784
2785 CodeEmitInfo* info = op->info();
2786 if (info != nullptr) {
2787 add_debug_info_for_null_check_here(info);
2788 }
2789
2790 __ load_klass(result, obj, rscratch1);
2791 }
2792
2793 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2794 ciMethod* method = op->profiled_method();
2795 int bci = op->profiled_bci();
2796 ciMethod* callee = op->profiled_callee();
2797 Register tmp_load_klass = rscratch1;
2798
2799 // Update counter for all call types
2800 ciMethodData* md = method->method_data_or_null();
2801 assert(md != nullptr, "Sanity");
2802 ciProfileData* data = md->bci_to_data(bci);
2803 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2804 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2805 Register mdo = op->mdo()->as_register();
2806 __ mov_metadata(mdo, md->constant_encoding());
2807 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2808 // Perform additional virtual call profiling for invokevirtual and
2809 // invokeinterface bytecodes
2810 if (op->should_profile_receiver_type()) {
2811 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2812 Register recv = op->recv()->as_register();
2813 assert_different_registers(mdo, recv);
2814 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2815 ciKlass* known_klass = op->known_holder();
2816 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2817 // We know the type that will be seen at this call site; we can
2818 // statically update the MethodData* rather than needing to do
2819 // dynamic tests on the receiver type
2820
2821 // NOTE: we should probably put a lock around this search to
2822 // avoid collisions by concurrent compilations
2823 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2824 uint i;
2825 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2826 ciKlass* receiver = vc_data->receiver(i);
2827 if (known_klass->equals(receiver)) {
2828 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2829 __ addptr(data_addr, DataLayout::counter_increment);
2830 return;
2831 }
2832 }
2833
2834 // Receiver type not found in profile data; select an empty slot
2835
2836 // Note that this is less efficient than it should be because it
2837 // always does a write to the receiver part of the
2838 // VirtualCallData rather than just the first time
2839 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2840 ciKlass* receiver = vc_data->receiver(i);
2841 if (receiver == nullptr) {
2842 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2843 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
2844 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2845 __ addptr(data_addr, DataLayout::counter_increment);
2846 return;
2847 }
2848 }
2849 } else {
2850 __ load_klass(recv, recv, tmp_load_klass);
2851 Label update_done;
2852 type_profile_helper(mdo, md, data, recv, &update_done);
2853 // Receiver did not match any saved receiver and there is no empty row for it.
2854 // Increment total counter to indicate polymorphic case.
2855 __ addptr(counter_addr, DataLayout::counter_increment);
2856
2857 __ bind(update_done);
2858 }
2859 } else {
2860 // Static call
2861 __ addptr(counter_addr, DataLayout::counter_increment);
2862 }
2863 }
2864
2865 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2866 Register obj = op->obj()->as_register();
2867 Register tmp = op->tmp()->as_pointer_register();
2868 Register tmp_load_klass = rscratch1;
2869 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2870 ciKlass* exact_klass = op->exact_klass();
2871 intptr_t current_klass = op->current_klass();
2872 bool not_null = op->not_null();
2873 bool no_conflict = op->no_conflict();
2874
2875 Label update, next, none;
2876
2877 bool do_null = !not_null;
2878 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2879 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2880
2881 assert(do_null || do_update, "why are we here?");
2882 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2883
2884 __ verify_oop(obj);
2885
2886 #ifdef ASSERT
2887 if (obj == tmp) {
2888 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2889 } else {
2890 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2891 }
2892 #endif
2893 if (do_null) {
2894 __ testptr(obj, obj);
2895 __ jccb(Assembler::notZero, update);
2896 if (!TypeEntries::was_null_seen(current_klass)) {
2897 __ testptr(mdo_addr, TypeEntries::null_seen);
2898 #ifndef ASSERT
2899 __ jccb(Assembler::notZero, next); // already set
2900 #else
2901 __ jcc(Assembler::notZero, next); // already set
2902 #endif
2903 // atomic update to prevent overwriting Klass* with 0
2904 __ lock();
2905 __ orptr(mdo_addr, TypeEntries::null_seen);
2906 }
2907 if (do_update) {
2908 #ifndef ASSERT
2909 __ jmpb(next);
2910 }
2911 #else
2912 __ jmp(next);
2913 }
2914 } else {
2915 __ testptr(obj, obj);
2916 __ jcc(Assembler::notZero, update);
2917 __ stop("unexpected null obj");
2918 #endif
2919 }
2920
2921 __ bind(update);
2922
2923 if (do_update) {
2924 #ifdef ASSERT
2925 if (exact_klass != nullptr) {
2926 Label ok;
2927 __ load_klass(tmp, obj, tmp_load_klass);
2928 __ push(tmp);
2929 __ mov_metadata(tmp, exact_klass->constant_encoding());
2930 __ cmpptr(tmp, Address(rsp, 0));
2931 __ jcc(Assembler::equal, ok);
2932 __ stop("exact klass and actual klass differ");
2933 __ bind(ok);
2934 __ pop(tmp);
2935 }
2936 #endif
2937 if (!no_conflict) {
2938 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2939 if (exact_klass != nullptr) {
2940 __ mov_metadata(tmp, exact_klass->constant_encoding());
2941 } else {
2942 __ load_klass(tmp, obj, tmp_load_klass);
2943 }
2944 __ mov(rscratch1, tmp); // save original value before XOR
2945 __ xorptr(tmp, mdo_addr);
2946 __ testptr(tmp, TypeEntries::type_klass_mask);
2947 // klass seen before, nothing to do. The unknown bit may have been
2948 // set already but no need to check.
2949 __ jccb(Assembler::zero, next);
2950
2951 __ testptr(tmp, TypeEntries::type_unknown);
2952 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2953
2954 if (TypeEntries::is_type_none(current_klass)) {
2955 __ testptr(mdo_addr, TypeEntries::type_mask);
2956 __ jccb(Assembler::zero, none);
2957 // There is a chance that the checks above (re-reading profiling
2958 // data from memory) fail if another thread has just set the
2959 // profiling to this obj's klass
2960 __ mov(tmp, rscratch1); // get back original value before XOR
2961 __ xorptr(tmp, mdo_addr);
2962 __ testptr(tmp, TypeEntries::type_klass_mask);
2963 __ jccb(Assembler::zero, next);
2964 }
2965 } else {
2966 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2967 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2968
2969 __ testptr(mdo_addr, TypeEntries::type_unknown);
2970 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2971 }
2972
2973 // different than before. Cannot keep accurate profile.
2974 __ orptr(mdo_addr, TypeEntries::type_unknown);
2975
2976 if (TypeEntries::is_type_none(current_klass)) {
2977 __ jmpb(next);
2978
2979 __ bind(none);
2980 // first time here. Set profile type.
2981 __ movptr(mdo_addr, tmp);
2982 #ifdef ASSERT
2983 __ andptr(tmp, TypeEntries::type_klass_mask);
2984 __ verify_klass_ptr(tmp);
2985 #endif
2986 }
2987 } else {
2988 // There's a single possible klass at this profile point
2989 assert(exact_klass != nullptr, "should be");
2990 if (TypeEntries::is_type_none(current_klass)) {
2991 __ mov_metadata(tmp, exact_klass->constant_encoding());
2992 __ xorptr(tmp, mdo_addr);
2993 __ testptr(tmp, TypeEntries::type_klass_mask);
2994 #ifdef ASSERT
2995 __ jcc(Assembler::zero, next);
2996
2997 {
2998 Label ok;
2999 __ push(tmp);
3000 __ testptr(mdo_addr, TypeEntries::type_mask);
3001 __ jcc(Assembler::zero, ok);
3002 // may have been set by another thread
3003 __ mov_metadata(tmp, exact_klass->constant_encoding());
3004 __ xorptr(tmp, mdo_addr);
3005 __ testptr(tmp, TypeEntries::type_mask);
3006 __ jcc(Assembler::zero, ok);
3007
3008 __ stop("unexpected profiling mismatch");
3009 __ bind(ok);
3010 __ pop(tmp);
3011 }
3012 #else
3013 __ jccb(Assembler::zero, next);
3014 #endif
3015 // first time here. Set profile type.
3016 __ movptr(mdo_addr, tmp);
3017 #ifdef ASSERT
3018 __ andptr(tmp, TypeEntries::type_klass_mask);
3019 __ verify_klass_ptr(tmp);
3020 #endif
3021 } else {
3022 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3023 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3024
3025 __ testptr(mdo_addr, TypeEntries::type_unknown);
3026 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3027
3028 __ orptr(mdo_addr, TypeEntries::type_unknown);
3029 }
3030 }
3031 }
3032 __ bind(next);
3033 }
3034
3035 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3036 Unimplemented();
3037 }
3038
3039
3040 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3041 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3042 }
3043
3044
3045 void LIR_Assembler::align_backward_branch_target() {
3046 __ align(BytesPerWord);
3047 }
3048
3049
3050 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3051 if (left->is_single_cpu()) {
3052 __ negl(left->as_register());
3053 move_regs(left->as_register(), dest->as_register());
3054
3055 } else if (left->is_double_cpu()) {
3056 Register lo = left->as_register_lo();
3057 Register dst = dest->as_register_lo();
3058 __ movptr(dst, lo);
3059 __ negptr(dst);
3060
3061 } else if (dest->is_single_xmm()) {
3062 assert(!tmp->is_valid(), "do not need temporary");
3063 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3064 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3065 }
3066 __ xorps(dest->as_xmm_float_reg(),
3067 ExternalAddress((address)float_signflip_pool),
3068 rscratch1);
3069 } else if (dest->is_double_xmm()) {
3070 assert(!tmp->is_valid(), "do not need temporary");
3071 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3072 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3073 }
3074 __ xorpd(dest->as_xmm_double_reg(),
3075 ExternalAddress((address)double_signflip_pool),
3076 rscratch1);
3077 } else {
3078 ShouldNotReachHere();
3079 }
3080 }
3081
3082
3083 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3084 assert(src->is_address(), "must be an address");
3085 assert(dest->is_register(), "must be a register");
3086
3087 PatchingStub* patch = nullptr;
3088 if (patch_code != lir_patch_none) {
3089 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3090 }
3091
3092 Register reg = dest->as_pointer_register();
3093 LIR_Address* addr = src->as_address_ptr();
3094 __ lea(reg, as_Address(addr));
3095
3096 if (patch != nullptr) {
3097 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3098 }
3099 }
3100
3101
3102
3103 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3104 assert(!tmp->is_valid(), "don't need temporary");
3105 __ call(RuntimeAddress(dest));
3106 if (info != nullptr) {
3107 add_call_info_here(info);
3108 }
3109 __ post_call_nop();
3110 }
3111
3112
3113 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3114 assert(type == T_LONG, "only for volatile long fields");
3115
3116 if (info != nullptr) {
3117 add_debug_info_for_null_check_here(info);
3118 }
3119
3120 if (src->is_double_xmm()) {
3121 if (dest->is_double_cpu()) {
3122 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3123 } else if (dest->is_double_stack()) {
3124 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3125 } else if (dest->is_address()) {
3126 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3127 } else {
3128 ShouldNotReachHere();
3129 }
3130
3131 } else if (dest->is_double_xmm()) {
3132 if (src->is_double_stack()) {
3133 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3134 } else if (src->is_address()) {
3135 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3136 } else {
3137 ShouldNotReachHere();
3138 }
3139
3140 } else {
3141 ShouldNotReachHere();
3142 }
3143 }
3144
3145 #ifdef ASSERT
3146 // emit run-time assertion
3147 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3148 assert(op->code() == lir_assert, "must be");
3149
3150 if (op->in_opr1()->is_valid()) {
3151 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3152 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3153 } else {
3154 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3155 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3156 }
3157
3158 Label ok;
3159 if (op->condition() != lir_cond_always) {
3160 Assembler::Condition acond = Assembler::zero;
3161 switch (op->condition()) {
3162 case lir_cond_equal: acond = Assembler::equal; break;
3163 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3164 case lir_cond_less: acond = Assembler::less; break;
3165 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3166 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3167 case lir_cond_greater: acond = Assembler::greater; break;
3168 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3169 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3170 default: ShouldNotReachHere();
3171 }
3172 __ jcc(acond, ok);
3173 }
3174 if (op->halt()) {
3175 const char* str = __ code_string(op->msg());
3176 __ stop(str);
3177 } else {
3178 breakpoint();
3179 }
3180 __ bind(ok);
3181 }
3182 #endif
3183
3184 void LIR_Assembler::membar() {
3185 // QQQ sparc TSO uses this,
3186 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3187 }
3188
3189 void LIR_Assembler::membar_acquire() {
3190 // No x86 machines currently require load fences
3191 }
3192
3193 void LIR_Assembler::membar_release() {
3194 // No x86 machines currently require store fences
3195 }
3196
3197 void LIR_Assembler::membar_loadload() {
3198 // no-op
3199 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3200 }
3201
3202 void LIR_Assembler::membar_storestore() {
3203 // no-op
3204 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3205 }
3206
3207 void LIR_Assembler::membar_loadstore() {
3208 // no-op
3209 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3210 }
3211
3212 void LIR_Assembler::membar_storeload() {
3213 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3214 }
3215
3216 void LIR_Assembler::on_spin_wait() {
3217 __ pause ();
3218 }
3219
3220 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3221 assert(result_reg->is_register(), "check");
3222 __ mov(result_reg->as_register(), r15_thread);
3223 }
3224
3225
3226 void LIR_Assembler::peephole(LIR_List*) {
3227 // do nothing for now
3228 }
3229
3230 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3231 assert(data == dest, "xchg/xadd uses only 2 operands");
3232
3233 if (data->type() == T_INT) {
3234 if (code == lir_xadd) {
3235 __ lock();
3236 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3237 } else {
3238 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3239 }
3240 } else if (data->is_oop()) {
3241 assert (code == lir_xchg, "xadd for oops");
3242 Register obj = data->as_register();
3243 if (UseCompressedOops) {
3244 __ encode_heap_oop(obj);
3245 __ xchgl(obj, as_Address(src->as_address_ptr()));
3246 __ decode_heap_oop(obj);
3247 } else {
3248 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3249 }
3250 } else if (data->type() == T_LONG) {
3251 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3252 if (code == lir_xadd) {
3253 __ lock();
3254 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3255 } else {
3256 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3257 }
3258 } else {
3259 ShouldNotReachHere();
3260 }
3261 }
3262
3263 #undef __