1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
61 operand[1] = hi;
62 return operand;
63 }
64
65 // Buffer for 128-bits masks used by SSE instructions.
66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
67
68 // Static initialization during VM startup.
69 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
70 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
71 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
72 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
73
74
75 NEEDS_CLEANUP // remove this definitions ?
76 const Register SYNC_header = rax; // synchronization header
77 const Register SHIFT_count = rcx; // where count for shift operations must be
78
79 #define __ _masm->
80
81
82 static void select_different_registers(Register preserve,
83 Register extra,
84 Register &tmp1,
85 Register &tmp2) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, extra);
91 tmp2 = extra;
92 }
93 assert_different_registers(preserve, tmp1, tmp2);
94 }
95
96
97
98 static void select_different_registers(Register preserve,
99 Register extra,
100 Register &tmp1,
101 Register &tmp2,
102 Register &tmp3) {
103 if (tmp1 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp1 = extra;
106 } else if (tmp2 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp2 = extra;
109 } else if (tmp3 == preserve) {
110 assert_different_registers(tmp1, tmp2, tmp3, extra);
111 tmp3 = extra;
112 }
113 assert_different_registers(preserve, tmp1, tmp2, tmp3);
114 }
115
116
117
118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
119 if (opr->is_constant()) {
120 LIR_Const* constant = opr->as_constant_ptr();
121 switch (constant->type()) {
122 case T_INT: {
123 return true;
124 }
125
126 default:
127 return false;
128 }
129 }
130 return false;
131 }
132
133
134 LIR_Opr LIR_Assembler::receiverOpr() {
135 return FrameMap::receiver_opr;
136 }
137
138 LIR_Opr LIR_Assembler::osrBufferPointer() {
139 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
140 }
141
142 //--------------fpu register translations-----------------------
143
144
145 address LIR_Assembler::float_constant(float f) {
146 address const_addr = __ float_constant(f);
147 if (const_addr == nullptr) {
148 bailout("const section overflow");
149 return __ code()->consts()->start();
150 } else {
151 return const_addr;
152 }
153 }
154
155
156 address LIR_Assembler::double_constant(double d) {
157 address const_addr = __ double_constant(d);
158 if (const_addr == nullptr) {
159 bailout("const section overflow");
160 return __ code()->consts()->start();
161 } else {
162 return const_addr;
163 }
164 }
165
166 void LIR_Assembler::breakpoint() {
167 __ int3();
168 }
169
170 void LIR_Assembler::push(LIR_Opr opr) {
171 if (opr->is_single_cpu()) {
172 __ push_reg(opr->as_register());
173 } else if (opr->is_double_cpu()) {
174 __ push_reg(opr->as_register_lo());
175 } else if (opr->is_stack()) {
176 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
177 } else if (opr->is_constant()) {
178 LIR_Const* const_opr = opr->as_constant_ptr();
179 if (const_opr->type() == T_OBJECT) {
180 __ push_oop(const_opr->as_jobject(), rscratch1);
181 } else if (const_opr->type() == T_INT) {
182 __ push_jint(const_opr->as_jint());
183 } else {
184 ShouldNotReachHere();
185 }
186
187 } else {
188 ShouldNotReachHere();
189 }
190 }
191
192 void LIR_Assembler::pop(LIR_Opr opr) {
193 if (opr->is_single_cpu()) {
194 __ pop_reg(opr->as_register());
195 } else {
196 ShouldNotReachHere();
197 }
198 }
199
200 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
201 return addr->base()->is_illegal() && addr->index()->is_illegal();
202 }
203
204 //-------------------------------------------
205
206 Address LIR_Assembler::as_Address(LIR_Address* addr) {
207 return as_Address(addr, rscratch1);
208 }
209
210 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
211 if (addr->base()->is_illegal()) {
212 assert(addr->index()->is_illegal(), "must be illegal too");
213 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
214 if (! __ reachable(laddr)) {
215 __ movptr(tmp, laddr.addr());
216 Address res(tmp, 0);
217 return res;
218 } else {
219 return __ as_Address(laddr);
220 }
221 }
222
223 Register base = addr->base()->as_pointer_register();
224
225 if (addr->index()->is_illegal()) {
226 return Address( base, addr->disp());
227 } else if (addr->index()->is_cpu_register()) {
228 Register index = addr->index()->as_pointer_register();
229 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
230 } else if (addr->index()->is_constant()) {
231 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
232 assert(Assembler::is_simm32(addr_offset), "must be");
233
234 return Address(base, addr_offset);
235 } else {
236 Unimplemented();
237 return Address();
238 }
239 }
240
241
242 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
243 Address base = as_Address(addr);
244 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
245 }
246
247
248 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
249 return as_Address(addr);
250 }
251
252
253 void LIR_Assembler::osr_entry() {
254 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
255 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
256 ValueStack* entry_state = osr_entry->state();
257 int number_of_locks = entry_state->locks_size();
258
259 // we jump here if osr happens with the interpreter
260 // state set up to continue at the beginning of the
261 // loop that triggered osr - in particular, we have
262 // the following registers setup:
263 //
264 // rcx: osr buffer
265 //
266
267 // build frame
268 ciMethod* m = compilation()->method();
269 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
270
271 // OSR buffer is
272 //
273 // locals[nlocals-1..0]
274 // monitors[0..number_of_locks]
275 //
276 // locals is a direct copy of the interpreter frame so in the osr buffer
277 // so first slot in the local array is the last local from the interpreter
278 // and last slot is local[0] (receiver) from the interpreter
279 //
280 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
281 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
282 // in the interpreter frame (the method lock if a sync method)
283
284 // Initialize monitors in the compiled activation.
285 // rcx: pointer to osr buffer
286 //
287 // All other registers are dead at this point and the locals will be
288 // copied into place by code emitted in the IR.
289
290 Register OSR_buf = osrBufferPointer()->as_pointer_register();
291 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
292 int monitor_offset = BytesPerWord * method()->max_locals() +
293 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
294 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
295 // the OSR buffer using 2 word entries: first the lock and then
296 // the oop.
297 for (int i = 0; i < number_of_locks; i++) {
298 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
299 #ifdef ASSERT
300 // verify the interpreter's monitor has a non-null object
301 {
302 Label L;
303 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
304 __ jcc(Assembler::notZero, L);
305 __ stop("locked object is null");
306 __ bind(L);
307 }
308 #endif
309 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
310 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
311 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
312 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
313 }
314 }
315 }
316
317
318 // inline cache check; done before the frame is built.
319 int LIR_Assembler::check_icache() {
320 return __ ic_check(CodeEntryAlignment);
321 }
322
323 void LIR_Assembler::clinit_barrier(ciMethod* method) {
324 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
325 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
326
327 Label L_skip_barrier;
328 Register klass = rscratch1;
329
330 __ mov_metadata(klass, method->holder()->constant_encoding());
331 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
332
333 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
334
335 __ bind(L_skip_barrier);
336 }
337
338 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
339 jobject o = nullptr;
340 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
341 __ movoop(reg, o);
342 patching_epilog(patch, lir_patch_normal, reg, info);
343 }
344
345 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
346 Metadata* o = nullptr;
347 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
348 __ mov_metadata(reg, o);
349 patching_epilog(patch, lir_patch_normal, reg, info);
350 }
351
352 // This specifies the rsp decrement needed to build the frame
353 int LIR_Assembler::initial_frame_size_in_bytes() const {
354 // if rounding, must let FrameMap know!
355
356 // The frame_map records size in slots (32bit word)
357
358 // subtract two words to account for return address and link
359 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
360 }
361
362
363 int LIR_Assembler::emit_exception_handler() {
364 // generate code for exception handler
365 address handler_base = __ start_a_stub(exception_handler_size());
366 if (handler_base == nullptr) {
367 // not enough space left for the handler
368 bailout("exception handler overflow");
369 return -1;
370 }
371
372 int offset = code_offset();
373
374 // the exception oop and pc are in rax, and rdx
375 // no other registers need to be preserved, so invalidate them
376 __ invalidate_registers(false, true, true, false, true, true);
377
378 // check that there is really an exception
379 __ verify_not_null_oop(rax);
380
381 // search an exception handler (rax: exception oop, rdx: throwing pc)
382 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
383 __ should_not_reach_here();
384 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
385 __ end_a_stub();
386
387 return offset;
388 }
389
390
391 // Emit the code to remove the frame from the stack in the exception
392 // unwind path.
393 int LIR_Assembler::emit_unwind_handler() {
394 #ifndef PRODUCT
395 if (CommentedAssembly) {
396 _masm->block_comment("Unwind handler");
397 }
398 #endif
399
400 int offset = code_offset();
401
402 // Fetch the exception from TLS and clear out exception related thread state
403 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
404 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
405 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
406
407 __ bind(_unwind_handler_entry);
408 __ verify_not_null_oop(rax);
409 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
410 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
411 }
412
413 // Perform needed unlocking
414 MonitorExitStub* stub = nullptr;
415 if (method()->is_synchronized()) {
416 monitor_address(0, FrameMap::rax_opr);
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 __ unlock_object(rdi, rsi, rax, *stub->entry());
419 __ bind(*stub->continuation());
420 }
421
422 if (compilation()->env()->dtrace_method_probes()) {
423 __ mov(rdi, r15_thread);
424 __ mov_metadata(rsi, method()->constant_encoding());
425 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
426 }
427
428 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
429 __ mov(rax, rbx); // Restore the exception
430 }
431
432 // remove the activation and dispatch to the unwind handler
433 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
434 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
435
436 // Emit the slow path assembly
437 if (stub != nullptr) {
438 stub->emit_code(this);
439 }
440
441 return offset;
442 }
443
444
445 int LIR_Assembler::emit_deopt_handler() {
446 // generate code for exception handler
447 address handler_base = __ start_a_stub(deopt_handler_size());
448 if (handler_base == nullptr) {
449 // not enough space left for the handler
450 bailout("deopt handler overflow");
451 return -1;
452 }
453
454 int offset = code_offset();
455 InternalAddress here(__ pc());
456
457 __ pushptr(here.addr(), rscratch1);
458 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 __ end_a_stub();
461
462 return offset;
463 }
464
465 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
466 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
467 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
468 assert(result->fpu() == 0, "result must already be on TOS");
469 }
470 if (InlineTypeReturnedAsFields) {
471 #ifndef _LP64
472 Unimplemented();
473 #endif
474 // Check if we are returning an non-null inline type and load its fields into registers
475 ciType* return_type = compilation()->method()->return_type();
476 if (return_type->is_inlinetype()) {
477 ciInlineKlass* vk = return_type->as_inline_klass();
478 if (vk->can_be_returned_as_fields()) {
479 address unpack_handler = vk->unpack_handler();
480 assert(unpack_handler != nullptr, "must be");
481 __ call(RuntimeAddress(unpack_handler));
482 }
483 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
484 Label skip;
485 Label not_null;
486 __ testptr(rax, rax);
487 __ jcc(Assembler::notZero, not_null);
488 // Returned value is null, zero all return registers because they may belong to oop fields
489 __ xorq(j_rarg1, j_rarg1);
490 __ xorq(j_rarg2, j_rarg2);
491 __ xorq(j_rarg3, j_rarg3);
492 __ xorq(j_rarg4, j_rarg4);
493 __ xorq(j_rarg5, j_rarg5);
494 __ jmp(skip);
495 __ bind(not_null);
496
497 // Check if we are returning an non-null inline type and load its fields into registers
498 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
499
500 // Load fields from a buffered value with an inline class specific handler
501 __ load_klass(rdi, rax, rscratch1);
502 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
503 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
504 // Unpack handler can be null if inline type is not scalarizable in returns
505 __ testptr(rdi, rdi);
506 __ jcc(Assembler::zero, skip);
507 __ call(rdi);
508
509 __ bind(skip);
510 }
511 // At this point, rax points to the value object (for interpreter or C1 caller).
512 // The fields of the object are copied into registers (for C2 caller).
513 }
514
515 // Pop the stack before the safepoint code
516 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
517
518 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
519 __ reserved_stack_check();
520 }
521
522 // Note: we do not need to round double result; float result has the right precision
523 // the poll sets the condition code, but no data registers
524
525 code_stub->set_safepoint_offset(__ offset());
526 __ relocate(relocInfo::poll_return_type);
527 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
528 __ ret(0);
529 }
530
531
532 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
533 return (__ store_inline_type_fields_to_buf(vk, false));
534 }
535
536 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
537 guarantee(info != nullptr, "Shouldn't be null");
538 int offset = __ offset();
539 const Register poll_addr = rscratch1;
540 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
541 add_debug_info_for_branch(info);
542 __ relocate(relocInfo::poll_type);
543 address pre_pc = __ pc();
544 __ testl(rax, Address(poll_addr, 0));
545 address post_pc = __ pc();
546 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
547 return offset;
548 }
549
550
551 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
552 if (from_reg != to_reg) __ mov(to_reg, from_reg);
553 }
554
555 void LIR_Assembler::swap_reg(Register a, Register b) {
556 __ xchgptr(a, b);
557 }
558
559
560 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
561 assert(src->is_constant(), "should not call otherwise");
562 assert(dest->is_register(), "should not call otherwise");
563 LIR_Const* c = src->as_constant_ptr();
564
565 switch (c->type()) {
566 case T_INT: {
567 assert(patch_code == lir_patch_none, "no patching handled here");
568 __ movl(dest->as_register(), c->as_jint());
569 break;
570 }
571
572 case T_ADDRESS: {
573 assert(patch_code == lir_patch_none, "no patching handled here");
574 __ movptr(dest->as_register(), c->as_jint());
575 break;
576 }
577
578 case T_LONG: {
579 assert(patch_code == lir_patch_none, "no patching handled here");
580 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
581 break;
582 }
583
584 case T_OBJECT: {
585 if (patch_code != lir_patch_none) {
586 jobject2reg_with_patching(dest->as_register(), info);
587 } else {
588 __ movoop(dest->as_register(), c->as_jobject());
589 }
590 break;
591 }
592
593 case T_METADATA: {
594 if (patch_code != lir_patch_none) {
595 klass2reg_with_patching(dest->as_register(), info);
596 } else {
597 __ mov_metadata(dest->as_register(), c->as_metadata());
598 }
599 break;
600 }
601
602 case T_FLOAT: {
603 if (dest->is_single_xmm()) {
604 if (UseAVX <= 2 && c->is_zero_float()) {
605 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
606 } else {
607 __ movflt(dest->as_xmm_float_reg(),
608 InternalAddress(float_constant(c->as_jfloat())));
609 }
610 } else {
611 ShouldNotReachHere();
612 }
613 break;
614 }
615
616 case T_DOUBLE: {
617 if (dest->is_double_xmm()) {
618 if (UseAVX <= 2 && c->is_zero_double()) {
619 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
620 } else {
621 __ movdbl(dest->as_xmm_double_reg(),
622 InternalAddress(double_constant(c->as_jdouble())));
623 }
624 } else {
625 ShouldNotReachHere();
626 }
627 break;
628 }
629
630 default:
631 ShouldNotReachHere();
632 }
633 }
634
635 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
636 assert(src->is_constant(), "should not call otherwise");
637 assert(dest->is_stack(), "should not call otherwise");
638 LIR_Const* c = src->as_constant_ptr();
639
640 switch (c->type()) {
641 case T_INT: // fall through
642 case T_FLOAT:
643 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
644 break;
645
646 case T_ADDRESS:
647 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
648 break;
649
650 case T_OBJECT:
651 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
652 break;
653
654 case T_LONG: // fall through
655 case T_DOUBLE:
656 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
657 lo_word_offset_in_bytes),
658 (intptr_t)c->as_jlong_bits(),
659 rscratch1);
660 break;
661
662 default:
663 ShouldNotReachHere();
664 }
665 }
666
667 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
668 assert(src->is_constant(), "should not call otherwise");
669 assert(dest->is_address(), "should not call otherwise");
670 LIR_Const* c = src->as_constant_ptr();
671 LIR_Address* addr = dest->as_address_ptr();
672
673 int null_check_here = code_offset();
674 switch (type) {
675 case T_INT: // fall through
676 case T_FLOAT:
677 __ movl(as_Address(addr), c->as_jint_bits());
678 break;
679
680 case T_ADDRESS:
681 __ movptr(as_Address(addr), c->as_jint_bits());
682 break;
683
684 case T_OBJECT: // fall through
685 case T_ARRAY:
686 if (c->as_jobject() == nullptr) {
687 if (UseCompressedOops && !wide) {
688 __ movl(as_Address(addr), NULL_WORD);
689 } else {
690 __ xorptr(rscratch1, rscratch1);
691 null_check_here = code_offset();
692 __ movptr(as_Address(addr), rscratch1);
693 }
694 } else {
695 if (is_literal_address(addr)) {
696 ShouldNotReachHere();
697 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
698 } else {
699 __ movoop(rscratch1, c->as_jobject());
700 if (UseCompressedOops && !wide) {
701 __ encode_heap_oop(rscratch1);
702 null_check_here = code_offset();
703 __ movl(as_Address_lo(addr), rscratch1);
704 } else {
705 null_check_here = code_offset();
706 __ movptr(as_Address_lo(addr), rscratch1);
707 }
708 }
709 }
710 break;
711
712 case T_LONG: // fall through
713 case T_DOUBLE:
714 if (is_literal_address(addr)) {
715 ShouldNotReachHere();
716 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
717 } else {
718 __ movptr(r10, (intptr_t)c->as_jlong_bits());
719 null_check_here = code_offset();
720 __ movptr(as_Address_lo(addr), r10);
721 }
722 break;
723
724 case T_BOOLEAN: // fall through
725 case T_BYTE:
726 __ movb(as_Address(addr), c->as_jint() & 0xFF);
727 break;
728
729 case T_CHAR: // fall through
730 case T_SHORT:
731 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
732 break;
733
734 default:
735 ShouldNotReachHere();
736 };
737
738 if (info != nullptr) {
739 add_debug_info_for_null_check(null_check_here, info);
740 }
741 }
742
743
744 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
745 assert(src->is_register(), "should not call otherwise");
746 assert(dest->is_register(), "should not call otherwise");
747
748 // move between cpu-registers
749 if (dest->is_single_cpu()) {
750 if (src->type() == T_LONG) {
751 // Can do LONG -> OBJECT
752 move_regs(src->as_register_lo(), dest->as_register());
753 return;
754 }
755 assert(src->is_single_cpu(), "must match");
756 if (src->type() == T_OBJECT) {
757 __ verify_oop(src->as_register());
758 }
759 move_regs(src->as_register(), dest->as_register());
760
761 } else if (dest->is_double_cpu()) {
762 if (is_reference_type(src->type())) {
763 // Surprising to me but we can see move of a long to t_object
764 __ verify_oop(src->as_register());
765 move_regs(src->as_register(), dest->as_register_lo());
766 return;
767 }
768 assert(src->is_double_cpu(), "must match");
769 Register f_lo = src->as_register_lo();
770 Register f_hi = src->as_register_hi();
771 Register t_lo = dest->as_register_lo();
772 Register t_hi = dest->as_register_hi();
773 assert(f_hi == f_lo, "must be same");
774 assert(t_hi == t_lo, "must be same");
775 move_regs(f_lo, t_lo);
776
777 // move between xmm-registers
778 } else if (dest->is_single_xmm()) {
779 assert(src->is_single_xmm(), "must match");
780 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
781 } else if (dest->is_double_xmm()) {
782 assert(src->is_double_xmm(), "must match");
783 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
784
785 } else {
786 ShouldNotReachHere();
787 }
788 }
789
790 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
791 assert(src->is_register(), "should not call otherwise");
792 assert(dest->is_stack(), "should not call otherwise");
793
794 if (src->is_single_cpu()) {
795 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
796 if (is_reference_type(type)) {
797 __ verify_oop(src->as_register());
798 __ movptr (dst, src->as_register());
799 } else if (type == T_METADATA || type == T_ADDRESS) {
800 __ movptr (dst, src->as_register());
801 } else {
802 __ movl (dst, src->as_register());
803 }
804
805 } else if (src->is_double_cpu()) {
806 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
807 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
808 __ movptr (dstLO, src->as_register_lo());
809
810 } else if (src->is_single_xmm()) {
811 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
812 __ movflt(dst_addr, src->as_xmm_float_reg());
813
814 } else if (src->is_double_xmm()) {
815 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
816 __ movdbl(dst_addr, src->as_xmm_double_reg());
817
818 } else {
819 ShouldNotReachHere();
820 }
821 }
822
823
824 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
825 LIR_Address* to_addr = dest->as_address_ptr();
826 PatchingStub* patch = nullptr;
827 Register compressed_src = rscratch1;
828
829 if (is_reference_type(type)) {
830 __ verify_oop(src->as_register());
831 if (UseCompressedOops && !wide) {
832 __ movptr(compressed_src, src->as_register());
833 __ encode_heap_oop(compressed_src);
834 if (patch_code != lir_patch_none) {
835 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
836 }
837 }
838 }
839
840 if (patch_code != lir_patch_none) {
841 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
842 Address toa = as_Address(to_addr);
843 assert(toa.disp() != 0, "must have");
844 }
845
846 int null_check_here = code_offset();
847 switch (type) {
848 case T_FLOAT: {
849 assert(src->is_single_xmm(), "not a float");
850 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
851 break;
852 }
853
854 case T_DOUBLE: {
855 assert(src->is_double_xmm(), "not a double");
856 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
857 break;
858 }
859
860 case T_ARRAY: // fall through
861 case T_OBJECT: // fall through
862 if (UseCompressedOops && !wide) {
863 __ movl(as_Address(to_addr), compressed_src);
864 } else {
865 __ movptr(as_Address(to_addr), src->as_register());
866 }
867 break;
868 case T_ADDRESS:
869 __ movptr(as_Address(to_addr), src->as_register());
870 break;
871 case T_INT:
872 __ movl(as_Address(to_addr), src->as_register());
873 break;
874
875 case T_LONG: {
876 Register from_lo = src->as_register_lo();
877 Register from_hi = src->as_register_hi();
878 __ movptr(as_Address_lo(to_addr), from_lo);
879 break;
880 }
881
882 case T_BYTE: // fall through
883 case T_BOOLEAN: {
884 Register src_reg = src->as_register();
885 Address dst_addr = as_Address(to_addr);
886 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
887 __ movb(dst_addr, src_reg);
888 break;
889 }
890
891 case T_CHAR: // fall through
892 case T_SHORT:
893 __ movw(as_Address(to_addr), src->as_register());
894 break;
895
896 default:
897 ShouldNotReachHere();
898 }
899 if (info != nullptr) {
900 add_debug_info_for_null_check(null_check_here, info);
901 }
902
903 if (patch_code != lir_patch_none) {
904 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
905 }
906 }
907
908
909 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
910 assert(src->is_stack(), "should not call otherwise");
911 assert(dest->is_register(), "should not call otherwise");
912
913 if (dest->is_single_cpu()) {
914 if (is_reference_type(type)) {
915 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
916 __ verify_oop(dest->as_register());
917 } else if (type == T_METADATA || type == T_ADDRESS) {
918 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
919 } else {
920 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
921 }
922
923 } else if (dest->is_double_cpu()) {
924 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
925 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
926 __ movptr(dest->as_register_lo(), src_addr_LO);
927
928 } else if (dest->is_single_xmm()) {
929 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
930 __ movflt(dest->as_xmm_float_reg(), src_addr);
931
932 } else if (dest->is_double_xmm()) {
933 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
934 __ movdbl(dest->as_xmm_double_reg(), src_addr);
935
936 } else {
937 ShouldNotReachHere();
938 }
939 }
940
941
942 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
943 if (src->is_single_stack()) {
944 if (is_reference_type(type)) {
945 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
946 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
947 } else {
948 //no pushl on 64bits
949 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
950 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
951 }
952
953 } else if (src->is_double_stack()) {
954 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
955 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
956
957 } else {
958 ShouldNotReachHere();
959 }
960 }
961
962
963 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
964 assert(src->is_address(), "should not call otherwise");
965 assert(dest->is_register(), "should not call otherwise");
966
967 LIR_Address* addr = src->as_address_ptr();
968 Address from_addr = as_Address(addr);
969
970 if (addr->base()->type() == T_OBJECT) {
971 __ verify_oop(addr->base()->as_pointer_register());
972 }
973
974 switch (type) {
975 case T_BOOLEAN: // fall through
976 case T_BYTE: // fall through
977 case T_CHAR: // fall through
978 case T_SHORT:
979 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
980 // on pre P6 processors we may get partial register stalls
981 // so blow away the value of to_rinfo before loading a
982 // partial word into it. Do it here so that it precedes
983 // the potential patch point below.
984 __ xorptr(dest->as_register(), dest->as_register());
985 }
986 break;
987 default:
988 break;
989 }
990
991 PatchingStub* patch = nullptr;
992 if (patch_code != lir_patch_none) {
993 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
994 assert(from_addr.disp() != 0, "must have");
995 }
996 if (info != nullptr) {
997 add_debug_info_for_null_check_here(info);
998 }
999
1000 switch (type) {
1001 case T_FLOAT: {
1002 if (dest->is_single_xmm()) {
1003 __ movflt(dest->as_xmm_float_reg(), from_addr);
1004 } else {
1005 ShouldNotReachHere();
1006 }
1007 break;
1008 }
1009
1010 case T_DOUBLE: {
1011 if (dest->is_double_xmm()) {
1012 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1013 } else {
1014 ShouldNotReachHere();
1015 }
1016 break;
1017 }
1018
1019 case T_OBJECT: // fall through
1020 case T_ARRAY: // fall through
1021 if (UseCompressedOops && !wide) {
1022 __ movl(dest->as_register(), from_addr);
1023 } else {
1024 __ movptr(dest->as_register(), from_addr);
1025 }
1026 break;
1027
1028 case T_ADDRESS:
1029 __ movptr(dest->as_register(), from_addr);
1030 break;
1031 case T_INT:
1032 __ movl(dest->as_register(), from_addr);
1033 break;
1034
1035 case T_LONG: {
1036 Register to_lo = dest->as_register_lo();
1037 Register to_hi = dest->as_register_hi();
1038 __ movptr(to_lo, as_Address_lo(addr));
1039 break;
1040 }
1041
1042 case T_BOOLEAN: // fall through
1043 case T_BYTE: {
1044 Register dest_reg = dest->as_register();
1045 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1046 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1047 __ movsbl(dest_reg, from_addr);
1048 } else {
1049 __ movb(dest_reg, from_addr);
1050 __ shll(dest_reg, 24);
1051 __ sarl(dest_reg, 24);
1052 }
1053 break;
1054 }
1055
1056 case T_CHAR: {
1057 Register dest_reg = dest->as_register();
1058 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1059 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1060 __ movzwl(dest_reg, from_addr);
1061 } else {
1062 __ movw(dest_reg, from_addr);
1063 }
1064 break;
1065 }
1066
1067 case T_SHORT: {
1068 Register dest_reg = dest->as_register();
1069 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1070 __ movswl(dest_reg, from_addr);
1071 } else {
1072 __ movw(dest_reg, from_addr);
1073 __ shll(dest_reg, 16);
1074 __ sarl(dest_reg, 16);
1075 }
1076 break;
1077 }
1078
1079 default:
1080 ShouldNotReachHere();
1081 }
1082
1083 if (patch != nullptr) {
1084 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1085 }
1086
1087 if (is_reference_type(type)) {
1088 if (UseCompressedOops && !wide) {
1089 __ decode_heap_oop(dest->as_register());
1090 }
1091
1092 __ verify_oop(dest->as_register());
1093 }
1094 }
1095
1096
1097 NEEDS_CLEANUP; // This could be static?
1098 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1099 int elem_size = type2aelembytes(type);
1100 switch (elem_size) {
1101 case 1: return Address::times_1;
1102 case 2: return Address::times_2;
1103 case 4: return Address::times_4;
1104 case 8: return Address::times_8;
1105 }
1106 ShouldNotReachHere();
1107 return Address::no_scale;
1108 }
1109
1110
1111 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1112 switch (op->code()) {
1113 case lir_idiv:
1114 case lir_irem:
1115 arithmetic_idiv(op->code(),
1116 op->in_opr1(),
1117 op->in_opr2(),
1118 op->in_opr3(),
1119 op->result_opr(),
1120 op->info());
1121 break;
1122 case lir_fmad:
1123 __ fmad(op->result_opr()->as_xmm_double_reg(),
1124 op->in_opr1()->as_xmm_double_reg(),
1125 op->in_opr2()->as_xmm_double_reg(),
1126 op->in_opr3()->as_xmm_double_reg());
1127 break;
1128 case lir_fmaf:
1129 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1130 op->in_opr1()->as_xmm_float_reg(),
1131 op->in_opr2()->as_xmm_float_reg(),
1132 op->in_opr3()->as_xmm_float_reg());
1133 break;
1134 default: ShouldNotReachHere(); break;
1135 }
1136 }
1137
1138 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1139 #ifdef ASSERT
1140 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1141 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1142 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1143 #endif
1144
1145 if (op->cond() == lir_cond_always) {
1146 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1147 __ jmp (*(op->label()));
1148 } else {
1149 Assembler::Condition acond = Assembler::zero;
1150 if (op->code() == lir_cond_float_branch) {
1151 assert(op->ublock() != nullptr, "must have unordered successor");
1152 __ jcc(Assembler::parity, *(op->ublock()->label()));
1153 switch(op->cond()) {
1154 case lir_cond_equal: acond = Assembler::equal; break;
1155 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1156 case lir_cond_less: acond = Assembler::below; break;
1157 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1158 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1159 case lir_cond_greater: acond = Assembler::above; break;
1160 default: ShouldNotReachHere();
1161 }
1162 } else {
1163 switch (op->cond()) {
1164 case lir_cond_equal: acond = Assembler::equal; break;
1165 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1166 case lir_cond_less: acond = Assembler::less; break;
1167 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1168 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1169 case lir_cond_greater: acond = Assembler::greater; break;
1170 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1171 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1172 default: ShouldNotReachHere();
1173 }
1174 }
1175 __ jcc(acond,*(op->label()));
1176 }
1177 }
1178
1179 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1180 LIR_Opr src = op->in_opr();
1181 LIR_Opr dest = op->result_opr();
1182
1183 switch (op->bytecode()) {
1184 case Bytecodes::_i2l:
1185 __ movl2ptr(dest->as_register_lo(), src->as_register());
1186 break;
1187
1188 case Bytecodes::_l2i:
1189 __ movl(dest->as_register(), src->as_register_lo());
1190 break;
1191
1192 case Bytecodes::_i2b:
1193 move_regs(src->as_register(), dest->as_register());
1194 __ sign_extend_byte(dest->as_register());
1195 break;
1196
1197 case Bytecodes::_i2c:
1198 move_regs(src->as_register(), dest->as_register());
1199 __ andl(dest->as_register(), 0xFFFF);
1200 break;
1201
1202 case Bytecodes::_i2s:
1203 move_regs(src->as_register(), dest->as_register());
1204 __ sign_extend_short(dest->as_register());
1205 break;
1206
1207 case Bytecodes::_f2d:
1208 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1209 break;
1210
1211 case Bytecodes::_d2f:
1212 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1213 break;
1214
1215 case Bytecodes::_i2f:
1216 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1217 break;
1218
1219 case Bytecodes::_i2d:
1220 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1221 break;
1222
1223 case Bytecodes::_l2f:
1224 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1225 break;
1226
1227 case Bytecodes::_l2d:
1228 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1229 break;
1230
1231 case Bytecodes::_f2i:
1232 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1233 break;
1234
1235 case Bytecodes::_d2i:
1236 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1237 break;
1238
1239 case Bytecodes::_f2l:
1240 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1241 break;
1242
1243 case Bytecodes::_d2l:
1244 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1245 break;
1246
1247 default: ShouldNotReachHere();
1248 }
1249 }
1250
1251 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1252 if (op->init_check()) {
1253 add_debug_info_for_null_check_here(op->stub()->info());
1254 // init_state needs acquire, but x86 is TSO, and so we are already good.
1255 __ cmpb(Address(op->klass()->as_register(),
1256 InstanceKlass::init_state_offset()),
1257 InstanceKlass::fully_initialized);
1258 __ jcc(Assembler::notEqual, *op->stub()->entry());
1259 }
1260 __ allocate_object(op->obj()->as_register(),
1261 op->tmp1()->as_register(),
1262 op->tmp2()->as_register(),
1263 op->header_size(),
1264 op->object_size(),
1265 op->klass()->as_register(),
1266 *op->stub()->entry());
1267 __ bind(*op->stub()->continuation());
1268 }
1269
1270 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1271 Register len = op->len()->as_register();
1272 __ movslq(len, len);
1273
1274 if (UseSlowPath || op->always_slow_path() ||
1275 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1276 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1277 __ jmp(*op->stub()->entry());
1278 } else {
1279 Register tmp1 = op->tmp1()->as_register();
1280 Register tmp2 = op->tmp2()->as_register();
1281 Register tmp3 = op->tmp3()->as_register();
1282 if (len == tmp1) {
1283 tmp1 = tmp3;
1284 } else if (len == tmp2) {
1285 tmp2 = tmp3;
1286 } else if (len == tmp3) {
1287 // everything is ok
1288 } else {
1289 __ mov(tmp3, len);
1290 }
1291 __ allocate_array(op->obj()->as_register(),
1292 len,
1293 tmp1,
1294 tmp2,
1295 arrayOopDesc::base_offset_in_bytes(op->type()),
1296 array_element_size(op->type()),
1297 op->klass()->as_register(),
1298 *op->stub()->entry(),
1299 op->zero_array());
1300 }
1301 __ bind(*op->stub()->continuation());
1302 }
1303
1304 void LIR_Assembler::type_profile_helper(Register mdo,
1305 ciMethodData *md, ciProfileData *data,
1306 Register recv, Label* update_done) {
1307 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1308 Label next_test;
1309 // See if the receiver is receiver[n].
1310 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1311 __ jccb(Assembler::notEqual, next_test);
1312 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1313 __ addptr(data_addr, DataLayout::counter_increment);
1314 __ jmp(*update_done);
1315 __ bind(next_test);
1316 }
1317
1318 // Didn't find receiver; find next empty slot and fill it in
1319 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1320 Label next_test;
1321 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1322 __ cmpptr(recv_addr, NULL_WORD);
1323 __ jccb(Assembler::notEqual, next_test);
1324 __ movptr(recv_addr, recv);
1325 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1326 __ jmp(*update_done);
1327 __ bind(next_test);
1328 }
1329 }
1330
1331 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1332 // we always need a stub for the failure case.
1333 CodeStub* stub = op->stub();
1334 Register obj = op->object()->as_register();
1335 Register k_RInfo = op->tmp1()->as_register();
1336 Register klass_RInfo = op->tmp2()->as_register();
1337 Register dst = op->result_opr()->as_register();
1338 ciKlass* k = op->klass();
1339 Register Rtmp1 = noreg;
1340 Register tmp_load_klass = rscratch1;
1341
1342 // check if it needs to be profiled
1343 ciMethodData* md = nullptr;
1344 ciProfileData* data = nullptr;
1345
1346 if (op->should_profile()) {
1347 ciMethod* method = op->profiled_method();
1348 assert(method != nullptr, "Should have method");
1349 int bci = op->profiled_bci();
1350 md = method->method_data_or_null();
1351 assert(md != nullptr, "Sanity");
1352 data = md->bci_to_data(bci);
1353 assert(data != nullptr, "need data for type check");
1354 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1355 }
1356 Label* success_target = success;
1357 Label* failure_target = failure;
1358
1359 if (obj == k_RInfo) {
1360 k_RInfo = dst;
1361 } else if (obj == klass_RInfo) {
1362 klass_RInfo = dst;
1363 }
1364 if (k->is_loaded() && !UseCompressedClassPointers) {
1365 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1366 } else {
1367 Rtmp1 = op->tmp3()->as_register();
1368 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1369 }
1370
1371 assert_different_registers(obj, k_RInfo, klass_RInfo);
1372
1373 if (op->need_null_check()) {
1374 __ testptr(obj, obj);
1375 if (op->should_profile()) {
1376 Label not_null;
1377 Register mdo = klass_RInfo;
1378 __ mov_metadata(mdo, md->constant_encoding());
1379 __ jccb(Assembler::notEqual, not_null);
1380 // Object is null; update MDO and exit
1381 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1382 int header_bits = BitData::null_seen_byte_constant();
1383 __ orb(data_addr, header_bits);
1384 __ jmp(*obj_is_null);
1385 __ bind(not_null);
1386
1387 Label update_done;
1388 Register recv = k_RInfo;
1389 __ load_klass(recv, obj, tmp_load_klass);
1390 type_profile_helper(mdo, md, data, recv, &update_done);
1391
1392 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1393 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1394
1395 __ bind(update_done);
1396 } else {
1397 __ jcc(Assembler::equal, *obj_is_null);
1398 }
1399 }
1400
1401 if (!k->is_loaded()) {
1402 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1403 } else {
1404 __ mov_metadata(k_RInfo, k->constant_encoding());
1405 }
1406 __ verify_oop(obj);
1407
1408 if (op->fast_check()) {
1409 // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
1410 // get object class
1411 // not a safepoint as obj null check happens earlier
1412 if (UseCompressedClassPointers) {
1413 __ load_klass(Rtmp1, obj, tmp_load_klass);
1414 __ cmpptr(k_RInfo, Rtmp1);
1415 } else {
1416 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1417 }
1418 __ jcc(Assembler::notEqual, *failure_target);
1419 // successful cast, fall through to profile or jump
1420 } else {
1421 // get object class
1422 // not a safepoint as obj null check happens earlier
1423 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1424 if (k->is_loaded()) {
1425 // See if we get an immediate positive hit
1426 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1427 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1428 __ jcc(Assembler::notEqual, *failure_target);
1429 // successful cast, fall through to profile or jump
1430 } else {
1431 // See if we get an immediate positive hit
1432 __ jcc(Assembler::equal, *success_target);
1433 // check for self
1434 __ cmpptr(klass_RInfo, k_RInfo);
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1455 __ pop_ppx(k_RInfo);
1456 // result is a boolean
1457 __ testl(k_RInfo, k_RInfo);
1458 __ jcc(Assembler::equal, *failure_target);
1459 // successful cast, fall through to profile or jump
1460 }
1461 }
1462 __ jmp(*success);
1463 }
1464
1465
1466 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1467 Register tmp_load_klass = rscratch1;
1468 LIR_Code code = op->code();
1469 if (code == lir_store_check) {
1470 Register value = op->object()->as_register();
1471 Register array = op->array()->as_register();
1472 Register k_RInfo = op->tmp1()->as_register();
1473 Register klass_RInfo = op->tmp2()->as_register();
1474 Register Rtmp1 = op->tmp3()->as_register();
1475
1476 CodeStub* stub = op->stub();
1477
1478 // check if it needs to be profiled
1479 ciMethodData* md = nullptr;
1480 ciProfileData* data = nullptr;
1481
1482 if (op->should_profile()) {
1483 ciMethod* method = op->profiled_method();
1484 assert(method != nullptr, "Should have method");
1485 int bci = op->profiled_bci();
1486 md = method->method_data_or_null();
1487 assert(md != nullptr, "Sanity");
1488 data = md->bci_to_data(bci);
1489 assert(data != nullptr, "need data for type check");
1490 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1491 }
1492 Label done;
1493 Label* success_target = &done;
1494 Label* failure_target = stub->entry();
1495
1496 __ testptr(value, value);
1497 if (op->should_profile()) {
1498 Label not_null;
1499 Register mdo = klass_RInfo;
1500 __ mov_metadata(mdo, md->constant_encoding());
1501 __ jccb(Assembler::notEqual, not_null);
1502 // Object is null; update MDO and exit
1503 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1504 int header_bits = BitData::null_seen_byte_constant();
1505 __ orb(data_addr, header_bits);
1506 __ jmp(done);
1507 __ bind(not_null);
1508
1509 Label update_done;
1510 Register recv = k_RInfo;
1511 __ load_klass(recv, value, tmp_load_klass);
1512 type_profile_helper(mdo, md, data, recv, &update_done);
1513
1514 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1515 __ addptr(counter_addr, DataLayout::counter_increment);
1516 __ bind(update_done);
1517 } else {
1518 __ jcc(Assembler::equal, done);
1519 }
1520
1521 add_debug_info_for_null_check_here(op->info_for_exception());
1522 __ load_klass(k_RInfo, array, tmp_load_klass);
1523 __ load_klass(klass_RInfo, value, tmp_load_klass);
1524
1525 // get instance klass (it's already uncompressed)
1526 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1527 // perform the fast part of the checking logic
1528 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1529 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1530 __ push_ppx(klass_RInfo);
1531 __ push_ppx(k_RInfo);
1532 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1533 __ pop_ppx(klass_RInfo);
1534 __ pop_ppx(k_RInfo);
1535 // result is a boolean
1536 __ testl(k_RInfo, k_RInfo);
1537 __ jcc(Assembler::equal, *failure_target);
1538 // fall through to the success case
1539
1540 __ bind(done);
1541 } else
1542 if (code == lir_checkcast) {
1543 Register obj = op->object()->as_register();
1544 Register dst = op->result_opr()->as_register();
1545 Label success;
1546 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1547 __ bind(success);
1548 if (dst != obj) {
1549 __ mov(dst, obj);
1550 }
1551 } else
1552 if (code == lir_instanceof) {
1553 Register obj = op->object()->as_register();
1554 Register dst = op->result_opr()->as_register();
1555 Label success, failure, done;
1556 emit_typecheck_helper(op, &success, &failure, &failure);
1557 __ bind(failure);
1558 __ xorptr(dst, dst);
1559 __ jmpb(done);
1560 __ bind(success);
1561 __ movptr(dst, 1);
1562 __ bind(done);
1563 } else {
1564 ShouldNotReachHere();
1565 }
1566
1567 }
1568
1569 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1570 // We are loading/storing from/to an array that *may* be a flat array (the
1571 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1572 // If this array is a flat array, take the slow path.
1573 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1574 if (!op->value()->is_illegal()) {
1575 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1576 // The array is not a flat array, but it might be null-free. If we are storing
1577 // a null into a null-free array, take the slow path (which will throw NPE).
1578 Label skip;
1579 __ cmpptr(op->value()->as_register(), NULL_WORD);
1580 __ jcc(Assembler::notEqual, skip);
1581 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582 __ bind(skip);
1583 }
1584 }
1585
1586 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1587 // We are storing into an array that *may* be null-free (the declared type is
1588 // Object[], abstract[], interface[] or VT.ref[]).
1589 Label test_mark_word;
1590 Register tmp = op->tmp()->as_register();
1591 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1592 __ testl(tmp, markWord::unlocked_value);
1593 __ jccb(Assembler::notZero, test_mark_word);
1594 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1595 __ bind(test_mark_word);
1596 __ testl(tmp, markWord::null_free_array_bit_in_place);
1597 }
1598
1599 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1600 Label L_oops_equal;
1601 Label L_oops_not_equal;
1602 Label L_end;
1603
1604 Register left = op->left()->as_register();
1605 Register right = op->right()->as_register();
1606
1607 __ cmpptr(left, right);
1608 __ jcc(Assembler::equal, L_oops_equal);
1609
1610 // (1) Null check -- if one of the operands is null, the other must not be null (because
1611 // the two references are not equal), so they are not substitutable,
1612 // FIXME: do null check only if the operand is nullable
1613 __ testptr(left, right);
1614 __ jcc(Assembler::zero, L_oops_not_equal);
1615
1616 ciKlass* left_klass = op->left_klass();
1617 ciKlass* right_klass = op->right_klass();
1618
1619 // (2) Inline type check -- if either of the operands is not a inline type,
1620 // they are not substitutable. We do this only if we are not sure that the
1621 // operands are inline type
1622 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1623 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1624 Register tmp1 = op->tmp1()->as_register();
1625 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1626 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1627 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1628 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1629 __ jcc(Assembler::notEqual, L_oops_not_equal);
1630 }
1631
1632 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1633 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1634 // No need to load klass -- the operands are statically known to be the same inline klass.
1635 __ jmp(*op->stub()->entry());
1636 } else {
1637 Register left_klass_op = op->left_klass_op()->as_register();
1638 Register right_klass_op = op->right_klass_op()->as_register();
1639
1640 if (UseCompressedClassPointers) {
1641 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1642 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1643 __ cmpl(left_klass_op, right_klass_op);
1644 } else {
1645 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1646 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1647 __ cmpptr(left_klass_op, right_klass_op);
1648 }
1649
1650 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1651 // fall through to L_oops_not_equal
1652 }
1653
1654 __ bind(L_oops_not_equal);
1655 move(op->not_equal_result(), op->result_opr());
1656 __ jmp(L_end);
1657
1658 __ bind(L_oops_equal);
1659 move(op->equal_result(), op->result_opr());
1660 __ jmp(L_end);
1661
1662 // We've returned from the stub. RAX contains 0x0 IFF the two
1663 // operands are not substitutable. (Don't compare against 0x1 in case the
1664 // C compiler is naughty)
1665 __ bind(*op->stub()->continuation());
1666 __ cmpl(rax, 0);
1667 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1668 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1669 // fall-through
1670 __ bind(L_end);
1671 }
1672
1673 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1674 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1675 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1676 Register newval = op->new_value()->as_register();
1677 Register cmpval = op->cmp_value()->as_register();
1678 assert(cmpval == rax, "wrong register");
1679 assert(newval != noreg, "new val must be register");
1680 assert(cmpval != newval, "cmp and new values must be in different registers");
1681 assert(cmpval != addr, "cmp and addr must be in different registers");
1682 assert(newval != addr, "new value and addr must be in different registers");
1683
1684 if (op->code() == lir_cas_obj) {
1685 if (UseCompressedOops) {
1686 __ encode_heap_oop(cmpval);
1687 __ mov(rscratch1, newval);
1688 __ encode_heap_oop(rscratch1);
1689 __ lock();
1690 // cmpval (rax) is implicitly used by this instruction
1691 __ cmpxchgl(rscratch1, Address(addr, 0));
1692 } else {
1693 __ lock();
1694 __ cmpxchgptr(newval, Address(addr, 0));
1695 }
1696 } else {
1697 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1698 __ lock();
1699 __ cmpxchgl(newval, Address(addr, 0));
1700 }
1701 } else if (op->code() == lir_cas_long) {
1702 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1703 Register newval = op->new_value()->as_register_lo();
1704 Register cmpval = op->cmp_value()->as_register_lo();
1705 assert(cmpval == rax, "wrong register");
1706 assert(newval != noreg, "new val must be register");
1707 assert(cmpval != newval, "cmp and new values must be in different registers");
1708 assert(cmpval != addr, "cmp and addr must be in different registers");
1709 assert(newval != addr, "new value and addr must be in different registers");
1710 __ lock();
1711 __ cmpxchgq(newval, Address(addr, 0));
1712 } else {
1713 Unimplemented();
1714 }
1715 }
1716
1717 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1718 assert(dst->is_cpu_register(), "must be");
1719 assert(dst->type() == src->type(), "must be");
1720
1721 if (src->is_cpu_register()) {
1722 reg2reg(src, dst);
1723 } else if (src->is_stack()) {
1724 stack2reg(src, dst, dst->type());
1725 } else if (src->is_constant()) {
1726 const2reg(src, dst, lir_patch_none, nullptr);
1727 } else {
1728 ShouldNotReachHere();
1729 }
1730 }
1731
1732 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1733 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1734 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1735
1736 Assembler::Condition acond, ncond;
1737 switch (condition) {
1738 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1739 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1740 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1741 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1742 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1743 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1744 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1745 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1746 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1747 ShouldNotReachHere();
1748 }
1749
1750 if (opr1->is_cpu_register()) {
1751 reg2reg(opr1, result);
1752 } else if (opr1->is_stack()) {
1753 stack2reg(opr1, result, result->type());
1754 } else if (opr1->is_constant()) {
1755 const2reg(opr1, result, lir_patch_none, nullptr);
1756 } else {
1757 ShouldNotReachHere();
1758 }
1759
1760 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1761 // optimized version that does not require a branch
1762 if (opr2->is_single_cpu()) {
1763 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1764 __ cmov(ncond, result->as_register(), opr2->as_register());
1765 } else if (opr2->is_double_cpu()) {
1766 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1767 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1768 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1769 } else if (opr2->is_single_stack()) {
1770 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1771 } else if (opr2->is_double_stack()) {
1772 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1773 } else {
1774 ShouldNotReachHere();
1775 }
1776
1777 } else {
1778 Label skip;
1779 __ jccb(acond, skip);
1780 if (opr2->is_cpu_register()) {
1781 reg2reg(opr2, result);
1782 } else if (opr2->is_stack()) {
1783 stack2reg(opr2, result, result->type());
1784 } else if (opr2->is_constant()) {
1785 const2reg(opr2, result, lir_patch_none, nullptr);
1786 } else {
1787 ShouldNotReachHere();
1788 }
1789 __ bind(skip);
1790 }
1791 }
1792
1793
1794 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1795 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1796
1797 if (left->is_single_cpu()) {
1798 assert(left == dest, "left and dest must be equal");
1799 Register lreg = left->as_register();
1800
1801 if (right->is_single_cpu()) {
1802 // cpu register - cpu register
1803 Register rreg = right->as_register();
1804 switch (code) {
1805 case lir_add: __ addl (lreg, rreg); break;
1806 case lir_sub: __ subl (lreg, rreg); break;
1807 case lir_mul: __ imull(lreg, rreg); break;
1808 default: ShouldNotReachHere();
1809 }
1810
1811 } else if (right->is_stack()) {
1812 // cpu register - stack
1813 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1814 switch (code) {
1815 case lir_add: __ addl(lreg, raddr); break;
1816 case lir_sub: __ subl(lreg, raddr); break;
1817 default: ShouldNotReachHere();
1818 }
1819
1820 } else if (right->is_constant()) {
1821 // cpu register - constant
1822 jint c = right->as_constant_ptr()->as_jint();
1823 switch (code) {
1824 case lir_add: {
1825 __ incrementl(lreg, c);
1826 break;
1827 }
1828 case lir_sub: {
1829 __ decrementl(lreg, c);
1830 break;
1831 }
1832 default: ShouldNotReachHere();
1833 }
1834
1835 } else {
1836 ShouldNotReachHere();
1837 }
1838
1839 } else if (left->is_double_cpu()) {
1840 assert(left == dest, "left and dest must be equal");
1841 Register lreg_lo = left->as_register_lo();
1842 Register lreg_hi = left->as_register_hi();
1843
1844 if (right->is_double_cpu()) {
1845 // cpu register - cpu register
1846 Register rreg_lo = right->as_register_lo();
1847 Register rreg_hi = right->as_register_hi();
1848 assert_different_registers(lreg_lo, rreg_lo);
1849 switch (code) {
1850 case lir_add:
1851 __ addptr(lreg_lo, rreg_lo);
1852 break;
1853 case lir_sub:
1854 __ subptr(lreg_lo, rreg_lo);
1855 break;
1856 case lir_mul:
1857 __ imulq(lreg_lo, rreg_lo);
1858 break;
1859 default:
1860 ShouldNotReachHere();
1861 }
1862
1863 } else if (right->is_constant()) {
1864 // cpu register - constant
1865 jlong c = right->as_constant_ptr()->as_jlong_bits();
1866 __ movptr(r10, (intptr_t) c);
1867 switch (code) {
1868 case lir_add:
1869 __ addptr(lreg_lo, r10);
1870 break;
1871 case lir_sub:
1872 __ subptr(lreg_lo, r10);
1873 break;
1874 default:
1875 ShouldNotReachHere();
1876 }
1877
1878 } else {
1879 ShouldNotReachHere();
1880 }
1881
1882 } else if (left->is_single_xmm()) {
1883 assert(left == dest, "left and dest must be equal");
1884 XMMRegister lreg = left->as_xmm_float_reg();
1885
1886 if (right->is_single_xmm()) {
1887 XMMRegister rreg = right->as_xmm_float_reg();
1888 switch (code) {
1889 case lir_add: __ addss(lreg, rreg); break;
1890 case lir_sub: __ subss(lreg, rreg); break;
1891 case lir_mul: __ mulss(lreg, rreg); break;
1892 case lir_div: __ divss(lreg, rreg); break;
1893 default: ShouldNotReachHere();
1894 }
1895 } else {
1896 Address raddr;
1897 if (right->is_single_stack()) {
1898 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1899 } else if (right->is_constant()) {
1900 // hack for now
1901 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1902 } else {
1903 ShouldNotReachHere();
1904 }
1905 switch (code) {
1906 case lir_add: __ addss(lreg, raddr); break;
1907 case lir_sub: __ subss(lreg, raddr); break;
1908 case lir_mul: __ mulss(lreg, raddr); break;
1909 case lir_div: __ divss(lreg, raddr); break;
1910 default: ShouldNotReachHere();
1911 }
1912 }
1913
1914 } else if (left->is_double_xmm()) {
1915 assert(left == dest, "left and dest must be equal");
1916
1917 XMMRegister lreg = left->as_xmm_double_reg();
1918 if (right->is_double_xmm()) {
1919 XMMRegister rreg = right->as_xmm_double_reg();
1920 switch (code) {
1921 case lir_add: __ addsd(lreg, rreg); break;
1922 case lir_sub: __ subsd(lreg, rreg); break;
1923 case lir_mul: __ mulsd(lreg, rreg); break;
1924 case lir_div: __ divsd(lreg, rreg); break;
1925 default: ShouldNotReachHere();
1926 }
1927 } else {
1928 Address raddr;
1929 if (right->is_double_stack()) {
1930 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1931 } else if (right->is_constant()) {
1932 // hack for now
1933 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1934 } else {
1935 ShouldNotReachHere();
1936 }
1937 switch (code) {
1938 case lir_add: __ addsd(lreg, raddr); break;
1939 case lir_sub: __ subsd(lreg, raddr); break;
1940 case lir_mul: __ mulsd(lreg, raddr); break;
1941 case lir_div: __ divsd(lreg, raddr); break;
1942 default: ShouldNotReachHere();
1943 }
1944 }
1945
1946 } else if (left->is_single_stack() || left->is_address()) {
1947 assert(left == dest, "left and dest must be equal");
1948
1949 Address laddr;
1950 if (left->is_single_stack()) {
1951 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1952 } else if (left->is_address()) {
1953 laddr = as_Address(left->as_address_ptr());
1954 } else {
1955 ShouldNotReachHere();
1956 }
1957
1958 if (right->is_single_cpu()) {
1959 Register rreg = right->as_register();
1960 switch (code) {
1961 case lir_add: __ addl(laddr, rreg); break;
1962 case lir_sub: __ subl(laddr, rreg); break;
1963 default: ShouldNotReachHere();
1964 }
1965 } else if (right->is_constant()) {
1966 jint c = right->as_constant_ptr()->as_jint();
1967 switch (code) {
1968 case lir_add: {
1969 __ incrementl(laddr, c);
1970 break;
1971 }
1972 case lir_sub: {
1973 __ decrementl(laddr, c);
1974 break;
1975 }
1976 default: ShouldNotReachHere();
1977 }
1978 } else {
1979 ShouldNotReachHere();
1980 }
1981
1982 } else {
1983 ShouldNotReachHere();
1984 }
1985 }
1986
1987
1988 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1989 if (value->is_double_xmm()) {
1990 switch(code) {
1991 case lir_abs :
1992 {
1993 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1994 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1995 }
1996 assert(!tmp->is_valid(), "do not need temporary");
1997 __ andpd(dest->as_xmm_double_reg(),
1998 ExternalAddress((address)double_signmask_pool),
1999 rscratch1);
2000 }
2001 break;
2002
2003 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2004 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2005 default : ShouldNotReachHere();
2006 }
2007
2008 } else if (code == lir_f2hf) {
2009 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2010 } else if (code == lir_hf2f) {
2011 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2012 } else {
2013 Unimplemented();
2014 }
2015 }
2016
2017 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2018 // assert(left->destroys_register(), "check");
2019 if (left->is_single_cpu()) {
2020 Register reg = left->as_register();
2021 if (right->is_constant()) {
2022 int val = right->as_constant_ptr()->as_jint();
2023 switch (code) {
2024 case lir_logic_and: __ andl (reg, val); break;
2025 case lir_logic_or: __ orl (reg, val); break;
2026 case lir_logic_xor: __ xorl (reg, val); break;
2027 default: ShouldNotReachHere();
2028 }
2029 } else if (right->is_stack()) {
2030 // added support for stack operands
2031 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2032 switch (code) {
2033 case lir_logic_and: __ andl (reg, raddr); break;
2034 case lir_logic_or: __ orl (reg, raddr); break;
2035 case lir_logic_xor: __ xorl (reg, raddr); break;
2036 default: ShouldNotReachHere();
2037 }
2038 } else {
2039 Register rright = right->as_register();
2040 switch (code) {
2041 case lir_logic_and: __ andptr (reg, rright); break;
2042 case lir_logic_or : __ orptr (reg, rright); break;
2043 case lir_logic_xor: __ xorptr (reg, rright); break;
2044 default: ShouldNotReachHere();
2045 }
2046 }
2047 move_regs(reg, dst->as_register());
2048 } else {
2049 Register l_lo = left->as_register_lo();
2050 Register l_hi = left->as_register_hi();
2051 if (right->is_constant()) {
2052 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2053 switch (code) {
2054 case lir_logic_and:
2055 __ andq(l_lo, rscratch1);
2056 break;
2057 case lir_logic_or:
2058 __ orq(l_lo, rscratch1);
2059 break;
2060 case lir_logic_xor:
2061 __ xorq(l_lo, rscratch1);
2062 break;
2063 default: ShouldNotReachHere();
2064 }
2065 } else {
2066 Register r_lo;
2067 if (is_reference_type(right->type())) {
2068 r_lo = right->as_register();
2069 } else {
2070 r_lo = right->as_register_lo();
2071 }
2072 switch (code) {
2073 case lir_logic_and:
2074 __ andptr(l_lo, r_lo);
2075 break;
2076 case lir_logic_or:
2077 __ orptr(l_lo, r_lo);
2078 break;
2079 case lir_logic_xor:
2080 __ xorptr(l_lo, r_lo);
2081 break;
2082 default: ShouldNotReachHere();
2083 }
2084 }
2085
2086 Register dst_lo = dst->as_register_lo();
2087 Register dst_hi = dst->as_register_hi();
2088
2089 move_regs(l_lo, dst_lo);
2090 }
2091 }
2092
2093
2094 // we assume that rax, and rdx can be overwritten
2095 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2096
2097 assert(left->is_single_cpu(), "left must be register");
2098 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2099 assert(result->is_single_cpu(), "result must be register");
2100
2101 // assert(left->destroys_register(), "check");
2102 // assert(right->destroys_register(), "check");
2103
2104 Register lreg = left->as_register();
2105 Register dreg = result->as_register();
2106
2107 if (right->is_constant()) {
2108 jint divisor = right->as_constant_ptr()->as_jint();
2109 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2110 if (code == lir_idiv) {
2111 assert(lreg == rax, "must be rax,");
2112 assert(temp->as_register() == rdx, "tmp register must be rdx");
2113 __ cdql(); // sign extend into rdx:rax
2114 if (divisor == 2) {
2115 __ subl(lreg, rdx);
2116 } else {
2117 __ andl(rdx, divisor - 1);
2118 __ addl(lreg, rdx);
2119 }
2120 __ sarl(lreg, log2i_exact(divisor));
2121 move_regs(lreg, dreg);
2122 } else if (code == lir_irem) {
2123 Label done;
2124 __ mov(dreg, lreg);
2125 __ andl(dreg, 0x80000000 | (divisor - 1));
2126 __ jcc(Assembler::positive, done);
2127 __ decrement(dreg);
2128 __ orl(dreg, ~(divisor - 1));
2129 __ increment(dreg);
2130 __ bind(done);
2131 } else {
2132 ShouldNotReachHere();
2133 }
2134 } else {
2135 Register rreg = right->as_register();
2136 assert(lreg == rax, "left register must be rax,");
2137 assert(rreg != rdx, "right register must not be rdx");
2138 assert(temp->as_register() == rdx, "tmp register must be rdx");
2139
2140 move_regs(lreg, rax);
2141
2142 int idivl_offset = __ corrected_idivl(rreg);
2143 if (ImplicitDiv0Checks) {
2144 add_debug_info_for_div0(idivl_offset, info);
2145 }
2146 if (code == lir_irem) {
2147 move_regs(rdx, dreg); // result is in rdx
2148 } else {
2149 move_regs(rax, dreg);
2150 }
2151 }
2152 }
2153
2154
2155 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2156 if (opr1->is_single_cpu()) {
2157 Register reg1 = opr1->as_register();
2158 if (opr2->is_single_cpu()) {
2159 // cpu register - cpu register
2160 if (is_reference_type(opr1->type())) {
2161 __ cmpoop(reg1, opr2->as_register());
2162 } else {
2163 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2164 __ cmpl(reg1, opr2->as_register());
2165 }
2166 } else if (opr2->is_stack()) {
2167 // cpu register - stack
2168 if (is_reference_type(opr1->type())) {
2169 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2170 } else {
2171 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2172 }
2173 } else if (opr2->is_constant()) {
2174 // cpu register - constant
2175 LIR_Const* c = opr2->as_constant_ptr();
2176 if (c->type() == T_INT) {
2177 jint i = c->as_jint();
2178 if (i == 0) {
2179 __ testl(reg1, reg1);
2180 } else {
2181 __ cmpl(reg1, i);
2182 }
2183 } else if (c->type() == T_METADATA) {
2184 // All we need for now is a comparison with null for equality.
2185 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2186 Metadata* m = c->as_metadata();
2187 if (m == nullptr) {
2188 __ testptr(reg1, reg1);
2189 } else {
2190 ShouldNotReachHere();
2191 }
2192 } else if (is_reference_type(c->type())) {
2193 // In 64bit oops are single register
2194 jobject o = c->as_jobject();
2195 if (o == nullptr) {
2196 __ testptr(reg1, reg1);
2197 } else {
2198 __ cmpoop(reg1, o, rscratch1);
2199 }
2200 } else {
2201 fatal("unexpected type: %s", basictype_to_str(c->type()));
2202 }
2203 // cpu register - address
2204 } else if (opr2->is_address()) {
2205 if (op->info() != nullptr) {
2206 add_debug_info_for_null_check_here(op->info());
2207 }
2208 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2209 } else {
2210 ShouldNotReachHere();
2211 }
2212
2213 } else if(opr1->is_double_cpu()) {
2214 Register xlo = opr1->as_register_lo();
2215 Register xhi = opr1->as_register_hi();
2216 if (opr2->is_double_cpu()) {
2217 __ cmpptr(xlo, opr2->as_register_lo());
2218 } else if (opr2->is_constant()) {
2219 // cpu register - constant 0
2220 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2221 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2222 } else {
2223 ShouldNotReachHere();
2224 }
2225
2226 } else if (opr1->is_single_xmm()) {
2227 XMMRegister reg1 = opr1->as_xmm_float_reg();
2228 if (opr2->is_single_xmm()) {
2229 // xmm register - xmm register
2230 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2231 } else if (opr2->is_stack()) {
2232 // xmm register - stack
2233 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2234 } else if (opr2->is_constant()) {
2235 // xmm register - constant
2236 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2237 } else if (opr2->is_address()) {
2238 // xmm register - address
2239 if (op->info() != nullptr) {
2240 add_debug_info_for_null_check_here(op->info());
2241 }
2242 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2243 } else {
2244 ShouldNotReachHere();
2245 }
2246
2247 } else if (opr1->is_double_xmm()) {
2248 XMMRegister reg1 = opr1->as_xmm_double_reg();
2249 if (opr2->is_double_xmm()) {
2250 // xmm register - xmm register
2251 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2252 } else if (opr2->is_stack()) {
2253 // xmm register - stack
2254 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2255 } else if (opr2->is_constant()) {
2256 // xmm register - constant
2257 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2258 } else if (opr2->is_address()) {
2259 // xmm register - address
2260 if (op->info() != nullptr) {
2261 add_debug_info_for_null_check_here(op->info());
2262 }
2263 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2264 } else {
2265 ShouldNotReachHere();
2266 }
2267
2268 } else if (opr1->is_address() && opr2->is_constant()) {
2269 LIR_Const* c = opr2->as_constant_ptr();
2270 if (is_reference_type(c->type())) {
2271 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2272 __ movoop(rscratch1, c->as_jobject());
2273 }
2274 if (op->info() != nullptr) {
2275 add_debug_info_for_null_check_here(op->info());
2276 }
2277 // special case: address - constant
2278 LIR_Address* addr = opr1->as_address_ptr();
2279 if (c->type() == T_INT) {
2280 __ cmpl(as_Address(addr), c->as_jint());
2281 } else if (is_reference_type(c->type())) {
2282 // %%% Make this explode if addr isn't reachable until we figure out a
2283 // better strategy by giving noreg as the temp for as_Address
2284 __ cmpoop(rscratch1, as_Address(addr, noreg));
2285 } else {
2286 ShouldNotReachHere();
2287 }
2288
2289 } else {
2290 ShouldNotReachHere();
2291 }
2292 }
2293
2294 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2295 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2296 if (left->is_single_xmm()) {
2297 assert(right->is_single_xmm(), "must match");
2298 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2299 } else if (left->is_double_xmm()) {
2300 assert(right->is_double_xmm(), "must match");
2301 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2302
2303 } else {
2304 ShouldNotReachHere();
2305 }
2306 } else {
2307 assert(code == lir_cmp_l2i, "check");
2308 Label done;
2309 Register dest = dst->as_register();
2310 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2311 __ movl(dest, -1);
2312 __ jccb(Assembler::less, done);
2313 __ setb(Assembler::notZero, dest);
2314 __ movzbl(dest, dest);
2315 __ bind(done);
2316 }
2317 }
2318
2319
2320 void LIR_Assembler::align_call(LIR_Code code) {
2321 // make sure that the displacement word of the call ends up word aligned
2322 int offset = __ offset();
2323 switch (code) {
2324 case lir_static_call:
2325 case lir_optvirtual_call:
2326 case lir_dynamic_call:
2327 offset += NativeCall::displacement_offset;
2328 break;
2329 case lir_icvirtual_call:
2330 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2331 break;
2332 default: ShouldNotReachHere();
2333 }
2334 __ align(BytesPerWord, offset);
2335 }
2336
2337
2338 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2339 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2340 "must be aligned");
2341 __ call(AddressLiteral(op->addr(), rtype));
2342 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2343 __ post_call_nop();
2344 }
2345
2346
2347 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2348 __ ic_call(op->addr());
2349 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2350 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2351 "must be aligned");
2352 __ post_call_nop();
2353 }
2354
2355
2356 void LIR_Assembler::emit_static_call_stub() {
2357 address call_pc = __ pc();
2358 address stub = __ start_a_stub(call_stub_size());
2359 if (stub == nullptr) {
2360 bailout("static call stub overflow");
2361 return;
2362 }
2363
2364 int start = __ offset();
2365
2366 // make sure that the displacement word of the call ends up word aligned
2367 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2368 __ relocate(static_stub_Relocation::spec(call_pc));
2369 __ mov_metadata(rbx, (Metadata*)nullptr);
2370 // must be set to -1 at code generation time
2371 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2372 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2373 __ jump(RuntimeAddress(__ pc()));
2374
2375 assert(__ offset() - start <= call_stub_size(), "stub too big");
2376 __ end_a_stub();
2377 }
2378
2379
2380 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2381 assert(exceptionOop->as_register() == rax, "must match");
2382 assert(exceptionPC->as_register() == rdx, "must match");
2383
2384 // exception object is not added to oop map by LinearScan
2385 // (LinearScan assumes that no oops are in fixed registers)
2386 info->add_register_oop(exceptionOop);
2387 StubId unwind_id;
2388
2389 // get current pc information
2390 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2391 int pc_for_athrow_offset = __ offset();
2392 InternalAddress pc_for_athrow(__ pc());
2393 __ lea(exceptionPC->as_register(), pc_for_athrow);
2394 add_call_info(pc_for_athrow_offset, info); // for exception handler
2395
2396 __ verify_not_null_oop(rax);
2397 // search an exception handler (rax: exception oop, rdx: throwing pc)
2398 if (compilation()->has_fpu_code()) {
2399 unwind_id = StubId::c1_handle_exception_id;
2400 } else {
2401 unwind_id = StubId::c1_handle_exception_nofpu_id;
2402 }
2403 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2404
2405 // enough room for two byte trap
2406 __ nop();
2407 }
2408
2409
2410 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2411 assert(exceptionOop->as_register() == rax, "must match");
2412
2413 __ jmp(_unwind_handler_entry);
2414 }
2415
2416
2417 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2418
2419 // optimized version for linear scan:
2420 // * count must be already in ECX (guaranteed by LinearScan)
2421 // * left and dest must be equal
2422 // * tmp must be unused
2423 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2424 assert(left == dest, "left and dest must be equal");
2425 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2426
2427 if (left->is_single_cpu()) {
2428 Register value = left->as_register();
2429 assert(value != SHIFT_count, "left cannot be ECX");
2430
2431 switch (code) {
2432 case lir_shl: __ shll(value); break;
2433 case lir_shr: __ sarl(value); break;
2434 case lir_ushr: __ shrl(value); break;
2435 default: ShouldNotReachHere();
2436 }
2437 } else if (left->is_double_cpu()) {
2438 Register lo = left->as_register_lo();
2439 Register hi = left->as_register_hi();
2440 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2441 switch (code) {
2442 case lir_shl: __ shlptr(lo); break;
2443 case lir_shr: __ sarptr(lo); break;
2444 case lir_ushr: __ shrptr(lo); break;
2445 default: ShouldNotReachHere();
2446 }
2447 } else {
2448 ShouldNotReachHere();
2449 }
2450 }
2451
2452
2453 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2454 if (dest->is_single_cpu()) {
2455 // first move left into dest so that left is not destroyed by the shift
2456 Register value = dest->as_register();
2457 count = count & 0x1F; // Java spec
2458
2459 move_regs(left->as_register(), value);
2460 switch (code) {
2461 case lir_shl: __ shll(value, count); break;
2462 case lir_shr: __ sarl(value, count); break;
2463 case lir_ushr: __ shrl(value, count); break;
2464 default: ShouldNotReachHere();
2465 }
2466 } else if (dest->is_double_cpu()) {
2467 // first move left into dest so that left is not destroyed by the shift
2468 Register value = dest->as_register_lo();
2469 count = count & 0x1F; // Java spec
2470
2471 move_regs(left->as_register_lo(), value);
2472 switch (code) {
2473 case lir_shl: __ shlptr(value, count); break;
2474 case lir_shr: __ sarptr(value, count); break;
2475 case lir_ushr: __ shrptr(value, count); break;
2476 default: ShouldNotReachHere();
2477 }
2478 } else {
2479 ShouldNotReachHere();
2480 }
2481 }
2482
2483
2484 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2485 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2486 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2487 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2488 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2489 }
2490
2491
2492 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2493 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2497 }
2498
2499
2500 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2501 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2505 }
2506
2507
2508 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2509 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2513 }
2514
2515
2516 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2517 if (null_check) {
2518 __ testptr(obj, obj);
2519 __ jcc(Assembler::zero, *slow_path->entry());
2520 }
2521 if (is_dest) {
2522 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2523 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2524 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2525 } else {
2526 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2527 }
2528 }
2529
2530
2531 // This code replaces a call to arraycopy; no exception may
2532 // be thrown in this code, they must be thrown in the System.arraycopy
2533 // activation frame; we could save some checks if this would not be the case
2534 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2535 ciArrayKlass* default_type = op->expected_type();
2536 Register src = op->src()->as_register();
2537 Register dst = op->dst()->as_register();
2538 Register src_pos = op->src_pos()->as_register();
2539 Register dst_pos = op->dst_pos()->as_register();
2540 Register length = op->length()->as_register();
2541 Register tmp = op->tmp()->as_register();
2542 Register tmp_load_klass = rscratch1;
2543 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2544
2545 CodeStub* stub = op->stub();
2546 int flags = op->flags();
2547 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2548 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2549
2550 if (flags & LIR_OpArrayCopy::always_slow_path) {
2551 __ jmp(*stub->entry());
2552 __ bind(*stub->continuation());
2553 return;
2554 }
2555
2556 // if we don't know anything, just go through the generic arraycopy
2557 if (default_type == nullptr) {
2558 // save outgoing arguments on stack in case call to System.arraycopy is needed
2559 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2560 // for interpreter calling conventions. Now we have to do it in new style conventions.
2561 // For the moment until C1 gets the new register allocator I just force all the
2562 // args to the right place (except the register args) and then on the back side
2563 // reload the register args properly if we go slow path. Yuck
2564
2565 // These are proper for the calling convention
2566 store_parameter(length, 2);
2567 store_parameter(dst_pos, 1);
2568 store_parameter(dst, 0);
2569
2570 // these are just temporary placements until we need to reload
2571 store_parameter(src_pos, 3);
2572 store_parameter(src, 4);
2573
2574 address copyfunc_addr = StubRoutines::generic_arraycopy();
2575 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2576
2577 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2578 // The arguments are in java calling convention so we can trivially shift them to C
2579 // convention
2580 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2581 __ mov(c_rarg0, j_rarg0);
2582 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2583 __ mov(c_rarg1, j_rarg1);
2584 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2585 __ mov(c_rarg2, j_rarg2);
2586 assert_different_registers(c_rarg3, j_rarg4);
2587 __ mov(c_rarg3, j_rarg3);
2588 #ifdef _WIN64
2589 // Allocate abi space for args but be sure to keep stack aligned
2590 __ subptr(rsp, 6*wordSize);
2591 store_parameter(j_rarg4, 4);
2592 #ifndef PRODUCT
2593 if (PrintC1Statistics) {
2594 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2595 }
2596 #endif
2597 __ call(RuntimeAddress(copyfunc_addr));
2598 __ addptr(rsp, 6*wordSize);
2599 #else
2600 __ mov(c_rarg4, j_rarg4);
2601 #ifndef PRODUCT
2602 if (PrintC1Statistics) {
2603 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2604 }
2605 #endif
2606 __ call(RuntimeAddress(copyfunc_addr));
2607 #endif // _WIN64
2608
2609 __ testl(rax, rax);
2610 __ jcc(Assembler::equal, *stub->continuation());
2611
2612 __ mov(tmp, rax);
2613 __ xorl(tmp, -1);
2614
2615 // Reload values from the stack so they are where the stub
2616 // expects them.
2617 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2618 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2619 __ movptr (length, Address(rsp, 2*BytesPerWord));
2620 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2621 __ movptr (src, Address(rsp, 4*BytesPerWord));
2622
2623 __ subl(length, tmp);
2624 __ addl(src_pos, tmp);
2625 __ addl(dst_pos, tmp);
2626 __ jmp(*stub->entry());
2627
2628 __ bind(*stub->continuation());
2629 return;
2630 }
2631
2632 // Handle inline type arrays
2633 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2634 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2635 }
2636 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2637 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2638 }
2639
2640 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2641
2642 int elem_size = type2aelembytes(basic_type);
2643 Address::ScaleFactor scale;
2644
2645 switch (elem_size) {
2646 case 1 :
2647 scale = Address::times_1;
2648 break;
2649 case 2 :
2650 scale = Address::times_2;
2651 break;
2652 case 4 :
2653 scale = Address::times_4;
2654 break;
2655 case 8 :
2656 scale = Address::times_8;
2657 break;
2658 default:
2659 scale = Address::no_scale;
2660 ShouldNotReachHere();
2661 }
2662
2663 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2664 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2665
2666 // length and pos's are all sign extended at this point on 64bit
2667
2668 // test for null
2669 if (flags & LIR_OpArrayCopy::src_null_check) {
2670 __ testptr(src, src);
2671 __ jcc(Assembler::zero, *stub->entry());
2672 }
2673 if (flags & LIR_OpArrayCopy::dst_null_check) {
2674 __ testptr(dst, dst);
2675 __ jcc(Assembler::zero, *stub->entry());
2676 }
2677
2678 // If the compiler was not able to prove that exact type of the source or the destination
2679 // of the arraycopy is an array type, check at runtime if the source or the destination is
2680 // an instance type.
2681 if (flags & LIR_OpArrayCopy::type_check) {
2682 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2683 __ load_klass(tmp, dst, tmp_load_klass);
2684 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2685 __ jcc(Assembler::greaterEqual, *stub->entry());
2686 }
2687
2688 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2689 __ load_klass(tmp, src, tmp_load_klass);
2690 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2691 __ jcc(Assembler::greaterEqual, *stub->entry());
2692 }
2693 }
2694
2695 // check if negative
2696 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2697 __ testl(src_pos, src_pos);
2698 __ jcc(Assembler::less, *stub->entry());
2699 }
2700 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2701 __ testl(dst_pos, dst_pos);
2702 __ jcc(Assembler::less, *stub->entry());
2703 }
2704
2705 if (flags & LIR_OpArrayCopy::src_range_check) {
2706 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2707 __ cmpl(tmp, src_length_addr);
2708 __ jcc(Assembler::above, *stub->entry());
2709 }
2710 if (flags & LIR_OpArrayCopy::dst_range_check) {
2711 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2712 __ cmpl(tmp, dst_length_addr);
2713 __ jcc(Assembler::above, *stub->entry());
2714 }
2715
2716 if (flags & LIR_OpArrayCopy::length_positive_check) {
2717 __ testl(length, length);
2718 __ jcc(Assembler::less, *stub->entry());
2719 }
2720
2721 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2722 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2723
2724 if (flags & LIR_OpArrayCopy::type_check) {
2725 // We don't know the array types are compatible
2726 if (basic_type != T_OBJECT) {
2727 // Simple test for basic type arrays
2728 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2729 __ jcc(Assembler::notEqual, *stub->entry());
2730 } else {
2731 // For object arrays, if src is a sub class of dst then we can
2732 // safely do the copy.
2733 Label cont, slow;
2734
2735 __ push_ppx(src);
2736 __ push_ppx(dst);
2737
2738 __ load_klass(src, src, tmp_load_klass);
2739 __ load_klass(dst, dst, tmp_load_klass);
2740
2741 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2742
2743 __ push_ppx(src);
2744 __ push_ppx(dst);
2745 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2746 __ pop_ppx(dst);
2747 __ pop_ppx(src);
2748
2749 __ testl(src, src);
2750 __ jcc(Assembler::notEqual, cont);
2751
2752 __ bind(slow);
2753 __ pop_ppx(dst);
2754 __ pop_ppx(src);
2755
2756 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2757 if (copyfunc_addr != nullptr) { // use stub if available
2758 // src is not a sub class of dst so we have to do a
2759 // per-element check.
2760
2761 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2762 if ((flags & mask) != mask) {
2763 // Check that at least both of them object arrays.
2764 assert(flags & mask, "one of the two should be known to be an object array");
2765
2766 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2767 __ load_klass(tmp, src, tmp_load_klass);
2768 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2769 __ load_klass(tmp, dst, tmp_load_klass);
2770 }
2771 int lh_offset = in_bytes(Klass::layout_helper_offset());
2772 Address klass_lh_addr(tmp, lh_offset);
2773 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2774 __ cmpl(klass_lh_addr, objArray_lh);
2775 __ jcc(Assembler::notEqual, *stub->entry());
2776 }
2777
2778 // Spill because stubs can use any register they like and it's
2779 // easier to restore just those that we care about.
2780 store_parameter(dst, 0);
2781 store_parameter(dst_pos, 1);
2782 store_parameter(length, 2);
2783 store_parameter(src_pos, 3);
2784 store_parameter(src, 4);
2785
2786 __ movl2ptr(length, length); //higher 32bits must be null
2787
2788 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2789 assert_different_registers(c_rarg0, dst, dst_pos, length);
2790 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2791 assert_different_registers(c_rarg1, dst, length);
2792
2793 __ mov(c_rarg2, length);
2794 assert_different_registers(c_rarg2, dst);
2795
2796 #ifdef _WIN64
2797 // Allocate abi space for args but be sure to keep stack aligned
2798 __ subptr(rsp, 6*wordSize);
2799 __ load_klass(c_rarg3, dst, tmp_load_klass);
2800 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2801 store_parameter(c_rarg3, 4);
2802 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2803 __ call(RuntimeAddress(copyfunc_addr));
2804 __ addptr(rsp, 6*wordSize);
2805 #else
2806 __ load_klass(c_rarg4, dst, tmp_load_klass);
2807 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2808 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2809 __ call(RuntimeAddress(copyfunc_addr));
2810 #endif
2811
2812 #ifndef PRODUCT
2813 if (PrintC1Statistics) {
2814 Label failed;
2815 __ testl(rax, rax);
2816 __ jcc(Assembler::notZero, failed);
2817 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2818 __ bind(failed);
2819 }
2820 #endif
2821
2822 __ testl(rax, rax);
2823 __ jcc(Assembler::zero, *stub->continuation());
2824
2825 #ifndef PRODUCT
2826 if (PrintC1Statistics) {
2827 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2828 }
2829 #endif
2830
2831 __ mov(tmp, rax);
2832
2833 __ xorl(tmp, -1);
2834
2835 // Restore previously spilled arguments
2836 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2837 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2838 __ movptr (length, Address(rsp, 2*BytesPerWord));
2839 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2840 __ movptr (src, Address(rsp, 4*BytesPerWord));
2841
2842
2843 __ subl(length, tmp);
2844 __ addl(src_pos, tmp);
2845 __ addl(dst_pos, tmp);
2846 }
2847
2848 __ jmp(*stub->entry());
2849
2850 __ bind(cont);
2851 __ pop(dst);
2852 __ pop(src);
2853 }
2854 }
2855
2856 #ifdef ASSERT
2857 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2858 // Sanity check the known type with the incoming class. For the
2859 // primitive case the types must match exactly with src.klass and
2860 // dst.klass each exactly matching the default type. For the
2861 // object array case, if no type check is needed then either the
2862 // dst type is exactly the expected type and the src type is a
2863 // subtype which we can't check or src is the same array as dst
2864 // but not necessarily exactly of type default_type.
2865 Label known_ok, halt;
2866
2867 __ mov_metadata(tmp, default_type->constant_encoding());
2868 if (UseCompressedClassPointers) {
2869 __ encode_klass_not_null(tmp, rscratch1);
2870 }
2871
2872 if (basic_type != T_OBJECT) {
2873 __ cmp_klass(tmp, dst, tmp2);
2874 __ jcc(Assembler::notEqual, halt);
2875 __ cmp_klass(tmp, src, tmp2);
2876 __ jcc(Assembler::equal, known_ok);
2877 } else {
2878 __ cmp_klass(tmp, dst, tmp2);
2879 __ jcc(Assembler::equal, known_ok);
2880 __ cmpptr(src, dst);
2881 __ jcc(Assembler::equal, known_ok);
2882 }
2883 __ bind(halt);
2884 __ stop("incorrect type information in arraycopy");
2885 __ bind(known_ok);
2886 }
2887 #endif
2888
2889 #ifndef PRODUCT
2890 if (PrintC1Statistics) {
2891 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2892 }
2893 #endif
2894
2895 assert_different_registers(c_rarg0, dst, dst_pos, length);
2896 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2897 assert_different_registers(c_rarg1, length);
2898 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2899 __ mov(c_rarg2, length);
2900
2901 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2902 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2903 const char *name;
2904 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2905 __ call_VM_leaf(entry, 0);
2906
2907 if (stub != nullptr) {
2908 __ bind(*stub->continuation());
2909 }
2910 }
2911
2912 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2913 assert(op->crc()->is_single_cpu(), "crc must be register");
2914 assert(op->val()->is_single_cpu(), "byte value must be register");
2915 assert(op->result_opr()->is_single_cpu(), "result must be register");
2916 Register crc = op->crc()->as_register();
2917 Register val = op->val()->as_register();
2918 Register res = op->result_opr()->as_register();
2919
2920 assert_different_registers(val, crc, res);
2921
2922 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2923 __ notl(crc); // ~crc
2924 __ update_byte_crc32(crc, val, res);
2925 __ notl(crc); // ~crc
2926 __ mov(res, crc);
2927 }
2928
2929 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2930 Register obj = op->obj_opr()->as_register(); // may not be an oop
2931 Register hdr = op->hdr_opr()->as_register();
2932 Register lock = op->lock_opr()->as_register();
2933 if (op->code() == lir_lock) {
2934 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2935 Register tmp = op->scratch_opr()->as_register();
2936 // add debug info for NullPointerException only if one is possible
2937 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2938 if (op->info() != nullptr) {
2939 add_debug_info_for_null_check(null_check_offset, op->info());
2940 }
2941 // done
2942 } else if (op->code() == lir_unlock) {
2943 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2944 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2945 } else {
2946 Unimplemented();
2947 }
2948 __ bind(*op->stub()->continuation());
2949 }
2950
2951 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2952 Register obj = op->obj()->as_pointer_register();
2953 Register result = op->result_opr()->as_pointer_register();
2954
2955 CodeEmitInfo* info = op->info();
2956 if (info != nullptr) {
2957 add_debug_info_for_null_check_here(info);
2958 }
2959
2960 __ load_klass(result, obj, rscratch1);
2961 }
2962
2963 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2964 ciMethod* method = op->profiled_method();
2965 int bci = op->profiled_bci();
2966 ciMethod* callee = op->profiled_callee();
2967 Register tmp_load_klass = rscratch1;
2968
2969 // Update counter for all call types
2970 ciMethodData* md = method->method_data_or_null();
2971 assert(md != nullptr, "Sanity");
2972 ciProfileData* data = md->bci_to_data(bci);
2973 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2974 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2975 Register mdo = op->mdo()->as_register();
2976 __ mov_metadata(mdo, md->constant_encoding());
2977 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2978 // Perform additional virtual call profiling for invokevirtual and
2979 // invokeinterface bytecodes
2980 if (op->should_profile_receiver_type()) {
2981 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2982 Register recv = op->recv()->as_register();
2983 assert_different_registers(mdo, recv);
2984 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2985 ciKlass* known_klass = op->known_holder();
2986 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2987 // We know the type that will be seen at this call site; we can
2988 // statically update the MethodData* rather than needing to do
2989 // dynamic tests on the receiver type
2990
2991 // NOTE: we should probably put a lock around this search to
2992 // avoid collisions by concurrent compilations
2993 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2994 uint i;
2995 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2996 ciKlass* receiver = vc_data->receiver(i);
2997 if (known_klass->equals(receiver)) {
2998 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2999 __ addptr(data_addr, DataLayout::counter_increment);
3000 return;
3001 }
3002 }
3003
3004 // Receiver type not found in profile data; select an empty slot
3005
3006 // Note that this is less efficient than it should be because it
3007 // always does a write to the receiver part of the
3008 // VirtualCallData rather than just the first time
3009 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3010 ciKlass* receiver = vc_data->receiver(i);
3011 if (receiver == nullptr) {
3012 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3013 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3014 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3015 __ addptr(data_addr, DataLayout::counter_increment);
3016 return;
3017 }
3018 }
3019 } else {
3020 __ load_klass(recv, recv, tmp_load_klass);
3021 Label update_done;
3022 type_profile_helper(mdo, md, data, recv, &update_done);
3023 // Receiver did not match any saved receiver and there is no empty row for it.
3024 // Increment total counter to indicate polymorphic case.
3025 __ addptr(counter_addr, DataLayout::counter_increment);
3026
3027 __ bind(update_done);
3028 }
3029 } else {
3030 // Static call
3031 __ addptr(counter_addr, DataLayout::counter_increment);
3032 }
3033 }
3034
3035 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3036 Register obj = op->obj()->as_register();
3037 Register tmp = op->tmp()->as_pointer_register();
3038 Register tmp_load_klass = rscratch1;
3039 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3040 ciKlass* exact_klass = op->exact_klass();
3041 intptr_t current_klass = op->current_klass();
3042 bool not_null = op->not_null();
3043 bool no_conflict = op->no_conflict();
3044
3045 Label update, next, none;
3046
3047 bool do_null = !not_null;
3048 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3049 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3050
3051 assert(do_null || do_update, "why are we here?");
3052 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3053
3054 __ verify_oop(obj);
3055
3056 #ifdef ASSERT
3057 if (obj == tmp) {
3058 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3059 } else {
3060 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3061 }
3062 #endif
3063 if (do_null) {
3064 __ testptr(obj, obj);
3065 __ jccb(Assembler::notZero, update);
3066 if (!TypeEntries::was_null_seen(current_klass)) {
3067 __ testptr(mdo_addr, TypeEntries::null_seen);
3068 #ifndef ASSERT
3069 __ jccb(Assembler::notZero, next); // already set
3070 #else
3071 __ jcc(Assembler::notZero, next); // already set
3072 #endif
3073 // atomic update to prevent overwriting Klass* with 0
3074 __ lock();
3075 __ orptr(mdo_addr, TypeEntries::null_seen);
3076 }
3077 if (do_update) {
3078 #ifndef ASSERT
3079 __ jmpb(next);
3080 }
3081 #else
3082 __ jmp(next);
3083 }
3084 } else {
3085 __ testptr(obj, obj);
3086 __ jcc(Assembler::notZero, update);
3087 __ stop("unexpected null obj");
3088 #endif
3089 }
3090
3091 __ bind(update);
3092
3093 if (do_update) {
3094 #ifdef ASSERT
3095 if (exact_klass != nullptr) {
3096 Label ok;
3097 __ load_klass(tmp, obj, tmp_load_klass);
3098 __ push_ppx(tmp);
3099 __ mov_metadata(tmp, exact_klass->constant_encoding());
3100 __ cmpptr(tmp, Address(rsp, 0));
3101 __ jcc(Assembler::equal, ok);
3102 __ stop("exact klass and actual klass differ");
3103 __ bind(ok);
3104 __ pop_ppx(tmp);
3105 }
3106 #endif
3107 if (!no_conflict) {
3108 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3109 if (exact_klass != nullptr) {
3110 __ mov_metadata(tmp, exact_klass->constant_encoding());
3111 } else {
3112 __ load_klass(tmp, obj, tmp_load_klass);
3113 }
3114 __ mov(rscratch1, tmp); // save original value before XOR
3115 __ xorptr(tmp, mdo_addr);
3116 __ testptr(tmp, TypeEntries::type_klass_mask);
3117 // klass seen before, nothing to do. The unknown bit may have been
3118 // set already but no need to check.
3119 __ jccb(Assembler::zero, next);
3120
3121 __ testptr(tmp, TypeEntries::type_unknown);
3122 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3123
3124 if (TypeEntries::is_type_none(current_klass)) {
3125 __ testptr(mdo_addr, TypeEntries::type_mask);
3126 __ jccb(Assembler::zero, none);
3127 // There is a chance that the checks above (re-reading profiling
3128 // data from memory) fail if another thread has just set the
3129 // profiling to this obj's klass
3130 __ mov(tmp, rscratch1); // get back original value before XOR
3131 __ xorptr(tmp, mdo_addr);
3132 __ testptr(tmp, TypeEntries::type_klass_mask);
3133 __ jccb(Assembler::zero, next);
3134 }
3135 } else {
3136 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3137 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3138
3139 __ testptr(mdo_addr, TypeEntries::type_unknown);
3140 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3141 }
3142
3143 // different than before. Cannot keep accurate profile.
3144 __ orptr(mdo_addr, TypeEntries::type_unknown);
3145
3146 if (TypeEntries::is_type_none(current_klass)) {
3147 __ jmpb(next);
3148
3149 __ bind(none);
3150 // first time here. Set profile type.
3151 __ movptr(mdo_addr, tmp);
3152 #ifdef ASSERT
3153 __ andptr(tmp, TypeEntries::type_klass_mask);
3154 __ verify_klass_ptr(tmp);
3155 #endif
3156 }
3157 } else {
3158 // There's a single possible klass at this profile point
3159 assert(exact_klass != nullptr, "should be");
3160 if (TypeEntries::is_type_none(current_klass)) {
3161 __ mov_metadata(tmp, exact_klass->constant_encoding());
3162 __ xorptr(tmp, mdo_addr);
3163 __ testptr(tmp, TypeEntries::type_klass_mask);
3164 #ifdef ASSERT
3165 __ jcc(Assembler::zero, next);
3166
3167 {
3168 Label ok;
3169 __ push_ppx(tmp);
3170 __ testptr(mdo_addr, TypeEntries::type_mask);
3171 __ jcc(Assembler::zero, ok);
3172 // may have been set by another thread
3173 __ mov_metadata(tmp, exact_klass->constant_encoding());
3174 __ xorptr(tmp, mdo_addr);
3175 __ testptr(tmp, TypeEntries::type_mask);
3176 __ jcc(Assembler::zero, ok);
3177
3178 __ stop("unexpected profiling mismatch");
3179 __ bind(ok);
3180 __ pop_ppx(tmp);
3181 }
3182 #else
3183 __ jccb(Assembler::zero, next);
3184 #endif
3185 // first time here. Set profile type.
3186 __ movptr(mdo_addr, tmp);
3187 #ifdef ASSERT
3188 __ andptr(tmp, TypeEntries::type_klass_mask);
3189 __ verify_klass_ptr(tmp);
3190 #endif
3191 } else {
3192 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3193 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3194
3195 __ testptr(mdo_addr, TypeEntries::type_unknown);
3196 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3197
3198 __ orptr(mdo_addr, TypeEntries::type_unknown);
3199 }
3200 }
3201 }
3202 __ bind(next);
3203 }
3204
3205 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3206 Register obj = op->obj()->as_register();
3207 Register tmp = op->tmp()->as_pointer_register();
3208 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3209 bool not_null = op->not_null();
3210 int flag = op->flag();
3211
3212 Label not_inline_type;
3213 if (!not_null) {
3214 __ testptr(obj, obj);
3215 __ jccb(Assembler::zero, not_inline_type);
3216 }
3217
3218 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3219
3220 __ orb(mdo_addr, flag);
3221
3222 __ bind(not_inline_type);
3223 }
3224
3225 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3226 Unimplemented();
3227 }
3228
3229
3230 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3231 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3232 }
3233
3234
3235 void LIR_Assembler::align_backward_branch_target() {
3236 __ align(BytesPerWord);
3237 }
3238
3239
3240 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3241 if (left->is_single_cpu()) {
3242 __ negl(left->as_register());
3243 move_regs(left->as_register(), dest->as_register());
3244
3245 } else if (left->is_double_cpu()) {
3246 Register lo = left->as_register_lo();
3247 Register dst = dest->as_register_lo();
3248 __ movptr(dst, lo);
3249 __ negptr(dst);
3250
3251 } else if (dest->is_single_xmm()) {
3252 assert(!tmp->is_valid(), "do not need temporary");
3253 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3254 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3255 }
3256 __ xorps(dest->as_xmm_float_reg(),
3257 ExternalAddress((address)float_signflip_pool),
3258 rscratch1);
3259 } else if (dest->is_double_xmm()) {
3260 assert(!tmp->is_valid(), "do not need temporary");
3261 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3262 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3263 }
3264 __ xorpd(dest->as_xmm_double_reg(),
3265 ExternalAddress((address)double_signflip_pool),
3266 rscratch1);
3267 } else {
3268 ShouldNotReachHere();
3269 }
3270 }
3271
3272
3273 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3274 assert(src->is_address(), "must be an address");
3275 assert(dest->is_register(), "must be a register");
3276
3277 PatchingStub* patch = nullptr;
3278 if (patch_code != lir_patch_none) {
3279 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3280 }
3281
3282 Register reg = dest->as_pointer_register();
3283 LIR_Address* addr = src->as_address_ptr();
3284 __ lea(reg, as_Address(addr));
3285
3286 if (patch != nullptr) {
3287 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3288 }
3289 }
3290
3291
3292
3293 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3294 assert(!tmp->is_valid(), "don't need temporary");
3295 __ call(RuntimeAddress(dest));
3296 if (info != nullptr) {
3297 add_call_info_here(info);
3298 }
3299 __ post_call_nop();
3300 }
3301
3302
3303 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3304 assert(type == T_LONG, "only for volatile long fields");
3305
3306 if (info != nullptr) {
3307 add_debug_info_for_null_check_here(info);
3308 }
3309
3310 if (src->is_double_xmm()) {
3311 if (dest->is_double_cpu()) {
3312 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3313 } else if (dest->is_double_stack()) {
3314 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3315 } else if (dest->is_address()) {
3316 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3317 } else {
3318 ShouldNotReachHere();
3319 }
3320
3321 } else if (dest->is_double_xmm()) {
3322 if (src->is_double_stack()) {
3323 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3324 } else if (src->is_address()) {
3325 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3326 } else {
3327 ShouldNotReachHere();
3328 }
3329
3330 } else {
3331 ShouldNotReachHere();
3332 }
3333 }
3334
3335 #ifdef ASSERT
3336 // emit run-time assertion
3337 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3338 assert(op->code() == lir_assert, "must be");
3339
3340 if (op->in_opr1()->is_valid()) {
3341 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3342 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3343 } else {
3344 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3345 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3346 }
3347
3348 Label ok;
3349 if (op->condition() != lir_cond_always) {
3350 Assembler::Condition acond = Assembler::zero;
3351 switch (op->condition()) {
3352 case lir_cond_equal: acond = Assembler::equal; break;
3353 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3354 case lir_cond_less: acond = Assembler::less; break;
3355 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3356 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3357 case lir_cond_greater: acond = Assembler::greater; break;
3358 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3359 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3360 default: ShouldNotReachHere();
3361 }
3362 __ jcc(acond, ok);
3363 }
3364 if (op->halt()) {
3365 const char* str = __ code_string(op->msg());
3366 __ stop(str);
3367 } else {
3368 breakpoint();
3369 }
3370 __ bind(ok);
3371 }
3372 #endif
3373
3374 void LIR_Assembler::membar() {
3375 // QQQ sparc TSO uses this,
3376 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3377 }
3378
3379 void LIR_Assembler::membar_acquire() {
3380 // No x86 machines currently require load fences
3381 }
3382
3383 void LIR_Assembler::membar_release() {
3384 // No x86 machines currently require store fences
3385 }
3386
3387 void LIR_Assembler::membar_loadload() {
3388 // no-op
3389 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3390 }
3391
3392 void LIR_Assembler::membar_storestore() {
3393 // no-op
3394 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3395 }
3396
3397 void LIR_Assembler::membar_loadstore() {
3398 // no-op
3399 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3400 }
3401
3402 void LIR_Assembler::membar_storeload() {
3403 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3404 }
3405
3406 void LIR_Assembler::on_spin_wait() {
3407 __ pause ();
3408 }
3409
3410 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3411 assert(result_reg->is_register(), "check");
3412 __ mov(result_reg->as_register(), r15_thread);
3413 }
3414
3415 void LIR_Assembler::check_orig_pc() {
3416 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3417 }
3418
3419 void LIR_Assembler::peephole(LIR_List*) {
3420 // do nothing for now
3421 }
3422
3423 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3424 assert(data == dest, "xchg/xadd uses only 2 operands");
3425
3426 if (data->type() == T_INT) {
3427 if (code == lir_xadd) {
3428 __ lock();
3429 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3430 } else {
3431 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3432 }
3433 } else if (data->is_oop()) {
3434 assert (code == lir_xchg, "xadd for oops");
3435 Register obj = data->as_register();
3436 if (UseCompressedOops) {
3437 __ encode_heap_oop(obj);
3438 __ xchgl(obj, as_Address(src->as_address_ptr()));
3439 __ decode_heap_oop(obj);
3440 } else {
3441 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3442 }
3443 } else if (data->type() == T_LONG) {
3444 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3445 if (code == lir_xadd) {
3446 __ lock();
3447 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3448 } else {
3449 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3450 }
3451 } else {
3452 ShouldNotReachHere();
3453 }
3454 }
3455
3456 #undef __