1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
61 operand[1] = hi;
62 return operand;
63 }
64
65 // Buffer for 128-bits masks used by SSE instructions.
66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
67
68 // Static initialization during VM startup.
69 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
70 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
71 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
72 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
73
74
75 NEEDS_CLEANUP // remove this definitions ?
76 const Register SYNC_header = rax; // synchronization header
77 const Register SHIFT_count = rcx; // where count for shift operations must be
78
79 #define __ _masm->
80
81
82 static void select_different_registers(Register preserve,
83 Register extra,
84 Register &tmp1,
85 Register &tmp2) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, extra);
91 tmp2 = extra;
92 }
93 assert_different_registers(preserve, tmp1, tmp2);
94 }
95
96
97
98 static void select_different_registers(Register preserve,
99 Register extra,
100 Register &tmp1,
101 Register &tmp2,
102 Register &tmp3) {
103 if (tmp1 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp1 = extra;
106 } else if (tmp2 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp2 = extra;
109 } else if (tmp3 == preserve) {
110 assert_different_registers(tmp1, tmp2, tmp3, extra);
111 tmp3 = extra;
112 }
113 assert_different_registers(preserve, tmp1, tmp2, tmp3);
114 }
115
116
117
118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
119 if (opr->is_constant()) {
120 LIR_Const* constant = opr->as_constant_ptr();
121 switch (constant->type()) {
122 case T_INT: {
123 return true;
124 }
125
126 default:
127 return false;
128 }
129 }
130 return false;
131 }
132
133
134 LIR_Opr LIR_Assembler::receiverOpr() {
135 return FrameMap::receiver_opr;
136 }
137
138 LIR_Opr LIR_Assembler::osrBufferPointer() {
139 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
140 }
141
142 //--------------fpu register translations-----------------------
143
144
145 address LIR_Assembler::float_constant(float f) {
146 address const_addr = __ float_constant(f);
147 if (const_addr == nullptr) {
148 bailout("const section overflow");
149 return __ code()->consts()->start();
150 } else {
151 return const_addr;
152 }
153 }
154
155
156 address LIR_Assembler::double_constant(double d) {
157 address const_addr = __ double_constant(d);
158 if (const_addr == nullptr) {
159 bailout("const section overflow");
160 return __ code()->consts()->start();
161 } else {
162 return const_addr;
163 }
164 }
165
166 void LIR_Assembler::breakpoint() {
167 __ int3();
168 }
169
170 void LIR_Assembler::push(LIR_Opr opr) {
171 if (opr->is_single_cpu()) {
172 __ push_reg(opr->as_register());
173 } else if (opr->is_double_cpu()) {
174 __ push_reg(opr->as_register_lo());
175 } else if (opr->is_stack()) {
176 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
177 } else if (opr->is_constant()) {
178 LIR_Const* const_opr = opr->as_constant_ptr();
179 if (const_opr->type() == T_OBJECT) {
180 __ push_oop(const_opr->as_jobject(), rscratch1);
181 } else if (const_opr->type() == T_INT) {
182 __ push_jint(const_opr->as_jint());
183 } else {
184 ShouldNotReachHere();
185 }
186
187 } else {
188 ShouldNotReachHere();
189 }
190 }
191
192 void LIR_Assembler::pop(LIR_Opr opr) {
193 if (opr->is_single_cpu()) {
194 __ pop_reg(opr->as_register());
195 } else {
196 ShouldNotReachHere();
197 }
198 }
199
200 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
201 return addr->base()->is_illegal() && addr->index()->is_illegal();
202 }
203
204 //-------------------------------------------
205
206 Address LIR_Assembler::as_Address(LIR_Address* addr) {
207 return as_Address(addr, rscratch1);
208 }
209
210 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
211 if (addr->base()->is_illegal()) {
212 assert(addr->index()->is_illegal(), "must be illegal too");
213 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
214 if (! __ reachable(laddr)) {
215 __ movptr(tmp, laddr.addr());
216 Address res(tmp, 0);
217 return res;
218 } else {
219 return __ as_Address(laddr);
220 }
221 }
222
223 Register base = addr->base()->as_pointer_register();
224
225 if (addr->index()->is_illegal()) {
226 return Address( base, addr->disp());
227 } else if (addr->index()->is_cpu_register()) {
228 Register index = addr->index()->as_pointer_register();
229 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
230 } else if (addr->index()->is_constant()) {
231 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
232 assert(Assembler::is_simm32(addr_offset), "must be");
233
234 return Address(base, addr_offset);
235 } else {
236 Unimplemented();
237 return Address();
238 }
239 }
240
241
242 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
243 Address base = as_Address(addr);
244 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
245 }
246
247
248 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
249 return as_Address(addr);
250 }
251
252
253 void LIR_Assembler::osr_entry() {
254 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
255 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
256 ValueStack* entry_state = osr_entry->state();
257 int number_of_locks = entry_state->locks_size();
258
259 // we jump here if osr happens with the interpreter
260 // state set up to continue at the beginning of the
261 // loop that triggered osr - in particular, we have
262 // the following registers setup:
263 //
264 // rcx: osr buffer
265 //
266
267 // build frame
268 ciMethod* m = compilation()->method();
269 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
270
271 // OSR buffer is
272 //
273 // locals[nlocals-1..0]
274 // monitors[0..number_of_locks]
275 //
276 // locals is a direct copy of the interpreter frame so in the osr buffer
277 // so first slot in the local array is the last local from the interpreter
278 // and last slot is local[0] (receiver) from the interpreter
279 //
280 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
281 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
282 // in the interpreter frame (the method lock if a sync method)
283
284 // Initialize monitors in the compiled activation.
285 // rcx: pointer to osr buffer
286 //
287 // All other registers are dead at this point and the locals will be
288 // copied into place by code emitted in the IR.
289
290 Register OSR_buf = osrBufferPointer()->as_pointer_register();
291 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
292 int monitor_offset = BytesPerWord * method()->max_locals() +
293 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
294 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
295 // the OSR buffer using 2 word entries: first the lock and then
296 // the oop.
297 for (int i = 0; i < number_of_locks; i++) {
298 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
299 #ifdef ASSERT
300 // verify the interpreter's monitor has a non-null object
301 {
302 Label L;
303 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
304 __ jcc(Assembler::notZero, L);
305 __ stop("locked object is null");
306 __ bind(L);
307 }
308 #endif
309 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
310 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
311 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
312 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
313 }
314 }
315 }
316
317
318 // inline cache check; done before the frame is built.
319 int LIR_Assembler::check_icache() {
320 return __ ic_check(CodeEntryAlignment);
321 }
322
323 void LIR_Assembler::clinit_barrier(ciMethod* method) {
324 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
325 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
326
327 Label L_skip_barrier;
328 Register klass = rscratch1;
329
330 __ mov_metadata(klass, method->holder()->constant_encoding());
331 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
332
333 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
334
335 __ bind(L_skip_barrier);
336 }
337
338 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
339 jobject o = nullptr;
340 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
341 __ movoop(reg, o);
342 patching_epilog(patch, lir_patch_normal, reg, info);
343 }
344
345 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
346 Metadata* o = nullptr;
347 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
348 __ mov_metadata(reg, o);
349 patching_epilog(patch, lir_patch_normal, reg, info);
350 }
351
352 // This specifies the rsp decrement needed to build the frame
353 int LIR_Assembler::initial_frame_size_in_bytes() const {
354 // if rounding, must let FrameMap know!
355
356 // The frame_map records size in slots (32bit word)
357
358 // subtract two words to account for return address and link
359 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
360 }
361
362
363 int LIR_Assembler::emit_exception_handler() {
364 // generate code for exception handler
365 address handler_base = __ start_a_stub(exception_handler_size());
366 if (handler_base == nullptr) {
367 // not enough space left for the handler
368 bailout("exception handler overflow");
369 return -1;
370 }
371
372 int offset = code_offset();
373
374 // the exception oop and pc are in rax, and rdx
375 // no other registers need to be preserved, so invalidate them
376 __ invalidate_registers(false, true, true, false, true, true);
377
378 // check that there is really an exception
379 __ verify_not_null_oop(rax);
380
381 // search an exception handler (rax: exception oop, rdx: throwing pc)
382 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id)));
383 __ should_not_reach_here();
384 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
385 __ end_a_stub();
386
387 return offset;
388 }
389
390
391 // Emit the code to remove the frame from the stack in the exception
392 // unwind path.
393 int LIR_Assembler::emit_unwind_handler() {
394 #ifndef PRODUCT
395 if (CommentedAssembly) {
396 _masm->block_comment("Unwind handler");
397 }
398 #endif
399
400 int offset = code_offset();
401
402 // Fetch the exception from TLS and clear out exception related thread state
403 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
404 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
405 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
406
407 __ bind(_unwind_handler_entry);
408 __ verify_not_null_oop(rax);
409 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
410 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
411 }
412
413 // Perform needed unlocking
414 MonitorExitStub* stub = nullptr;
415 if (method()->is_synchronized()) {
416 monitor_address(0, FrameMap::rax_opr);
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 if (LockingMode == LM_MONITOR) {
419 __ jmp(*stub->entry());
420 } else {
421 __ unlock_object(rdi, rsi, rax, *stub->entry());
422 }
423 __ bind(*stub->continuation());
424 }
425
426 if (compilation()->env()->dtrace_method_probes()) {
427 __ mov(rdi, r15_thread);
428 __ mov_metadata(rsi, method()->constant_encoding());
429 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
430 }
431
432 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
433 __ mov(rax, rbx); // Restore the exception
434 }
435
436 // remove the activation and dispatch to the unwind handler
437 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
438 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
439
440 // Emit the slow path assembly
441 if (stub != nullptr) {
442 stub->emit_code(this);
443 }
444
445 return offset;
446 }
447
448
449 int LIR_Assembler::emit_deopt_handler() {
450 // generate code for exception handler
451 address handler_base = __ start_a_stub(deopt_handler_size());
452 if (handler_base == nullptr) {
453 // not enough space left for the handler
454 bailout("deopt handler overflow");
455 return -1;
456 }
457
458 int offset = code_offset();
459 InternalAddress here(__ pc());
460
461 __ pushptr(here.addr(), rscratch1);
462 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 __ end_a_stub();
465
466 return offset;
467 }
468
469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
470 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
471 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
472 assert(result->fpu() == 0, "result must already be on TOS");
473 }
474 if (InlineTypeReturnedAsFields) {
475 #ifndef _LP64
476 Unimplemented();
477 #endif
478 // Check if we are returning an non-null inline type and load its fields into registers
479 ciType* return_type = compilation()->method()->return_type();
480 if (return_type->is_inlinetype()) {
481 ciInlineKlass* vk = return_type->as_inline_klass();
482 if (vk->can_be_returned_as_fields()) {
483 address unpack_handler = vk->unpack_handler();
484 assert(unpack_handler != nullptr, "must be");
485 __ call(RuntimeAddress(unpack_handler));
486 }
487 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
488 Label skip;
489 Label not_null;
490 __ testptr(rax, rax);
491 __ jcc(Assembler::notZero, not_null);
492 // Returned value is null, zero all return registers because they may belong to oop fields
493 __ xorq(j_rarg1, j_rarg1);
494 __ xorq(j_rarg2, j_rarg2);
495 __ xorq(j_rarg3, j_rarg3);
496 __ xorq(j_rarg4, j_rarg4);
497 __ xorq(j_rarg5, j_rarg5);
498 __ jmp(skip);
499 __ bind(not_null);
500
501 // Check if we are returning an non-null inline type and load its fields into registers
502 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
503
504 // Load fields from a buffered value with an inline class specific handler
505 __ load_klass(rdi, rax, rscratch1);
506 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
507 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
508 // Unpack handler can be null if inline type is not scalarizable in returns
509 __ testptr(rdi, rdi);
510 __ jcc(Assembler::zero, skip);
511 __ call(rdi);
512
513 __ bind(skip);
514 }
515 // At this point, rax points to the value object (for interpreter or C1 caller).
516 // The fields of the object are copied into registers (for C2 caller).
517 }
518
519 // Pop the stack before the safepoint code
520 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
521
522 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
523 __ reserved_stack_check();
524 }
525
526 // Note: we do not need to round double result; float result has the right precision
527 // the poll sets the condition code, but no data registers
528
529 code_stub->set_safepoint_offset(__ offset());
530 __ relocate(relocInfo::poll_return_type);
531 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
532 __ ret(0);
533 }
534
535
536 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
537 return (__ store_inline_type_fields_to_buf(vk, false));
538 }
539
540 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
541 guarantee(info != nullptr, "Shouldn't be null");
542 int offset = __ offset();
543 const Register poll_addr = rscratch1;
544 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
545 add_debug_info_for_branch(info);
546 __ relocate(relocInfo::poll_type);
547 address pre_pc = __ pc();
548 __ testl(rax, Address(poll_addr, 0));
549 address post_pc = __ pc();
550 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
551 return offset;
552 }
553
554
555 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
556 if (from_reg != to_reg) __ mov(to_reg, from_reg);
557 }
558
559 void LIR_Assembler::swap_reg(Register a, Register b) {
560 __ xchgptr(a, b);
561 }
562
563
564 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
565 assert(src->is_constant(), "should not call otherwise");
566 assert(dest->is_register(), "should not call otherwise");
567 LIR_Const* c = src->as_constant_ptr();
568
569 switch (c->type()) {
570 case T_INT: {
571 assert(patch_code == lir_patch_none, "no patching handled here");
572 __ movl(dest->as_register(), c->as_jint());
573 break;
574 }
575
576 case T_ADDRESS: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ movptr(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_LONG: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
585 break;
586 }
587
588 case T_OBJECT: {
589 if (patch_code != lir_patch_none) {
590 jobject2reg_with_patching(dest->as_register(), info);
591 } else {
592 __ movoop(dest->as_register(), c->as_jobject());
593 }
594 break;
595 }
596
597 case T_METADATA: {
598 if (patch_code != lir_patch_none) {
599 klass2reg_with_patching(dest->as_register(), info);
600 } else {
601 __ mov_metadata(dest->as_register(), c->as_metadata());
602 }
603 break;
604 }
605
606 case T_FLOAT: {
607 if (dest->is_single_xmm()) {
608 if (UseAVX <= 2 && c->is_zero_float()) {
609 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
610 } else {
611 __ movflt(dest->as_xmm_float_reg(),
612 InternalAddress(float_constant(c->as_jfloat())));
613 }
614 } else {
615 ShouldNotReachHere();
616 }
617 break;
618 }
619
620 case T_DOUBLE: {
621 if (dest->is_double_xmm()) {
622 if (UseAVX <= 2 && c->is_zero_double()) {
623 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
624 } else {
625 __ movdbl(dest->as_xmm_double_reg(),
626 InternalAddress(double_constant(c->as_jdouble())));
627 }
628 } else {
629 ShouldNotReachHere();
630 }
631 break;
632 }
633
634 default:
635 ShouldNotReachHere();
636 }
637 }
638
639 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
640 assert(src->is_constant(), "should not call otherwise");
641 assert(dest->is_stack(), "should not call otherwise");
642 LIR_Const* c = src->as_constant_ptr();
643
644 switch (c->type()) {
645 case T_INT: // fall through
646 case T_FLOAT:
647 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
648 break;
649
650 case T_ADDRESS:
651 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
652 break;
653
654 case T_OBJECT:
655 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
656 break;
657
658 case T_LONG: // fall through
659 case T_DOUBLE:
660 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
661 lo_word_offset_in_bytes),
662 (intptr_t)c->as_jlong_bits(),
663 rscratch1);
664 break;
665
666 default:
667 ShouldNotReachHere();
668 }
669 }
670
671 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
672 assert(src->is_constant(), "should not call otherwise");
673 assert(dest->is_address(), "should not call otherwise");
674 LIR_Const* c = src->as_constant_ptr();
675 LIR_Address* addr = dest->as_address_ptr();
676
677 int null_check_here = code_offset();
678 switch (type) {
679 case T_INT: // fall through
680 case T_FLOAT:
681 __ movl(as_Address(addr), c->as_jint_bits());
682 break;
683
684 case T_ADDRESS:
685 __ movptr(as_Address(addr), c->as_jint_bits());
686 break;
687
688 case T_OBJECT: // fall through
689 case T_ARRAY:
690 if (c->as_jobject() == nullptr) {
691 if (UseCompressedOops && !wide) {
692 __ movl(as_Address(addr), NULL_WORD);
693 } else {
694 __ xorptr(rscratch1, rscratch1);
695 null_check_here = code_offset();
696 __ movptr(as_Address(addr), rscratch1);
697 }
698 } else {
699 if (is_literal_address(addr)) {
700 ShouldNotReachHere();
701 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
702 } else {
703 __ movoop(rscratch1, c->as_jobject());
704 if (UseCompressedOops && !wide) {
705 __ encode_heap_oop(rscratch1);
706 null_check_here = code_offset();
707 __ movl(as_Address_lo(addr), rscratch1);
708 } else {
709 null_check_here = code_offset();
710 __ movptr(as_Address_lo(addr), rscratch1);
711 }
712 }
713 }
714 break;
715
716 case T_LONG: // fall through
717 case T_DOUBLE:
718 if (is_literal_address(addr)) {
719 ShouldNotReachHere();
720 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
721 } else {
722 __ movptr(r10, (intptr_t)c->as_jlong_bits());
723 null_check_here = code_offset();
724 __ movptr(as_Address_lo(addr), r10);
725 }
726 break;
727
728 case T_BOOLEAN: // fall through
729 case T_BYTE:
730 __ movb(as_Address(addr), c->as_jint() & 0xFF);
731 break;
732
733 case T_CHAR: // fall through
734 case T_SHORT:
735 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
736 break;
737
738 default:
739 ShouldNotReachHere();
740 };
741
742 if (info != nullptr) {
743 add_debug_info_for_null_check(null_check_here, info);
744 }
745 }
746
747
748 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
749 assert(src->is_register(), "should not call otherwise");
750 assert(dest->is_register(), "should not call otherwise");
751
752 // move between cpu-registers
753 if (dest->is_single_cpu()) {
754 if (src->type() == T_LONG) {
755 // Can do LONG -> OBJECT
756 move_regs(src->as_register_lo(), dest->as_register());
757 return;
758 }
759 assert(src->is_single_cpu(), "must match");
760 if (src->type() == T_OBJECT) {
761 __ verify_oop(src->as_register());
762 }
763 move_regs(src->as_register(), dest->as_register());
764
765 } else if (dest->is_double_cpu()) {
766 if (is_reference_type(src->type())) {
767 // Surprising to me but we can see move of a long to t_object
768 __ verify_oop(src->as_register());
769 move_regs(src->as_register(), dest->as_register_lo());
770 return;
771 }
772 assert(src->is_double_cpu(), "must match");
773 Register f_lo = src->as_register_lo();
774 Register f_hi = src->as_register_hi();
775 Register t_lo = dest->as_register_lo();
776 Register t_hi = dest->as_register_hi();
777 assert(f_hi == f_lo, "must be same");
778 assert(t_hi == t_lo, "must be same");
779 move_regs(f_lo, t_lo);
780
781 // move between xmm-registers
782 } else if (dest->is_single_xmm()) {
783 assert(src->is_single_xmm(), "must match");
784 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
785 } else if (dest->is_double_xmm()) {
786 assert(src->is_double_xmm(), "must match");
787 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
788
789 } else {
790 ShouldNotReachHere();
791 }
792 }
793
794 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
795 assert(src->is_register(), "should not call otherwise");
796 assert(dest->is_stack(), "should not call otherwise");
797
798 if (src->is_single_cpu()) {
799 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
800 if (is_reference_type(type)) {
801 __ verify_oop(src->as_register());
802 __ movptr (dst, src->as_register());
803 } else if (type == T_METADATA || type == T_ADDRESS) {
804 __ movptr (dst, src->as_register());
805 } else {
806 __ movl (dst, src->as_register());
807 }
808
809 } else if (src->is_double_cpu()) {
810 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
811 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
812 __ movptr (dstLO, src->as_register_lo());
813
814 } else if (src->is_single_xmm()) {
815 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
816 __ movflt(dst_addr, src->as_xmm_float_reg());
817
818 } else if (src->is_double_xmm()) {
819 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
820 __ movdbl(dst_addr, src->as_xmm_double_reg());
821
822 } else {
823 ShouldNotReachHere();
824 }
825 }
826
827
828 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
829 LIR_Address* to_addr = dest->as_address_ptr();
830 PatchingStub* patch = nullptr;
831 Register compressed_src = rscratch1;
832
833 if (is_reference_type(type)) {
834 __ verify_oop(src->as_register());
835 if (UseCompressedOops && !wide) {
836 __ movptr(compressed_src, src->as_register());
837 __ encode_heap_oop(compressed_src);
838 if (patch_code != lir_patch_none) {
839 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
840 }
841 }
842 }
843
844 if (patch_code != lir_patch_none) {
845 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
846 Address toa = as_Address(to_addr);
847 assert(toa.disp() != 0, "must have");
848 }
849
850 int null_check_here = code_offset();
851 switch (type) {
852 case T_FLOAT: {
853 assert(src->is_single_xmm(), "not a float");
854 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
855 break;
856 }
857
858 case T_DOUBLE: {
859 assert(src->is_double_xmm(), "not a double");
860 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
861 break;
862 }
863
864 case T_ARRAY: // fall through
865 case T_OBJECT: // fall through
866 if (UseCompressedOops && !wide) {
867 __ movl(as_Address(to_addr), compressed_src);
868 } else {
869 __ movptr(as_Address(to_addr), src->as_register());
870 }
871 break;
872 case T_ADDRESS:
873 __ movptr(as_Address(to_addr), src->as_register());
874 break;
875 case T_INT:
876 __ movl(as_Address(to_addr), src->as_register());
877 break;
878
879 case T_LONG: {
880 Register from_lo = src->as_register_lo();
881 Register from_hi = src->as_register_hi();
882 __ movptr(as_Address_lo(to_addr), from_lo);
883 break;
884 }
885
886 case T_BYTE: // fall through
887 case T_BOOLEAN: {
888 Register src_reg = src->as_register();
889 Address dst_addr = as_Address(to_addr);
890 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
891 __ movb(dst_addr, src_reg);
892 break;
893 }
894
895 case T_CHAR: // fall through
896 case T_SHORT:
897 __ movw(as_Address(to_addr), src->as_register());
898 break;
899
900 default:
901 ShouldNotReachHere();
902 }
903 if (info != nullptr) {
904 add_debug_info_for_null_check(null_check_here, info);
905 }
906
907 if (patch_code != lir_patch_none) {
908 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
909 }
910 }
911
912
913 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
914 assert(src->is_stack(), "should not call otherwise");
915 assert(dest->is_register(), "should not call otherwise");
916
917 if (dest->is_single_cpu()) {
918 if (is_reference_type(type)) {
919 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
920 __ verify_oop(dest->as_register());
921 } else if (type == T_METADATA || type == T_ADDRESS) {
922 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
923 } else {
924 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
925 }
926
927 } else if (dest->is_double_cpu()) {
928 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
929 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
930 __ movptr(dest->as_register_lo(), src_addr_LO);
931
932 } else if (dest->is_single_xmm()) {
933 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
934 __ movflt(dest->as_xmm_float_reg(), src_addr);
935
936 } else if (dest->is_double_xmm()) {
937 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
938 __ movdbl(dest->as_xmm_double_reg(), src_addr);
939
940 } else {
941 ShouldNotReachHere();
942 }
943 }
944
945
946 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
947 if (src->is_single_stack()) {
948 if (is_reference_type(type)) {
949 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
950 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
951 } else {
952 //no pushl on 64bits
953 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
954 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
955 }
956
957 } else if (src->is_double_stack()) {
958 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
959 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
960
961 } else {
962 ShouldNotReachHere();
963 }
964 }
965
966
967 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
968 assert(src->is_address(), "should not call otherwise");
969 assert(dest->is_register(), "should not call otherwise");
970
971 LIR_Address* addr = src->as_address_ptr();
972 Address from_addr = as_Address(addr);
973
974 if (addr->base()->type() == T_OBJECT) {
975 __ verify_oop(addr->base()->as_pointer_register());
976 }
977
978 switch (type) {
979 case T_BOOLEAN: // fall through
980 case T_BYTE: // fall through
981 case T_CHAR: // fall through
982 case T_SHORT:
983 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
984 // on pre P6 processors we may get partial register stalls
985 // so blow away the value of to_rinfo before loading a
986 // partial word into it. Do it here so that it precedes
987 // the potential patch point below.
988 __ xorptr(dest->as_register(), dest->as_register());
989 }
990 break;
991 default:
992 break;
993 }
994
995 PatchingStub* patch = nullptr;
996 if (patch_code != lir_patch_none) {
997 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
998 assert(from_addr.disp() != 0, "must have");
999 }
1000 if (info != nullptr) {
1001 add_debug_info_for_null_check_here(info);
1002 }
1003
1004 switch (type) {
1005 case T_FLOAT: {
1006 if (dest->is_single_xmm()) {
1007 __ movflt(dest->as_xmm_float_reg(), from_addr);
1008 } else {
1009 ShouldNotReachHere();
1010 }
1011 break;
1012 }
1013
1014 case T_DOUBLE: {
1015 if (dest->is_double_xmm()) {
1016 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1017 } else {
1018 ShouldNotReachHere();
1019 }
1020 break;
1021 }
1022
1023 case T_OBJECT: // fall through
1024 case T_ARRAY: // fall through
1025 if (UseCompressedOops && !wide) {
1026 __ movl(dest->as_register(), from_addr);
1027 } else {
1028 __ movptr(dest->as_register(), from_addr);
1029 }
1030 break;
1031
1032 case T_ADDRESS:
1033 __ movptr(dest->as_register(), from_addr);
1034 break;
1035 case T_INT:
1036 __ movl(dest->as_register(), from_addr);
1037 break;
1038
1039 case T_LONG: {
1040 Register to_lo = dest->as_register_lo();
1041 Register to_hi = dest->as_register_hi();
1042 __ movptr(to_lo, as_Address_lo(addr));
1043 break;
1044 }
1045
1046 case T_BOOLEAN: // fall through
1047 case T_BYTE: {
1048 Register dest_reg = dest->as_register();
1049 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1050 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1051 __ movsbl(dest_reg, from_addr);
1052 } else {
1053 __ movb(dest_reg, from_addr);
1054 __ shll(dest_reg, 24);
1055 __ sarl(dest_reg, 24);
1056 }
1057 break;
1058 }
1059
1060 case T_CHAR: {
1061 Register dest_reg = dest->as_register();
1062 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1063 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1064 __ movzwl(dest_reg, from_addr);
1065 } else {
1066 __ movw(dest_reg, from_addr);
1067 }
1068 break;
1069 }
1070
1071 case T_SHORT: {
1072 Register dest_reg = dest->as_register();
1073 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1074 __ movswl(dest_reg, from_addr);
1075 } else {
1076 __ movw(dest_reg, from_addr);
1077 __ shll(dest_reg, 16);
1078 __ sarl(dest_reg, 16);
1079 }
1080 break;
1081 }
1082
1083 default:
1084 ShouldNotReachHere();
1085 }
1086
1087 if (patch != nullptr) {
1088 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1089 }
1090
1091 if (is_reference_type(type)) {
1092 if (UseCompressedOops && !wide) {
1093 __ decode_heap_oop(dest->as_register());
1094 }
1095
1096 __ verify_oop(dest->as_register());
1097 }
1098 }
1099
1100
1101 NEEDS_CLEANUP; // This could be static?
1102 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1103 int elem_size = type2aelembytes(type);
1104 switch (elem_size) {
1105 case 1: return Address::times_1;
1106 case 2: return Address::times_2;
1107 case 4: return Address::times_4;
1108 case 8: return Address::times_8;
1109 }
1110 ShouldNotReachHere();
1111 return Address::no_scale;
1112 }
1113
1114
1115 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1116 switch (op->code()) {
1117 case lir_idiv:
1118 case lir_irem:
1119 arithmetic_idiv(op->code(),
1120 op->in_opr1(),
1121 op->in_opr2(),
1122 op->in_opr3(),
1123 op->result_opr(),
1124 op->info());
1125 break;
1126 case lir_fmad:
1127 __ fmad(op->result_opr()->as_xmm_double_reg(),
1128 op->in_opr1()->as_xmm_double_reg(),
1129 op->in_opr2()->as_xmm_double_reg(),
1130 op->in_opr3()->as_xmm_double_reg());
1131 break;
1132 case lir_fmaf:
1133 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1134 op->in_opr1()->as_xmm_float_reg(),
1135 op->in_opr2()->as_xmm_float_reg(),
1136 op->in_opr3()->as_xmm_float_reg());
1137 break;
1138 default: ShouldNotReachHere(); break;
1139 }
1140 }
1141
1142 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1143 #ifdef ASSERT
1144 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1145 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1146 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1147 #endif
1148
1149 if (op->cond() == lir_cond_always) {
1150 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1151 __ jmp (*(op->label()));
1152 } else {
1153 Assembler::Condition acond = Assembler::zero;
1154 if (op->code() == lir_cond_float_branch) {
1155 assert(op->ublock() != nullptr, "must have unordered successor");
1156 __ jcc(Assembler::parity, *(op->ublock()->label()));
1157 switch(op->cond()) {
1158 case lir_cond_equal: acond = Assembler::equal; break;
1159 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1160 case lir_cond_less: acond = Assembler::below; break;
1161 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1162 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1163 case lir_cond_greater: acond = Assembler::above; break;
1164 default: ShouldNotReachHere();
1165 }
1166 } else {
1167 switch (op->cond()) {
1168 case lir_cond_equal: acond = Assembler::equal; break;
1169 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1170 case lir_cond_less: acond = Assembler::less; break;
1171 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1172 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1173 case lir_cond_greater: acond = Assembler::greater; break;
1174 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1175 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1176 default: ShouldNotReachHere();
1177 }
1178 }
1179 __ jcc(acond,*(op->label()));
1180 }
1181 }
1182
1183 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1184 LIR_Opr src = op->in_opr();
1185 LIR_Opr dest = op->result_opr();
1186
1187 switch (op->bytecode()) {
1188 case Bytecodes::_i2l:
1189 __ movl2ptr(dest->as_register_lo(), src->as_register());
1190 break;
1191
1192 case Bytecodes::_l2i:
1193 __ movl(dest->as_register(), src->as_register_lo());
1194 break;
1195
1196 case Bytecodes::_i2b:
1197 move_regs(src->as_register(), dest->as_register());
1198 __ sign_extend_byte(dest->as_register());
1199 break;
1200
1201 case Bytecodes::_i2c:
1202 move_regs(src->as_register(), dest->as_register());
1203 __ andl(dest->as_register(), 0xFFFF);
1204 break;
1205
1206 case Bytecodes::_i2s:
1207 move_regs(src->as_register(), dest->as_register());
1208 __ sign_extend_short(dest->as_register());
1209 break;
1210
1211 case Bytecodes::_f2d:
1212 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1213 break;
1214
1215 case Bytecodes::_d2f:
1216 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1217 break;
1218
1219 case Bytecodes::_i2f:
1220 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1221 break;
1222
1223 case Bytecodes::_i2d:
1224 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1225 break;
1226
1227 case Bytecodes::_l2f:
1228 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1229 break;
1230
1231 case Bytecodes::_l2d:
1232 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1233 break;
1234
1235 case Bytecodes::_f2i:
1236 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1237 break;
1238
1239 case Bytecodes::_d2i:
1240 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1241 break;
1242
1243 case Bytecodes::_f2l:
1244 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1245 break;
1246
1247 case Bytecodes::_d2l:
1248 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1249 break;
1250
1251 default: ShouldNotReachHere();
1252 }
1253 }
1254
1255 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1256 if (op->init_check()) {
1257 add_debug_info_for_null_check_here(op->stub()->info());
1258 // init_state needs acquire, but x86 is TSO, and so we are already good.
1259 __ cmpb(Address(op->klass()->as_register(),
1260 InstanceKlass::init_state_offset()),
1261 InstanceKlass::fully_initialized);
1262 __ jcc(Assembler::notEqual, *op->stub()->entry());
1263 }
1264 __ allocate_object(op->obj()->as_register(),
1265 op->tmp1()->as_register(),
1266 op->tmp2()->as_register(),
1267 op->header_size(),
1268 op->object_size(),
1269 op->klass()->as_register(),
1270 *op->stub()->entry());
1271 __ bind(*op->stub()->continuation());
1272 }
1273
1274 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1275 Register len = op->len()->as_register();
1276 __ movslq(len, len);
1277
1278 if (UseSlowPath || op->always_slow_path() ||
1279 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1280 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1281 __ jmp(*op->stub()->entry());
1282 } else {
1283 Register tmp1 = op->tmp1()->as_register();
1284 Register tmp2 = op->tmp2()->as_register();
1285 Register tmp3 = op->tmp3()->as_register();
1286 if (len == tmp1) {
1287 tmp1 = tmp3;
1288 } else if (len == tmp2) {
1289 tmp2 = tmp3;
1290 } else if (len == tmp3) {
1291 // everything is ok
1292 } else {
1293 __ mov(tmp3, len);
1294 }
1295 __ allocate_array(op->obj()->as_register(),
1296 len,
1297 tmp1,
1298 tmp2,
1299 arrayOopDesc::base_offset_in_bytes(op->type()),
1300 array_element_size(op->type()),
1301 op->klass()->as_register(),
1302 *op->stub()->entry(),
1303 op->zero_array());
1304 }
1305 __ bind(*op->stub()->continuation());
1306 }
1307
1308 void LIR_Assembler::type_profile_helper(Register mdo,
1309 ciMethodData *md, ciProfileData *data,
1310 Register recv, Label* update_done) {
1311 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1312 Label next_test;
1313 // See if the receiver is receiver[n].
1314 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1315 __ jccb(Assembler::notEqual, next_test);
1316 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1317 __ addptr(data_addr, DataLayout::counter_increment);
1318 __ jmp(*update_done);
1319 __ bind(next_test);
1320 }
1321
1322 // Didn't find receiver; find next empty slot and fill it in
1323 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1324 Label next_test;
1325 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1326 __ cmpptr(recv_addr, NULL_WORD);
1327 __ jccb(Assembler::notEqual, next_test);
1328 __ movptr(recv_addr, recv);
1329 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1330 __ jmp(*update_done);
1331 __ bind(next_test);
1332 }
1333 }
1334
1335 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1336 // we always need a stub for the failure case.
1337 CodeStub* stub = op->stub();
1338 Register obj = op->object()->as_register();
1339 Register k_RInfo = op->tmp1()->as_register();
1340 Register klass_RInfo = op->tmp2()->as_register();
1341 Register dst = op->result_opr()->as_register();
1342 ciKlass* k = op->klass();
1343 Register Rtmp1 = noreg;
1344 Register tmp_load_klass = rscratch1;
1345
1346 // check if it needs to be profiled
1347 ciMethodData* md = nullptr;
1348 ciProfileData* data = nullptr;
1349
1350 if (op->should_profile()) {
1351 ciMethod* method = op->profiled_method();
1352 assert(method != nullptr, "Should have method");
1353 int bci = op->profiled_bci();
1354 md = method->method_data_or_null();
1355 assert(md != nullptr, "Sanity");
1356 data = md->bci_to_data(bci);
1357 assert(data != nullptr, "need data for type check");
1358 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1359 }
1360 Label* success_target = success;
1361 Label* failure_target = failure;
1362
1363 if (obj == k_RInfo) {
1364 k_RInfo = dst;
1365 } else if (obj == klass_RInfo) {
1366 klass_RInfo = dst;
1367 }
1368 if (k->is_loaded() && !UseCompressedClassPointers) {
1369 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1370 } else {
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373 }
1374
1375 assert_different_registers(obj, k_RInfo, klass_RInfo);
1376
1377 if (op->need_null_check()) {
1378 __ testptr(obj, obj);
1379 if (op->should_profile()) {
1380 Label not_null;
1381 Register mdo = klass_RInfo;
1382 __ mov_metadata(mdo, md->constant_encoding());
1383 __ jccb(Assembler::notEqual, not_null);
1384 // Object is null; update MDO and exit
1385 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1386 int header_bits = BitData::null_seen_byte_constant();
1387 __ orb(data_addr, header_bits);
1388 __ jmp(*obj_is_null);
1389 __ bind(not_null);
1390
1391 Label update_done;
1392 Register recv = k_RInfo;
1393 __ load_klass(recv, obj, tmp_load_klass);
1394 type_profile_helper(mdo, md, data, recv, &update_done);
1395
1396 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1397 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1398
1399 __ bind(update_done);
1400 } else {
1401 __ jcc(Assembler::equal, *obj_is_null);
1402 }
1403 }
1404
1405 if (!k->is_loaded()) {
1406 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1407 } else {
1408 __ mov_metadata(k_RInfo, k->constant_encoding());
1409 }
1410 __ verify_oop(obj);
1411
1412 if (op->fast_check()) {
1413 // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 if (UseCompressedClassPointers) {
1417 __ load_klass(Rtmp1, obj, tmp_load_klass);
1418 __ cmpptr(k_RInfo, Rtmp1);
1419 } else {
1420 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1421 }
1422 __ jcc(Assembler::notEqual, *failure_target);
1423 // successful cast, fall through to profile or jump
1424 } else {
1425 // get object class
1426 // not a safepoint as obj null check happens earlier
1427 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1428 if (k->is_loaded()) {
1429 // See if we get an immediate positive hit
1430 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1431 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1432 __ jcc(Assembler::notEqual, *failure_target);
1433 // successful cast, fall through to profile or jump
1434 } else {
1435 // See if we get an immediate positive hit
1436 __ jcc(Assembler::equal, *success_target);
1437 // check for self
1438 __ cmpptr(klass_RInfo, k_RInfo);
1439 __ jcc(Assembler::equal, *success_target);
1440
1441 __ push(klass_RInfo);
1442 __ push(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1444 __ pop(klass_RInfo);
1445 __ pop(klass_RInfo);
1446 // result is a boolean
1447 __ testl(klass_RInfo, klass_RInfo);
1448 __ jcc(Assembler::equal, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 } else {
1452 // perform the fast part of the checking logic
1453 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1454 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1455 __ push(klass_RInfo);
1456 __ push(k_RInfo);
1457 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1458 __ pop(klass_RInfo);
1459 __ pop(k_RInfo);
1460 // result is a boolean
1461 __ testl(k_RInfo, k_RInfo);
1462 __ jcc(Assembler::equal, *failure_target);
1463 // successful cast, fall through to profile or jump
1464 }
1465 }
1466 __ jmp(*success);
1467 }
1468
1469
1470 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1471 Register tmp_load_klass = rscratch1;
1472 LIR_Code code = op->code();
1473 if (code == lir_store_check) {
1474 Register value = op->object()->as_register();
1475 Register array = op->array()->as_register();
1476 Register k_RInfo = op->tmp1()->as_register();
1477 Register klass_RInfo = op->tmp2()->as_register();
1478 Register Rtmp1 = op->tmp3()->as_register();
1479
1480 CodeStub* stub = op->stub();
1481
1482 // check if it needs to be profiled
1483 ciMethodData* md = nullptr;
1484 ciProfileData* data = nullptr;
1485
1486 if (op->should_profile()) {
1487 ciMethod* method = op->profiled_method();
1488 assert(method != nullptr, "Should have method");
1489 int bci = op->profiled_bci();
1490 md = method->method_data_or_null();
1491 assert(md != nullptr, "Sanity");
1492 data = md->bci_to_data(bci);
1493 assert(data != nullptr, "need data for type check");
1494 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1495 }
1496 Label done;
1497 Label* success_target = &done;
1498 Label* failure_target = stub->entry();
1499
1500 __ testptr(value, value);
1501 if (op->should_profile()) {
1502 Label not_null;
1503 Register mdo = klass_RInfo;
1504 __ mov_metadata(mdo, md->constant_encoding());
1505 __ jccb(Assembler::notEqual, not_null);
1506 // Object is null; update MDO and exit
1507 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1508 int header_bits = BitData::null_seen_byte_constant();
1509 __ orb(data_addr, header_bits);
1510 __ jmp(done);
1511 __ bind(not_null);
1512
1513 Label update_done;
1514 Register recv = k_RInfo;
1515 __ load_klass(recv, value, tmp_load_klass);
1516 type_profile_helper(mdo, md, data, recv, &update_done);
1517
1518 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1519 __ addptr(counter_addr, DataLayout::counter_increment);
1520 __ bind(update_done);
1521 } else {
1522 __ jcc(Assembler::equal, done);
1523 }
1524
1525 add_debug_info_for_null_check_here(op->info_for_exception());
1526 __ load_klass(k_RInfo, array, tmp_load_klass);
1527 __ load_klass(klass_RInfo, value, tmp_load_klass);
1528
1529 // get instance klass (it's already uncompressed)
1530 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1531 // perform the fast part of the checking logic
1532 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1533 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1534 __ push(klass_RInfo);
1535 __ push(k_RInfo);
1536 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1537 __ pop(klass_RInfo);
1538 __ pop(k_RInfo);
1539 // result is a boolean
1540 __ testl(k_RInfo, k_RInfo);
1541 __ jcc(Assembler::equal, *failure_target);
1542 // fall through to the success case
1543
1544 __ bind(done);
1545 } else
1546 if (code == lir_checkcast) {
1547 Register obj = op->object()->as_register();
1548 Register dst = op->result_opr()->as_register();
1549 Label success;
1550 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1551 __ bind(success);
1552 if (dst != obj) {
1553 __ mov(dst, obj);
1554 }
1555 } else
1556 if (code == lir_instanceof) {
1557 Register obj = op->object()->as_register();
1558 Register dst = op->result_opr()->as_register();
1559 Label success, failure, done;
1560 emit_typecheck_helper(op, &success, &failure, &failure);
1561 __ bind(failure);
1562 __ xorptr(dst, dst);
1563 __ jmpb(done);
1564 __ bind(success);
1565 __ movptr(dst, 1);
1566 __ bind(done);
1567 } else {
1568 ShouldNotReachHere();
1569 }
1570
1571 }
1572
1573 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1574 // We are loading/storing from/to an array that *may* be a flat array (the
1575 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1576 // If this array is a flat array, take the slow path.
1577 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1578 if (!op->value()->is_illegal()) {
1579 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1580 // The array is not a flat array, but it might be null-free. If we are storing
1581 // a null into a null-free array, take the slow path (which will throw NPE).
1582 Label skip;
1583 __ cmpptr(op->value()->as_register(), NULL_WORD);
1584 __ jcc(Assembler::notEqual, skip);
1585 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1586 __ bind(skip);
1587 }
1588 }
1589
1590 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1591 // We are storing into an array that *may* be null-free (the declared type is
1592 // Object[], abstract[], interface[] or VT.ref[]).
1593 Label test_mark_word;
1594 Register tmp = op->tmp()->as_register();
1595 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1596 __ testl(tmp, markWord::unlocked_value);
1597 __ jccb(Assembler::notZero, test_mark_word);
1598 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1599 __ bind(test_mark_word);
1600 __ testl(tmp, markWord::null_free_array_bit_in_place);
1601 }
1602
1603 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1604 Label L_oops_equal;
1605 Label L_oops_not_equal;
1606 Label L_end;
1607
1608 Register left = op->left()->as_register();
1609 Register right = op->right()->as_register();
1610
1611 __ cmpptr(left, right);
1612 __ jcc(Assembler::equal, L_oops_equal);
1613
1614 // (1) Null check -- if one of the operands is null, the other must not be null (because
1615 // the two references are not equal), so they are not substitutable,
1616 // FIXME: do null check only if the operand is nullable
1617 __ testptr(left, right);
1618 __ jcc(Assembler::zero, L_oops_not_equal);
1619
1620 ciKlass* left_klass = op->left_klass();
1621 ciKlass* right_klass = op->right_klass();
1622
1623 // (2) Inline type check -- if either of the operands is not a inline type,
1624 // they are not substitutable. We do this only if we are not sure that the
1625 // operands are inline type
1626 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1627 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1628 Register tmp1 = op->tmp1()->as_register();
1629 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1630 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1631 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1632 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1633 __ jcc(Assembler::notEqual, L_oops_not_equal);
1634 }
1635
1636 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1637 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1638 // No need to load klass -- the operands are statically known to be the same inline klass.
1639 __ jmp(*op->stub()->entry());
1640 } else {
1641 Register left_klass_op = op->left_klass_op()->as_register();
1642 Register right_klass_op = op->right_klass_op()->as_register();
1643
1644 if (UseCompressedClassPointers) {
1645 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1646 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1647 __ cmpl(left_klass_op, right_klass_op);
1648 } else {
1649 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1650 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651 __ cmpptr(left_klass_op, right_klass_op);
1652 }
1653
1654 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1655 // fall through to L_oops_not_equal
1656 }
1657
1658 __ bind(L_oops_not_equal);
1659 move(op->not_equal_result(), op->result_opr());
1660 __ jmp(L_end);
1661
1662 __ bind(L_oops_equal);
1663 move(op->equal_result(), op->result_opr());
1664 __ jmp(L_end);
1665
1666 // We've returned from the stub. RAX contains 0x0 IFF the two
1667 // operands are not substitutable. (Don't compare against 0x1 in case the
1668 // C compiler is naughty)
1669 __ bind(*op->stub()->continuation());
1670 __ cmpl(rax, 0);
1671 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1672 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1673 // fall-through
1674 __ bind(L_end);
1675 }
1676
1677 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1678 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1679 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1680 Register newval = op->new_value()->as_register();
1681 Register cmpval = op->cmp_value()->as_register();
1682 assert(cmpval == rax, "wrong register");
1683 assert(newval != noreg, "new val must be register");
1684 assert(cmpval != newval, "cmp and new values must be in different registers");
1685 assert(cmpval != addr, "cmp and addr must be in different registers");
1686 assert(newval != addr, "new value and addr must be in different registers");
1687
1688 if (op->code() == lir_cas_obj) {
1689 if (UseCompressedOops) {
1690 __ encode_heap_oop(cmpval);
1691 __ mov(rscratch1, newval);
1692 __ encode_heap_oop(rscratch1);
1693 __ lock();
1694 // cmpval (rax) is implicitly used by this instruction
1695 __ cmpxchgl(rscratch1, Address(addr, 0));
1696 } else {
1697 __ lock();
1698 __ cmpxchgptr(newval, Address(addr, 0));
1699 }
1700 } else {
1701 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1702 __ lock();
1703 __ cmpxchgl(newval, Address(addr, 0));
1704 }
1705 } else if (op->code() == lir_cas_long) {
1706 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1707 Register newval = op->new_value()->as_register_lo();
1708 Register cmpval = op->cmp_value()->as_register_lo();
1709 assert(cmpval == rax, "wrong register");
1710 assert(newval != noreg, "new val must be register");
1711 assert(cmpval != newval, "cmp and new values must be in different registers");
1712 assert(cmpval != addr, "cmp and addr must be in different registers");
1713 assert(newval != addr, "new value and addr must be in different registers");
1714 __ lock();
1715 __ cmpxchgq(newval, Address(addr, 0));
1716 } else {
1717 Unimplemented();
1718 }
1719 }
1720
1721 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1722 assert(dst->is_cpu_register(), "must be");
1723 assert(dst->type() == src->type(), "must be");
1724
1725 if (src->is_cpu_register()) {
1726 reg2reg(src, dst);
1727 } else if (src->is_stack()) {
1728 stack2reg(src, dst, dst->type());
1729 } else if (src->is_constant()) {
1730 const2reg(src, dst, lir_patch_none, nullptr);
1731 } else {
1732 ShouldNotReachHere();
1733 }
1734 }
1735
1736 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1737 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1738 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1739
1740 Assembler::Condition acond, ncond;
1741 switch (condition) {
1742 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1743 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1744 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1745 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1746 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1747 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1748 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1749 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1750 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1751 ShouldNotReachHere();
1752 }
1753
1754 if (opr1->is_cpu_register()) {
1755 reg2reg(opr1, result);
1756 } else if (opr1->is_stack()) {
1757 stack2reg(opr1, result, result->type());
1758 } else if (opr1->is_constant()) {
1759 const2reg(opr1, result, lir_patch_none, nullptr);
1760 } else {
1761 ShouldNotReachHere();
1762 }
1763
1764 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1765 // optimized version that does not require a branch
1766 if (opr2->is_single_cpu()) {
1767 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1768 __ cmov(ncond, result->as_register(), opr2->as_register());
1769 } else if (opr2->is_double_cpu()) {
1770 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1771 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1772 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1773 } else if (opr2->is_single_stack()) {
1774 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1775 } else if (opr2->is_double_stack()) {
1776 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1777 } else {
1778 ShouldNotReachHere();
1779 }
1780
1781 } else {
1782 Label skip;
1783 __ jccb(acond, skip);
1784 if (opr2->is_cpu_register()) {
1785 reg2reg(opr2, result);
1786 } else if (opr2->is_stack()) {
1787 stack2reg(opr2, result, result->type());
1788 } else if (opr2->is_constant()) {
1789 const2reg(opr2, result, lir_patch_none, nullptr);
1790 } else {
1791 ShouldNotReachHere();
1792 }
1793 __ bind(skip);
1794 }
1795 }
1796
1797
1798 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1799 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1800
1801 if (left->is_single_cpu()) {
1802 assert(left == dest, "left and dest must be equal");
1803 Register lreg = left->as_register();
1804
1805 if (right->is_single_cpu()) {
1806 // cpu register - cpu register
1807 Register rreg = right->as_register();
1808 switch (code) {
1809 case lir_add: __ addl (lreg, rreg); break;
1810 case lir_sub: __ subl (lreg, rreg); break;
1811 case lir_mul: __ imull(lreg, rreg); break;
1812 default: ShouldNotReachHere();
1813 }
1814
1815 } else if (right->is_stack()) {
1816 // cpu register - stack
1817 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1818 switch (code) {
1819 case lir_add: __ addl(lreg, raddr); break;
1820 case lir_sub: __ subl(lreg, raddr); break;
1821 default: ShouldNotReachHere();
1822 }
1823
1824 } else if (right->is_constant()) {
1825 // cpu register - constant
1826 jint c = right->as_constant_ptr()->as_jint();
1827 switch (code) {
1828 case lir_add: {
1829 __ incrementl(lreg, c);
1830 break;
1831 }
1832 case lir_sub: {
1833 __ decrementl(lreg, c);
1834 break;
1835 }
1836 default: ShouldNotReachHere();
1837 }
1838
1839 } else {
1840 ShouldNotReachHere();
1841 }
1842
1843 } else if (left->is_double_cpu()) {
1844 assert(left == dest, "left and dest must be equal");
1845 Register lreg_lo = left->as_register_lo();
1846 Register lreg_hi = left->as_register_hi();
1847
1848 if (right->is_double_cpu()) {
1849 // cpu register - cpu register
1850 Register rreg_lo = right->as_register_lo();
1851 Register rreg_hi = right->as_register_hi();
1852 assert_different_registers(lreg_lo, rreg_lo);
1853 switch (code) {
1854 case lir_add:
1855 __ addptr(lreg_lo, rreg_lo);
1856 break;
1857 case lir_sub:
1858 __ subptr(lreg_lo, rreg_lo);
1859 break;
1860 case lir_mul:
1861 __ imulq(lreg_lo, rreg_lo);
1862 break;
1863 default:
1864 ShouldNotReachHere();
1865 }
1866
1867 } else if (right->is_constant()) {
1868 // cpu register - constant
1869 jlong c = right->as_constant_ptr()->as_jlong_bits();
1870 __ movptr(r10, (intptr_t) c);
1871 switch (code) {
1872 case lir_add:
1873 __ addptr(lreg_lo, r10);
1874 break;
1875 case lir_sub:
1876 __ subptr(lreg_lo, r10);
1877 break;
1878 default:
1879 ShouldNotReachHere();
1880 }
1881
1882 } else {
1883 ShouldNotReachHere();
1884 }
1885
1886 } else if (left->is_single_xmm()) {
1887 assert(left == dest, "left and dest must be equal");
1888 XMMRegister lreg = left->as_xmm_float_reg();
1889
1890 if (right->is_single_xmm()) {
1891 XMMRegister rreg = right->as_xmm_float_reg();
1892 switch (code) {
1893 case lir_add: __ addss(lreg, rreg); break;
1894 case lir_sub: __ subss(lreg, rreg); break;
1895 case lir_mul: __ mulss(lreg, rreg); break;
1896 case lir_div: __ divss(lreg, rreg); break;
1897 default: ShouldNotReachHere();
1898 }
1899 } else {
1900 Address raddr;
1901 if (right->is_single_stack()) {
1902 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1903 } else if (right->is_constant()) {
1904 // hack for now
1905 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1906 } else {
1907 ShouldNotReachHere();
1908 }
1909 switch (code) {
1910 case lir_add: __ addss(lreg, raddr); break;
1911 case lir_sub: __ subss(lreg, raddr); break;
1912 case lir_mul: __ mulss(lreg, raddr); break;
1913 case lir_div: __ divss(lreg, raddr); break;
1914 default: ShouldNotReachHere();
1915 }
1916 }
1917
1918 } else if (left->is_double_xmm()) {
1919 assert(left == dest, "left and dest must be equal");
1920
1921 XMMRegister lreg = left->as_xmm_double_reg();
1922 if (right->is_double_xmm()) {
1923 XMMRegister rreg = right->as_xmm_double_reg();
1924 switch (code) {
1925 case lir_add: __ addsd(lreg, rreg); break;
1926 case lir_sub: __ subsd(lreg, rreg); break;
1927 case lir_mul: __ mulsd(lreg, rreg); break;
1928 case lir_div: __ divsd(lreg, rreg); break;
1929 default: ShouldNotReachHere();
1930 }
1931 } else {
1932 Address raddr;
1933 if (right->is_double_stack()) {
1934 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1935 } else if (right->is_constant()) {
1936 // hack for now
1937 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1938 } else {
1939 ShouldNotReachHere();
1940 }
1941 switch (code) {
1942 case lir_add: __ addsd(lreg, raddr); break;
1943 case lir_sub: __ subsd(lreg, raddr); break;
1944 case lir_mul: __ mulsd(lreg, raddr); break;
1945 case lir_div: __ divsd(lreg, raddr); break;
1946 default: ShouldNotReachHere();
1947 }
1948 }
1949
1950 } else if (left->is_single_stack() || left->is_address()) {
1951 assert(left == dest, "left and dest must be equal");
1952
1953 Address laddr;
1954 if (left->is_single_stack()) {
1955 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1956 } else if (left->is_address()) {
1957 laddr = as_Address(left->as_address_ptr());
1958 } else {
1959 ShouldNotReachHere();
1960 }
1961
1962 if (right->is_single_cpu()) {
1963 Register rreg = right->as_register();
1964 switch (code) {
1965 case lir_add: __ addl(laddr, rreg); break;
1966 case lir_sub: __ subl(laddr, rreg); break;
1967 default: ShouldNotReachHere();
1968 }
1969 } else if (right->is_constant()) {
1970 jint c = right->as_constant_ptr()->as_jint();
1971 switch (code) {
1972 case lir_add: {
1973 __ incrementl(laddr, c);
1974 break;
1975 }
1976 case lir_sub: {
1977 __ decrementl(laddr, c);
1978 break;
1979 }
1980 default: ShouldNotReachHere();
1981 }
1982 } else {
1983 ShouldNotReachHere();
1984 }
1985
1986 } else {
1987 ShouldNotReachHere();
1988 }
1989 }
1990
1991
1992 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1993 if (value->is_double_xmm()) {
1994 switch(code) {
1995 case lir_abs :
1996 {
1997 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1998 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1999 }
2000 assert(!tmp->is_valid(), "do not need temporary");
2001 __ andpd(dest->as_xmm_double_reg(),
2002 ExternalAddress((address)double_signmask_pool),
2003 rscratch1);
2004 }
2005 break;
2006
2007 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2008 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2009 default : ShouldNotReachHere();
2010 }
2011
2012 } else if (code == lir_f2hf) {
2013 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2014 } else if (code == lir_hf2f) {
2015 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2016 } else {
2017 Unimplemented();
2018 }
2019 }
2020
2021 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2022 // assert(left->destroys_register(), "check");
2023 if (left->is_single_cpu()) {
2024 Register reg = left->as_register();
2025 if (right->is_constant()) {
2026 int val = right->as_constant_ptr()->as_jint();
2027 switch (code) {
2028 case lir_logic_and: __ andl (reg, val); break;
2029 case lir_logic_or: __ orl (reg, val); break;
2030 case lir_logic_xor: __ xorl (reg, val); break;
2031 default: ShouldNotReachHere();
2032 }
2033 } else if (right->is_stack()) {
2034 // added support for stack operands
2035 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2036 switch (code) {
2037 case lir_logic_and: __ andl (reg, raddr); break;
2038 case lir_logic_or: __ orl (reg, raddr); break;
2039 case lir_logic_xor: __ xorl (reg, raddr); break;
2040 default: ShouldNotReachHere();
2041 }
2042 } else {
2043 Register rright = right->as_register();
2044 switch (code) {
2045 case lir_logic_and: __ andptr (reg, rright); break;
2046 case lir_logic_or : __ orptr (reg, rright); break;
2047 case lir_logic_xor: __ xorptr (reg, rright); break;
2048 default: ShouldNotReachHere();
2049 }
2050 }
2051 move_regs(reg, dst->as_register());
2052 } else {
2053 Register l_lo = left->as_register_lo();
2054 Register l_hi = left->as_register_hi();
2055 if (right->is_constant()) {
2056 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2057 switch (code) {
2058 case lir_logic_and:
2059 __ andq(l_lo, rscratch1);
2060 break;
2061 case lir_logic_or:
2062 __ orq(l_lo, rscratch1);
2063 break;
2064 case lir_logic_xor:
2065 __ xorq(l_lo, rscratch1);
2066 break;
2067 default: ShouldNotReachHere();
2068 }
2069 } else {
2070 Register r_lo;
2071 if (is_reference_type(right->type())) {
2072 r_lo = right->as_register();
2073 } else {
2074 r_lo = right->as_register_lo();
2075 }
2076 switch (code) {
2077 case lir_logic_and:
2078 __ andptr(l_lo, r_lo);
2079 break;
2080 case lir_logic_or:
2081 __ orptr(l_lo, r_lo);
2082 break;
2083 case lir_logic_xor:
2084 __ xorptr(l_lo, r_lo);
2085 break;
2086 default: ShouldNotReachHere();
2087 }
2088 }
2089
2090 Register dst_lo = dst->as_register_lo();
2091 Register dst_hi = dst->as_register_hi();
2092
2093 move_regs(l_lo, dst_lo);
2094 }
2095 }
2096
2097
2098 // we assume that rax, and rdx can be overwritten
2099 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2100
2101 assert(left->is_single_cpu(), "left must be register");
2102 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2103 assert(result->is_single_cpu(), "result must be register");
2104
2105 // assert(left->destroys_register(), "check");
2106 // assert(right->destroys_register(), "check");
2107
2108 Register lreg = left->as_register();
2109 Register dreg = result->as_register();
2110
2111 if (right->is_constant()) {
2112 jint divisor = right->as_constant_ptr()->as_jint();
2113 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2114 if (code == lir_idiv) {
2115 assert(lreg == rax, "must be rax,");
2116 assert(temp->as_register() == rdx, "tmp register must be rdx");
2117 __ cdql(); // sign extend into rdx:rax
2118 if (divisor == 2) {
2119 __ subl(lreg, rdx);
2120 } else {
2121 __ andl(rdx, divisor - 1);
2122 __ addl(lreg, rdx);
2123 }
2124 __ sarl(lreg, log2i_exact(divisor));
2125 move_regs(lreg, dreg);
2126 } else if (code == lir_irem) {
2127 Label done;
2128 __ mov(dreg, lreg);
2129 __ andl(dreg, 0x80000000 | (divisor - 1));
2130 __ jcc(Assembler::positive, done);
2131 __ decrement(dreg);
2132 __ orl(dreg, ~(divisor - 1));
2133 __ increment(dreg);
2134 __ bind(done);
2135 } else {
2136 ShouldNotReachHere();
2137 }
2138 } else {
2139 Register rreg = right->as_register();
2140 assert(lreg == rax, "left register must be rax,");
2141 assert(rreg != rdx, "right register must not be rdx");
2142 assert(temp->as_register() == rdx, "tmp register must be rdx");
2143
2144 move_regs(lreg, rax);
2145
2146 int idivl_offset = __ corrected_idivl(rreg);
2147 if (ImplicitDiv0Checks) {
2148 add_debug_info_for_div0(idivl_offset, info);
2149 }
2150 if (code == lir_irem) {
2151 move_regs(rdx, dreg); // result is in rdx
2152 } else {
2153 move_regs(rax, dreg);
2154 }
2155 }
2156 }
2157
2158
2159 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2160 if (opr1->is_single_cpu()) {
2161 Register reg1 = opr1->as_register();
2162 if (opr2->is_single_cpu()) {
2163 // cpu register - cpu register
2164 if (is_reference_type(opr1->type())) {
2165 __ cmpoop(reg1, opr2->as_register());
2166 } else {
2167 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2168 __ cmpl(reg1, opr2->as_register());
2169 }
2170 } else if (opr2->is_stack()) {
2171 // cpu register - stack
2172 if (is_reference_type(opr1->type())) {
2173 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2174 } else {
2175 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2176 }
2177 } else if (opr2->is_constant()) {
2178 // cpu register - constant
2179 LIR_Const* c = opr2->as_constant_ptr();
2180 if (c->type() == T_INT) {
2181 jint i = c->as_jint();
2182 if (i == 0) {
2183 __ testl(reg1, reg1);
2184 } else {
2185 __ cmpl(reg1, i);
2186 }
2187 } else if (c->type() == T_METADATA) {
2188 // All we need for now is a comparison with null for equality.
2189 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2190 Metadata* m = c->as_metadata();
2191 if (m == nullptr) {
2192 __ testptr(reg1, reg1);
2193 } else {
2194 ShouldNotReachHere();
2195 }
2196 } else if (is_reference_type(c->type())) {
2197 // In 64bit oops are single register
2198 jobject o = c->as_jobject();
2199 if (o == nullptr) {
2200 __ testptr(reg1, reg1);
2201 } else {
2202 __ cmpoop(reg1, o, rscratch1);
2203 }
2204 } else {
2205 fatal("unexpected type: %s", basictype_to_str(c->type()));
2206 }
2207 // cpu register - address
2208 } else if (opr2->is_address()) {
2209 if (op->info() != nullptr) {
2210 add_debug_info_for_null_check_here(op->info());
2211 }
2212 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2213 } else {
2214 ShouldNotReachHere();
2215 }
2216
2217 } else if(opr1->is_double_cpu()) {
2218 Register xlo = opr1->as_register_lo();
2219 Register xhi = opr1->as_register_hi();
2220 if (opr2->is_double_cpu()) {
2221 __ cmpptr(xlo, opr2->as_register_lo());
2222 } else if (opr2->is_constant()) {
2223 // cpu register - constant 0
2224 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2225 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2226 } else {
2227 ShouldNotReachHere();
2228 }
2229
2230 } else if (opr1->is_single_xmm()) {
2231 XMMRegister reg1 = opr1->as_xmm_float_reg();
2232 if (opr2->is_single_xmm()) {
2233 // xmm register - xmm register
2234 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2235 } else if (opr2->is_stack()) {
2236 // xmm register - stack
2237 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2238 } else if (opr2->is_constant()) {
2239 // xmm register - constant
2240 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2241 } else if (opr2->is_address()) {
2242 // xmm register - address
2243 if (op->info() != nullptr) {
2244 add_debug_info_for_null_check_here(op->info());
2245 }
2246 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2247 } else {
2248 ShouldNotReachHere();
2249 }
2250
2251 } else if (opr1->is_double_xmm()) {
2252 XMMRegister reg1 = opr1->as_xmm_double_reg();
2253 if (opr2->is_double_xmm()) {
2254 // xmm register - xmm register
2255 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2256 } else if (opr2->is_stack()) {
2257 // xmm register - stack
2258 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2259 } else if (opr2->is_constant()) {
2260 // xmm register - constant
2261 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2262 } else if (opr2->is_address()) {
2263 // xmm register - address
2264 if (op->info() != nullptr) {
2265 add_debug_info_for_null_check_here(op->info());
2266 }
2267 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2268 } else {
2269 ShouldNotReachHere();
2270 }
2271
2272 } else if (opr1->is_address() && opr2->is_constant()) {
2273 LIR_Const* c = opr2->as_constant_ptr();
2274 if (is_reference_type(c->type())) {
2275 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2276 __ movoop(rscratch1, c->as_jobject());
2277 }
2278 if (op->info() != nullptr) {
2279 add_debug_info_for_null_check_here(op->info());
2280 }
2281 // special case: address - constant
2282 LIR_Address* addr = opr1->as_address_ptr();
2283 if (c->type() == T_INT) {
2284 __ cmpl(as_Address(addr), c->as_jint());
2285 } else if (is_reference_type(c->type())) {
2286 // %%% Make this explode if addr isn't reachable until we figure out a
2287 // better strategy by giving noreg as the temp for as_Address
2288 __ cmpoop(rscratch1, as_Address(addr, noreg));
2289 } else {
2290 ShouldNotReachHere();
2291 }
2292
2293 } else {
2294 ShouldNotReachHere();
2295 }
2296 }
2297
2298 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2299 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2300 if (left->is_single_xmm()) {
2301 assert(right->is_single_xmm(), "must match");
2302 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2303 } else if (left->is_double_xmm()) {
2304 assert(right->is_double_xmm(), "must match");
2305 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2306
2307 } else {
2308 ShouldNotReachHere();
2309 }
2310 } else {
2311 assert(code == lir_cmp_l2i, "check");
2312 Label done;
2313 Register dest = dst->as_register();
2314 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2315 __ movl(dest, -1);
2316 __ jccb(Assembler::less, done);
2317 __ setb(Assembler::notZero, dest);
2318 __ movzbl(dest, dest);
2319 __ bind(done);
2320 }
2321 }
2322
2323
2324 void LIR_Assembler::align_call(LIR_Code code) {
2325 // make sure that the displacement word of the call ends up word aligned
2326 int offset = __ offset();
2327 switch (code) {
2328 case lir_static_call:
2329 case lir_optvirtual_call:
2330 case lir_dynamic_call:
2331 offset += NativeCall::displacement_offset;
2332 break;
2333 case lir_icvirtual_call:
2334 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2335 break;
2336 default: ShouldNotReachHere();
2337 }
2338 __ align(BytesPerWord, offset);
2339 }
2340
2341
2342 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2343 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2344 "must be aligned");
2345 __ call(AddressLiteral(op->addr(), rtype));
2346 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2347 __ post_call_nop();
2348 }
2349
2350
2351 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2352 __ ic_call(op->addr());
2353 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2354 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2355 "must be aligned");
2356 __ post_call_nop();
2357 }
2358
2359
2360 void LIR_Assembler::emit_static_call_stub() {
2361 address call_pc = __ pc();
2362 address stub = __ start_a_stub(call_stub_size());
2363 if (stub == nullptr) {
2364 bailout("static call stub overflow");
2365 return;
2366 }
2367
2368 int start = __ offset();
2369
2370 // make sure that the displacement word of the call ends up word aligned
2371 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2372 __ relocate(static_stub_Relocation::spec(call_pc));
2373 __ mov_metadata(rbx, (Metadata*)nullptr);
2374 // must be set to -1 at code generation time
2375 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2376 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2377 __ jump(RuntimeAddress(__ pc()));
2378
2379 assert(__ offset() - start <= call_stub_size(), "stub too big");
2380 __ end_a_stub();
2381 }
2382
2383
2384 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2385 assert(exceptionOop->as_register() == rax, "must match");
2386 assert(exceptionPC->as_register() == rdx, "must match");
2387
2388 // exception object is not added to oop map by LinearScan
2389 // (LinearScan assumes that no oops are in fixed registers)
2390 info->add_register_oop(exceptionOop);
2391 C1StubId unwind_id;
2392
2393 // get current pc information
2394 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2395 int pc_for_athrow_offset = __ offset();
2396 InternalAddress pc_for_athrow(__ pc());
2397 __ lea(exceptionPC->as_register(), pc_for_athrow);
2398 add_call_info(pc_for_athrow_offset, info); // for exception handler
2399
2400 __ verify_not_null_oop(rax);
2401 // search an exception handler (rax: exception oop, rdx: throwing pc)
2402 if (compilation()->has_fpu_code()) {
2403 unwind_id = C1StubId::handle_exception_id;
2404 } else {
2405 unwind_id = C1StubId::handle_exception_nofpu_id;
2406 }
2407 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2408
2409 // enough room for two byte trap
2410 __ nop();
2411 }
2412
2413
2414 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2415 assert(exceptionOop->as_register() == rax, "must match");
2416
2417 __ jmp(_unwind_handler_entry);
2418 }
2419
2420
2421 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2422
2423 // optimized version for linear scan:
2424 // * count must be already in ECX (guaranteed by LinearScan)
2425 // * left and dest must be equal
2426 // * tmp must be unused
2427 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2428 assert(left == dest, "left and dest must be equal");
2429 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2430
2431 if (left->is_single_cpu()) {
2432 Register value = left->as_register();
2433 assert(value != SHIFT_count, "left cannot be ECX");
2434
2435 switch (code) {
2436 case lir_shl: __ shll(value); break;
2437 case lir_shr: __ sarl(value); break;
2438 case lir_ushr: __ shrl(value); break;
2439 default: ShouldNotReachHere();
2440 }
2441 } else if (left->is_double_cpu()) {
2442 Register lo = left->as_register_lo();
2443 Register hi = left->as_register_hi();
2444 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2445 switch (code) {
2446 case lir_shl: __ shlptr(lo); break;
2447 case lir_shr: __ sarptr(lo); break;
2448 case lir_ushr: __ shrptr(lo); break;
2449 default: ShouldNotReachHere();
2450 }
2451 } else {
2452 ShouldNotReachHere();
2453 }
2454 }
2455
2456
2457 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2458 if (dest->is_single_cpu()) {
2459 // first move left into dest so that left is not destroyed by the shift
2460 Register value = dest->as_register();
2461 count = count & 0x1F; // Java spec
2462
2463 move_regs(left->as_register(), value);
2464 switch (code) {
2465 case lir_shl: __ shll(value, count); break;
2466 case lir_shr: __ sarl(value, count); break;
2467 case lir_ushr: __ shrl(value, count); break;
2468 default: ShouldNotReachHere();
2469 }
2470 } else if (dest->is_double_cpu()) {
2471 // first move left into dest so that left is not destroyed by the shift
2472 Register value = dest->as_register_lo();
2473 count = count & 0x1F; // Java spec
2474
2475 move_regs(left->as_register_lo(), value);
2476 switch (code) {
2477 case lir_shl: __ shlptr(value, count); break;
2478 case lir_shr: __ sarptr(value, count); break;
2479 case lir_ushr: __ shrptr(value, count); break;
2480 default: ShouldNotReachHere();
2481 }
2482 } else {
2483 ShouldNotReachHere();
2484 }
2485 }
2486
2487
2488 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2489 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2490 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2491 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2492 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2493 }
2494
2495
2496 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2497 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2498 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2499 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2500 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2501 }
2502
2503
2504 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2505 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2506 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2507 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2508 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2509 }
2510
2511
2512 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2513 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2514 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2515 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2516 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2517 }
2518
2519
2520 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2521 if (null_check) {
2522 __ testptr(obj, obj);
2523 __ jcc(Assembler::zero, *slow_path->entry());
2524 }
2525 if (is_dest) {
2526 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2527 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2528 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2529 } else {
2530 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2531 }
2532 }
2533
2534
2535 // This code replaces a call to arraycopy; no exception may
2536 // be thrown in this code, they must be thrown in the System.arraycopy
2537 // activation frame; we could save some checks if this would not be the case
2538 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2539 ciArrayKlass* default_type = op->expected_type();
2540 Register src = op->src()->as_register();
2541 Register dst = op->dst()->as_register();
2542 Register src_pos = op->src_pos()->as_register();
2543 Register dst_pos = op->dst_pos()->as_register();
2544 Register length = op->length()->as_register();
2545 Register tmp = op->tmp()->as_register();
2546 Register tmp_load_klass = rscratch1;
2547 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2548
2549 CodeStub* stub = op->stub();
2550 int flags = op->flags();
2551 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2552 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2553
2554 if (flags & LIR_OpArrayCopy::always_slow_path) {
2555 __ jmp(*stub->entry());
2556 __ bind(*stub->continuation());
2557 return;
2558 }
2559
2560 // if we don't know anything, just go through the generic arraycopy
2561 if (default_type == nullptr) {
2562 // save outgoing arguments on stack in case call to System.arraycopy is needed
2563 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2564 // for interpreter calling conventions. Now we have to do it in new style conventions.
2565 // For the moment until C1 gets the new register allocator I just force all the
2566 // args to the right place (except the register args) and then on the back side
2567 // reload the register args properly if we go slow path. Yuck
2568
2569 // These are proper for the calling convention
2570 store_parameter(length, 2);
2571 store_parameter(dst_pos, 1);
2572 store_parameter(dst, 0);
2573
2574 // these are just temporary placements until we need to reload
2575 store_parameter(src_pos, 3);
2576 store_parameter(src, 4);
2577
2578 address copyfunc_addr = StubRoutines::generic_arraycopy();
2579 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2580
2581 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2582 // The arguments are in java calling convention so we can trivially shift them to C
2583 // convention
2584 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2585 __ mov(c_rarg0, j_rarg0);
2586 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2587 __ mov(c_rarg1, j_rarg1);
2588 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2589 __ mov(c_rarg2, j_rarg2);
2590 assert_different_registers(c_rarg3, j_rarg4);
2591 __ mov(c_rarg3, j_rarg3);
2592 #ifdef _WIN64
2593 // Allocate abi space for args but be sure to keep stack aligned
2594 __ subptr(rsp, 6*wordSize);
2595 store_parameter(j_rarg4, 4);
2596 #ifndef PRODUCT
2597 if (PrintC1Statistics) {
2598 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2599 }
2600 #endif
2601 __ call(RuntimeAddress(copyfunc_addr));
2602 __ addptr(rsp, 6*wordSize);
2603 #else
2604 __ mov(c_rarg4, j_rarg4);
2605 #ifndef PRODUCT
2606 if (PrintC1Statistics) {
2607 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2608 }
2609 #endif
2610 __ call(RuntimeAddress(copyfunc_addr));
2611 #endif // _WIN64
2612
2613 __ testl(rax, rax);
2614 __ jcc(Assembler::equal, *stub->continuation());
2615
2616 __ mov(tmp, rax);
2617 __ xorl(tmp, -1);
2618
2619 // Reload values from the stack so they are where the stub
2620 // expects them.
2621 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2622 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2623 __ movptr (length, Address(rsp, 2*BytesPerWord));
2624 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2625 __ movptr (src, Address(rsp, 4*BytesPerWord));
2626
2627 __ subl(length, tmp);
2628 __ addl(src_pos, tmp);
2629 __ addl(dst_pos, tmp);
2630 __ jmp(*stub->entry());
2631
2632 __ bind(*stub->continuation());
2633 return;
2634 }
2635
2636 // Handle inline type arrays
2637 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2638 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2639 }
2640 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2641 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2642 }
2643
2644 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2645
2646 int elem_size = type2aelembytes(basic_type);
2647 Address::ScaleFactor scale;
2648
2649 switch (elem_size) {
2650 case 1 :
2651 scale = Address::times_1;
2652 break;
2653 case 2 :
2654 scale = Address::times_2;
2655 break;
2656 case 4 :
2657 scale = Address::times_4;
2658 break;
2659 case 8 :
2660 scale = Address::times_8;
2661 break;
2662 default:
2663 scale = Address::no_scale;
2664 ShouldNotReachHere();
2665 }
2666
2667 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2668 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2669
2670 // length and pos's are all sign extended at this point on 64bit
2671
2672 // test for null
2673 if (flags & LIR_OpArrayCopy::src_null_check) {
2674 __ testptr(src, src);
2675 __ jcc(Assembler::zero, *stub->entry());
2676 }
2677 if (flags & LIR_OpArrayCopy::dst_null_check) {
2678 __ testptr(dst, dst);
2679 __ jcc(Assembler::zero, *stub->entry());
2680 }
2681
2682 // If the compiler was not able to prove that exact type of the source or the destination
2683 // of the arraycopy is an array type, check at runtime if the source or the destination is
2684 // an instance type.
2685 if (flags & LIR_OpArrayCopy::type_check) {
2686 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2687 __ load_klass(tmp, dst, tmp_load_klass);
2688 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2689 __ jcc(Assembler::greaterEqual, *stub->entry());
2690 }
2691
2692 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2693 __ load_klass(tmp, src, tmp_load_klass);
2694 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2695 __ jcc(Assembler::greaterEqual, *stub->entry());
2696 }
2697 }
2698
2699 // check if negative
2700 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2701 __ testl(src_pos, src_pos);
2702 __ jcc(Assembler::less, *stub->entry());
2703 }
2704 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2705 __ testl(dst_pos, dst_pos);
2706 __ jcc(Assembler::less, *stub->entry());
2707 }
2708
2709 if (flags & LIR_OpArrayCopy::src_range_check) {
2710 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2711 __ cmpl(tmp, src_length_addr);
2712 __ jcc(Assembler::above, *stub->entry());
2713 }
2714 if (flags & LIR_OpArrayCopy::dst_range_check) {
2715 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2716 __ cmpl(tmp, dst_length_addr);
2717 __ jcc(Assembler::above, *stub->entry());
2718 }
2719
2720 if (flags & LIR_OpArrayCopy::length_positive_check) {
2721 __ testl(length, length);
2722 __ jcc(Assembler::less, *stub->entry());
2723 }
2724
2725 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2726 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2727
2728 if (flags & LIR_OpArrayCopy::type_check) {
2729 // We don't know the array types are compatible
2730 if (basic_type != T_OBJECT) {
2731 // Simple test for basic type arrays
2732 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2733 __ jcc(Assembler::notEqual, *stub->entry());
2734 } else {
2735 // For object arrays, if src is a sub class of dst then we can
2736 // safely do the copy.
2737 Label cont, slow;
2738
2739 __ push(src);
2740 __ push(dst);
2741
2742 __ load_klass(src, src, tmp_load_klass);
2743 __ load_klass(dst, dst, tmp_load_klass);
2744
2745 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2746
2747 __ push(src);
2748 __ push(dst);
2749 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
2750 __ pop(dst);
2751 __ pop(src);
2752
2753 __ testl(src, src);
2754 __ jcc(Assembler::notEqual, cont);
2755
2756 __ bind(slow);
2757 __ pop(dst);
2758 __ pop(src);
2759
2760 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2761 if (copyfunc_addr != nullptr) { // use stub if available
2762 // src is not a sub class of dst so we have to do a
2763 // per-element check.
2764
2765 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2766 if ((flags & mask) != mask) {
2767 // Check that at least both of them object arrays.
2768 assert(flags & mask, "one of the two should be known to be an object array");
2769
2770 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2771 __ load_klass(tmp, src, tmp_load_klass);
2772 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2773 __ load_klass(tmp, dst, tmp_load_klass);
2774 }
2775 int lh_offset = in_bytes(Klass::layout_helper_offset());
2776 Address klass_lh_addr(tmp, lh_offset);
2777 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2778 __ cmpl(klass_lh_addr, objArray_lh);
2779 __ jcc(Assembler::notEqual, *stub->entry());
2780 }
2781
2782 // Spill because stubs can use any register they like and it's
2783 // easier to restore just those that we care about.
2784 store_parameter(dst, 0);
2785 store_parameter(dst_pos, 1);
2786 store_parameter(length, 2);
2787 store_parameter(src_pos, 3);
2788 store_parameter(src, 4);
2789
2790 __ movl2ptr(length, length); //higher 32bits must be null
2791
2792 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2793 assert_different_registers(c_rarg0, dst, dst_pos, length);
2794 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2795 assert_different_registers(c_rarg1, dst, length);
2796
2797 __ mov(c_rarg2, length);
2798 assert_different_registers(c_rarg2, dst);
2799
2800 #ifdef _WIN64
2801 // Allocate abi space for args but be sure to keep stack aligned
2802 __ subptr(rsp, 6*wordSize);
2803 __ load_klass(c_rarg3, dst, tmp_load_klass);
2804 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2805 store_parameter(c_rarg3, 4);
2806 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2807 __ call(RuntimeAddress(copyfunc_addr));
2808 __ addptr(rsp, 6*wordSize);
2809 #else
2810 __ load_klass(c_rarg4, dst, tmp_load_klass);
2811 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2812 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2813 __ call(RuntimeAddress(copyfunc_addr));
2814 #endif
2815
2816 #ifndef PRODUCT
2817 if (PrintC1Statistics) {
2818 Label failed;
2819 __ testl(rax, rax);
2820 __ jcc(Assembler::notZero, failed);
2821 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2822 __ bind(failed);
2823 }
2824 #endif
2825
2826 __ testl(rax, rax);
2827 __ jcc(Assembler::zero, *stub->continuation());
2828
2829 #ifndef PRODUCT
2830 if (PrintC1Statistics) {
2831 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2832 }
2833 #endif
2834
2835 __ mov(tmp, rax);
2836
2837 __ xorl(tmp, -1);
2838
2839 // Restore previously spilled arguments
2840 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2841 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2842 __ movptr (length, Address(rsp, 2*BytesPerWord));
2843 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2844 __ movptr (src, Address(rsp, 4*BytesPerWord));
2845
2846
2847 __ subl(length, tmp);
2848 __ addl(src_pos, tmp);
2849 __ addl(dst_pos, tmp);
2850 }
2851
2852 __ jmp(*stub->entry());
2853
2854 __ bind(cont);
2855 __ pop(dst);
2856 __ pop(src);
2857 }
2858 }
2859
2860 #ifdef ASSERT
2861 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2862 // Sanity check the known type with the incoming class. For the
2863 // primitive case the types must match exactly with src.klass and
2864 // dst.klass each exactly matching the default type. For the
2865 // object array case, if no type check is needed then either the
2866 // dst type is exactly the expected type and the src type is a
2867 // subtype which we can't check or src is the same array as dst
2868 // but not necessarily exactly of type default_type.
2869 Label known_ok, halt;
2870
2871 __ mov_metadata(tmp, default_type->constant_encoding());
2872 if (UseCompressedClassPointers) {
2873 __ encode_klass_not_null(tmp, rscratch1);
2874 }
2875
2876 if (basic_type != T_OBJECT) {
2877 __ cmp_klass(tmp, dst, tmp2);
2878 __ jcc(Assembler::notEqual, halt);
2879 __ cmp_klass(tmp, src, tmp2);
2880 __ jcc(Assembler::equal, known_ok);
2881 } else {
2882 __ cmp_klass(tmp, dst, tmp2);
2883 __ jcc(Assembler::equal, known_ok);
2884 __ cmpptr(src, dst);
2885 __ jcc(Assembler::equal, known_ok);
2886 }
2887 __ bind(halt);
2888 __ stop("incorrect type information in arraycopy");
2889 __ bind(known_ok);
2890 }
2891 #endif
2892
2893 #ifndef PRODUCT
2894 if (PrintC1Statistics) {
2895 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2896 }
2897 #endif
2898
2899 assert_different_registers(c_rarg0, dst, dst_pos, length);
2900 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2901 assert_different_registers(c_rarg1, length);
2902 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2903 __ mov(c_rarg2, length);
2904
2905 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2906 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2907 const char *name;
2908 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2909 __ call_VM_leaf(entry, 0);
2910
2911 if (stub != nullptr) {
2912 __ bind(*stub->continuation());
2913 }
2914 }
2915
2916 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2917 assert(op->crc()->is_single_cpu(), "crc must be register");
2918 assert(op->val()->is_single_cpu(), "byte value must be register");
2919 assert(op->result_opr()->is_single_cpu(), "result must be register");
2920 Register crc = op->crc()->as_register();
2921 Register val = op->val()->as_register();
2922 Register res = op->result_opr()->as_register();
2923
2924 assert_different_registers(val, crc, res);
2925
2926 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2927 __ notl(crc); // ~crc
2928 __ update_byte_crc32(crc, val, res);
2929 __ notl(crc); // ~crc
2930 __ mov(res, crc);
2931 }
2932
2933 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2934 Register obj = op->obj_opr()->as_register(); // may not be an oop
2935 Register hdr = op->hdr_opr()->as_register();
2936 Register lock = op->lock_opr()->as_register();
2937 if (LockingMode == LM_MONITOR) {
2938 if (op->info() != nullptr) {
2939 add_debug_info_for_null_check_here(op->info());
2940 __ null_check(obj);
2941 }
2942 __ jmp(*op->stub()->entry());
2943 } else if (op->code() == lir_lock) {
2944 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2945 Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
2946 // add debug info for NullPointerException only if one is possible
2947 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2948 if (op->info() != nullptr) {
2949 add_debug_info_for_null_check(null_check_offset, op->info());
2950 }
2951 // done
2952 } else if (op->code() == lir_unlock) {
2953 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2954 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2955 } else {
2956 Unimplemented();
2957 }
2958 __ bind(*op->stub()->continuation());
2959 }
2960
2961 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2962 Register obj = op->obj()->as_pointer_register();
2963 Register result = op->result_opr()->as_pointer_register();
2964
2965 CodeEmitInfo* info = op->info();
2966 if (info != nullptr) {
2967 add_debug_info_for_null_check_here(info);
2968 }
2969
2970 __ load_klass(result, obj, rscratch1);
2971 }
2972
2973 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2974 ciMethod* method = op->profiled_method();
2975 int bci = op->profiled_bci();
2976 ciMethod* callee = op->profiled_callee();
2977 Register tmp_load_klass = rscratch1;
2978
2979 // Update counter for all call types
2980 ciMethodData* md = method->method_data_or_null();
2981 assert(md != nullptr, "Sanity");
2982 ciProfileData* data = md->bci_to_data(bci);
2983 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2984 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2985 Register mdo = op->mdo()->as_register();
2986 __ mov_metadata(mdo, md->constant_encoding());
2987 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2988 // Perform additional virtual call profiling for invokevirtual and
2989 // invokeinterface bytecodes
2990 if (op->should_profile_receiver_type()) {
2991 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2992 Register recv = op->recv()->as_register();
2993 assert_different_registers(mdo, recv);
2994 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2995 ciKlass* known_klass = op->known_holder();
2996 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2997 // We know the type that will be seen at this call site; we can
2998 // statically update the MethodData* rather than needing to do
2999 // dynamic tests on the receiver type
3000
3001 // NOTE: we should probably put a lock around this search to
3002 // avoid collisions by concurrent compilations
3003 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3004 uint i;
3005 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3006 ciKlass* receiver = vc_data->receiver(i);
3007 if (known_klass->equals(receiver)) {
3008 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3009 __ addptr(data_addr, DataLayout::counter_increment);
3010 return;
3011 }
3012 }
3013
3014 // Receiver type not found in profile data; select an empty slot
3015
3016 // Note that this is less efficient than it should be because it
3017 // always does a write to the receiver part of the
3018 // VirtualCallData rather than just the first time
3019 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3020 ciKlass* receiver = vc_data->receiver(i);
3021 if (receiver == nullptr) {
3022 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3023 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3024 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3025 __ addptr(data_addr, DataLayout::counter_increment);
3026 return;
3027 }
3028 }
3029 } else {
3030 __ load_klass(recv, recv, tmp_load_klass);
3031 Label update_done;
3032 type_profile_helper(mdo, md, data, recv, &update_done);
3033 // Receiver did not match any saved receiver and there is no empty row for it.
3034 // Increment total counter to indicate polymorphic case.
3035 __ addptr(counter_addr, DataLayout::counter_increment);
3036
3037 __ bind(update_done);
3038 }
3039 } else {
3040 // Static call
3041 __ addptr(counter_addr, DataLayout::counter_increment);
3042 }
3043 }
3044
3045 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3046 Register obj = op->obj()->as_register();
3047 Register tmp = op->tmp()->as_pointer_register();
3048 Register tmp_load_klass = rscratch1;
3049 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3050 ciKlass* exact_klass = op->exact_klass();
3051 intptr_t current_klass = op->current_klass();
3052 bool not_null = op->not_null();
3053 bool no_conflict = op->no_conflict();
3054
3055 Label update, next, none;
3056
3057 bool do_null = !not_null;
3058 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3059 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3060
3061 assert(do_null || do_update, "why are we here?");
3062 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3063
3064 __ verify_oop(obj);
3065
3066 #ifdef ASSERT
3067 if (obj == tmp) {
3068 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3069 } else {
3070 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3071 }
3072 #endif
3073 if (do_null) {
3074 __ testptr(obj, obj);
3075 __ jccb(Assembler::notZero, update);
3076 if (!TypeEntries::was_null_seen(current_klass)) {
3077 __ testptr(mdo_addr, TypeEntries::null_seen);
3078 #ifndef ASSERT
3079 __ jccb(Assembler::notZero, next); // already set
3080 #else
3081 __ jcc(Assembler::notZero, next); // already set
3082 #endif
3083 // atomic update to prevent overwriting Klass* with 0
3084 __ lock();
3085 __ orptr(mdo_addr, TypeEntries::null_seen);
3086 }
3087 if (do_update) {
3088 #ifndef ASSERT
3089 __ jmpb(next);
3090 }
3091 #else
3092 __ jmp(next);
3093 }
3094 } else {
3095 __ testptr(obj, obj);
3096 __ jcc(Assembler::notZero, update);
3097 __ stop("unexpected null obj");
3098 #endif
3099 }
3100
3101 __ bind(update);
3102
3103 if (do_update) {
3104 #ifdef ASSERT
3105 if (exact_klass != nullptr) {
3106 Label ok;
3107 __ load_klass(tmp, obj, tmp_load_klass);
3108 __ push(tmp);
3109 __ mov_metadata(tmp, exact_klass->constant_encoding());
3110 __ cmpptr(tmp, Address(rsp, 0));
3111 __ jcc(Assembler::equal, ok);
3112 __ stop("exact klass and actual klass differ");
3113 __ bind(ok);
3114 __ pop(tmp);
3115 }
3116 #endif
3117 if (!no_conflict) {
3118 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3119 if (exact_klass != nullptr) {
3120 __ mov_metadata(tmp, exact_klass->constant_encoding());
3121 } else {
3122 __ load_klass(tmp, obj, tmp_load_klass);
3123 }
3124 __ mov(rscratch1, tmp); // save original value before XOR
3125 __ xorptr(tmp, mdo_addr);
3126 __ testptr(tmp, TypeEntries::type_klass_mask);
3127 // klass seen before, nothing to do. The unknown bit may have been
3128 // set already but no need to check.
3129 __ jccb(Assembler::zero, next);
3130
3131 __ testptr(tmp, TypeEntries::type_unknown);
3132 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3133
3134 if (TypeEntries::is_type_none(current_klass)) {
3135 __ testptr(mdo_addr, TypeEntries::type_mask);
3136 __ jccb(Assembler::zero, none);
3137 // There is a chance that the checks above (re-reading profiling
3138 // data from memory) fail if another thread has just set the
3139 // profiling to this obj's klass
3140 __ mov(tmp, rscratch1); // get back original value before XOR
3141 __ xorptr(tmp, mdo_addr);
3142 __ testptr(tmp, TypeEntries::type_klass_mask);
3143 __ jccb(Assembler::zero, next);
3144 }
3145 } else {
3146 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3147 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3148
3149 __ testptr(mdo_addr, TypeEntries::type_unknown);
3150 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3151 }
3152
3153 // different than before. Cannot keep accurate profile.
3154 __ orptr(mdo_addr, TypeEntries::type_unknown);
3155
3156 if (TypeEntries::is_type_none(current_klass)) {
3157 __ jmpb(next);
3158
3159 __ bind(none);
3160 // first time here. Set profile type.
3161 __ movptr(mdo_addr, tmp);
3162 #ifdef ASSERT
3163 __ andptr(tmp, TypeEntries::type_klass_mask);
3164 __ verify_klass_ptr(tmp);
3165 #endif
3166 }
3167 } else {
3168 // There's a single possible klass at this profile point
3169 assert(exact_klass != nullptr, "should be");
3170 if (TypeEntries::is_type_none(current_klass)) {
3171 __ mov_metadata(tmp, exact_klass->constant_encoding());
3172 __ xorptr(tmp, mdo_addr);
3173 __ testptr(tmp, TypeEntries::type_klass_mask);
3174 #ifdef ASSERT
3175 __ jcc(Assembler::zero, next);
3176
3177 {
3178 Label ok;
3179 __ push(tmp);
3180 __ testptr(mdo_addr, TypeEntries::type_mask);
3181 __ jcc(Assembler::zero, ok);
3182 // may have been set by another thread
3183 __ mov_metadata(tmp, exact_klass->constant_encoding());
3184 __ xorptr(tmp, mdo_addr);
3185 __ testptr(tmp, TypeEntries::type_mask);
3186 __ jcc(Assembler::zero, ok);
3187
3188 __ stop("unexpected profiling mismatch");
3189 __ bind(ok);
3190 __ pop(tmp);
3191 }
3192 #else
3193 __ jccb(Assembler::zero, next);
3194 #endif
3195 // first time here. Set profile type.
3196 __ movptr(mdo_addr, tmp);
3197 #ifdef ASSERT
3198 __ andptr(tmp, TypeEntries::type_klass_mask);
3199 __ verify_klass_ptr(tmp);
3200 #endif
3201 } else {
3202 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3203 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3204
3205 __ testptr(mdo_addr, TypeEntries::type_unknown);
3206 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3207
3208 __ orptr(mdo_addr, TypeEntries::type_unknown);
3209 }
3210 }
3211 }
3212 __ bind(next);
3213 }
3214
3215 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3216 Register obj = op->obj()->as_register();
3217 Register tmp = op->tmp()->as_pointer_register();
3218 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3219 bool not_null = op->not_null();
3220 int flag = op->flag();
3221
3222 Label not_inline_type;
3223 if (!not_null) {
3224 __ testptr(obj, obj);
3225 __ jccb(Assembler::zero, not_inline_type);
3226 }
3227
3228 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3229
3230 __ orb(mdo_addr, flag);
3231
3232 __ bind(not_inline_type);
3233 }
3234
3235 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3236 Unimplemented();
3237 }
3238
3239
3240 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3241 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3242 }
3243
3244
3245 void LIR_Assembler::align_backward_branch_target() {
3246 __ align(BytesPerWord);
3247 }
3248
3249
3250 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3251 if (left->is_single_cpu()) {
3252 __ negl(left->as_register());
3253 move_regs(left->as_register(), dest->as_register());
3254
3255 } else if (left->is_double_cpu()) {
3256 Register lo = left->as_register_lo();
3257 Register dst = dest->as_register_lo();
3258 __ movptr(dst, lo);
3259 __ negptr(dst);
3260
3261 } else if (dest->is_single_xmm()) {
3262 assert(!tmp->is_valid(), "do not need temporary");
3263 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3264 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3265 }
3266 __ xorps(dest->as_xmm_float_reg(),
3267 ExternalAddress((address)float_signflip_pool),
3268 rscratch1);
3269 } else if (dest->is_double_xmm()) {
3270 assert(!tmp->is_valid(), "do not need temporary");
3271 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3272 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3273 }
3274 __ xorpd(dest->as_xmm_double_reg(),
3275 ExternalAddress((address)double_signflip_pool),
3276 rscratch1);
3277 } else {
3278 ShouldNotReachHere();
3279 }
3280 }
3281
3282
3283 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3284 assert(src->is_address(), "must be an address");
3285 assert(dest->is_register(), "must be a register");
3286
3287 PatchingStub* patch = nullptr;
3288 if (patch_code != lir_patch_none) {
3289 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3290 }
3291
3292 Register reg = dest->as_pointer_register();
3293 LIR_Address* addr = src->as_address_ptr();
3294 __ lea(reg, as_Address(addr));
3295
3296 if (patch != nullptr) {
3297 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3298 }
3299 }
3300
3301
3302
3303 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3304 assert(!tmp->is_valid(), "don't need temporary");
3305 __ call(RuntimeAddress(dest));
3306 if (info != nullptr) {
3307 add_call_info_here(info);
3308 }
3309 __ post_call_nop();
3310 }
3311
3312
3313 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3314 assert(type == T_LONG, "only for volatile long fields");
3315
3316 if (info != nullptr) {
3317 add_debug_info_for_null_check_here(info);
3318 }
3319
3320 if (src->is_double_xmm()) {
3321 if (dest->is_double_cpu()) {
3322 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3323 } else if (dest->is_double_stack()) {
3324 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3325 } else if (dest->is_address()) {
3326 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3327 } else {
3328 ShouldNotReachHere();
3329 }
3330
3331 } else if (dest->is_double_xmm()) {
3332 if (src->is_double_stack()) {
3333 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3334 } else if (src->is_address()) {
3335 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3336 } else {
3337 ShouldNotReachHere();
3338 }
3339
3340 } else {
3341 ShouldNotReachHere();
3342 }
3343 }
3344
3345 #ifdef ASSERT
3346 // emit run-time assertion
3347 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3348 assert(op->code() == lir_assert, "must be");
3349
3350 if (op->in_opr1()->is_valid()) {
3351 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3352 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3353 } else {
3354 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3355 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3356 }
3357
3358 Label ok;
3359 if (op->condition() != lir_cond_always) {
3360 Assembler::Condition acond = Assembler::zero;
3361 switch (op->condition()) {
3362 case lir_cond_equal: acond = Assembler::equal; break;
3363 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3364 case lir_cond_less: acond = Assembler::less; break;
3365 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3366 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3367 case lir_cond_greater: acond = Assembler::greater; break;
3368 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3369 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3370 default: ShouldNotReachHere();
3371 }
3372 __ jcc(acond, ok);
3373 }
3374 if (op->halt()) {
3375 const char* str = __ code_string(op->msg());
3376 __ stop(str);
3377 } else {
3378 breakpoint();
3379 }
3380 __ bind(ok);
3381 }
3382 #endif
3383
3384 void LIR_Assembler::membar() {
3385 // QQQ sparc TSO uses this,
3386 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3387 }
3388
3389 void LIR_Assembler::membar_acquire() {
3390 // No x86 machines currently require load fences
3391 }
3392
3393 void LIR_Assembler::membar_release() {
3394 // No x86 machines currently require store fences
3395 }
3396
3397 void LIR_Assembler::membar_loadload() {
3398 // no-op
3399 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3400 }
3401
3402 void LIR_Assembler::membar_storestore() {
3403 // no-op
3404 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3405 }
3406
3407 void LIR_Assembler::membar_loadstore() {
3408 // no-op
3409 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3410 }
3411
3412 void LIR_Assembler::membar_storeload() {
3413 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3414 }
3415
3416 void LIR_Assembler::on_spin_wait() {
3417 __ pause ();
3418 }
3419
3420 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3421 assert(result_reg->is_register(), "check");
3422 __ mov(result_reg->as_register(), r15_thread);
3423 }
3424
3425 void LIR_Assembler::check_orig_pc() {
3426 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3427 }
3428
3429 void LIR_Assembler::peephole(LIR_List*) {
3430 // do nothing for now
3431 }
3432
3433 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3434 assert(data == dest, "xchg/xadd uses only 2 operands");
3435
3436 if (data->type() == T_INT) {
3437 if (code == lir_xadd) {
3438 __ lock();
3439 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3440 } else {
3441 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3442 }
3443 } else if (data->is_oop()) {
3444 assert (code == lir_xchg, "xadd for oops");
3445 Register obj = data->as_register();
3446 if (UseCompressedOops) {
3447 __ encode_heap_oop(obj);
3448 __ xchgl(obj, as_Address(src->as_address_ptr()));
3449 __ decode_heap_oop(obj);
3450 } else {
3451 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3452 }
3453 } else if (data->type() == T_LONG) {
3454 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3455 if (code == lir_xadd) {
3456 __ lock();
3457 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3458 } else {
3459 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3460 }
3461 } else {
3462 ShouldNotReachHere();
3463 }
3464 }
3465
3466 #undef __