1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
63 operand[0] = lo;
64 operand[1] = hi;
65 return operand;
66 }
67
68 // Buffer for 128-bits masks used by SSE instructions.
69 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
70
71 // Static initialization during VM startup.
72 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
73 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
74 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
75 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
76
77 #if INCLUDE_CDS
78 // publish external addresses defined in this file
79 void LIR_Assembler::init_AOTAddressTable(GrowableArray<address>& external_addresses) {
80 #define ADD(addr) external_addresses.append((address)(addr));
81 ADD(float_signmask_pool);
82 ADD(double_signmask_pool);
83 ADD(float_signflip_pool);
84 ADD(double_signflip_pool);
85 #undef ADD
86 }
87 #endif // INCLUDE_CDS
88
89 NEEDS_CLEANUP // remove this definitions ?
90 const Register SYNC_header = rax; // synchronization header
91 const Register SHIFT_count = rcx; // where count for shift operations must be
92
93 #define __ _masm->
94
95 static void select_different_registers(Register preserve,
96 Register extra,
97 Register &tmp1,
98 Register &tmp2,
99 Register &tmp3) {
100 if (tmp1 == preserve) {
101 assert_different_registers(tmp1, tmp2, tmp3, extra);
102 tmp1 = extra;
103 } else if (tmp2 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp2 = extra;
106 } else if (tmp3 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp3 = extra;
109 }
110 assert_different_registers(preserve, tmp1, tmp2, tmp3);
111 }
112
113
114
115 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
116 if (opr->is_constant()) {
117 LIR_Const* constant = opr->as_constant_ptr();
118 switch (constant->type()) {
119 case T_INT: {
120 return true;
121 }
122
123 default:
124 return false;
125 }
126 }
127 return false;
128 }
129
130
131 LIR_Opr LIR_Assembler::receiverOpr() {
132 return FrameMap::receiver_opr;
133 }
134
135 LIR_Opr LIR_Assembler::osrBufferPointer() {
136 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
137 }
138
139 //--------------fpu register translations-----------------------
140
141
142 address LIR_Assembler::float_constant(float f) {
143 address const_addr = __ float_constant(f);
144 if (const_addr == nullptr) {
145 bailout("const section overflow");
146 return __ code()->consts()->start();
147 } else {
148 return const_addr;
149 }
150 }
151
152
153 address LIR_Assembler::double_constant(double d) {
154 address const_addr = __ double_constant(d);
155 if (const_addr == nullptr) {
156 bailout("const section overflow");
157 return __ code()->consts()->start();
158 } else {
159 return const_addr;
160 }
161 }
162
163 void LIR_Assembler::breakpoint() {
164 __ int3();
165 }
166
167 void LIR_Assembler::push(LIR_Opr opr) {
168 if (opr->is_single_cpu()) {
169 __ push_reg(opr->as_register());
170 } else if (opr->is_double_cpu()) {
171 __ push_reg(opr->as_register_lo());
172 } else if (opr->is_stack()) {
173 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
174 } else if (opr->is_constant()) {
175 LIR_Const* const_opr = opr->as_constant_ptr();
176 if (const_opr->type() == T_OBJECT) {
177 __ push_oop(const_opr->as_jobject(), rscratch1);
178 } else if (const_opr->type() == T_INT) {
179 __ push_jint(const_opr->as_jint());
180 } else {
181 ShouldNotReachHere();
182 }
183
184 } else {
185 ShouldNotReachHere();
186 }
187 }
188
189 void LIR_Assembler::pop(LIR_Opr opr) {
190 if (opr->is_single_cpu()) {
191 __ pop_reg(opr->as_register());
192 } else {
193 ShouldNotReachHere();
194 }
195 }
196
197 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
198 return addr->base()->is_illegal() && addr->index()->is_illegal();
199 }
200
201 //-------------------------------------------
202
203 Address LIR_Assembler::as_Address(LIR_Address* addr) {
204 return as_Address(addr, rscratch1);
205 }
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
208 if (addr->base()->is_illegal()) {
209 assert(addr->index()->is_illegal(), "must be illegal too");
210 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
211 if (! __ reachable(laddr)) {
212 __ movptr(tmp, laddr.addr());
213 Address res(tmp, 0);
214 return res;
215 } else {
216 return __ as_Address(laddr);
217 }
218 }
219
220 Register base = addr->base()->as_pointer_register();
221
222 if (addr->index()->is_illegal()) {
223 return Address( base, addr->disp());
224 } else if (addr->index()->is_cpu_register()) {
225 Register index = addr->index()->as_pointer_register();
226 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
227 } else if (addr->index()->is_constant()) {
228 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
229 assert(Assembler::is_simm32(addr_offset), "must be");
230
231 return Address(base, addr_offset);
232 } else {
233 Unimplemented();
234 return Address();
235 }
236 }
237
238
239 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
240 Address base = as_Address(addr);
241 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
242 }
243
244
245 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
246 return as_Address(addr);
247 }
248
249
250 void LIR_Assembler::osr_entry() {
251 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
252 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
253 ValueStack* entry_state = osr_entry->state();
254 int number_of_locks = entry_state->locks_size();
255
256 // we jump here if osr happens with the interpreter
257 // state set up to continue at the beginning of the
258 // loop that triggered osr - in particular, we have
259 // the following registers setup:
260 //
261 // rcx: osr buffer
262 //
263
264 // build frame
265 ciMethod* m = compilation()->method();
266 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
267
268 // OSR buffer is
269 //
270 // locals[nlocals-1..0]
271 // monitors[0..number_of_locks]
272 //
273 // locals is a direct copy of the interpreter frame so in the osr buffer
274 // so first slot in the local array is the last local from the interpreter
275 // and last slot is local[0] (receiver) from the interpreter
276 //
277 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
278 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
279 // in the interpreter frame (the method lock if a sync method)
280
281 // Initialize monitors in the compiled activation.
282 // rcx: pointer to osr buffer
283 //
284 // All other registers are dead at this point and the locals will be
285 // copied into place by code emitted in the IR.
286
287 Register OSR_buf = osrBufferPointer()->as_pointer_register();
288 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
289 int monitor_offset = BytesPerWord * method()->max_locals() +
290 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
291 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
292 // the OSR buffer using 2 word entries: first the lock and then
293 // the oop.
294 for (int i = 0; i < number_of_locks; i++) {
295 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
296 #ifdef ASSERT
297 // verify the interpreter's monitor has a non-null object
298 {
299 Label L;
300 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
301 __ jcc(Assembler::notZero, L);
302 __ stop("locked object is null");
303 __ bind(L);
304 }
305 #endif
306 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
307 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
308 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
309 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
310 }
311 }
312 }
313
314
315 // inline cache check; done before the frame is built.
316 int LIR_Assembler::check_icache() {
317 return __ ic_check(CodeEntryAlignment);
318 }
319
320 void LIR_Assembler::clinit_barrier(ciMethod* method) {
321 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
322 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
323
324 Label L_skip_barrier;
325 Register klass = rscratch1;
326
327 __ mov_metadata(klass, method->holder()->constant_encoding());
328 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
329
330 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
331
332 __ bind(L_skip_barrier);
333 }
334
335 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
336 jobject o = nullptr;
337 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
338 __ movoop(reg, o);
339 patching_epilog(patch, lir_patch_normal, reg, info);
340 }
341
342 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
343 Metadata* o = nullptr;
344 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
345 __ mov_metadata(reg, o);
346 patching_epilog(patch, lir_patch_normal, reg, info);
347 }
348
349 // This specifies the rsp decrement needed to build the frame
350 int LIR_Assembler::initial_frame_size_in_bytes() const {
351 // if rounding, must let FrameMap know!
352
353 // The frame_map records size in slots (32bit word)
354
355 // subtract two words to account for return address and link
356 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
357 }
358
359
360 int LIR_Assembler::emit_exception_handler() {
361 // generate code for exception handler
362 address handler_base = __ start_a_stub(exception_handler_size());
363 if (handler_base == nullptr) {
364 // not enough space left for the handler
365 bailout("exception handler overflow");
366 return -1;
367 }
368
369 int offset = code_offset();
370
371 // the exception oop and pc are in rax, and rdx
372 // no other registers need to be preserved, so invalidate them
373 __ invalidate_registers(false, true, true, false, true, true);
374
375 // check that there is really an exception
376 __ verify_not_null_oop(rax);
377
378 // search an exception handler (rax: exception oop, rdx: throwing pc)
379 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
380 __ should_not_reach_here();
381 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
382 __ end_a_stub();
383
384 return offset;
385 }
386
387
388 // Emit the code to remove the frame from the stack in the exception
389 // unwind path.
390 int LIR_Assembler::emit_unwind_handler() {
391 #ifndef PRODUCT
392 if (CommentedAssembly) {
393 _masm->block_comment("Unwind handler");
394 }
395 #endif
396
397 int offset = code_offset();
398
399 // Fetch the exception from TLS and clear out exception related thread state
400 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
401 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
402 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
403
404 __ bind(_unwind_handler_entry);
405 __ verify_not_null_oop(rax);
406 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
407 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
408 }
409
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
451 int offset = code_offset();
452
453 Label start;
454 __ bind(start);
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning an non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning an non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
561 __ xchgptr(a, b);
562 }
563
564
565 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
566 assert(src->is_constant(), "should not call otherwise");
567 assert(dest->is_register(), "should not call otherwise");
568 LIR_Const* c = src->as_constant_ptr();
569
570 switch (c->type()) {
571 case T_INT: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movl(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_ADDRESS: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 __ movptr(dest->as_register(), c->as_jint());
580 break;
581 }
582
583 case T_LONG: {
584 #if INCLUDE_CDS
585 if (AOTCodeCache::is_on_for_dump()) {
586 address b = c->as_pointer();
587 if (b == (address)ThreadIdentifier::unsafe_offset()) {
588 __ lea(dest->as_register_lo(), ExternalAddress(b));
589 break;
590 }
591 }
592 #endif
593 assert(patch_code == lir_patch_none, "no patching handled here");
594 #if INCLUDE_CDS
595 if (AOTCodeCache::is_on_for_dump()) {
596 address b = c->as_pointer();
597 if (b == (address)ThreadIdentifier::unsafe_offset()) {
598 __ lea(dest->as_register_lo(), ExternalAddress(b));
599 break;
600 }
601 if (AOTRuntimeConstants::contains(b)) {
602 __ load_aotrc_address(dest->as_register_lo(), b);
603 break;
604 }
605 }
606 #endif
607 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
608 break;
609 }
610
611 case T_OBJECT: {
612 if (patch_code != lir_patch_none) {
613 jobject2reg_with_patching(dest->as_register(), info);
614 } else {
615 __ movoop(dest->as_register(), c->as_jobject());
616 }
617 break;
618 }
619
620 case T_METADATA: {
621 if (patch_code != lir_patch_none) {
622 klass2reg_with_patching(dest->as_register(), info);
623 } else {
624 __ mov_metadata(dest->as_register(), c->as_metadata());
625 }
626 break;
627 }
628
629 case T_FLOAT: {
630 if (dest->is_single_xmm()) {
631 if (UseAVX <= 2 && c->is_zero_float()) {
632 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
633 } else {
634 __ movflt(dest->as_xmm_float_reg(),
635 InternalAddress(float_constant(c->as_jfloat())));
636 }
637 } else {
638 ShouldNotReachHere();
639 }
640 break;
641 }
642
643 case T_DOUBLE: {
644 if (dest->is_double_xmm()) {
645 if (UseAVX <= 2 && c->is_zero_double()) {
646 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
647 } else {
648 __ movdbl(dest->as_xmm_double_reg(),
649 InternalAddress(double_constant(c->as_jdouble())));
650 }
651 } else {
652 ShouldNotReachHere();
653 }
654 break;
655 }
656
657 default:
658 ShouldNotReachHere();
659 }
660 }
661
662 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
663 assert(src->is_constant(), "should not call otherwise");
664 assert(dest->is_stack(), "should not call otherwise");
665 LIR_Const* c = src->as_constant_ptr();
666
667 switch (c->type()) {
668 case T_INT: // fall through
669 case T_FLOAT:
670 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
671 break;
672
673 case T_ADDRESS:
674 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
675 break;
676
677 case T_OBJECT:
678 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
679 break;
680
681 case T_LONG: // fall through
682 case T_DOUBLE:
683 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
684 lo_word_offset_in_bytes),
685 (intptr_t)c->as_jlong_bits(),
686 rscratch1);
687 break;
688
689 default:
690 ShouldNotReachHere();
691 }
692 }
693
694 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
695 assert(src->is_constant(), "should not call otherwise");
696 assert(dest->is_address(), "should not call otherwise");
697 LIR_Const* c = src->as_constant_ptr();
698 LIR_Address* addr = dest->as_address_ptr();
699
700 int null_check_here = code_offset();
701 switch (type) {
702 case T_INT: // fall through
703 case T_FLOAT:
704 __ movl(as_Address(addr), c->as_jint_bits());
705 break;
706
707 case T_ADDRESS:
708 __ movptr(as_Address(addr), c->as_jint_bits());
709 break;
710
711 case T_OBJECT: // fall through
712 case T_ARRAY:
713 if (c->as_jobject() == nullptr) {
714 if (UseCompressedOops && !wide) {
715 __ movl(as_Address(addr), NULL_WORD);
716 } else {
717 __ xorptr(rscratch1, rscratch1);
718 null_check_here = code_offset();
719 __ movptr(as_Address(addr), rscratch1);
720 }
721 } else {
722 if (is_literal_address(addr)) {
723 ShouldNotReachHere();
724 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
725 } else {
726 __ movoop(rscratch1, c->as_jobject());
727 if (UseCompressedOops && !wide) {
728 __ encode_heap_oop(rscratch1);
729 null_check_here = code_offset();
730 __ movl(as_Address_lo(addr), rscratch1);
731 } else {
732 null_check_here = code_offset();
733 __ movptr(as_Address_lo(addr), rscratch1);
734 }
735 }
736 }
737 break;
738
739 case T_LONG: // fall through
740 case T_DOUBLE:
741 if (is_literal_address(addr)) {
742 ShouldNotReachHere();
743 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
744 } else {
745 __ movptr(r10, (intptr_t)c->as_jlong_bits());
746 null_check_here = code_offset();
747 __ movptr(as_Address_lo(addr), r10);
748 }
749 break;
750
751 case T_BOOLEAN: // fall through
752 case T_BYTE:
753 __ movb(as_Address(addr), c->as_jint() & 0xFF);
754 break;
755
756 case T_CHAR: // fall through
757 case T_SHORT:
758 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
759 break;
760
761 default:
762 ShouldNotReachHere();
763 };
764
765 if (info != nullptr) {
766 add_debug_info_for_null_check(null_check_here, info);
767 }
768 }
769
770
771 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
772 assert(src->is_register(), "should not call otherwise");
773 assert(dest->is_register(), "should not call otherwise");
774
775 // move between cpu-registers
776 if (dest->is_single_cpu()) {
777 if (src->type() == T_LONG) {
778 // Can do LONG -> OBJECT
779 move_regs(src->as_register_lo(), dest->as_register());
780 return;
781 }
782 assert(src->is_single_cpu(), "must match");
783 if (src->type() == T_OBJECT) {
784 __ verify_oop(src->as_register());
785 }
786 move_regs(src->as_register(), dest->as_register());
787
788 } else if (dest->is_double_cpu()) {
789 if (is_reference_type(src->type())) {
790 // Surprising to me but we can see move of a long to t_object
791 __ verify_oop(src->as_register());
792 move_regs(src->as_register(), dest->as_register_lo());
793 return;
794 }
795 assert(src->is_double_cpu(), "must match");
796 Register f_lo = src->as_register_lo();
797 Register f_hi = src->as_register_hi();
798 Register t_lo = dest->as_register_lo();
799 Register t_hi = dest->as_register_hi();
800 assert(f_hi == f_lo, "must be same");
801 assert(t_hi == t_lo, "must be same");
802 move_regs(f_lo, t_lo);
803
804 // move between xmm-registers
805 } else if (dest->is_single_xmm()) {
806 assert(src->is_single_xmm(), "must match");
807 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
808 } else if (dest->is_double_xmm()) {
809 assert(src->is_double_xmm(), "must match");
810 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
811
812 } else {
813 ShouldNotReachHere();
814 }
815 }
816
817 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
818 assert(src->is_register(), "should not call otherwise");
819 assert(dest->is_stack(), "should not call otherwise");
820
821 if (src->is_single_cpu()) {
822 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
823 if (is_reference_type(type)) {
824 __ verify_oop(src->as_register());
825 __ movptr (dst, src->as_register());
826 } else if (type == T_METADATA || type == T_ADDRESS) {
827 __ movptr (dst, src->as_register());
828 } else {
829 __ movl (dst, src->as_register());
830 }
831
832 } else if (src->is_double_cpu()) {
833 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
834 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
835 __ movptr (dstLO, src->as_register_lo());
836
837 } else if (src->is_single_xmm()) {
838 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
839 __ movflt(dst_addr, src->as_xmm_float_reg());
840
841 } else if (src->is_double_xmm()) {
842 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
843 __ movdbl(dst_addr, src->as_xmm_double_reg());
844
845 } else {
846 ShouldNotReachHere();
847 }
848 }
849
850
851 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
852 LIR_Address* to_addr = dest->as_address_ptr();
853 PatchingStub* patch = nullptr;
854 Register compressed_src = rscratch1;
855
856 if (is_reference_type(type)) {
857 __ verify_oop(src->as_register());
858 if (UseCompressedOops && !wide) {
859 __ movptr(compressed_src, src->as_register());
860 __ encode_heap_oop(compressed_src);
861 if (patch_code != lir_patch_none) {
862 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
863 }
864 }
865 }
866
867 if (patch_code != lir_patch_none) {
868 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
869 Address toa = as_Address(to_addr);
870 assert(toa.disp() != 0, "must have");
871 }
872
873 int null_check_here = code_offset();
874 switch (type) {
875 case T_FLOAT: {
876 assert(src->is_single_xmm(), "not a float");
877 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
878 break;
879 }
880
881 case T_DOUBLE: {
882 assert(src->is_double_xmm(), "not a double");
883 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
884 break;
885 }
886
887 case T_ARRAY: // fall through
888 case T_OBJECT: // fall through
889 if (UseCompressedOops && !wide) {
890 __ movl(as_Address(to_addr), compressed_src);
891 } else {
892 __ movptr(as_Address(to_addr), src->as_register());
893 }
894 break;
895 case T_ADDRESS:
896 __ movptr(as_Address(to_addr), src->as_register());
897 break;
898 case T_INT:
899 __ movl(as_Address(to_addr), src->as_register());
900 break;
901
902 case T_LONG: {
903 Register from_lo = src->as_register_lo();
904 Register from_hi = src->as_register_hi();
905 __ movptr(as_Address_lo(to_addr), from_lo);
906 break;
907 }
908
909 case T_BYTE: // fall through
910 case T_BOOLEAN: {
911 Register src_reg = src->as_register();
912 Address dst_addr = as_Address(to_addr);
913 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
914 __ movb(dst_addr, src_reg);
915 break;
916 }
917
918 case T_CHAR: // fall through
919 case T_SHORT:
920 __ movw(as_Address(to_addr), src->as_register());
921 break;
922
923 default:
924 ShouldNotReachHere();
925 }
926 if (info != nullptr) {
927 add_debug_info_for_null_check(null_check_here, info);
928 }
929
930 if (patch_code != lir_patch_none) {
931 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
932 }
933 }
934
935
936 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
937 assert(src->is_stack(), "should not call otherwise");
938 assert(dest->is_register(), "should not call otherwise");
939
940 if (dest->is_single_cpu()) {
941 if (is_reference_type(type)) {
942 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
943 __ verify_oop(dest->as_register());
944 } else if (type == T_METADATA || type == T_ADDRESS) {
945 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
946 } else {
947 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
948 }
949
950 } else if (dest->is_double_cpu()) {
951 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
952 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
953 __ movptr(dest->as_register_lo(), src_addr_LO);
954
955 } else if (dest->is_single_xmm()) {
956 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
957 __ movflt(dest->as_xmm_float_reg(), src_addr);
958
959 } else if (dest->is_double_xmm()) {
960 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
961 __ movdbl(dest->as_xmm_double_reg(), src_addr);
962
963 } else {
964 ShouldNotReachHere();
965 }
966 }
967
968
969 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
970 if (src->is_single_stack()) {
971 if (is_reference_type(type)) {
972 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
973 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
974 } else {
975 //no pushl on 64bits
976 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
977 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
978 }
979
980 } else if (src->is_double_stack()) {
981 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
982 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
983
984 } else {
985 ShouldNotReachHere();
986 }
987 }
988
989
990 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
991 assert(src->is_address(), "should not call otherwise");
992 assert(dest->is_register(), "should not call otherwise");
993
994 LIR_Address* addr = src->as_address_ptr();
995 Address from_addr = as_Address(addr);
996
997 if (addr->base()->type() == T_OBJECT) {
998 __ verify_oop(addr->base()->as_pointer_register());
999 }
1000
1001 switch (type) {
1002 case T_BOOLEAN: // fall through
1003 case T_BYTE: // fall through
1004 case T_CHAR: // fall through
1005 case T_SHORT:
1006 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1007 // on pre P6 processors we may get partial register stalls
1008 // so blow away the value of to_rinfo before loading a
1009 // partial word into it. Do it here so that it precedes
1010 // the potential patch point below.
1011 __ xorptr(dest->as_register(), dest->as_register());
1012 }
1013 break;
1014 default:
1015 break;
1016 }
1017
1018 PatchingStub* patch = nullptr;
1019 if (patch_code != lir_patch_none) {
1020 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1021 assert(from_addr.disp() != 0, "must have");
1022 }
1023 if (info != nullptr) {
1024 add_debug_info_for_null_check_here(info);
1025 }
1026
1027 switch (type) {
1028 case T_FLOAT: {
1029 if (dest->is_single_xmm()) {
1030 __ movflt(dest->as_xmm_float_reg(), from_addr);
1031 } else {
1032 ShouldNotReachHere();
1033 }
1034 break;
1035 }
1036
1037 case T_DOUBLE: {
1038 if (dest->is_double_xmm()) {
1039 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1040 } else {
1041 ShouldNotReachHere();
1042 }
1043 break;
1044 }
1045
1046 case T_OBJECT: // fall through
1047 case T_ARRAY: // fall through
1048 if (UseCompressedOops && !wide) {
1049 __ movl(dest->as_register(), from_addr);
1050 } else {
1051 __ movptr(dest->as_register(), from_addr);
1052 }
1053 break;
1054
1055 case T_ADDRESS:
1056 __ movptr(dest->as_register(), from_addr);
1057 break;
1058 case T_INT:
1059 __ movl(dest->as_register(), from_addr);
1060 break;
1061
1062 case T_LONG: {
1063 Register to_lo = dest->as_register_lo();
1064 Register to_hi = dest->as_register_hi();
1065 __ movptr(to_lo, as_Address_lo(addr));
1066 break;
1067 }
1068
1069 case T_BOOLEAN: // fall through
1070 case T_BYTE: {
1071 Register dest_reg = dest->as_register();
1072 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1073 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1074 __ movsbl(dest_reg, from_addr);
1075 } else {
1076 __ movb(dest_reg, from_addr);
1077 __ shll(dest_reg, 24);
1078 __ sarl(dest_reg, 24);
1079 }
1080 break;
1081 }
1082
1083 case T_CHAR: {
1084 Register dest_reg = dest->as_register();
1085 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1086 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1087 __ movzwl(dest_reg, from_addr);
1088 } else {
1089 __ movw(dest_reg, from_addr);
1090 }
1091 break;
1092 }
1093
1094 case T_SHORT: {
1095 Register dest_reg = dest->as_register();
1096 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1097 __ movswl(dest_reg, from_addr);
1098 } else {
1099 __ movw(dest_reg, from_addr);
1100 __ shll(dest_reg, 16);
1101 __ sarl(dest_reg, 16);
1102 }
1103 break;
1104 }
1105
1106 default:
1107 ShouldNotReachHere();
1108 }
1109
1110 if (patch != nullptr) {
1111 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1112 }
1113
1114 if (is_reference_type(type)) {
1115 if (UseCompressedOops && !wide) {
1116 __ decode_heap_oop(dest->as_register());
1117 }
1118
1119 __ verify_oop(dest->as_register());
1120 }
1121 }
1122
1123
1124 NEEDS_CLEANUP; // This could be static?
1125 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1126 int elem_size = type2aelembytes(type);
1127 switch (elem_size) {
1128 case 1: return Address::times_1;
1129 case 2: return Address::times_2;
1130 case 4: return Address::times_4;
1131 case 8: return Address::times_8;
1132 }
1133 ShouldNotReachHere();
1134 return Address::no_scale;
1135 }
1136
1137
1138 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1139 switch (op->code()) {
1140 case lir_idiv:
1141 case lir_irem:
1142 arithmetic_idiv(op->code(),
1143 op->in_opr1(),
1144 op->in_opr2(),
1145 op->in_opr3(),
1146 op->result_opr(),
1147 op->info());
1148 break;
1149 case lir_fmad:
1150 __ fmad(op->result_opr()->as_xmm_double_reg(),
1151 op->in_opr1()->as_xmm_double_reg(),
1152 op->in_opr2()->as_xmm_double_reg(),
1153 op->in_opr3()->as_xmm_double_reg());
1154 break;
1155 case lir_fmaf:
1156 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1157 op->in_opr1()->as_xmm_float_reg(),
1158 op->in_opr2()->as_xmm_float_reg(),
1159 op->in_opr3()->as_xmm_float_reg());
1160 break;
1161 default: ShouldNotReachHere(); break;
1162 }
1163 }
1164
1165 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1166 #ifdef ASSERT
1167 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1168 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1169 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1170 #endif
1171
1172 if (op->cond() == lir_cond_always) {
1173 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1174 __ jmp (*(op->label()));
1175 } else {
1176 Assembler::Condition acond = Assembler::zero;
1177 if (op->code() == lir_cond_float_branch) {
1178 assert(op->ublock() != nullptr, "must have unordered successor");
1179 __ jcc(Assembler::parity, *(op->ublock()->label()));
1180 switch(op->cond()) {
1181 case lir_cond_equal: acond = Assembler::equal; break;
1182 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1183 case lir_cond_less: acond = Assembler::below; break;
1184 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1185 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1186 case lir_cond_greater: acond = Assembler::above; break;
1187 default: ShouldNotReachHere();
1188 }
1189 } else {
1190 switch (op->cond()) {
1191 case lir_cond_equal: acond = Assembler::equal; break;
1192 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1193 case lir_cond_less: acond = Assembler::less; break;
1194 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1195 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1196 case lir_cond_greater: acond = Assembler::greater; break;
1197 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1198 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1199 default: ShouldNotReachHere();
1200 }
1201 }
1202 __ jcc(acond,*(op->label()));
1203 }
1204 }
1205
1206 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1207 LIR_Opr src = op->in_opr();
1208 LIR_Opr dest = op->result_opr();
1209
1210 switch (op->bytecode()) {
1211 case Bytecodes::_i2l:
1212 __ movl2ptr(dest->as_register_lo(), src->as_register());
1213 break;
1214
1215 case Bytecodes::_l2i:
1216 __ movl(dest->as_register(), src->as_register_lo());
1217 break;
1218
1219 case Bytecodes::_i2b:
1220 move_regs(src->as_register(), dest->as_register());
1221 __ sign_extend_byte(dest->as_register());
1222 break;
1223
1224 case Bytecodes::_i2c:
1225 move_regs(src->as_register(), dest->as_register());
1226 __ andl(dest->as_register(), 0xFFFF);
1227 break;
1228
1229 case Bytecodes::_i2s:
1230 move_regs(src->as_register(), dest->as_register());
1231 __ sign_extend_short(dest->as_register());
1232 break;
1233
1234 case Bytecodes::_f2d:
1235 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1236 break;
1237
1238 case Bytecodes::_d2f:
1239 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1240 break;
1241
1242 case Bytecodes::_i2f:
1243 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1244 break;
1245
1246 case Bytecodes::_i2d:
1247 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1248 break;
1249
1250 case Bytecodes::_l2f:
1251 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1252 break;
1253
1254 case Bytecodes::_l2d:
1255 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1256 break;
1257
1258 case Bytecodes::_f2i:
1259 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1260 break;
1261
1262 case Bytecodes::_d2i:
1263 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1264 break;
1265
1266 case Bytecodes::_f2l:
1267 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1268 break;
1269
1270 case Bytecodes::_d2l:
1271 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1272 break;
1273
1274 default: ShouldNotReachHere();
1275 }
1276 }
1277
1278 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1279 if (op->init_check()) {
1280 add_debug_info_for_null_check_here(op->stub()->info());
1281 // init_state needs acquire, but x86 is TSO, and so we are already good.
1282 __ cmpb(Address(op->klass()->as_register(),
1283 InstanceKlass::init_state_offset()),
1284 InstanceKlass::fully_initialized);
1285 __ jcc(Assembler::notEqual, *op->stub()->entry());
1286 }
1287 __ allocate_object(op->obj()->as_register(),
1288 op->tmp1()->as_register(),
1289 op->tmp2()->as_register(),
1290 op->header_size(),
1291 op->object_size(),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298 Register len = op->len()->as_register();
1299 __ movslq(len, len);
1300
1301 if (UseSlowPath || op->always_slow_path() ||
1302 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1304 __ jmp(*op->stub()->entry());
1305 } else {
1306 Register tmp1 = op->tmp1()->as_register();
1307 Register tmp2 = op->tmp2()->as_register();
1308 Register tmp3 = op->tmp3()->as_register();
1309 if (len == tmp1) {
1310 tmp1 = tmp3;
1311 } else if (len == tmp2) {
1312 tmp2 = tmp3;
1313 } else if (len == tmp3) {
1314 // everything is ok
1315 } else {
1316 __ mov(tmp3, len);
1317 }
1318 __ allocate_array(op->obj()->as_register(),
1319 len,
1320 tmp1,
1321 tmp2,
1322 arrayOopDesc::base_offset_in_bytes(op->type()),
1323 array_element_size(op->type()),
1324 op->klass()->as_register(),
1325 *op->stub()->entry(),
1326 op->zero_array());
1327 }
1328 __ bind(*op->stub()->continuation());
1329 }
1330
1331 void LIR_Assembler::type_profile_helper(Register mdo,
1332 ciMethodData *md, ciProfileData *data,
1333 Register recv) {
1334 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1335 __ profile_receiver_type(recv, mdo, mdp_offset);
1336 }
1337
1338 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1339 // we always need a stub for the failure case.
1340 CodeStub* stub = op->stub();
1341 Register obj = op->object()->as_register();
1342 Register k_RInfo = op->tmp1()->as_register();
1343 Register klass_RInfo = op->tmp2()->as_register();
1344 Register dst = op->result_opr()->as_register();
1345 ciKlass* k = op->klass();
1346 Register Rtmp1 = noreg;
1347 Register tmp_load_klass = rscratch1;
1348
1349 // check if it needs to be profiled
1350 ciMethodData* md = nullptr;
1351 ciProfileData* data = nullptr;
1352
1353 if (op->should_profile()) {
1354 ciMethod* method = op->profiled_method();
1355 assert(method != nullptr, "Should have method");
1356 int bci = op->profiled_bci();
1357 md = method->method_data_or_null();
1358 assert(md != nullptr, "Sanity");
1359 data = md->bci_to_data(bci);
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373
1374 assert_different_registers(obj, k_RInfo, klass_RInfo);
1375
1376 if (op->need_null_check()) {
1377 __ testptr(obj, obj);
1378 if (op->should_profile()) {
1379 Label not_null;
1380 Register mdo = klass_RInfo;
1381 __ mov_metadata(mdo, md->constant_encoding());
1382 __ jccb(Assembler::notEqual, not_null);
1383 // Object is null; update MDO and exit
1384 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385 int header_bits = BitData::null_seen_byte_constant();
1386 __ orb(data_addr, header_bits);
1387 __ jmp(*obj_is_null);
1388 __ bind(not_null);
1389
1390 Register recv = k_RInfo;
1391 __ load_klass(recv, obj, tmp_load_klass);
1392 type_profile_helper(mdo, md, data, recv);
1393 } else {
1394 __ jcc(Assembler::equal, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(Rtmp1, obj, tmp_load_klass);
1410 __ cmpptr(k_RInfo, Rtmp1);
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431 __ cmpptr(klass_RInfo, tmp_load_klass);
1432 } else {
1433 __ cmpptr(klass_RInfo, k_RInfo);
1434 }
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1455 __ pop_ppx(k_RInfo);
1456 // result is a boolean
1457 __ testl(k_RInfo, k_RInfo);
1458 __ jcc(Assembler::equal, *failure_target);
1459 // successful cast, fall through to profile or jump
1460 }
1461 }
1462 __ jmp(*success);
1463 }
1464
1465
1466 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1467 Register tmp_load_klass = rscratch1;
1468 LIR_Code code = op->code();
1469 if (code == lir_store_check) {
1470 Register value = op->object()->as_register();
1471 Register array = op->array()->as_register();
1472 Register k_RInfo = op->tmp1()->as_register();
1473 Register klass_RInfo = op->tmp2()->as_register();
1474 Register Rtmp1 = op->tmp3()->as_register();
1475
1476 CodeStub* stub = op->stub();
1477
1478 // check if it needs to be profiled
1479 ciMethodData* md = nullptr;
1480 ciProfileData* data = nullptr;
1481
1482 if (op->should_profile()) {
1483 ciMethod* method = op->profiled_method();
1484 assert(method != nullptr, "Should have method");
1485 int bci = op->profiled_bci();
1486 md = method->method_data_or_null();
1487 assert(md != nullptr, "Sanity");
1488 data = md->bci_to_data(bci);
1489 assert(data != nullptr, "need data for type check");
1490 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1491 }
1492 Label done;
1493 Label* success_target = &done;
1494 Label* failure_target = stub->entry();
1495
1496 __ testptr(value, value);
1497 if (op->should_profile()) {
1498 Label not_null;
1499 Register mdo = klass_RInfo;
1500 __ mov_metadata(mdo, md->constant_encoding());
1501 __ jccb(Assembler::notEqual, not_null);
1502 // Object is null; update MDO and exit
1503 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1504 int header_bits = BitData::null_seen_byte_constant();
1505 __ orb(data_addr, header_bits);
1506 __ jmp(done);
1507 __ bind(not_null);
1508
1509 Register recv = k_RInfo;
1510 __ load_klass(recv, value, tmp_load_klass);
1511 type_profile_helper(mdo, md, data, recv);
1512 } else {
1513 __ jcc(Assembler::equal, done);
1514 }
1515
1516 add_debug_info_for_null_check_here(op->info_for_exception());
1517 __ load_klass(k_RInfo, array, tmp_load_klass);
1518 __ load_klass(klass_RInfo, value, tmp_load_klass);
1519
1520 // get instance klass (it's already uncompressed)
1521 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1522 // perform the fast part of the checking logic
1523 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1524 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1525 __ push_ppx(klass_RInfo);
1526 __ push_ppx(k_RInfo);
1527 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1528 __ pop_ppx(klass_RInfo);
1529 __ pop_ppx(k_RInfo);
1530 // result is a boolean
1531 __ testl(k_RInfo, k_RInfo);
1532 __ jcc(Assembler::equal, *failure_target);
1533 // fall through to the success case
1534
1535 __ bind(done);
1536 } else
1537 if (code == lir_checkcast) {
1538 Register obj = op->object()->as_register();
1539 Register dst = op->result_opr()->as_register();
1540 Label success;
1541 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1542 __ bind(success);
1543 if (dst != obj) {
1544 __ mov(dst, obj);
1545 }
1546 } else
1547 if (code == lir_instanceof) {
1548 Register obj = op->object()->as_register();
1549 Register dst = op->result_opr()->as_register();
1550 Label success, failure, done;
1551 emit_typecheck_helper(op, &success, &failure, &failure);
1552 __ bind(failure);
1553 __ xorptr(dst, dst);
1554 __ jmpb(done);
1555 __ bind(success);
1556 __ movptr(dst, 1);
1557 __ bind(done);
1558 } else {
1559 ShouldNotReachHere();
1560 }
1561
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be a flat array (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is a flat array, take the slow path.
1568 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 }
1570
1571 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1572 // We are storing into an array that *may* be null-free (the declared type is
1573 // Object[], abstract[], interface[] or VT.ref[]).
1574 Label test_mark_word;
1575 Register tmp = op->tmp()->as_register();
1576 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1577 __ testl(tmp, markWord::unlocked_value);
1578 __ jccb(Assembler::notZero, test_mark_word);
1579 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1580 __ bind(test_mark_word);
1581 __ testl(tmp, markWord::null_free_array_bit_in_place);
1582 }
1583
1584 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1585 Label L_oops_equal;
1586 Label L_oops_not_equal;
1587 Label L_end;
1588
1589 Register left = op->left()->as_register();
1590 Register right = op->right()->as_register();
1591
1592 __ cmpptr(left, right);
1593 __ jcc(Assembler::equal, L_oops_equal);
1594
1595 // (1) Null check -- if one of the operands is null, the other must not be null (because
1596 // the two references are not equal), so they are not substitutable,
1597 __ testptr(left, left);
1598 __ jcc(Assembler::zero, L_oops_not_equal);
1599 __ testptr(right, right);
1600 __ jcc(Assembler::zero, L_oops_not_equal);
1601
1602 ciKlass* left_klass = op->left_klass();
1603 ciKlass* right_klass = op->right_klass();
1604
1605 // (2) Inline type check -- if either of the operands is not a inline type,
1606 // they are not substitutable. We do this only if we are not sure that the
1607 // operands are inline type
1608 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1609 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1610 Register tmp = op->tmp1()->as_register();
1611 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1612 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1613 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1614 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1615 __ jcc(Assembler::notEqual, L_oops_not_equal);
1616 }
1617
1618 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1619 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1620 // No need to load klass -- the operands are statically known to be the same inline klass.
1621 __ jmp(*op->stub()->entry());
1622 } else {
1623 Register tmp1 = op->tmp1()->as_register();
1624 Register tmp2 = op->tmp2()->as_register();
1625 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1626 __ jmp (*op->stub()->entry()); // -> do slow check
1627 } else {
1628 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1629 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1630 }
1631 // fall through to L_oops_not_equal
1632 }
1633
1634 __ bind(L_oops_not_equal);
1635 move(op->not_equal_result(), op->result_opr());
1636 __ jmp(L_end);
1637
1638 __ bind(L_oops_equal);
1639 move(op->equal_result(), op->result_opr());
1640 __ jmp(L_end);
1641
1642 // We've returned from the stub. RAX contains 0x0 IFF the two
1643 // operands are not substitutable. (Don't compare against 0x1 in case the
1644 // C compiler is naughty)
1645 __ bind(*op->stub()->continuation());
1646 __ cmpl(rax, 0);
1647 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1648 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1649 // fall-through
1650 __ bind(L_end);
1651 }
1652
1653 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1654 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1655 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1656 Register newval = op->new_value()->as_register();
1657 Register cmpval = op->cmp_value()->as_register();
1658 assert(cmpval == rax, "wrong register");
1659 assert(newval != noreg, "new val must be register");
1660 assert(cmpval != newval, "cmp and new values must be in different registers");
1661 assert(cmpval != addr, "cmp and addr must be in different registers");
1662 assert(newval != addr, "new value and addr must be in different registers");
1663
1664 if (op->code() == lir_cas_obj) {
1665 if (UseCompressedOops) {
1666 __ encode_heap_oop(cmpval);
1667 __ mov(rscratch1, newval);
1668 __ encode_heap_oop(rscratch1);
1669 __ lock();
1670 // cmpval (rax) is implicitly used by this instruction
1671 __ cmpxchgl(rscratch1, Address(addr, 0));
1672 } else {
1673 __ lock();
1674 __ cmpxchgptr(newval, Address(addr, 0));
1675 }
1676 } else {
1677 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1678 __ lock();
1679 __ cmpxchgl(newval, Address(addr, 0));
1680 }
1681 } else if (op->code() == lir_cas_long) {
1682 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1683 Register newval = op->new_value()->as_register_lo();
1684 Register cmpval = op->cmp_value()->as_register_lo();
1685 assert(cmpval == rax, "wrong register");
1686 assert(newval != noreg, "new val must be register");
1687 assert(cmpval != newval, "cmp and new values must be in different registers");
1688 assert(cmpval != addr, "cmp and addr must be in different registers");
1689 assert(newval != addr, "new value and addr must be in different registers");
1690 __ lock();
1691 __ cmpxchgq(newval, Address(addr, 0));
1692 } else {
1693 Unimplemented();
1694 }
1695 }
1696
1697 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1698 assert(dst->is_cpu_register(), "must be");
1699 assert(dst->type() == src->type(), "must be");
1700
1701 if (src->is_cpu_register()) {
1702 reg2reg(src, dst);
1703 } else if (src->is_stack()) {
1704 stack2reg(src, dst, dst->type());
1705 } else if (src->is_constant()) {
1706 const2reg(src, dst, lir_patch_none, nullptr);
1707 } else {
1708 ShouldNotReachHere();
1709 }
1710 }
1711
1712 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1713 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1714 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1715
1716 Assembler::Condition acond, ncond;
1717 switch (condition) {
1718 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1719 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1720 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1721 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1722 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1723 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1724 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1725 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1726 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1727 ShouldNotReachHere();
1728 }
1729
1730 if (opr1->is_cpu_register()) {
1731 reg2reg(opr1, result);
1732 } else if (opr1->is_stack()) {
1733 stack2reg(opr1, result, result->type());
1734 } else if (opr1->is_constant()) {
1735 const2reg(opr1, result, lir_patch_none, nullptr);
1736 } else {
1737 ShouldNotReachHere();
1738 }
1739
1740 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1741 // optimized version that does not require a branch
1742 if (opr2->is_single_cpu()) {
1743 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1744 __ cmov(ncond, result->as_register(), opr2->as_register());
1745 } else if (opr2->is_double_cpu()) {
1746 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1747 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1748 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1749 } else if (opr2->is_single_stack()) {
1750 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1751 } else if (opr2->is_double_stack()) {
1752 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1753 } else {
1754 ShouldNotReachHere();
1755 }
1756
1757 } else {
1758 Label skip;
1759 __ jccb(acond, skip);
1760 if (opr2->is_cpu_register()) {
1761 reg2reg(opr2, result);
1762 } else if (opr2->is_stack()) {
1763 stack2reg(opr2, result, result->type());
1764 } else if (opr2->is_constant()) {
1765 const2reg(opr2, result, lir_patch_none, nullptr);
1766 } else {
1767 ShouldNotReachHere();
1768 }
1769 __ bind(skip);
1770 }
1771 }
1772
1773
1774 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1775 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1776
1777 if (left->is_single_cpu()) {
1778 assert(left == dest, "left and dest must be equal");
1779 Register lreg = left->as_register();
1780
1781 if (right->is_single_cpu()) {
1782 // cpu register - cpu register
1783 Register rreg = right->as_register();
1784 switch (code) {
1785 case lir_add: __ addl (lreg, rreg); break;
1786 case lir_sub: __ subl (lreg, rreg); break;
1787 case lir_mul: __ imull(lreg, rreg); break;
1788 default: ShouldNotReachHere();
1789 }
1790
1791 } else if (right->is_stack()) {
1792 // cpu register - stack
1793 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1794 switch (code) {
1795 case lir_add: __ addl(lreg, raddr); break;
1796 case lir_sub: __ subl(lreg, raddr); break;
1797 default: ShouldNotReachHere();
1798 }
1799
1800 } else if (right->is_constant()) {
1801 // cpu register - constant
1802 jint c = right->as_constant_ptr()->as_jint();
1803 switch (code) {
1804 case lir_add: {
1805 __ incrementl(lreg, c);
1806 break;
1807 }
1808 case lir_sub: {
1809 __ decrementl(lreg, c);
1810 break;
1811 }
1812 default: ShouldNotReachHere();
1813 }
1814
1815 } else {
1816 ShouldNotReachHere();
1817 }
1818
1819 } else if (left->is_double_cpu()) {
1820 assert(left == dest, "left and dest must be equal");
1821 Register lreg_lo = left->as_register_lo();
1822 Register lreg_hi = left->as_register_hi();
1823
1824 if (right->is_double_cpu()) {
1825 // cpu register - cpu register
1826 Register rreg_lo = right->as_register_lo();
1827 Register rreg_hi = right->as_register_hi();
1828 assert_different_registers(lreg_lo, rreg_lo);
1829 switch (code) {
1830 case lir_add:
1831 __ addptr(lreg_lo, rreg_lo);
1832 break;
1833 case lir_sub:
1834 __ subptr(lreg_lo, rreg_lo);
1835 break;
1836 case lir_mul:
1837 __ imulq(lreg_lo, rreg_lo);
1838 break;
1839 default:
1840 ShouldNotReachHere();
1841 }
1842
1843 } else if (right->is_constant()) {
1844 // cpu register - constant
1845 jlong c = right->as_constant_ptr()->as_jlong_bits();
1846 __ movptr(r10, (intptr_t) c);
1847 switch (code) {
1848 case lir_add:
1849 __ addptr(lreg_lo, r10);
1850 break;
1851 case lir_sub:
1852 __ subptr(lreg_lo, r10);
1853 break;
1854 default:
1855 ShouldNotReachHere();
1856 }
1857
1858 } else {
1859 ShouldNotReachHere();
1860 }
1861
1862 } else if (left->is_single_xmm()) {
1863 assert(left == dest, "left and dest must be equal");
1864 XMMRegister lreg = left->as_xmm_float_reg();
1865
1866 if (right->is_single_xmm()) {
1867 XMMRegister rreg = right->as_xmm_float_reg();
1868 switch (code) {
1869 case lir_add: __ addss(lreg, rreg); break;
1870 case lir_sub: __ subss(lreg, rreg); break;
1871 case lir_mul: __ mulss(lreg, rreg); break;
1872 case lir_div: __ divss(lreg, rreg); break;
1873 default: ShouldNotReachHere();
1874 }
1875 } else {
1876 Address raddr;
1877 if (right->is_single_stack()) {
1878 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1879 } else if (right->is_constant()) {
1880 // hack for now
1881 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1882 } else {
1883 ShouldNotReachHere();
1884 }
1885 switch (code) {
1886 case lir_add: __ addss(lreg, raddr); break;
1887 case lir_sub: __ subss(lreg, raddr); break;
1888 case lir_mul: __ mulss(lreg, raddr); break;
1889 case lir_div: __ divss(lreg, raddr); break;
1890 default: ShouldNotReachHere();
1891 }
1892 }
1893
1894 } else if (left->is_double_xmm()) {
1895 assert(left == dest, "left and dest must be equal");
1896
1897 XMMRegister lreg = left->as_xmm_double_reg();
1898 if (right->is_double_xmm()) {
1899 XMMRegister rreg = right->as_xmm_double_reg();
1900 switch (code) {
1901 case lir_add: __ addsd(lreg, rreg); break;
1902 case lir_sub: __ subsd(lreg, rreg); break;
1903 case lir_mul: __ mulsd(lreg, rreg); break;
1904 case lir_div: __ divsd(lreg, rreg); break;
1905 default: ShouldNotReachHere();
1906 }
1907 } else {
1908 Address raddr;
1909 if (right->is_double_stack()) {
1910 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1911 } else if (right->is_constant()) {
1912 // hack for now
1913 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1914 } else {
1915 ShouldNotReachHere();
1916 }
1917 switch (code) {
1918 case lir_add: __ addsd(lreg, raddr); break;
1919 case lir_sub: __ subsd(lreg, raddr); break;
1920 case lir_mul: __ mulsd(lreg, raddr); break;
1921 case lir_div: __ divsd(lreg, raddr); break;
1922 default: ShouldNotReachHere();
1923 }
1924 }
1925
1926 } else if (left->is_single_stack() || left->is_address()) {
1927 assert(left == dest, "left and dest must be equal");
1928
1929 Address laddr;
1930 if (left->is_single_stack()) {
1931 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1932 } else if (left->is_address()) {
1933 laddr = as_Address(left->as_address_ptr());
1934 } else {
1935 ShouldNotReachHere();
1936 }
1937
1938 if (right->is_single_cpu()) {
1939 Register rreg = right->as_register();
1940 switch (code) {
1941 case lir_add: __ addl(laddr, rreg); break;
1942 case lir_sub: __ subl(laddr, rreg); break;
1943 default: ShouldNotReachHere();
1944 }
1945 } else if (right->is_constant()) {
1946 jint c = right->as_constant_ptr()->as_jint();
1947 switch (code) {
1948 case lir_add: {
1949 __ incrementl(laddr, c);
1950 break;
1951 }
1952 case lir_sub: {
1953 __ decrementl(laddr, c);
1954 break;
1955 }
1956 default: ShouldNotReachHere();
1957 }
1958 } else {
1959 ShouldNotReachHere();
1960 }
1961
1962 } else {
1963 ShouldNotReachHere();
1964 }
1965 }
1966
1967
1968 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1969 if (value->is_double_xmm()) {
1970 switch(code) {
1971 case lir_abs :
1972 {
1973 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1974 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1975 }
1976 assert(!tmp->is_valid(), "do not need temporary");
1977 __ andpd(dest->as_xmm_double_reg(),
1978 ExternalAddress((address)double_signmask_pool),
1979 rscratch1);
1980 }
1981 break;
1982
1983 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1984 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1985 default : ShouldNotReachHere();
1986 }
1987
1988 } else if (code == lir_f2hf) {
1989 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1990 } else if (code == lir_hf2f) {
1991 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1992 } else {
1993 Unimplemented();
1994 }
1995 }
1996
1997 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1998 // assert(left->destroys_register(), "check");
1999 if (left->is_single_cpu()) {
2000 Register reg = left->as_register();
2001 if (right->is_constant()) {
2002 int val = right->as_constant_ptr()->as_jint();
2003 switch (code) {
2004 case lir_logic_and: __ andl (reg, val); break;
2005 case lir_logic_or: __ orl (reg, val); break;
2006 case lir_logic_xor: __ xorl (reg, val); break;
2007 default: ShouldNotReachHere();
2008 }
2009 } else if (right->is_stack()) {
2010 // added support for stack operands
2011 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2012 switch (code) {
2013 case lir_logic_and: __ andl (reg, raddr); break;
2014 case lir_logic_or: __ orl (reg, raddr); break;
2015 case lir_logic_xor: __ xorl (reg, raddr); break;
2016 default: ShouldNotReachHere();
2017 }
2018 } else {
2019 Register rright = right->as_register();
2020 switch (code) {
2021 case lir_logic_and: __ andptr (reg, rright); break;
2022 case lir_logic_or : __ orptr (reg, rright); break;
2023 case lir_logic_xor: __ xorptr (reg, rright); break;
2024 default: ShouldNotReachHere();
2025 }
2026 }
2027 move_regs(reg, dst->as_register());
2028 } else {
2029 Register l_lo = left->as_register_lo();
2030 Register l_hi = left->as_register_hi();
2031 if (right->is_constant()) {
2032 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2033 switch (code) {
2034 case lir_logic_and:
2035 __ andq(l_lo, rscratch1);
2036 break;
2037 case lir_logic_or:
2038 __ orq(l_lo, rscratch1);
2039 break;
2040 case lir_logic_xor:
2041 __ xorq(l_lo, rscratch1);
2042 break;
2043 default: ShouldNotReachHere();
2044 }
2045 } else {
2046 Register r_lo;
2047 if (is_reference_type(right->type())) {
2048 r_lo = right->as_register();
2049 } else {
2050 r_lo = right->as_register_lo();
2051 }
2052 switch (code) {
2053 case lir_logic_and:
2054 __ andptr(l_lo, r_lo);
2055 break;
2056 case lir_logic_or:
2057 __ orptr(l_lo, r_lo);
2058 break;
2059 case lir_logic_xor:
2060 __ xorptr(l_lo, r_lo);
2061 break;
2062 default: ShouldNotReachHere();
2063 }
2064 }
2065
2066 Register dst_lo = dst->as_register_lo();
2067 Register dst_hi = dst->as_register_hi();
2068
2069 move_regs(l_lo, dst_lo);
2070 }
2071 }
2072
2073
2074 // we assume that rax, and rdx can be overwritten
2075 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2076
2077 assert(left->is_single_cpu(), "left must be register");
2078 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2079 assert(result->is_single_cpu(), "result must be register");
2080
2081 // assert(left->destroys_register(), "check");
2082 // assert(right->destroys_register(), "check");
2083
2084 Register lreg = left->as_register();
2085 Register dreg = result->as_register();
2086
2087 if (right->is_constant()) {
2088 jint divisor = right->as_constant_ptr()->as_jint();
2089 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2090 if (code == lir_idiv) {
2091 assert(lreg == rax, "must be rax,");
2092 assert(temp->as_register() == rdx, "tmp register must be rdx");
2093 __ cdql(); // sign extend into rdx:rax
2094 if (divisor == 2) {
2095 __ subl(lreg, rdx);
2096 } else {
2097 __ andl(rdx, divisor - 1);
2098 __ addl(lreg, rdx);
2099 }
2100 __ sarl(lreg, log2i_exact(divisor));
2101 move_regs(lreg, dreg);
2102 } else if (code == lir_irem) {
2103 Label done;
2104 __ mov(dreg, lreg);
2105 __ andl(dreg, 0x80000000 | (divisor - 1));
2106 __ jcc(Assembler::positive, done);
2107 __ decrement(dreg);
2108 __ orl(dreg, ~(divisor - 1));
2109 __ increment(dreg);
2110 __ bind(done);
2111 } else {
2112 ShouldNotReachHere();
2113 }
2114 } else {
2115 Register rreg = right->as_register();
2116 assert(lreg == rax, "left register must be rax,");
2117 assert(rreg != rdx, "right register must not be rdx");
2118 assert(temp->as_register() == rdx, "tmp register must be rdx");
2119
2120 move_regs(lreg, rax);
2121
2122 int idivl_offset = __ corrected_idivl(rreg);
2123 if (ImplicitDiv0Checks) {
2124 add_debug_info_for_div0(idivl_offset, info);
2125 }
2126 if (code == lir_irem) {
2127 move_regs(rdx, dreg); // result is in rdx
2128 } else {
2129 move_regs(rax, dreg);
2130 }
2131 }
2132 }
2133
2134
2135 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2136 if (opr1->is_single_cpu()) {
2137 Register reg1 = opr1->as_register();
2138 if (opr2->is_single_cpu()) {
2139 // cpu register - cpu register
2140 if (is_reference_type(opr1->type())) {
2141 __ cmpoop(reg1, opr2->as_register());
2142 } else {
2143 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2144 __ cmpl(reg1, opr2->as_register());
2145 }
2146 } else if (opr2->is_stack()) {
2147 // cpu register - stack
2148 if (is_reference_type(opr1->type())) {
2149 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2150 } else {
2151 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2152 }
2153 } else if (opr2->is_constant()) {
2154 // cpu register - constant
2155 LIR_Const* c = opr2->as_constant_ptr();
2156 if (c->type() == T_INT) {
2157 jint i = c->as_jint();
2158 if (i == 0) {
2159 __ testl(reg1, reg1);
2160 } else {
2161 __ cmpl(reg1, i);
2162 }
2163 } else if (c->type() == T_METADATA) {
2164 // All we need for now is a comparison with null for equality.
2165 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2166 Metadata* m = c->as_metadata();
2167 if (m == nullptr) {
2168 __ testptr(reg1, reg1);
2169 } else {
2170 ShouldNotReachHere();
2171 }
2172 } else if (is_reference_type(c->type())) {
2173 // In 64bit oops are single register
2174 jobject o = c->as_jobject();
2175 if (o == nullptr) {
2176 __ testptr(reg1, reg1);
2177 } else {
2178 __ cmpoop(reg1, o, rscratch1);
2179 }
2180 } else {
2181 fatal("unexpected type: %s", basictype_to_str(c->type()));
2182 }
2183 // cpu register - address
2184 } else if (opr2->is_address()) {
2185 if (op->info() != nullptr) {
2186 add_debug_info_for_null_check_here(op->info());
2187 }
2188 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2189 } else {
2190 ShouldNotReachHere();
2191 }
2192
2193 } else if(opr1->is_double_cpu()) {
2194 Register xlo = opr1->as_register_lo();
2195 Register xhi = opr1->as_register_hi();
2196 if (opr2->is_double_cpu()) {
2197 __ cmpptr(xlo, opr2->as_register_lo());
2198 } else if (opr2->is_constant()) {
2199 // cpu register - constant 0
2200 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2201 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2202 } else {
2203 ShouldNotReachHere();
2204 }
2205
2206 } else if (opr1->is_single_xmm()) {
2207 XMMRegister reg1 = opr1->as_xmm_float_reg();
2208 if (opr2->is_single_xmm()) {
2209 // xmm register - xmm register
2210 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2211 } else if (opr2->is_stack()) {
2212 // xmm register - stack
2213 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2214 } else if (opr2->is_constant()) {
2215 // xmm register - constant
2216 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2217 } else if (opr2->is_address()) {
2218 // xmm register - address
2219 if (op->info() != nullptr) {
2220 add_debug_info_for_null_check_here(op->info());
2221 }
2222 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2223 } else {
2224 ShouldNotReachHere();
2225 }
2226
2227 } else if (opr1->is_double_xmm()) {
2228 XMMRegister reg1 = opr1->as_xmm_double_reg();
2229 if (opr2->is_double_xmm()) {
2230 // xmm register - xmm register
2231 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2232 } else if (opr2->is_stack()) {
2233 // xmm register - stack
2234 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2235 } else if (opr2->is_constant()) {
2236 // xmm register - constant
2237 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2238 } else if (opr2->is_address()) {
2239 // xmm register - address
2240 if (op->info() != nullptr) {
2241 add_debug_info_for_null_check_here(op->info());
2242 }
2243 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2244 } else {
2245 ShouldNotReachHere();
2246 }
2247
2248 } else if (opr1->is_address() && opr2->is_constant()) {
2249 LIR_Const* c = opr2->as_constant_ptr();
2250 if (is_reference_type(c->type())) {
2251 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2252 __ movoop(rscratch1, c->as_jobject());
2253 }
2254 if (op->info() != nullptr) {
2255 add_debug_info_for_null_check_here(op->info());
2256 }
2257 // special case: address - constant
2258 LIR_Address* addr = opr1->as_address_ptr();
2259 if (c->type() == T_INT) {
2260 __ cmpl(as_Address(addr), c->as_jint());
2261 } else if (is_reference_type(c->type())) {
2262 // %%% Make this explode if addr isn't reachable until we figure out a
2263 // better strategy by giving noreg as the temp for as_Address
2264 __ cmpoop(rscratch1, as_Address(addr, noreg));
2265 } else {
2266 ShouldNotReachHere();
2267 }
2268
2269 } else {
2270 ShouldNotReachHere();
2271 }
2272 }
2273
2274 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2275 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2276 if (left->is_single_xmm()) {
2277 assert(right->is_single_xmm(), "must match");
2278 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2279 } else if (left->is_double_xmm()) {
2280 assert(right->is_double_xmm(), "must match");
2281 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2282
2283 } else {
2284 ShouldNotReachHere();
2285 }
2286 } else {
2287 assert(code == lir_cmp_l2i, "check");
2288 Label done;
2289 Register dest = dst->as_register();
2290 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2291 __ movl(dest, -1);
2292 __ jccb(Assembler::less, done);
2293 __ setb(Assembler::notZero, dest);
2294 __ movzbl(dest, dest);
2295 __ bind(done);
2296 }
2297 }
2298
2299
2300 void LIR_Assembler::align_call(LIR_Code code) {
2301 // make sure that the displacement word of the call ends up word aligned
2302 int offset = __ offset();
2303 switch (code) {
2304 case lir_static_call:
2305 case lir_optvirtual_call:
2306 case lir_dynamic_call:
2307 offset += NativeCall::displacement_offset;
2308 break;
2309 case lir_icvirtual_call:
2310 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2311 break;
2312 default: ShouldNotReachHere();
2313 }
2314 __ align(BytesPerWord, offset);
2315 }
2316
2317
2318 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2319 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2320 "must be aligned");
2321 __ call(AddressLiteral(op->addr(), rtype));
2322 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2323 __ post_call_nop();
2324 }
2325
2326
2327 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2328 __ ic_call(op->addr());
2329 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2330 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2331 "must be aligned");
2332 __ post_call_nop();
2333 }
2334
2335
2336 void LIR_Assembler::emit_static_call_stub() {
2337 address call_pc = __ pc();
2338 address stub = __ start_a_stub(call_stub_size());
2339 if (stub == nullptr) {
2340 bailout("static call stub overflow");
2341 return;
2342 }
2343
2344 int start = __ offset();
2345
2346 // make sure that the displacement word of the call ends up word aligned
2347 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2348 __ relocate(static_stub_Relocation::spec(call_pc));
2349 __ mov_metadata(rbx, (Metadata*)nullptr);
2350 // must be set to -1 at code generation time
2351 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2352 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2353 __ jump(RuntimeAddress(__ pc()));
2354
2355 assert(__ offset() - start <= call_stub_size(), "stub too big");
2356 __ end_a_stub();
2357 }
2358
2359
2360 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2361 assert(exceptionOop->as_register() == rax, "must match");
2362 assert(exceptionPC->as_register() == rdx, "must match");
2363
2364 // exception object is not added to oop map by LinearScan
2365 // (LinearScan assumes that no oops are in fixed registers)
2366 info->add_register_oop(exceptionOop);
2367 StubId unwind_id;
2368
2369 // get current pc information
2370 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2371 int pc_for_athrow_offset = __ offset();
2372 InternalAddress pc_for_athrow(__ pc());
2373 __ lea(exceptionPC->as_register(), pc_for_athrow);
2374 add_call_info(pc_for_athrow_offset, info); // for exception handler
2375
2376 __ verify_not_null_oop(rax);
2377 // search an exception handler (rax: exception oop, rdx: throwing pc)
2378 if (compilation()->has_fpu_code()) {
2379 unwind_id = StubId::c1_handle_exception_id;
2380 } else {
2381 unwind_id = StubId::c1_handle_exception_nofpu_id;
2382 }
2383 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2384
2385 // enough room for two byte trap
2386 __ nop();
2387 }
2388
2389
2390 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2391 assert(exceptionOop->as_register() == rax, "must match");
2392
2393 __ jmp(_unwind_handler_entry);
2394 }
2395
2396
2397 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2398
2399 // optimized version for linear scan:
2400 // * count must be already in ECX (guaranteed by LinearScan)
2401 // * left and dest must be equal
2402 // * tmp must be unused
2403 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2404 assert(left == dest, "left and dest must be equal");
2405 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2406
2407 if (left->is_single_cpu()) {
2408 Register value = left->as_register();
2409 assert(value != SHIFT_count, "left cannot be ECX");
2410
2411 switch (code) {
2412 case lir_shl: __ shll(value); break;
2413 case lir_shr: __ sarl(value); break;
2414 case lir_ushr: __ shrl(value); break;
2415 default: ShouldNotReachHere();
2416 }
2417 } else if (left->is_double_cpu()) {
2418 Register lo = left->as_register_lo();
2419 Register hi = left->as_register_hi();
2420 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2421 switch (code) {
2422 case lir_shl: __ shlptr(lo); break;
2423 case lir_shr: __ sarptr(lo); break;
2424 case lir_ushr: __ shrptr(lo); break;
2425 default: ShouldNotReachHere();
2426 }
2427 } else {
2428 ShouldNotReachHere();
2429 }
2430 }
2431
2432
2433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2434 if (dest->is_single_cpu()) {
2435 // first move left into dest so that left is not destroyed by the shift
2436 Register value = dest->as_register();
2437 count = count & 0x1F; // Java spec
2438
2439 move_regs(left->as_register(), value);
2440 switch (code) {
2441 case lir_shl: __ shll(value, count); break;
2442 case lir_shr: __ sarl(value, count); break;
2443 case lir_ushr: __ shrl(value, count); break;
2444 default: ShouldNotReachHere();
2445 }
2446 } else if (dest->is_double_cpu()) {
2447 // first move left into dest so that left is not destroyed by the shift
2448 Register value = dest->as_register_lo();
2449 count = count & 0x1F; // Java spec
2450
2451 move_regs(left->as_register_lo(), value);
2452 switch (code) {
2453 case lir_shl: __ shlptr(value, count); break;
2454 case lir_shr: __ sarptr(value, count); break;
2455 case lir_ushr: __ shrptr(value, count); break;
2456 default: ShouldNotReachHere();
2457 }
2458 } else {
2459 ShouldNotReachHere();
2460 }
2461 }
2462
2463
2464 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2465 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2466 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2467 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2468 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2469 }
2470
2471
2472 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2473 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2474 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2475 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2476 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2477 }
2478
2479
2480 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2481 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2482 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2483 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2484 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2485 }
2486
2487
2488 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2489 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2490 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2491 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2492 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2493 }
2494
2495
2496 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2497 if (null_check) {
2498 __ testptr(obj, obj);
2499 __ jcc(Assembler::zero, *slow_path->entry());
2500 }
2501 if (is_dest) {
2502 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2503 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2504 } else {
2505 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2506 }
2507 }
2508
2509
2510 // This code replaces a call to arraycopy; no exception may
2511 // be thrown in this code, they must be thrown in the System.arraycopy
2512 // activation frame; we could save some checks if this would not be the case
2513 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2514 ciArrayKlass* default_type = op->expected_type();
2515 Register src = op->src()->as_register();
2516 Register dst = op->dst()->as_register();
2517 Register src_pos = op->src_pos()->as_register();
2518 Register dst_pos = op->dst_pos()->as_register();
2519 Register length = op->length()->as_register();
2520 Register tmp = op->tmp()->as_register();
2521 Register tmp_load_klass = rscratch1;
2522 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2523
2524 CodeStub* stub = op->stub();
2525 int flags = op->flags();
2526 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2527 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2528
2529 if (flags & LIR_OpArrayCopy::always_slow_path) {
2530 __ jmp(*stub->entry());
2531 __ bind(*stub->continuation());
2532 return;
2533 }
2534
2535 // if we don't know anything, just go through the generic arraycopy
2536 if (default_type == nullptr) {
2537 // save outgoing arguments on stack in case call to System.arraycopy is needed
2538 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2539 // for interpreter calling conventions. Now we have to do it in new style conventions.
2540 // For the moment until C1 gets the new register allocator I just force all the
2541 // args to the right place (except the register args) and then on the back side
2542 // reload the register args properly if we go slow path. Yuck
2543
2544 // These are proper for the calling convention
2545 store_parameter(length, 2);
2546 store_parameter(dst_pos, 1);
2547 store_parameter(dst, 0);
2548
2549 // these are just temporary placements until we need to reload
2550 store_parameter(src_pos, 3);
2551 store_parameter(src, 4);
2552
2553 address copyfunc_addr = StubRoutines::generic_arraycopy();
2554 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2555
2556 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2557 // The arguments are in java calling convention so we can trivially shift them to C
2558 // convention
2559 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2560 __ mov(c_rarg0, j_rarg0);
2561 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2562 __ mov(c_rarg1, j_rarg1);
2563 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2564 __ mov(c_rarg2, j_rarg2);
2565 assert_different_registers(c_rarg3, j_rarg4);
2566 __ mov(c_rarg3, j_rarg3);
2567 #ifdef _WIN64
2568 // Allocate abi space for args but be sure to keep stack aligned
2569 __ subptr(rsp, 6*wordSize);
2570 store_parameter(j_rarg4, 4);
2571 #ifndef PRODUCT
2572 if (PrintC1Statistics) {
2573 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2574 }
2575 #endif
2576 __ call(RuntimeAddress(copyfunc_addr));
2577 __ addptr(rsp, 6*wordSize);
2578 #else
2579 __ mov(c_rarg4, j_rarg4);
2580 #ifndef PRODUCT
2581 if (PrintC1Statistics) {
2582 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2583 }
2584 #endif
2585 __ call(RuntimeAddress(copyfunc_addr));
2586 #endif // _WIN64
2587
2588 __ testl(rax, rax);
2589 __ jcc(Assembler::equal, *stub->continuation());
2590
2591 __ mov(tmp, rax);
2592 __ xorl(tmp, -1);
2593
2594 // Reload values from the stack so they are where the stub
2595 // expects them.
2596 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2597 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2598 __ movptr (length, Address(rsp, 2*BytesPerWord));
2599 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2600 __ movptr (src, Address(rsp, 4*BytesPerWord));
2601
2602 __ subl(length, tmp);
2603 __ addl(src_pos, tmp);
2604 __ addl(dst_pos, tmp);
2605 __ jmp(*stub->entry());
2606
2607 __ bind(*stub->continuation());
2608 return;
2609 }
2610
2611 // Handle inline type arrays
2612 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2613 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2614 }
2615 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2616 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2617 }
2618
2619 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2620
2621 int elem_size = type2aelembytes(basic_type);
2622 Address::ScaleFactor scale;
2623
2624 switch (elem_size) {
2625 case 1 :
2626 scale = Address::times_1;
2627 break;
2628 case 2 :
2629 scale = Address::times_2;
2630 break;
2631 case 4 :
2632 scale = Address::times_4;
2633 break;
2634 case 8 :
2635 scale = Address::times_8;
2636 break;
2637 default:
2638 scale = Address::no_scale;
2639 ShouldNotReachHere();
2640 }
2641
2642 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2643 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2644
2645 // length and pos's are all sign extended at this point on 64bit
2646
2647 // test for null
2648 if (flags & LIR_OpArrayCopy::src_null_check) {
2649 __ testptr(src, src);
2650 __ jcc(Assembler::zero, *stub->entry());
2651 }
2652 if (flags & LIR_OpArrayCopy::dst_null_check) {
2653 __ testptr(dst, dst);
2654 __ jcc(Assembler::zero, *stub->entry());
2655 }
2656
2657 // If the compiler was not able to prove that exact type of the source or the destination
2658 // of the arraycopy is an array type, check at runtime if the source or the destination is
2659 // an instance type.
2660 if (flags & LIR_OpArrayCopy::type_check) {
2661 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2662 __ load_klass(tmp, dst, tmp_load_klass);
2663 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2664 __ jcc(Assembler::greaterEqual, *stub->entry());
2665 }
2666
2667 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2668 __ load_klass(tmp, src, tmp_load_klass);
2669 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2670 __ jcc(Assembler::greaterEqual, *stub->entry());
2671 }
2672 }
2673
2674 // check if negative
2675 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2676 __ testl(src_pos, src_pos);
2677 __ jcc(Assembler::less, *stub->entry());
2678 }
2679 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2680 __ testl(dst_pos, dst_pos);
2681 __ jcc(Assembler::less, *stub->entry());
2682 }
2683
2684 if (flags & LIR_OpArrayCopy::src_range_check) {
2685 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2686 __ cmpl(tmp, src_length_addr);
2687 __ jcc(Assembler::above, *stub->entry());
2688 }
2689 if (flags & LIR_OpArrayCopy::dst_range_check) {
2690 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2691 __ cmpl(tmp, dst_length_addr);
2692 __ jcc(Assembler::above, *stub->entry());
2693 }
2694
2695 if (flags & LIR_OpArrayCopy::length_positive_check) {
2696 __ testl(length, length);
2697 __ jcc(Assembler::less, *stub->entry());
2698 }
2699
2700 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2701 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2702
2703 if (flags & LIR_OpArrayCopy::type_check) {
2704 // We don't know the array types are compatible
2705 if (basic_type != T_OBJECT) {
2706 // Simple test for basic type arrays
2707 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2708 __ jcc(Assembler::notEqual, *stub->entry());
2709 } else {
2710 // For object arrays, if src is a sub class of dst then we can
2711 // safely do the copy.
2712 Label cont, slow;
2713
2714 __ push_ppx(src);
2715 __ push_ppx(dst);
2716
2717 __ load_klass(src, src, tmp_load_klass);
2718 __ load_klass(dst, dst, tmp_load_klass);
2719
2720 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2721
2722 __ push_ppx(src);
2723 __ push_ppx(dst);
2724 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2725 __ pop_ppx(dst);
2726 __ pop_ppx(src);
2727
2728 __ testl(src, src);
2729 __ jcc(Assembler::notEqual, cont);
2730
2731 __ bind(slow);
2732 __ pop_ppx(dst);
2733 __ pop_ppx(src);
2734
2735 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2736 if (copyfunc_addr != nullptr) { // use stub if available
2737 // src is not a sub class of dst so we have to do a
2738 // per-element check.
2739
2740 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2741 if ((flags & mask) != mask) {
2742 // Check that at least both of them object arrays.
2743 assert(flags & mask, "one of the two should be known to be an object array");
2744
2745 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2746 __ load_klass(tmp, src, tmp_load_klass);
2747 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2748 __ load_klass(tmp, dst, tmp_load_klass);
2749 }
2750 int lh_offset = in_bytes(Klass::layout_helper_offset());
2751 Address klass_lh_addr(tmp, lh_offset);
2752 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2753 __ cmpl(klass_lh_addr, objArray_lh);
2754 __ jcc(Assembler::notEqual, *stub->entry());
2755 }
2756
2757 // Spill because stubs can use any register they like and it's
2758 // easier to restore just those that we care about.
2759 store_parameter(dst, 0);
2760 store_parameter(dst_pos, 1);
2761 store_parameter(length, 2);
2762 store_parameter(src_pos, 3);
2763 store_parameter(src, 4);
2764
2765 __ movl2ptr(length, length); //higher 32bits must be null
2766
2767 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2768 assert_different_registers(c_rarg0, dst, dst_pos, length);
2769 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2770 assert_different_registers(c_rarg1, dst, length);
2771
2772 __ mov(c_rarg2, length);
2773 assert_different_registers(c_rarg2, dst);
2774
2775 #ifdef _WIN64
2776 // Allocate abi space for args but be sure to keep stack aligned
2777 __ subptr(rsp, 6*wordSize);
2778 __ load_klass(c_rarg3, dst, tmp_load_klass);
2779 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2780 store_parameter(c_rarg3, 4);
2781 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2782 __ call(RuntimeAddress(copyfunc_addr));
2783 __ addptr(rsp, 6*wordSize);
2784 #else
2785 __ load_klass(c_rarg4, dst, tmp_load_klass);
2786 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2787 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2788 __ call(RuntimeAddress(copyfunc_addr));
2789 #endif
2790
2791 #ifndef PRODUCT
2792 if (PrintC1Statistics) {
2793 Label failed;
2794 __ testl(rax, rax);
2795 __ jcc(Assembler::notZero, failed);
2796 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2797 __ bind(failed);
2798 }
2799 #endif
2800
2801 __ testl(rax, rax);
2802 __ jcc(Assembler::zero, *stub->continuation());
2803
2804 #ifndef PRODUCT
2805 if (PrintC1Statistics) {
2806 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2807 }
2808 #endif
2809
2810 __ mov(tmp, rax);
2811
2812 __ xorl(tmp, -1);
2813
2814 // Restore previously spilled arguments
2815 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2816 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2817 __ movptr (length, Address(rsp, 2*BytesPerWord));
2818 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2819 __ movptr (src, Address(rsp, 4*BytesPerWord));
2820
2821
2822 __ subl(length, tmp);
2823 __ addl(src_pos, tmp);
2824 __ addl(dst_pos, tmp);
2825 }
2826
2827 __ jmp(*stub->entry());
2828
2829 __ bind(cont);
2830 __ pop(dst);
2831 __ pop(src);
2832 }
2833 }
2834
2835 #ifdef ASSERT
2836 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2837 // Sanity check the known type with the incoming class. For the
2838 // primitive case the types must match exactly with src.klass and
2839 // dst.klass each exactly matching the default type. For the
2840 // object array case, if no type check is needed then either the
2841 // dst type is exactly the expected type and the src type is a
2842 // subtype which we can't check or src is the same array as dst
2843 // but not necessarily exactly of type default_type.
2844 Label known_ok, halt;
2845 __ mov_metadata(tmp, default_type->constant_encoding());
2846 __ encode_klass_not_null(tmp, rscratch1);
2847
2848 if (basic_type != T_OBJECT) {
2849 __ cmp_klass(tmp, dst, tmp2);
2850 __ jcc(Assembler::notEqual, halt);
2851 __ cmp_klass(tmp, src, tmp2);
2852 __ jcc(Assembler::equal, known_ok);
2853 } else {
2854 __ cmp_klass(tmp, dst, tmp2);
2855 __ jcc(Assembler::equal, known_ok);
2856 __ cmpptr(src, dst);
2857 __ jcc(Assembler::equal, known_ok);
2858 }
2859 __ bind(halt);
2860 __ stop("incorrect type information in arraycopy");
2861 __ bind(known_ok);
2862 }
2863 #endif
2864
2865 #ifndef PRODUCT
2866 if (PrintC1Statistics) {
2867 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2868 }
2869 #endif
2870
2871 assert_different_registers(c_rarg0, dst, dst_pos, length);
2872 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2873 assert_different_registers(c_rarg1, length);
2874 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2875 __ mov(c_rarg2, length);
2876
2877 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2878 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2879 const char *name;
2880 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2881 __ call_VM_leaf(entry, 0);
2882
2883 if (stub != nullptr) {
2884 __ bind(*stub->continuation());
2885 }
2886 }
2887
2888 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2889 assert(op->crc()->is_single_cpu(), "crc must be register");
2890 assert(op->val()->is_single_cpu(), "byte value must be register");
2891 assert(op->result_opr()->is_single_cpu(), "result must be register");
2892 Register crc = op->crc()->as_register();
2893 Register val = op->val()->as_register();
2894 Register res = op->result_opr()->as_register();
2895
2896 assert_different_registers(val, crc, res);
2897
2898 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2899 __ notl(crc); // ~crc
2900 __ update_byte_crc32(crc, val, res);
2901 __ notl(crc); // ~crc
2902 __ mov(res, crc);
2903 }
2904
2905 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2906 Register obj = op->obj_opr()->as_register(); // may not be an oop
2907 Register hdr = op->hdr_opr()->as_register();
2908 Register lock = op->lock_opr()->as_register();
2909 if (op->code() == lir_lock) {
2910 Register tmp = op->scratch_opr()->as_register();
2911 // add debug info for NullPointerException only if one is possible
2912 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2913 if (op->info() != nullptr) {
2914 add_debug_info_for_null_check(null_check_offset, op->info());
2915 }
2916 // done
2917 } else if (op->code() == lir_unlock) {
2918 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2919 } else {
2920 Unimplemented();
2921 }
2922 __ bind(*op->stub()->continuation());
2923 }
2924
2925 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2926 Register obj = op->obj()->as_pointer_register();
2927 Register result = op->result_opr()->as_pointer_register();
2928
2929 CodeEmitInfo* info = op->info();
2930 if (info != nullptr) {
2931 add_debug_info_for_null_check_here(info);
2932 }
2933
2934 __ load_klass(result, obj, rscratch1);
2935 }
2936
2937 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2938 ciMethod* method = op->profiled_method();
2939 int bci = op->profiled_bci();
2940 ciMethod* callee = op->profiled_callee();
2941 Register tmp_load_klass = rscratch1;
2942
2943 // Update counter for all call types
2944 ciMethodData* md = method->method_data_or_null();
2945 assert(md != nullptr, "Sanity");
2946 ciProfileData* data = md->bci_to_data(bci);
2947 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2948 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2949 Register mdo = op->mdo()->as_register();
2950 __ mov_metadata(mdo, md->constant_encoding());
2951 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2952 // Perform additional virtual call profiling for invokevirtual and
2953 // invokeinterface bytecodes
2954 if (op->should_profile_receiver_type()) {
2955 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2956 Register recv = op->recv()->as_register();
2957 assert_different_registers(mdo, recv);
2958 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2959 ciKlass* known_klass = op->known_holder();
2960 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2961 // We know the type that will be seen at this call site; we can
2962 // statically update the MethodData* rather than needing to do
2963 // dynamic tests on the receiver type.
2964 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2965 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2966 ciKlass* receiver = vc_data->receiver(i);
2967 if (known_klass->equals(receiver)) {
2968 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2969 __ addptr(data_addr, DataLayout::counter_increment);
2970 return;
2971 }
2972 }
2973 // Receiver type is not found in profile data.
2974 // Fall back to runtime helper to handle the rest at runtime.
2975 __ mov_metadata(recv, known_klass->constant_encoding());
2976 } else {
2977 __ load_klass(recv, recv, tmp_load_klass);
2978 }
2979 type_profile_helper(mdo, md, data, recv);
2980 } else {
2981 // Static call
2982 __ addptr(counter_addr, DataLayout::counter_increment);
2983 }
2984 }
2985
2986 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2987 Register obj = op->obj()->as_register();
2988 Register tmp = op->tmp()->as_pointer_register();
2989 Register tmp_load_klass = rscratch1;
2990 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2991 ciKlass* exact_klass = op->exact_klass();
2992 intptr_t current_klass = op->current_klass();
2993 bool not_null = op->not_null();
2994 bool no_conflict = op->no_conflict();
2995
2996 Label update, next, none;
2997
2998 bool do_null = !not_null;
2999 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3000 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3001
3002 assert(do_null || do_update, "why are we here?");
3003 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3004
3005 __ verify_oop(obj);
3006
3007 #ifdef ASSERT
3008 if (obj == tmp) {
3009 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3010 } else {
3011 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3012 }
3013 #endif
3014 if (do_null) {
3015 __ testptr(obj, obj);
3016 __ jccb(Assembler::notZero, update);
3017 if (!TypeEntries::was_null_seen(current_klass)) {
3018 __ testptr(mdo_addr, TypeEntries::null_seen);
3019 #ifndef ASSERT
3020 __ jccb(Assembler::notZero, next); // already set
3021 #else
3022 __ jcc(Assembler::notZero, next); // already set
3023 #endif
3024 // atomic update to prevent overwriting Klass* with 0
3025 __ lock();
3026 __ orptr(mdo_addr, TypeEntries::null_seen);
3027 }
3028 if (do_update) {
3029 #ifndef ASSERT
3030 __ jmpb(next);
3031 }
3032 #else
3033 __ jmp(next);
3034 }
3035 } else {
3036 __ testptr(obj, obj);
3037 __ jcc(Assembler::notZero, update);
3038 __ stop("unexpected null obj");
3039 #endif
3040 }
3041
3042 __ bind(update);
3043
3044 if (do_update) {
3045 #ifdef ASSERT
3046 if (exact_klass != nullptr) {
3047 Label ok;
3048 __ load_klass(tmp, obj, tmp_load_klass);
3049 __ push_ppx(tmp);
3050 __ mov_metadata(tmp, exact_klass->constant_encoding());
3051 __ cmpptr(tmp, Address(rsp, 0));
3052 __ jcc(Assembler::equal, ok);
3053 __ stop("exact klass and actual klass differ");
3054 __ bind(ok);
3055 __ pop_ppx(tmp);
3056 }
3057 #endif
3058 if (!no_conflict) {
3059 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3060 if (exact_klass != nullptr) {
3061 __ mov_metadata(tmp, exact_klass->constant_encoding());
3062 } else {
3063 __ load_klass(tmp, obj, tmp_load_klass);
3064 }
3065 __ mov(rscratch1, tmp); // save original value before XOR
3066 __ xorptr(tmp, mdo_addr);
3067 __ testptr(tmp, TypeEntries::type_klass_mask);
3068 // klass seen before, nothing to do. The unknown bit may have been
3069 // set already but no need to check.
3070 __ jccb(Assembler::zero, next);
3071
3072 __ testptr(tmp, TypeEntries::type_unknown);
3073 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3074
3075 if (TypeEntries::is_type_none(current_klass)) {
3076 __ testptr(mdo_addr, TypeEntries::type_mask);
3077 __ jccb(Assembler::zero, none);
3078 // There is a chance that the checks above (re-reading profiling
3079 // data from memory) fail if another thread has just set the
3080 // profiling to this obj's klass
3081 __ mov(tmp, rscratch1); // get back original value before XOR
3082 __ xorptr(tmp, mdo_addr);
3083 __ testptr(tmp, TypeEntries::type_klass_mask);
3084 __ jccb(Assembler::zero, next);
3085 }
3086 } else {
3087 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3088 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3089
3090 __ testptr(mdo_addr, TypeEntries::type_unknown);
3091 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3092 }
3093
3094 // different than before. Cannot keep accurate profile.
3095 __ orptr(mdo_addr, TypeEntries::type_unknown);
3096
3097 if (TypeEntries::is_type_none(current_klass)) {
3098 __ jmpb(next);
3099
3100 __ bind(none);
3101 // first time here. Set profile type.
3102 __ movptr(mdo_addr, tmp);
3103 #ifdef ASSERT
3104 __ andptr(tmp, TypeEntries::type_klass_mask);
3105 __ verify_klass_ptr(tmp);
3106 #endif
3107 }
3108 } else {
3109 // There's a single possible klass at this profile point
3110 assert(exact_klass != nullptr, "should be");
3111 if (TypeEntries::is_type_none(current_klass)) {
3112 __ mov_metadata(tmp, exact_klass->constant_encoding());
3113 __ xorptr(tmp, mdo_addr);
3114 __ testptr(tmp, TypeEntries::type_klass_mask);
3115 #ifdef ASSERT
3116 __ jcc(Assembler::zero, next);
3117
3118 {
3119 Label ok;
3120 __ push_ppx(tmp);
3121 __ testptr(mdo_addr, TypeEntries::type_mask);
3122 __ jcc(Assembler::zero, ok);
3123 // may have been set by another thread
3124 __ mov_metadata(tmp, exact_klass->constant_encoding());
3125 __ xorptr(tmp, mdo_addr);
3126 __ testptr(tmp, TypeEntries::type_mask);
3127 __ jcc(Assembler::zero, ok);
3128
3129 __ stop("unexpected profiling mismatch");
3130 __ bind(ok);
3131 __ pop_ppx(tmp);
3132 }
3133 #else
3134 __ jccb(Assembler::zero, next);
3135 #endif
3136 // first time here. Set profile type.
3137 __ movptr(mdo_addr, tmp);
3138 #ifdef ASSERT
3139 __ andptr(tmp, TypeEntries::type_klass_mask);
3140 __ verify_klass_ptr(tmp);
3141 #endif
3142 } else {
3143 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3144 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3145
3146 __ testptr(mdo_addr, TypeEntries::type_unknown);
3147 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3148
3149 __ orptr(mdo_addr, TypeEntries::type_unknown);
3150 }
3151 }
3152 }
3153 __ bind(next);
3154 }
3155
3156 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3157 Register obj = op->obj()->as_register();
3158 Register tmp = op->tmp()->as_pointer_register();
3159 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3160 bool not_null = op->not_null();
3161 int flag = op->flag();
3162
3163 Label not_inline_type;
3164 if (!not_null) {
3165 __ testptr(obj, obj);
3166 __ jccb(Assembler::zero, not_inline_type);
3167 }
3168
3169 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3170
3171 __ orb(mdo_addr, flag);
3172
3173 __ bind(not_inline_type);
3174 }
3175
3176
3177 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3178 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3179 }
3180
3181
3182 void LIR_Assembler::align_backward_branch_target() {
3183 __ align(BytesPerWord);
3184 }
3185
3186
3187 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3188 if (left->is_single_cpu()) {
3189 __ negl(left->as_register());
3190 move_regs(left->as_register(), dest->as_register());
3191
3192 } else if (left->is_double_cpu()) {
3193 Register lo = left->as_register_lo();
3194 Register dst = dest->as_register_lo();
3195 __ movptr(dst, lo);
3196 __ negptr(dst);
3197
3198 } else if (dest->is_single_xmm()) {
3199 assert(!tmp->is_valid(), "do not need temporary");
3200 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3201 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3202 }
3203 __ xorps(dest->as_xmm_float_reg(),
3204 ExternalAddress((address)float_signflip_pool),
3205 rscratch1);
3206 } else if (dest->is_double_xmm()) {
3207 assert(!tmp->is_valid(), "do not need temporary");
3208 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3209 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3210 }
3211 __ xorpd(dest->as_xmm_double_reg(),
3212 ExternalAddress((address)double_signflip_pool),
3213 rscratch1);
3214 } else {
3215 ShouldNotReachHere();
3216 }
3217 }
3218
3219
3220 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3221 assert(src->is_address(), "must be an address");
3222 assert(dest->is_register(), "must be a register");
3223
3224 PatchingStub* patch = nullptr;
3225 if (patch_code != lir_patch_none) {
3226 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3227 }
3228
3229 Register reg = dest->as_pointer_register();
3230 LIR_Address* addr = src->as_address_ptr();
3231 __ lea(reg, as_Address(addr));
3232
3233 if (patch != nullptr) {
3234 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3235 }
3236 }
3237
3238
3239
3240 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3241 assert(!tmp->is_valid(), "don't need temporary");
3242 __ call(RuntimeAddress(dest));
3243 if (info != nullptr) {
3244 add_call_info_here(info);
3245 }
3246 __ post_call_nop();
3247 }
3248
3249
3250 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3251 assert(type == T_LONG, "only for volatile long fields");
3252
3253 if (info != nullptr) {
3254 add_debug_info_for_null_check_here(info);
3255 }
3256
3257 if (src->is_double_xmm()) {
3258 if (dest->is_double_cpu()) {
3259 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3260 } else if (dest->is_double_stack()) {
3261 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3262 } else if (dest->is_address()) {
3263 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3264 } else {
3265 ShouldNotReachHere();
3266 }
3267
3268 } else if (dest->is_double_xmm()) {
3269 if (src->is_double_stack()) {
3270 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3271 } else if (src->is_address()) {
3272 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3273 } else {
3274 ShouldNotReachHere();
3275 }
3276
3277 } else {
3278 ShouldNotReachHere();
3279 }
3280 }
3281
3282 #ifdef ASSERT
3283 // emit run-time assertion
3284 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3285 assert(op->code() == lir_assert, "must be");
3286
3287 if (op->in_opr1()->is_valid()) {
3288 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3289 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3290 } else {
3291 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3292 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3293 }
3294
3295 Label ok;
3296 if (op->condition() != lir_cond_always) {
3297 Assembler::Condition acond = Assembler::zero;
3298 switch (op->condition()) {
3299 case lir_cond_equal: acond = Assembler::equal; break;
3300 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3301 case lir_cond_less: acond = Assembler::less; break;
3302 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3303 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3304 case lir_cond_greater: acond = Assembler::greater; break;
3305 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3306 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3307 default: ShouldNotReachHere();
3308 }
3309 __ jcc(acond, ok);
3310 }
3311 if (op->halt()) {
3312 const char* str = __ code_string(op->msg());
3313 __ stop(str);
3314 } else {
3315 breakpoint();
3316 }
3317 __ bind(ok);
3318 }
3319 #endif
3320
3321 void LIR_Assembler::membar() {
3322 // QQQ sparc TSO uses this,
3323 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3324 }
3325
3326 void LIR_Assembler::membar_acquire() {
3327 // No x86 machines currently require load fences
3328 }
3329
3330 void LIR_Assembler::membar_release() {
3331 // No x86 machines currently require store fences
3332 }
3333
3334 void LIR_Assembler::membar_loadload() {
3335 // no-op
3336 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3337 }
3338
3339 void LIR_Assembler::membar_storestore() {
3340 // no-op
3341 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3342 }
3343
3344 void LIR_Assembler::membar_loadstore() {
3345 // no-op
3346 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3347 }
3348
3349 void LIR_Assembler::membar_storeload() {
3350 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3351 }
3352
3353 void LIR_Assembler::on_spin_wait() {
3354 __ pause ();
3355 }
3356
3357 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3358 assert(result_reg->is_register(), "check");
3359 __ mov(result_reg->as_register(), r15_thread);
3360 }
3361
3362 void LIR_Assembler::check_orig_pc() {
3363 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3364 }
3365
3366 void LIR_Assembler::peephole(LIR_List*) {
3367 // do nothing for now
3368 }
3369
3370 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3371 assert(data == dest, "xchg/xadd uses only 2 operands");
3372
3373 if (data->type() == T_INT) {
3374 if (code == lir_xadd) {
3375 __ lock();
3376 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3377 } else {
3378 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3379 }
3380 } else if (data->is_oop()) {
3381 assert (code == lir_xchg, "xadd for oops");
3382 Register obj = data->as_register();
3383 if (UseCompressedOops) {
3384 __ encode_heap_oop(obj);
3385 __ xchgl(obj, as_Address(src->as_address_ptr()));
3386 __ decode_heap_oop(obj);
3387 } else {
3388 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3389 }
3390 } else if (data->type() == T_LONG) {
3391 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3392 if (code == lir_xadd) {
3393 __ lock();
3394 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3395 } else {
3396 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3397 }
3398 } else {
3399 ShouldNotReachHere();
3400 }
3401 }
3402
3403 #undef __