1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/powerOfTwo.hpp"
49 #include "vmreg_x86.inline.hpp"
50
51
52 // These masks are used to provide 128-bit aligned bitmasks to the XMM
53 // instructions, to allow sign-masking or sign-bit flipping. They allow
54 // fast versions of NegF/NegD and AbsF/AbsD.
55
56 // Note: 'double' and 'long long' have 32-bits alignment on x86.
57 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
58 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
59 // of 128-bits operands for SSE instructions.
60 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
61 // Store the value to a 128-bits operand.
62 operand[0] = lo;
63 operand[1] = hi;
64 return operand;
65 }
66
67 // Buffer for 128-bits masks used by SSE instructions.
68 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
69
70 // Static initialization during VM startup.
71 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
72 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
73 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
74 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
75
76
77 NEEDS_CLEANUP // remove this definitions ?
78 const Register SYNC_header = rax; // synchronization header
79 const Register SHIFT_count = rcx; // where count for shift operations must be
80
81 #define __ _masm->
82
83
84 static void select_different_registers(Register preserve,
85 Register extra,
86 Register &tmp1,
87 Register &tmp2) {
88 if (tmp1 == preserve) {
89 assert_different_registers(tmp1, tmp2, extra);
90 tmp1 = extra;
91 } else if (tmp2 == preserve) {
92 assert_different_registers(tmp1, tmp2, extra);
93 tmp2 = extra;
94 }
95 assert_different_registers(preserve, tmp1, tmp2);
96 }
97
98
99
100 static void select_different_registers(Register preserve,
101 Register extra,
102 Register &tmp1,
103 Register &tmp2,
104 Register &tmp3) {
105 if (tmp1 == preserve) {
106 assert_different_registers(tmp1, tmp2, tmp3, extra);
107 tmp1 = extra;
108 } else if (tmp2 == preserve) {
109 assert_different_registers(tmp1, tmp2, tmp3, extra);
110 tmp2 = extra;
111 } else if (tmp3 == preserve) {
112 assert_different_registers(tmp1, tmp2, tmp3, extra);
113 tmp3 = extra;
114 }
115 assert_different_registers(preserve, tmp1, tmp2, tmp3);
116 }
117
118
119
120 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
121 if (opr->is_constant()) {
122 LIR_Const* constant = opr->as_constant_ptr();
123 switch (constant->type()) {
124 case T_INT: {
125 return true;
126 }
127
128 default:
129 return false;
130 }
131 }
132 return false;
133 }
134
135
136 LIR_Opr LIR_Assembler::receiverOpr() {
137 return FrameMap::receiver_opr;
138 }
139
140 LIR_Opr LIR_Assembler::osrBufferPointer() {
141 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
142 }
143
144 //--------------fpu register translations-----------------------
145
146
147 address LIR_Assembler::float_constant(float f) {
148 address const_addr = __ float_constant(f);
149 if (const_addr == nullptr) {
150 bailout("const section overflow");
151 return __ code()->consts()->start();
152 } else {
153 return const_addr;
154 }
155 }
156
157
158 address LIR_Assembler::double_constant(double d) {
159 address const_addr = __ double_constant(d);
160 if (const_addr == nullptr) {
161 bailout("const section overflow");
162 return __ code()->consts()->start();
163 } else {
164 return const_addr;
165 }
166 }
167
168 void LIR_Assembler::breakpoint() {
169 __ int3();
170 }
171
172 void LIR_Assembler::push(LIR_Opr opr) {
173 if (opr->is_single_cpu()) {
174 __ push_reg(opr->as_register());
175 } else if (opr->is_double_cpu()) {
176 __ push_reg(opr->as_register_lo());
177 } else if (opr->is_stack()) {
178 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
179 } else if (opr->is_constant()) {
180 LIR_Const* const_opr = opr->as_constant_ptr();
181 if (const_opr->type() == T_OBJECT) {
182 __ push_oop(const_opr->as_jobject(), rscratch1);
183 } else if (const_opr->type() == T_INT) {
184 __ push_jint(const_opr->as_jint());
185 } else {
186 ShouldNotReachHere();
187 }
188
189 } else {
190 ShouldNotReachHere();
191 }
192 }
193
194 void LIR_Assembler::pop(LIR_Opr opr) {
195 if (opr->is_single_cpu()) {
196 __ pop_reg(opr->as_register());
197 } else {
198 ShouldNotReachHere();
199 }
200 }
201
202 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
203 return addr->base()->is_illegal() && addr->index()->is_illegal();
204 }
205
206 //-------------------------------------------
207
208 Address LIR_Assembler::as_Address(LIR_Address* addr) {
209 return as_Address(addr, rscratch1);
210 }
211
212 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
213 if (addr->base()->is_illegal()) {
214 assert(addr->index()->is_illegal(), "must be illegal too");
215 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
216 if (! __ reachable(laddr)) {
217 __ movptr(tmp, laddr.addr());
218 Address res(tmp, 0);
219 return res;
220 } else {
221 return __ as_Address(laddr);
222 }
223 }
224
225 Register base = addr->base()->as_pointer_register();
226
227 if (addr->index()->is_illegal()) {
228 return Address( base, addr->disp());
229 } else if (addr->index()->is_cpu_register()) {
230 Register index = addr->index()->as_pointer_register();
231 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
232 } else if (addr->index()->is_constant()) {
233 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
234 assert(Assembler::is_simm32(addr_offset), "must be");
235
236 return Address(base, addr_offset);
237 } else {
238 Unimplemented();
239 return Address();
240 }
241 }
242
243
244 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
245 Address base = as_Address(addr);
246 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
247 }
248
249
250 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
251 return as_Address(addr);
252 }
253
254
255 void LIR_Assembler::osr_entry() {
256 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
257 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
258 ValueStack* entry_state = osr_entry->state();
259 int number_of_locks = entry_state->locks_size();
260
261 // we jump here if osr happens with the interpreter
262 // state set up to continue at the beginning of the
263 // loop that triggered osr - in particular, we have
264 // the following registers setup:
265 //
266 // rcx: osr buffer
267 //
268
269 // build frame
270 ciMethod* m = compilation()->method();
271 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
272
273 // OSR buffer is
274 //
275 // locals[nlocals-1..0]
276 // monitors[0..number_of_locks]
277 //
278 // locals is a direct copy of the interpreter frame so in the osr buffer
279 // so first slot in the local array is the last local from the interpreter
280 // and last slot is local[0] (receiver) from the interpreter
281 //
282 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
283 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
284 // in the interpreter frame (the method lock if a sync method)
285
286 // Initialize monitors in the compiled activation.
287 // rcx: pointer to osr buffer
288 //
289 // All other registers are dead at this point and the locals will be
290 // copied into place by code emitted in the IR.
291
292 Register OSR_buf = osrBufferPointer()->as_pointer_register();
293 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
294 int monitor_offset = BytesPerWord * method()->max_locals() +
295 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
296 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
297 // the OSR buffer using 2 word entries: first the lock and then
298 // the oop.
299 for (int i = 0; i < number_of_locks; i++) {
300 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
301 #ifdef ASSERT
302 // verify the interpreter's monitor has a non-null object
303 {
304 Label L;
305 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
306 __ jcc(Assembler::notZero, L);
307 __ stop("locked object is null");
308 __ bind(L);
309 }
310 #endif
311 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
312 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
313 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
314 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
315 }
316 }
317 }
318
319
320 // inline cache check; done before the frame is built.
321 int LIR_Assembler::check_icache() {
322 return __ ic_check(CodeEntryAlignment);
323 }
324
325 void LIR_Assembler::clinit_barrier(ciMethod* method) {
326 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
327 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
328
329 Label L_skip_barrier;
330 Register klass = rscratch1;
331
332 __ mov_metadata(klass, method->holder()->constant_encoding());
333 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
334
335 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
336
337 __ bind(L_skip_barrier);
338 }
339
340 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
341 jobject o = nullptr;
342 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
343 __ movoop(reg, o);
344 patching_epilog(patch, lir_patch_normal, reg, info);
345 }
346
347 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
348 Metadata* o = nullptr;
349 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
350 __ mov_metadata(reg, o);
351 patching_epilog(patch, lir_patch_normal, reg, info);
352 }
353
354 // This specifies the rsp decrement needed to build the frame
355 int LIR_Assembler::initial_frame_size_in_bytes() const {
356 // if rounding, must let FrameMap know!
357
358 // The frame_map records size in slots (32bit word)
359
360 // subtract two words to account for return address and link
361 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
362 }
363
364
365 int LIR_Assembler::emit_exception_handler() {
366 // generate code for exception handler
367 address handler_base = __ start_a_stub(exception_handler_size());
368 if (handler_base == nullptr) {
369 // not enough space left for the handler
370 bailout("exception handler overflow");
371 return -1;
372 }
373
374 int offset = code_offset();
375
376 // the exception oop and pc are in rax, and rdx
377 // no other registers need to be preserved, so invalidate them
378 __ invalidate_registers(false, true, true, false, true, true);
379
380 // check that there is really an exception
381 __ verify_not_null_oop(rax);
382
383 // search an exception handler (rax: exception oop, rdx: throwing pc)
384 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
385 __ should_not_reach_here();
386 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
387 __ end_a_stub();
388
389 return offset;
390 }
391
392
393 // Emit the code to remove the frame from the stack in the exception
394 // unwind path.
395 int LIR_Assembler::emit_unwind_handler() {
396 #ifndef PRODUCT
397 if (CommentedAssembly) {
398 _masm->block_comment("Unwind handler");
399 }
400 #endif
401
402 int offset = code_offset();
403
404 // Fetch the exception from TLS and clear out exception related thread state
405 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
406 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
407 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
408
409 __ bind(_unwind_handler_entry);
410 __ verify_not_null_oop(rax);
411 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
412 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
413 }
414
415 // Perform needed unlocking
416 MonitorExitStub* stub = nullptr;
417 if (method()->is_synchronized()) {
418 monitor_address(0, FrameMap::rax_opr);
419 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
420 __ unlock_object(rdi, rsi, rax, *stub->entry());
421 __ bind(*stub->continuation());
422 }
423
424 if (compilation()->env()->dtrace_method_probes()) {
425 __ mov(rdi, r15_thread);
426 __ mov_metadata(rsi, method()->constant_encoding());
427 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
428 }
429
430 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
431 __ mov(rax, rbx); // Restore the exception
432 }
433
434 // remove the activation and dispatch to the unwind handler
435 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
436 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
437
438 // Emit the slow path assembly
439 if (stub != nullptr) {
440 stub->emit_code(this);
441 }
442
443 return offset;
444 }
445
446
447 int LIR_Assembler::emit_deopt_handler() {
448 // generate code for exception handler
449 address handler_base = __ start_a_stub(deopt_handler_size());
450 if (handler_base == nullptr) {
451 // not enough space left for the handler
452 bailout("deopt handler overflow");
453 return -1;
454 }
455
456 int offset = code_offset();
457
458 Label start;
459 __ bind(start);
460
461 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
462
463 int entry_offset = __ offset();
464
465 __ jmp(start);
466
467 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
468 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
469 "out of bounds read in post-call NOP check");
470 __ end_a_stub();
471
472 return entry_offset;
473 }
474
475 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
476 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
477 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
478 assert(result->fpu() == 0, "result must already be on TOS");
479 }
480 if (InlineTypeReturnedAsFields) {
481 #ifndef _LP64
482 Unimplemented();
483 #endif
484 // Check if we are returning an non-null inline type and load its fields into registers
485 ciType* return_type = compilation()->method()->return_type();
486 if (return_type->is_inlinetype()) {
487 ciInlineKlass* vk = return_type->as_inline_klass();
488 if (vk->can_be_returned_as_fields()) {
489 address unpack_handler = vk->unpack_handler();
490 assert(unpack_handler != nullptr, "must be");
491 __ call(RuntimeAddress(unpack_handler));
492 }
493 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
494 Label skip;
495 Label not_null;
496 __ testptr(rax, rax);
497 __ jcc(Assembler::notZero, not_null);
498 // Returned value is null, zero all return registers because they may belong to oop fields
499 __ xorq(j_rarg1, j_rarg1);
500 __ xorq(j_rarg2, j_rarg2);
501 __ xorq(j_rarg3, j_rarg3);
502 __ xorq(j_rarg4, j_rarg4);
503 __ xorq(j_rarg5, j_rarg5);
504 __ jmp(skip);
505 __ bind(not_null);
506
507 // Check if we are returning an non-null inline type and load its fields into registers
508 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
509
510 // Load fields from a buffered value with an inline class specific handler
511 __ load_klass(rdi, rax, rscratch1);
512 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
513 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
514 // Unpack handler can be null if inline type is not scalarizable in returns
515 __ testptr(rdi, rdi);
516 __ jcc(Assembler::zero, skip);
517 __ call(rdi);
518
519 __ bind(skip);
520 }
521 // At this point, rax points to the value object (for interpreter or C1 caller).
522 // The fields of the object are copied into registers (for C2 caller).
523 }
524
525 // Pop the stack before the safepoint code
526 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
527
528 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
529 __ reserved_stack_check();
530 }
531
532 // Note: we do not need to round double result; float result has the right precision
533 // the poll sets the condition code, but no data registers
534
535 code_stub->set_safepoint_offset(__ offset());
536 __ relocate(relocInfo::poll_return_type);
537 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
538 __ ret(0);
539 }
540
541
542 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
543 return (__ store_inline_type_fields_to_buf(vk, false));
544 }
545
546 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
547 guarantee(info != nullptr, "Shouldn't be null");
548 int offset = __ offset();
549 const Register poll_addr = rscratch1;
550 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
551 add_debug_info_for_branch(info);
552 __ relocate(relocInfo::poll_type);
553 address pre_pc = __ pc();
554 __ testl(rax, Address(poll_addr, 0));
555 address post_pc = __ pc();
556 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
557 return offset;
558 }
559
560
561 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
562 if (from_reg != to_reg) __ mov(to_reg, from_reg);
563 }
564
565 void LIR_Assembler::swap_reg(Register a, Register b) {
566 __ xchgptr(a, b);
567 }
568
569
570 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
571 assert(src->is_constant(), "should not call otherwise");
572 assert(dest->is_register(), "should not call otherwise");
573 LIR_Const* c = src->as_constant_ptr();
574
575 switch (c->type()) {
576 case T_INT: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ movl(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_ADDRESS: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ movptr(dest->as_register(), c->as_jint());
585 break;
586 }
587
588 case T_LONG: {
589 assert(patch_code == lir_patch_none, "no patching handled here");
590 #if INCLUDE_CDS
591 if (AOTCodeCache::is_on_for_dump()) {
592 address b = c->as_pointer();
593 if (AOTRuntimeConstants::contains(b)) {
594 __ load_aotrc_address(dest->as_register_lo(), b);
595 break;
596 }
597 }
598 #endif
599 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
600 break;
601 }
602
603 case T_OBJECT: {
604 if (patch_code != lir_patch_none) {
605 jobject2reg_with_patching(dest->as_register(), info);
606 } else {
607 __ movoop(dest->as_register(), c->as_jobject());
608 }
609 break;
610 }
611
612 case T_METADATA: {
613 if (patch_code != lir_patch_none) {
614 klass2reg_with_patching(dest->as_register(), info);
615 } else {
616 __ mov_metadata(dest->as_register(), c->as_metadata());
617 }
618 break;
619 }
620
621 case T_FLOAT: {
622 if (dest->is_single_xmm()) {
623 if (UseAVX <= 2 && c->is_zero_float()) {
624 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
625 } else {
626 __ movflt(dest->as_xmm_float_reg(),
627 InternalAddress(float_constant(c->as_jfloat())));
628 }
629 } else {
630 ShouldNotReachHere();
631 }
632 break;
633 }
634
635 case T_DOUBLE: {
636 if (dest->is_double_xmm()) {
637 if (UseAVX <= 2 && c->is_zero_double()) {
638 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
639 } else {
640 __ movdbl(dest->as_xmm_double_reg(),
641 InternalAddress(double_constant(c->as_jdouble())));
642 }
643 } else {
644 ShouldNotReachHere();
645 }
646 break;
647 }
648
649 default:
650 ShouldNotReachHere();
651 }
652 }
653
654 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
655 assert(src->is_constant(), "should not call otherwise");
656 assert(dest->is_stack(), "should not call otherwise");
657 LIR_Const* c = src->as_constant_ptr();
658
659 switch (c->type()) {
660 case T_INT: // fall through
661 case T_FLOAT:
662 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
663 break;
664
665 case T_ADDRESS:
666 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
667 break;
668
669 case T_OBJECT:
670 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
671 break;
672
673 case T_LONG: // fall through
674 case T_DOUBLE:
675 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
676 lo_word_offset_in_bytes),
677 (intptr_t)c->as_jlong_bits(),
678 rscratch1);
679 break;
680
681 default:
682 ShouldNotReachHere();
683 }
684 }
685
686 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
687 assert(src->is_constant(), "should not call otherwise");
688 assert(dest->is_address(), "should not call otherwise");
689 LIR_Const* c = src->as_constant_ptr();
690 LIR_Address* addr = dest->as_address_ptr();
691
692 int null_check_here = code_offset();
693 switch (type) {
694 case T_INT: // fall through
695 case T_FLOAT:
696 __ movl(as_Address(addr), c->as_jint_bits());
697 break;
698
699 case T_ADDRESS:
700 __ movptr(as_Address(addr), c->as_jint_bits());
701 break;
702
703 case T_OBJECT: // fall through
704 case T_ARRAY:
705 if (c->as_jobject() == nullptr) {
706 if (UseCompressedOops && !wide) {
707 __ movl(as_Address(addr), NULL_WORD);
708 } else {
709 __ xorptr(rscratch1, rscratch1);
710 null_check_here = code_offset();
711 __ movptr(as_Address(addr), rscratch1);
712 }
713 } else {
714 if (is_literal_address(addr)) {
715 ShouldNotReachHere();
716 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
717 } else {
718 __ movoop(rscratch1, c->as_jobject());
719 if (UseCompressedOops && !wide) {
720 __ encode_heap_oop(rscratch1);
721 null_check_here = code_offset();
722 __ movl(as_Address_lo(addr), rscratch1);
723 } else {
724 null_check_here = code_offset();
725 __ movptr(as_Address_lo(addr), rscratch1);
726 }
727 }
728 }
729 break;
730
731 case T_LONG: // fall through
732 case T_DOUBLE:
733 if (is_literal_address(addr)) {
734 ShouldNotReachHere();
735 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
736 } else {
737 __ movptr(r10, (intptr_t)c->as_jlong_bits());
738 null_check_here = code_offset();
739 __ movptr(as_Address_lo(addr), r10);
740 }
741 break;
742
743 case T_BOOLEAN: // fall through
744 case T_BYTE:
745 __ movb(as_Address(addr), c->as_jint() & 0xFF);
746 break;
747
748 case T_CHAR: // fall through
749 case T_SHORT:
750 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
751 break;
752
753 default:
754 ShouldNotReachHere();
755 };
756
757 if (info != nullptr) {
758 add_debug_info_for_null_check(null_check_here, info);
759 }
760 }
761
762
763 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
764 assert(src->is_register(), "should not call otherwise");
765 assert(dest->is_register(), "should not call otherwise");
766
767 // move between cpu-registers
768 if (dest->is_single_cpu()) {
769 if (src->type() == T_LONG) {
770 // Can do LONG -> OBJECT
771 move_regs(src->as_register_lo(), dest->as_register());
772 return;
773 }
774 assert(src->is_single_cpu(), "must match");
775 if (src->type() == T_OBJECT) {
776 __ verify_oop(src->as_register());
777 }
778 move_regs(src->as_register(), dest->as_register());
779
780 } else if (dest->is_double_cpu()) {
781 if (is_reference_type(src->type())) {
782 // Surprising to me but we can see move of a long to t_object
783 __ verify_oop(src->as_register());
784 move_regs(src->as_register(), dest->as_register_lo());
785 return;
786 }
787 assert(src->is_double_cpu(), "must match");
788 Register f_lo = src->as_register_lo();
789 Register f_hi = src->as_register_hi();
790 Register t_lo = dest->as_register_lo();
791 Register t_hi = dest->as_register_hi();
792 assert(f_hi == f_lo, "must be same");
793 assert(t_hi == t_lo, "must be same");
794 move_regs(f_lo, t_lo);
795
796 // move between xmm-registers
797 } else if (dest->is_single_xmm()) {
798 assert(src->is_single_xmm(), "must match");
799 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
800 } else if (dest->is_double_xmm()) {
801 assert(src->is_double_xmm(), "must match");
802 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
803
804 } else {
805 ShouldNotReachHere();
806 }
807 }
808
809 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
810 assert(src->is_register(), "should not call otherwise");
811 assert(dest->is_stack(), "should not call otherwise");
812
813 if (src->is_single_cpu()) {
814 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
815 if (is_reference_type(type)) {
816 __ verify_oop(src->as_register());
817 __ movptr (dst, src->as_register());
818 } else if (type == T_METADATA || type == T_ADDRESS) {
819 __ movptr (dst, src->as_register());
820 } else {
821 __ movl (dst, src->as_register());
822 }
823
824 } else if (src->is_double_cpu()) {
825 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
826 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
827 __ movptr (dstLO, src->as_register_lo());
828
829 } else if (src->is_single_xmm()) {
830 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
831 __ movflt(dst_addr, src->as_xmm_float_reg());
832
833 } else if (src->is_double_xmm()) {
834 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
835 __ movdbl(dst_addr, src->as_xmm_double_reg());
836
837 } else {
838 ShouldNotReachHere();
839 }
840 }
841
842
843 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
844 LIR_Address* to_addr = dest->as_address_ptr();
845 PatchingStub* patch = nullptr;
846 Register compressed_src = rscratch1;
847
848 if (is_reference_type(type)) {
849 __ verify_oop(src->as_register());
850 if (UseCompressedOops && !wide) {
851 __ movptr(compressed_src, src->as_register());
852 __ encode_heap_oop(compressed_src);
853 if (patch_code != lir_patch_none) {
854 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
855 }
856 }
857 }
858
859 if (patch_code != lir_patch_none) {
860 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
861 Address toa = as_Address(to_addr);
862 assert(toa.disp() != 0, "must have");
863 }
864
865 int null_check_here = code_offset();
866 switch (type) {
867 case T_FLOAT: {
868 assert(src->is_single_xmm(), "not a float");
869 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
870 break;
871 }
872
873 case T_DOUBLE: {
874 assert(src->is_double_xmm(), "not a double");
875 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
876 break;
877 }
878
879 case T_ARRAY: // fall through
880 case T_OBJECT: // fall through
881 if (UseCompressedOops && !wide) {
882 __ movl(as_Address(to_addr), compressed_src);
883 } else {
884 __ movptr(as_Address(to_addr), src->as_register());
885 }
886 break;
887 case T_ADDRESS:
888 __ movptr(as_Address(to_addr), src->as_register());
889 break;
890 case T_INT:
891 __ movl(as_Address(to_addr), src->as_register());
892 break;
893
894 case T_LONG: {
895 Register from_lo = src->as_register_lo();
896 Register from_hi = src->as_register_hi();
897 __ movptr(as_Address_lo(to_addr), from_lo);
898 break;
899 }
900
901 case T_BYTE: // fall through
902 case T_BOOLEAN: {
903 Register src_reg = src->as_register();
904 Address dst_addr = as_Address(to_addr);
905 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
906 __ movb(dst_addr, src_reg);
907 break;
908 }
909
910 case T_CHAR: // fall through
911 case T_SHORT:
912 __ movw(as_Address(to_addr), src->as_register());
913 break;
914
915 default:
916 ShouldNotReachHere();
917 }
918 if (info != nullptr) {
919 add_debug_info_for_null_check(null_check_here, info);
920 }
921
922 if (patch_code != lir_patch_none) {
923 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
924 }
925 }
926
927
928 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
929 assert(src->is_stack(), "should not call otherwise");
930 assert(dest->is_register(), "should not call otherwise");
931
932 if (dest->is_single_cpu()) {
933 if (is_reference_type(type)) {
934 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
935 __ verify_oop(dest->as_register());
936 } else if (type == T_METADATA || type == T_ADDRESS) {
937 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
938 } else {
939 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
940 }
941
942 } else if (dest->is_double_cpu()) {
943 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
944 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
945 __ movptr(dest->as_register_lo(), src_addr_LO);
946
947 } else if (dest->is_single_xmm()) {
948 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
949 __ movflt(dest->as_xmm_float_reg(), src_addr);
950
951 } else if (dest->is_double_xmm()) {
952 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
953 __ movdbl(dest->as_xmm_double_reg(), src_addr);
954
955 } else {
956 ShouldNotReachHere();
957 }
958 }
959
960
961 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
962 if (src->is_single_stack()) {
963 if (is_reference_type(type)) {
964 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
965 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
966 } else {
967 //no pushl on 64bits
968 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
969 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
970 }
971
972 } else if (src->is_double_stack()) {
973 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
974 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
975
976 } else {
977 ShouldNotReachHere();
978 }
979 }
980
981
982 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
983 assert(src->is_address(), "should not call otherwise");
984 assert(dest->is_register(), "should not call otherwise");
985
986 LIR_Address* addr = src->as_address_ptr();
987 Address from_addr = as_Address(addr);
988
989 if (addr->base()->type() == T_OBJECT) {
990 __ verify_oop(addr->base()->as_pointer_register());
991 }
992
993 switch (type) {
994 case T_BOOLEAN: // fall through
995 case T_BYTE: // fall through
996 case T_CHAR: // fall through
997 case T_SHORT:
998 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
999 // on pre P6 processors we may get partial register stalls
1000 // so blow away the value of to_rinfo before loading a
1001 // partial word into it. Do it here so that it precedes
1002 // the potential patch point below.
1003 __ xorptr(dest->as_register(), dest->as_register());
1004 }
1005 break;
1006 default:
1007 break;
1008 }
1009
1010 PatchingStub* patch = nullptr;
1011 if (patch_code != lir_patch_none) {
1012 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1013 assert(from_addr.disp() != 0, "must have");
1014 }
1015 if (info != nullptr) {
1016 add_debug_info_for_null_check_here(info);
1017 }
1018
1019 switch (type) {
1020 case T_FLOAT: {
1021 if (dest->is_single_xmm()) {
1022 __ movflt(dest->as_xmm_float_reg(), from_addr);
1023 } else {
1024 ShouldNotReachHere();
1025 }
1026 break;
1027 }
1028
1029 case T_DOUBLE: {
1030 if (dest->is_double_xmm()) {
1031 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1032 } else {
1033 ShouldNotReachHere();
1034 }
1035 break;
1036 }
1037
1038 case T_OBJECT: // fall through
1039 case T_ARRAY: // fall through
1040 if (UseCompressedOops && !wide) {
1041 __ movl(dest->as_register(), from_addr);
1042 } else {
1043 __ movptr(dest->as_register(), from_addr);
1044 }
1045 break;
1046
1047 case T_ADDRESS:
1048 __ movptr(dest->as_register(), from_addr);
1049 break;
1050 case T_INT:
1051 __ movl(dest->as_register(), from_addr);
1052 break;
1053
1054 case T_LONG: {
1055 Register to_lo = dest->as_register_lo();
1056 Register to_hi = dest->as_register_hi();
1057 __ movptr(to_lo, as_Address_lo(addr));
1058 break;
1059 }
1060
1061 case T_BOOLEAN: // fall through
1062 case T_BYTE: {
1063 Register dest_reg = dest->as_register();
1064 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1065 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1066 __ movsbl(dest_reg, from_addr);
1067 } else {
1068 __ movb(dest_reg, from_addr);
1069 __ shll(dest_reg, 24);
1070 __ sarl(dest_reg, 24);
1071 }
1072 break;
1073 }
1074
1075 case T_CHAR: {
1076 Register dest_reg = dest->as_register();
1077 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1078 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1079 __ movzwl(dest_reg, from_addr);
1080 } else {
1081 __ movw(dest_reg, from_addr);
1082 }
1083 break;
1084 }
1085
1086 case T_SHORT: {
1087 Register dest_reg = dest->as_register();
1088 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1089 __ movswl(dest_reg, from_addr);
1090 } else {
1091 __ movw(dest_reg, from_addr);
1092 __ shll(dest_reg, 16);
1093 __ sarl(dest_reg, 16);
1094 }
1095 break;
1096 }
1097
1098 default:
1099 ShouldNotReachHere();
1100 }
1101
1102 if (patch != nullptr) {
1103 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1104 }
1105
1106 if (is_reference_type(type)) {
1107 if (UseCompressedOops && !wide) {
1108 __ decode_heap_oop(dest->as_register());
1109 }
1110
1111 __ verify_oop(dest->as_register());
1112 }
1113 }
1114
1115
1116 NEEDS_CLEANUP; // This could be static?
1117 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1118 int elem_size = type2aelembytes(type);
1119 switch (elem_size) {
1120 case 1: return Address::times_1;
1121 case 2: return Address::times_2;
1122 case 4: return Address::times_4;
1123 case 8: return Address::times_8;
1124 }
1125 ShouldNotReachHere();
1126 return Address::no_scale;
1127 }
1128
1129
1130 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1131 switch (op->code()) {
1132 case lir_idiv:
1133 case lir_irem:
1134 arithmetic_idiv(op->code(),
1135 op->in_opr1(),
1136 op->in_opr2(),
1137 op->in_opr3(),
1138 op->result_opr(),
1139 op->info());
1140 break;
1141 case lir_fmad:
1142 __ fmad(op->result_opr()->as_xmm_double_reg(),
1143 op->in_opr1()->as_xmm_double_reg(),
1144 op->in_opr2()->as_xmm_double_reg(),
1145 op->in_opr3()->as_xmm_double_reg());
1146 break;
1147 case lir_fmaf:
1148 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1149 op->in_opr1()->as_xmm_float_reg(),
1150 op->in_opr2()->as_xmm_float_reg(),
1151 op->in_opr3()->as_xmm_float_reg());
1152 break;
1153 default: ShouldNotReachHere(); break;
1154 }
1155 }
1156
1157 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1158 #ifdef ASSERT
1159 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1160 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1161 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1162 #endif
1163
1164 if (op->cond() == lir_cond_always) {
1165 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1166 __ jmp (*(op->label()));
1167 } else {
1168 Assembler::Condition acond = Assembler::zero;
1169 if (op->code() == lir_cond_float_branch) {
1170 assert(op->ublock() != nullptr, "must have unordered successor");
1171 __ jcc(Assembler::parity, *(op->ublock()->label()));
1172 switch(op->cond()) {
1173 case lir_cond_equal: acond = Assembler::equal; break;
1174 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1175 case lir_cond_less: acond = Assembler::below; break;
1176 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1177 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1178 case lir_cond_greater: acond = Assembler::above; break;
1179 default: ShouldNotReachHere();
1180 }
1181 } else {
1182 switch (op->cond()) {
1183 case lir_cond_equal: acond = Assembler::equal; break;
1184 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1185 case lir_cond_less: acond = Assembler::less; break;
1186 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1187 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1188 case lir_cond_greater: acond = Assembler::greater; break;
1189 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1190 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1191 default: ShouldNotReachHere();
1192 }
1193 }
1194 __ jcc(acond,*(op->label()));
1195 }
1196 }
1197
1198 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1199 LIR_Opr src = op->in_opr();
1200 LIR_Opr dest = op->result_opr();
1201
1202 switch (op->bytecode()) {
1203 case Bytecodes::_i2l:
1204 __ movl2ptr(dest->as_register_lo(), src->as_register());
1205 break;
1206
1207 case Bytecodes::_l2i:
1208 __ movl(dest->as_register(), src->as_register_lo());
1209 break;
1210
1211 case Bytecodes::_i2b:
1212 move_regs(src->as_register(), dest->as_register());
1213 __ sign_extend_byte(dest->as_register());
1214 break;
1215
1216 case Bytecodes::_i2c:
1217 move_regs(src->as_register(), dest->as_register());
1218 __ andl(dest->as_register(), 0xFFFF);
1219 break;
1220
1221 case Bytecodes::_i2s:
1222 move_regs(src->as_register(), dest->as_register());
1223 __ sign_extend_short(dest->as_register());
1224 break;
1225
1226 case Bytecodes::_f2d:
1227 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1228 break;
1229
1230 case Bytecodes::_d2f:
1231 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1232 break;
1233
1234 case Bytecodes::_i2f:
1235 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1236 break;
1237
1238 case Bytecodes::_i2d:
1239 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1240 break;
1241
1242 case Bytecodes::_l2f:
1243 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1244 break;
1245
1246 case Bytecodes::_l2d:
1247 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1248 break;
1249
1250 case Bytecodes::_f2i:
1251 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1252 break;
1253
1254 case Bytecodes::_d2i:
1255 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1256 break;
1257
1258 case Bytecodes::_f2l:
1259 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1260 break;
1261
1262 case Bytecodes::_d2l:
1263 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1264 break;
1265
1266 default: ShouldNotReachHere();
1267 }
1268 }
1269
1270 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1271 if (op->init_check()) {
1272 add_debug_info_for_null_check_here(op->stub()->info());
1273 // init_state needs acquire, but x86 is TSO, and so we are already good.
1274 __ cmpb(Address(op->klass()->as_register(),
1275 InstanceKlass::init_state_offset()),
1276 InstanceKlass::fully_initialized);
1277 __ jcc(Assembler::notEqual, *op->stub()->entry());
1278 }
1279 __ allocate_object(op->obj()->as_register(),
1280 op->tmp1()->as_register(),
1281 op->tmp2()->as_register(),
1282 op->header_size(),
1283 op->object_size(),
1284 op->klass()->as_register(),
1285 *op->stub()->entry());
1286 __ bind(*op->stub()->continuation());
1287 }
1288
1289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1290 Register len = op->len()->as_register();
1291 __ movslq(len, len);
1292
1293 if (UseSlowPath || op->always_slow_path() ||
1294 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1295 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1296 __ jmp(*op->stub()->entry());
1297 } else {
1298 Register tmp1 = op->tmp1()->as_register();
1299 Register tmp2 = op->tmp2()->as_register();
1300 Register tmp3 = op->tmp3()->as_register();
1301 if (len == tmp1) {
1302 tmp1 = tmp3;
1303 } else if (len == tmp2) {
1304 tmp2 = tmp3;
1305 } else if (len == tmp3) {
1306 // everything is ok
1307 } else {
1308 __ mov(tmp3, len);
1309 }
1310 __ allocate_array(op->obj()->as_register(),
1311 len,
1312 tmp1,
1313 tmp2,
1314 arrayOopDesc::base_offset_in_bytes(op->type()),
1315 array_element_size(op->type()),
1316 op->klass()->as_register(),
1317 *op->stub()->entry(),
1318 op->zero_array());
1319 }
1320 __ bind(*op->stub()->continuation());
1321 }
1322
1323 void LIR_Assembler::type_profile_helper(Register mdo,
1324 ciMethodData *md, ciProfileData *data,
1325 Register recv) {
1326 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1327 __ profile_receiver_type(recv, mdo, mdp_offset);
1328 }
1329
1330 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1331 // we always need a stub for the failure case.
1332 CodeStub* stub = op->stub();
1333 Register obj = op->object()->as_register();
1334 Register k_RInfo = op->tmp1()->as_register();
1335 Register klass_RInfo = op->tmp2()->as_register();
1336 Register dst = op->result_opr()->as_register();
1337 ciKlass* k = op->klass();
1338 Register Rtmp1 = noreg;
1339 Register tmp_load_klass = rscratch1;
1340
1341 // check if it needs to be profiled
1342 ciMethodData* md = nullptr;
1343 ciProfileData* data = nullptr;
1344
1345 if (op->should_profile()) {
1346 ciMethod* method = op->profiled_method();
1347 assert(method != nullptr, "Should have method");
1348 int bci = op->profiled_bci();
1349 md = method->method_data_or_null();
1350 assert(md != nullptr, "Sanity");
1351 data = md->bci_to_data(bci);
1352 assert(data != nullptr, "need data for type check");
1353 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1354 }
1355 Label* success_target = success;
1356 Label* failure_target = failure;
1357
1358 if (obj == k_RInfo) {
1359 k_RInfo = dst;
1360 } else if (obj == klass_RInfo) {
1361 klass_RInfo = dst;
1362 }
1363 if (k->is_loaded() && !UseCompressedClassPointers) {
1364 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1365 } else {
1366 Rtmp1 = op->tmp3()->as_register();
1367 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1368 }
1369
1370 assert_different_registers(obj, k_RInfo, klass_RInfo);
1371
1372 if (op->need_null_check()) {
1373 __ testptr(obj, obj);
1374 if (op->should_profile()) {
1375 Label not_null;
1376 Register mdo = klass_RInfo;
1377 __ mov_metadata(mdo, md->constant_encoding());
1378 __ jccb(Assembler::notEqual, not_null);
1379 // Object is null; update MDO and exit
1380 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1381 int header_bits = BitData::null_seen_byte_constant();
1382 __ orb(data_addr, header_bits);
1383 __ jmp(*obj_is_null);
1384 __ bind(not_null);
1385
1386 Register recv = k_RInfo;
1387 __ load_klass(recv, obj, tmp_load_klass);
1388 type_profile_helper(mdo, md, data, recv);
1389 } else {
1390 __ jcc(Assembler::equal, *obj_is_null);
1391 }
1392 }
1393
1394 if (!k->is_loaded()) {
1395 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1396 } else {
1397 __ mov_metadata(k_RInfo, k->constant_encoding());
1398 }
1399 __ verify_oop(obj);
1400
1401 if (op->fast_check()) {
1402 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1403 // get object class
1404 // not a safepoint as obj null check happens earlier
1405 if (UseCompressedClassPointers) {
1406 __ load_klass(Rtmp1, obj, tmp_load_klass);
1407 __ cmpptr(k_RInfo, Rtmp1);
1408 } else {
1409 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1410 }
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431 __ cmpptr(klass_RInfo, tmp_load_klass);
1432 } else {
1433 __ cmpptr(klass_RInfo, k_RInfo);
1434 }
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1455 __ pop_ppx(k_RInfo);
1456 // result is a boolean
1457 __ testl(k_RInfo, k_RInfo);
1458 __ jcc(Assembler::equal, *failure_target);
1459 // successful cast, fall through to profile or jump
1460 }
1461 }
1462 __ jmp(*success);
1463 }
1464
1465
1466 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1467 Register tmp_load_klass = rscratch1;
1468 LIR_Code code = op->code();
1469 if (code == lir_store_check) {
1470 Register value = op->object()->as_register();
1471 Register array = op->array()->as_register();
1472 Register k_RInfo = op->tmp1()->as_register();
1473 Register klass_RInfo = op->tmp2()->as_register();
1474 Register Rtmp1 = op->tmp3()->as_register();
1475
1476 CodeStub* stub = op->stub();
1477
1478 // check if it needs to be profiled
1479 ciMethodData* md = nullptr;
1480 ciProfileData* data = nullptr;
1481
1482 if (op->should_profile()) {
1483 ciMethod* method = op->profiled_method();
1484 assert(method != nullptr, "Should have method");
1485 int bci = op->profiled_bci();
1486 md = method->method_data_or_null();
1487 assert(md != nullptr, "Sanity");
1488 data = md->bci_to_data(bci);
1489 assert(data != nullptr, "need data for type check");
1490 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1491 }
1492 Label done;
1493 Label* success_target = &done;
1494 Label* failure_target = stub->entry();
1495
1496 __ testptr(value, value);
1497 if (op->should_profile()) {
1498 Label not_null;
1499 Register mdo = klass_RInfo;
1500 __ mov_metadata(mdo, md->constant_encoding());
1501 __ jccb(Assembler::notEqual, not_null);
1502 // Object is null; update MDO and exit
1503 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1504 int header_bits = BitData::null_seen_byte_constant();
1505 __ orb(data_addr, header_bits);
1506 __ jmp(done);
1507 __ bind(not_null);
1508
1509 Register recv = k_RInfo;
1510 __ load_klass(recv, value, tmp_load_klass);
1511 type_profile_helper(mdo, md, data, recv);
1512 } else {
1513 __ jcc(Assembler::equal, done);
1514 }
1515
1516 add_debug_info_for_null_check_here(op->info_for_exception());
1517 __ load_klass(k_RInfo, array, tmp_load_klass);
1518 __ load_klass(klass_RInfo, value, tmp_load_klass);
1519
1520 // get instance klass (it's already uncompressed)
1521 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1522 // perform the fast part of the checking logic
1523 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1524 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1525 __ push_ppx(klass_RInfo);
1526 __ push_ppx(k_RInfo);
1527 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1528 __ pop_ppx(klass_RInfo);
1529 __ pop_ppx(k_RInfo);
1530 // result is a boolean
1531 __ testl(k_RInfo, k_RInfo);
1532 __ jcc(Assembler::equal, *failure_target);
1533 // fall through to the success case
1534
1535 __ bind(done);
1536 } else
1537 if (code == lir_checkcast) {
1538 Register obj = op->object()->as_register();
1539 Register dst = op->result_opr()->as_register();
1540 Label success;
1541 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1542 __ bind(success);
1543 if (dst != obj) {
1544 __ mov(dst, obj);
1545 }
1546 } else
1547 if (code == lir_instanceof) {
1548 Register obj = op->object()->as_register();
1549 Register dst = op->result_opr()->as_register();
1550 Label success, failure, done;
1551 emit_typecheck_helper(op, &success, &failure, &failure);
1552 __ bind(failure);
1553 __ xorptr(dst, dst);
1554 __ jmpb(done);
1555 __ bind(success);
1556 __ movptr(dst, 1);
1557 __ bind(done);
1558 } else {
1559 ShouldNotReachHere();
1560 }
1561
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be a flat array (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is a flat array, take the slow path.
1568 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 if (!op->value()->is_illegal()) {
1570 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1571 // The array is not a flat array, but it might be null-free. If we are storing
1572 // a null into a null-free array, take the slow path (which will throw NPE).
1573 Label skip;
1574 __ cmpptr(op->value()->as_register(), NULL_WORD);
1575 __ jcc(Assembler::notEqual, skip);
1576 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1577 __ bind(skip);
1578 }
1579 }
1580
1581 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1582 // We are storing into an array that *may* be null-free (the declared type is
1583 // Object[], abstract[], interface[] or VT.ref[]).
1584 Label test_mark_word;
1585 Register tmp = op->tmp()->as_register();
1586 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1587 __ testl(tmp, markWord::unlocked_value);
1588 __ jccb(Assembler::notZero, test_mark_word);
1589 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1590 __ bind(test_mark_word);
1591 __ testl(tmp, markWord::null_free_array_bit_in_place);
1592 }
1593
1594 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1595 Label L_oops_equal;
1596 Label L_oops_not_equal;
1597 Label L_end;
1598
1599 Register left = op->left()->as_register();
1600 Register right = op->right()->as_register();
1601
1602 __ cmpptr(left, right);
1603 __ jcc(Assembler::equal, L_oops_equal);
1604
1605 // (1) Null check -- if one of the operands is null, the other must not be null (because
1606 // the two references are not equal), so they are not substitutable,
1607 // FIXME: do null check only if the operand is nullable
1608 __ testptr(left, right);
1609 __ jcc(Assembler::zero, L_oops_not_equal);
1610
1611 ciKlass* left_klass = op->left_klass();
1612 ciKlass* right_klass = op->right_klass();
1613
1614 // (2) Inline type check -- if either of the operands is not a inline type,
1615 // they are not substitutable. We do this only if we are not sure that the
1616 // operands are inline type
1617 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1618 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1619 Register tmp1 = op->tmp1()->as_register();
1620 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1621 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1622 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1623 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1624 __ jcc(Assembler::notEqual, L_oops_not_equal);
1625 }
1626
1627 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1628 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1629 // No need to load klass -- the operands are statically known to be the same inline klass.
1630 __ jmp(*op->stub()->entry());
1631 } else {
1632 Register left_klass_op = op->left_klass_op()->as_register();
1633 Register right_klass_op = op->right_klass_op()->as_register();
1634
1635 if (UseCompressedClassPointers) {
1636 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1637 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1638 __ cmpl(left_klass_op, right_klass_op);
1639 } else {
1640 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1641 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1642 __ cmpptr(left_klass_op, right_klass_op);
1643 }
1644
1645 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1646 // fall through to L_oops_not_equal
1647 }
1648
1649 __ bind(L_oops_not_equal);
1650 move(op->not_equal_result(), op->result_opr());
1651 __ jmp(L_end);
1652
1653 __ bind(L_oops_equal);
1654 move(op->equal_result(), op->result_opr());
1655 __ jmp(L_end);
1656
1657 // We've returned from the stub. RAX contains 0x0 IFF the two
1658 // operands are not substitutable. (Don't compare against 0x1 in case the
1659 // C compiler is naughty)
1660 __ bind(*op->stub()->continuation());
1661 __ cmpl(rax, 0);
1662 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1663 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1664 // fall-through
1665 __ bind(L_end);
1666 }
1667
1668 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1669 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1670 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1671 Register newval = op->new_value()->as_register();
1672 Register cmpval = op->cmp_value()->as_register();
1673 assert(cmpval == rax, "wrong register");
1674 assert(newval != noreg, "new val must be register");
1675 assert(cmpval != newval, "cmp and new values must be in different registers");
1676 assert(cmpval != addr, "cmp and addr must be in different registers");
1677 assert(newval != addr, "new value and addr must be in different registers");
1678
1679 if (op->code() == lir_cas_obj) {
1680 if (UseCompressedOops) {
1681 __ encode_heap_oop(cmpval);
1682 __ mov(rscratch1, newval);
1683 __ encode_heap_oop(rscratch1);
1684 __ lock();
1685 // cmpval (rax) is implicitly used by this instruction
1686 __ cmpxchgl(rscratch1, Address(addr, 0));
1687 } else {
1688 __ lock();
1689 __ cmpxchgptr(newval, Address(addr, 0));
1690 }
1691 } else {
1692 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1693 __ lock();
1694 __ cmpxchgl(newval, Address(addr, 0));
1695 }
1696 } else if (op->code() == lir_cas_long) {
1697 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1698 Register newval = op->new_value()->as_register_lo();
1699 Register cmpval = op->cmp_value()->as_register_lo();
1700 assert(cmpval == rax, "wrong register");
1701 assert(newval != noreg, "new val must be register");
1702 assert(cmpval != newval, "cmp and new values must be in different registers");
1703 assert(cmpval != addr, "cmp and addr must be in different registers");
1704 assert(newval != addr, "new value and addr must be in different registers");
1705 __ lock();
1706 __ cmpxchgq(newval, Address(addr, 0));
1707 } else {
1708 Unimplemented();
1709 }
1710 }
1711
1712 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1713 assert(dst->is_cpu_register(), "must be");
1714 assert(dst->type() == src->type(), "must be");
1715
1716 if (src->is_cpu_register()) {
1717 reg2reg(src, dst);
1718 } else if (src->is_stack()) {
1719 stack2reg(src, dst, dst->type());
1720 } else if (src->is_constant()) {
1721 const2reg(src, dst, lir_patch_none, nullptr);
1722 } else {
1723 ShouldNotReachHere();
1724 }
1725 }
1726
1727 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1728 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1729 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1730
1731 Assembler::Condition acond, ncond;
1732 switch (condition) {
1733 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1734 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1735 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1736 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1737 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1738 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1739 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1740 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1741 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1742 ShouldNotReachHere();
1743 }
1744
1745 if (opr1->is_cpu_register()) {
1746 reg2reg(opr1, result);
1747 } else if (opr1->is_stack()) {
1748 stack2reg(opr1, result, result->type());
1749 } else if (opr1->is_constant()) {
1750 const2reg(opr1, result, lir_patch_none, nullptr);
1751 } else {
1752 ShouldNotReachHere();
1753 }
1754
1755 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1756 // optimized version that does not require a branch
1757 if (opr2->is_single_cpu()) {
1758 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1759 __ cmov(ncond, result->as_register(), opr2->as_register());
1760 } else if (opr2->is_double_cpu()) {
1761 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1762 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1763 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1764 } else if (opr2->is_single_stack()) {
1765 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1766 } else if (opr2->is_double_stack()) {
1767 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1768 } else {
1769 ShouldNotReachHere();
1770 }
1771
1772 } else {
1773 Label skip;
1774 __ jccb(acond, skip);
1775 if (opr2->is_cpu_register()) {
1776 reg2reg(opr2, result);
1777 } else if (opr2->is_stack()) {
1778 stack2reg(opr2, result, result->type());
1779 } else if (opr2->is_constant()) {
1780 const2reg(opr2, result, lir_patch_none, nullptr);
1781 } else {
1782 ShouldNotReachHere();
1783 }
1784 __ bind(skip);
1785 }
1786 }
1787
1788
1789 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1790 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1791
1792 if (left->is_single_cpu()) {
1793 assert(left == dest, "left and dest must be equal");
1794 Register lreg = left->as_register();
1795
1796 if (right->is_single_cpu()) {
1797 // cpu register - cpu register
1798 Register rreg = right->as_register();
1799 switch (code) {
1800 case lir_add: __ addl (lreg, rreg); break;
1801 case lir_sub: __ subl (lreg, rreg); break;
1802 case lir_mul: __ imull(lreg, rreg); break;
1803 default: ShouldNotReachHere();
1804 }
1805
1806 } else if (right->is_stack()) {
1807 // cpu register - stack
1808 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1809 switch (code) {
1810 case lir_add: __ addl(lreg, raddr); break;
1811 case lir_sub: __ subl(lreg, raddr); break;
1812 default: ShouldNotReachHere();
1813 }
1814
1815 } else if (right->is_constant()) {
1816 // cpu register - constant
1817 jint c = right->as_constant_ptr()->as_jint();
1818 switch (code) {
1819 case lir_add: {
1820 __ incrementl(lreg, c);
1821 break;
1822 }
1823 case lir_sub: {
1824 __ decrementl(lreg, c);
1825 break;
1826 }
1827 default: ShouldNotReachHere();
1828 }
1829
1830 } else {
1831 ShouldNotReachHere();
1832 }
1833
1834 } else if (left->is_double_cpu()) {
1835 assert(left == dest, "left and dest must be equal");
1836 Register lreg_lo = left->as_register_lo();
1837 Register lreg_hi = left->as_register_hi();
1838
1839 if (right->is_double_cpu()) {
1840 // cpu register - cpu register
1841 Register rreg_lo = right->as_register_lo();
1842 Register rreg_hi = right->as_register_hi();
1843 assert_different_registers(lreg_lo, rreg_lo);
1844 switch (code) {
1845 case lir_add:
1846 __ addptr(lreg_lo, rreg_lo);
1847 break;
1848 case lir_sub:
1849 __ subptr(lreg_lo, rreg_lo);
1850 break;
1851 case lir_mul:
1852 __ imulq(lreg_lo, rreg_lo);
1853 break;
1854 default:
1855 ShouldNotReachHere();
1856 }
1857
1858 } else if (right->is_constant()) {
1859 // cpu register - constant
1860 jlong c = right->as_constant_ptr()->as_jlong_bits();
1861 __ movptr(r10, (intptr_t) c);
1862 switch (code) {
1863 case lir_add:
1864 __ addptr(lreg_lo, r10);
1865 break;
1866 case lir_sub:
1867 __ subptr(lreg_lo, r10);
1868 break;
1869 default:
1870 ShouldNotReachHere();
1871 }
1872
1873 } else {
1874 ShouldNotReachHere();
1875 }
1876
1877 } else if (left->is_single_xmm()) {
1878 assert(left == dest, "left and dest must be equal");
1879 XMMRegister lreg = left->as_xmm_float_reg();
1880
1881 if (right->is_single_xmm()) {
1882 XMMRegister rreg = right->as_xmm_float_reg();
1883 switch (code) {
1884 case lir_add: __ addss(lreg, rreg); break;
1885 case lir_sub: __ subss(lreg, rreg); break;
1886 case lir_mul: __ mulss(lreg, rreg); break;
1887 case lir_div: __ divss(lreg, rreg); break;
1888 default: ShouldNotReachHere();
1889 }
1890 } else {
1891 Address raddr;
1892 if (right->is_single_stack()) {
1893 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1894 } else if (right->is_constant()) {
1895 // hack for now
1896 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1897 } else {
1898 ShouldNotReachHere();
1899 }
1900 switch (code) {
1901 case lir_add: __ addss(lreg, raddr); break;
1902 case lir_sub: __ subss(lreg, raddr); break;
1903 case lir_mul: __ mulss(lreg, raddr); break;
1904 case lir_div: __ divss(lreg, raddr); break;
1905 default: ShouldNotReachHere();
1906 }
1907 }
1908
1909 } else if (left->is_double_xmm()) {
1910 assert(left == dest, "left and dest must be equal");
1911
1912 XMMRegister lreg = left->as_xmm_double_reg();
1913 if (right->is_double_xmm()) {
1914 XMMRegister rreg = right->as_xmm_double_reg();
1915 switch (code) {
1916 case lir_add: __ addsd(lreg, rreg); break;
1917 case lir_sub: __ subsd(lreg, rreg); break;
1918 case lir_mul: __ mulsd(lreg, rreg); break;
1919 case lir_div: __ divsd(lreg, rreg); break;
1920 default: ShouldNotReachHere();
1921 }
1922 } else {
1923 Address raddr;
1924 if (right->is_double_stack()) {
1925 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1926 } else if (right->is_constant()) {
1927 // hack for now
1928 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1929 } else {
1930 ShouldNotReachHere();
1931 }
1932 switch (code) {
1933 case lir_add: __ addsd(lreg, raddr); break;
1934 case lir_sub: __ subsd(lreg, raddr); break;
1935 case lir_mul: __ mulsd(lreg, raddr); break;
1936 case lir_div: __ divsd(lreg, raddr); break;
1937 default: ShouldNotReachHere();
1938 }
1939 }
1940
1941 } else if (left->is_single_stack() || left->is_address()) {
1942 assert(left == dest, "left and dest must be equal");
1943
1944 Address laddr;
1945 if (left->is_single_stack()) {
1946 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1947 } else if (left->is_address()) {
1948 laddr = as_Address(left->as_address_ptr());
1949 } else {
1950 ShouldNotReachHere();
1951 }
1952
1953 if (right->is_single_cpu()) {
1954 Register rreg = right->as_register();
1955 switch (code) {
1956 case lir_add: __ addl(laddr, rreg); break;
1957 case lir_sub: __ subl(laddr, rreg); break;
1958 default: ShouldNotReachHere();
1959 }
1960 } else if (right->is_constant()) {
1961 jint c = right->as_constant_ptr()->as_jint();
1962 switch (code) {
1963 case lir_add: {
1964 __ incrementl(laddr, c);
1965 break;
1966 }
1967 case lir_sub: {
1968 __ decrementl(laddr, c);
1969 break;
1970 }
1971 default: ShouldNotReachHere();
1972 }
1973 } else {
1974 ShouldNotReachHere();
1975 }
1976
1977 } else {
1978 ShouldNotReachHere();
1979 }
1980 }
1981
1982
1983 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1984 if (value->is_double_xmm()) {
1985 switch(code) {
1986 case lir_abs :
1987 {
1988 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1989 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1990 }
1991 assert(!tmp->is_valid(), "do not need temporary");
1992 __ andpd(dest->as_xmm_double_reg(),
1993 ExternalAddress((address)double_signmask_pool),
1994 rscratch1);
1995 }
1996 break;
1997
1998 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1999 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2000 default : ShouldNotReachHere();
2001 }
2002
2003 } else if (code == lir_f2hf) {
2004 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2005 } else if (code == lir_hf2f) {
2006 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2007 } else {
2008 Unimplemented();
2009 }
2010 }
2011
2012 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2013 // assert(left->destroys_register(), "check");
2014 if (left->is_single_cpu()) {
2015 Register reg = left->as_register();
2016 if (right->is_constant()) {
2017 int val = right->as_constant_ptr()->as_jint();
2018 switch (code) {
2019 case lir_logic_and: __ andl (reg, val); break;
2020 case lir_logic_or: __ orl (reg, val); break;
2021 case lir_logic_xor: __ xorl (reg, val); break;
2022 default: ShouldNotReachHere();
2023 }
2024 } else if (right->is_stack()) {
2025 // added support for stack operands
2026 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2027 switch (code) {
2028 case lir_logic_and: __ andl (reg, raddr); break;
2029 case lir_logic_or: __ orl (reg, raddr); break;
2030 case lir_logic_xor: __ xorl (reg, raddr); break;
2031 default: ShouldNotReachHere();
2032 }
2033 } else {
2034 Register rright = right->as_register();
2035 switch (code) {
2036 case lir_logic_and: __ andptr (reg, rright); break;
2037 case lir_logic_or : __ orptr (reg, rright); break;
2038 case lir_logic_xor: __ xorptr (reg, rright); break;
2039 default: ShouldNotReachHere();
2040 }
2041 }
2042 move_regs(reg, dst->as_register());
2043 } else {
2044 Register l_lo = left->as_register_lo();
2045 Register l_hi = left->as_register_hi();
2046 if (right->is_constant()) {
2047 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2048 switch (code) {
2049 case lir_logic_and:
2050 __ andq(l_lo, rscratch1);
2051 break;
2052 case lir_logic_or:
2053 __ orq(l_lo, rscratch1);
2054 break;
2055 case lir_logic_xor:
2056 __ xorq(l_lo, rscratch1);
2057 break;
2058 default: ShouldNotReachHere();
2059 }
2060 } else {
2061 Register r_lo;
2062 if (is_reference_type(right->type())) {
2063 r_lo = right->as_register();
2064 } else {
2065 r_lo = right->as_register_lo();
2066 }
2067 switch (code) {
2068 case lir_logic_and:
2069 __ andptr(l_lo, r_lo);
2070 break;
2071 case lir_logic_or:
2072 __ orptr(l_lo, r_lo);
2073 break;
2074 case lir_logic_xor:
2075 __ xorptr(l_lo, r_lo);
2076 break;
2077 default: ShouldNotReachHere();
2078 }
2079 }
2080
2081 Register dst_lo = dst->as_register_lo();
2082 Register dst_hi = dst->as_register_hi();
2083
2084 move_regs(l_lo, dst_lo);
2085 }
2086 }
2087
2088
2089 // we assume that rax, and rdx can be overwritten
2090 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2091
2092 assert(left->is_single_cpu(), "left must be register");
2093 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2094 assert(result->is_single_cpu(), "result must be register");
2095
2096 // assert(left->destroys_register(), "check");
2097 // assert(right->destroys_register(), "check");
2098
2099 Register lreg = left->as_register();
2100 Register dreg = result->as_register();
2101
2102 if (right->is_constant()) {
2103 jint divisor = right->as_constant_ptr()->as_jint();
2104 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2105 if (code == lir_idiv) {
2106 assert(lreg == rax, "must be rax,");
2107 assert(temp->as_register() == rdx, "tmp register must be rdx");
2108 __ cdql(); // sign extend into rdx:rax
2109 if (divisor == 2) {
2110 __ subl(lreg, rdx);
2111 } else {
2112 __ andl(rdx, divisor - 1);
2113 __ addl(lreg, rdx);
2114 }
2115 __ sarl(lreg, log2i_exact(divisor));
2116 move_regs(lreg, dreg);
2117 } else if (code == lir_irem) {
2118 Label done;
2119 __ mov(dreg, lreg);
2120 __ andl(dreg, 0x80000000 | (divisor - 1));
2121 __ jcc(Assembler::positive, done);
2122 __ decrement(dreg);
2123 __ orl(dreg, ~(divisor - 1));
2124 __ increment(dreg);
2125 __ bind(done);
2126 } else {
2127 ShouldNotReachHere();
2128 }
2129 } else {
2130 Register rreg = right->as_register();
2131 assert(lreg == rax, "left register must be rax,");
2132 assert(rreg != rdx, "right register must not be rdx");
2133 assert(temp->as_register() == rdx, "tmp register must be rdx");
2134
2135 move_regs(lreg, rax);
2136
2137 int idivl_offset = __ corrected_idivl(rreg);
2138 if (ImplicitDiv0Checks) {
2139 add_debug_info_for_div0(idivl_offset, info);
2140 }
2141 if (code == lir_irem) {
2142 move_regs(rdx, dreg); // result is in rdx
2143 } else {
2144 move_regs(rax, dreg);
2145 }
2146 }
2147 }
2148
2149
2150 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2151 if (opr1->is_single_cpu()) {
2152 Register reg1 = opr1->as_register();
2153 if (opr2->is_single_cpu()) {
2154 // cpu register - cpu register
2155 if (is_reference_type(opr1->type())) {
2156 __ cmpoop(reg1, opr2->as_register());
2157 } else {
2158 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2159 __ cmpl(reg1, opr2->as_register());
2160 }
2161 } else if (opr2->is_stack()) {
2162 // cpu register - stack
2163 if (is_reference_type(opr1->type())) {
2164 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2165 } else {
2166 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2167 }
2168 } else if (opr2->is_constant()) {
2169 // cpu register - constant
2170 LIR_Const* c = opr2->as_constant_ptr();
2171 if (c->type() == T_INT) {
2172 jint i = c->as_jint();
2173 if (i == 0) {
2174 __ testl(reg1, reg1);
2175 } else {
2176 __ cmpl(reg1, i);
2177 }
2178 } else if (c->type() == T_METADATA) {
2179 // All we need for now is a comparison with null for equality.
2180 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2181 Metadata* m = c->as_metadata();
2182 if (m == nullptr) {
2183 __ testptr(reg1, reg1);
2184 } else {
2185 ShouldNotReachHere();
2186 }
2187 } else if (is_reference_type(c->type())) {
2188 // In 64bit oops are single register
2189 jobject o = c->as_jobject();
2190 if (o == nullptr) {
2191 __ testptr(reg1, reg1);
2192 } else {
2193 __ cmpoop(reg1, o, rscratch1);
2194 }
2195 } else {
2196 fatal("unexpected type: %s", basictype_to_str(c->type()));
2197 }
2198 // cpu register - address
2199 } else if (opr2->is_address()) {
2200 if (op->info() != nullptr) {
2201 add_debug_info_for_null_check_here(op->info());
2202 }
2203 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2204 } else {
2205 ShouldNotReachHere();
2206 }
2207
2208 } else if(opr1->is_double_cpu()) {
2209 Register xlo = opr1->as_register_lo();
2210 Register xhi = opr1->as_register_hi();
2211 if (opr2->is_double_cpu()) {
2212 __ cmpptr(xlo, opr2->as_register_lo());
2213 } else if (opr2->is_constant()) {
2214 // cpu register - constant 0
2215 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2216 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2217 } else {
2218 ShouldNotReachHere();
2219 }
2220
2221 } else if (opr1->is_single_xmm()) {
2222 XMMRegister reg1 = opr1->as_xmm_float_reg();
2223 if (opr2->is_single_xmm()) {
2224 // xmm register - xmm register
2225 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2226 } else if (opr2->is_stack()) {
2227 // xmm register - stack
2228 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2229 } else if (opr2->is_constant()) {
2230 // xmm register - constant
2231 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2232 } else if (opr2->is_address()) {
2233 // xmm register - address
2234 if (op->info() != nullptr) {
2235 add_debug_info_for_null_check_here(op->info());
2236 }
2237 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2238 } else {
2239 ShouldNotReachHere();
2240 }
2241
2242 } else if (opr1->is_double_xmm()) {
2243 XMMRegister reg1 = opr1->as_xmm_double_reg();
2244 if (opr2->is_double_xmm()) {
2245 // xmm register - xmm register
2246 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2247 } else if (opr2->is_stack()) {
2248 // xmm register - stack
2249 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2250 } else if (opr2->is_constant()) {
2251 // xmm register - constant
2252 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2253 } else if (opr2->is_address()) {
2254 // xmm register - address
2255 if (op->info() != nullptr) {
2256 add_debug_info_for_null_check_here(op->info());
2257 }
2258 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2259 } else {
2260 ShouldNotReachHere();
2261 }
2262
2263 } else if (opr1->is_address() && opr2->is_constant()) {
2264 LIR_Const* c = opr2->as_constant_ptr();
2265 if (is_reference_type(c->type())) {
2266 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2267 __ movoop(rscratch1, c->as_jobject());
2268 }
2269 if (op->info() != nullptr) {
2270 add_debug_info_for_null_check_here(op->info());
2271 }
2272 // special case: address - constant
2273 LIR_Address* addr = opr1->as_address_ptr();
2274 if (c->type() == T_INT) {
2275 __ cmpl(as_Address(addr), c->as_jint());
2276 } else if (is_reference_type(c->type())) {
2277 // %%% Make this explode if addr isn't reachable until we figure out a
2278 // better strategy by giving noreg as the temp for as_Address
2279 __ cmpoop(rscratch1, as_Address(addr, noreg));
2280 } else {
2281 ShouldNotReachHere();
2282 }
2283
2284 } else {
2285 ShouldNotReachHere();
2286 }
2287 }
2288
2289 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2290 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2291 if (left->is_single_xmm()) {
2292 assert(right->is_single_xmm(), "must match");
2293 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2294 } else if (left->is_double_xmm()) {
2295 assert(right->is_double_xmm(), "must match");
2296 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2297
2298 } else {
2299 ShouldNotReachHere();
2300 }
2301 } else {
2302 assert(code == lir_cmp_l2i, "check");
2303 Label done;
2304 Register dest = dst->as_register();
2305 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2306 __ movl(dest, -1);
2307 __ jccb(Assembler::less, done);
2308 __ setb(Assembler::notZero, dest);
2309 __ movzbl(dest, dest);
2310 __ bind(done);
2311 }
2312 }
2313
2314
2315 void LIR_Assembler::align_call(LIR_Code code) {
2316 // make sure that the displacement word of the call ends up word aligned
2317 int offset = __ offset();
2318 switch (code) {
2319 case lir_static_call:
2320 case lir_optvirtual_call:
2321 case lir_dynamic_call:
2322 offset += NativeCall::displacement_offset;
2323 break;
2324 case lir_icvirtual_call:
2325 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2326 break;
2327 default: ShouldNotReachHere();
2328 }
2329 __ align(BytesPerWord, offset);
2330 }
2331
2332
2333 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2334 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2335 "must be aligned");
2336 __ call(AddressLiteral(op->addr(), rtype));
2337 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2338 __ post_call_nop();
2339 }
2340
2341
2342 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2343 __ ic_call(op->addr());
2344 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2345 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2346 "must be aligned");
2347 __ post_call_nop();
2348 }
2349
2350
2351 void LIR_Assembler::emit_static_call_stub() {
2352 address call_pc = __ pc();
2353 address stub = __ start_a_stub(call_stub_size());
2354 if (stub == nullptr) {
2355 bailout("static call stub overflow");
2356 return;
2357 }
2358
2359 int start = __ offset();
2360
2361 // make sure that the displacement word of the call ends up word aligned
2362 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2363 __ relocate(static_stub_Relocation::spec(call_pc));
2364 __ mov_metadata(rbx, (Metadata*)nullptr);
2365 // must be set to -1 at code generation time
2366 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2367 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2368 __ jump(RuntimeAddress(__ pc()));
2369
2370 assert(__ offset() - start <= call_stub_size(), "stub too big");
2371 __ end_a_stub();
2372 }
2373
2374
2375 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2376 assert(exceptionOop->as_register() == rax, "must match");
2377 assert(exceptionPC->as_register() == rdx, "must match");
2378
2379 // exception object is not added to oop map by LinearScan
2380 // (LinearScan assumes that no oops are in fixed registers)
2381 info->add_register_oop(exceptionOop);
2382 StubId unwind_id;
2383
2384 // get current pc information
2385 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2386 int pc_for_athrow_offset = __ offset();
2387 InternalAddress pc_for_athrow(__ pc());
2388 __ lea(exceptionPC->as_register(), pc_for_athrow);
2389 add_call_info(pc_for_athrow_offset, info); // for exception handler
2390
2391 __ verify_not_null_oop(rax);
2392 // search an exception handler (rax: exception oop, rdx: throwing pc)
2393 if (compilation()->has_fpu_code()) {
2394 unwind_id = StubId::c1_handle_exception_id;
2395 } else {
2396 unwind_id = StubId::c1_handle_exception_nofpu_id;
2397 }
2398 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2399
2400 // enough room for two byte trap
2401 __ nop();
2402 }
2403
2404
2405 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2406 assert(exceptionOop->as_register() == rax, "must match");
2407
2408 __ jmp(_unwind_handler_entry);
2409 }
2410
2411
2412 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2413
2414 // optimized version for linear scan:
2415 // * count must be already in ECX (guaranteed by LinearScan)
2416 // * left and dest must be equal
2417 // * tmp must be unused
2418 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2419 assert(left == dest, "left and dest must be equal");
2420 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2421
2422 if (left->is_single_cpu()) {
2423 Register value = left->as_register();
2424 assert(value != SHIFT_count, "left cannot be ECX");
2425
2426 switch (code) {
2427 case lir_shl: __ shll(value); break;
2428 case lir_shr: __ sarl(value); break;
2429 case lir_ushr: __ shrl(value); break;
2430 default: ShouldNotReachHere();
2431 }
2432 } else if (left->is_double_cpu()) {
2433 Register lo = left->as_register_lo();
2434 Register hi = left->as_register_hi();
2435 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2436 switch (code) {
2437 case lir_shl: __ shlptr(lo); break;
2438 case lir_shr: __ sarptr(lo); break;
2439 case lir_ushr: __ shrptr(lo); break;
2440 default: ShouldNotReachHere();
2441 }
2442 } else {
2443 ShouldNotReachHere();
2444 }
2445 }
2446
2447
2448 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2449 if (dest->is_single_cpu()) {
2450 // first move left into dest so that left is not destroyed by the shift
2451 Register value = dest->as_register();
2452 count = count & 0x1F; // Java spec
2453
2454 move_regs(left->as_register(), value);
2455 switch (code) {
2456 case lir_shl: __ shll(value, count); break;
2457 case lir_shr: __ sarl(value, count); break;
2458 case lir_ushr: __ shrl(value, count); break;
2459 default: ShouldNotReachHere();
2460 }
2461 } else if (dest->is_double_cpu()) {
2462 // first move left into dest so that left is not destroyed by the shift
2463 Register value = dest->as_register_lo();
2464 count = count & 0x1F; // Java spec
2465
2466 move_regs(left->as_register_lo(), value);
2467 switch (code) {
2468 case lir_shl: __ shlptr(value, count); break;
2469 case lir_shr: __ sarptr(value, count); break;
2470 case lir_ushr: __ shrptr(value, count); break;
2471 default: ShouldNotReachHere();
2472 }
2473 } else {
2474 ShouldNotReachHere();
2475 }
2476 }
2477
2478
2479 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2480 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2481 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2482 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2483 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2484 }
2485
2486
2487 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2488 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2489 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2490 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2491 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2492 }
2493
2494
2495 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2496 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2497 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2498 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2499 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2500 }
2501
2502
2503 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2504 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2505 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2506 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2507 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2508 }
2509
2510
2511 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2512 if (null_check) {
2513 __ testptr(obj, obj);
2514 __ jcc(Assembler::zero, *slow_path->entry());
2515 }
2516 if (is_dest) {
2517 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2518 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2519 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2520 } else {
2521 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2522 }
2523 }
2524
2525
2526 // This code replaces a call to arraycopy; no exception may
2527 // be thrown in this code, they must be thrown in the System.arraycopy
2528 // activation frame; we could save some checks if this would not be the case
2529 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2530 ciArrayKlass* default_type = op->expected_type();
2531 Register src = op->src()->as_register();
2532 Register dst = op->dst()->as_register();
2533 Register src_pos = op->src_pos()->as_register();
2534 Register dst_pos = op->dst_pos()->as_register();
2535 Register length = op->length()->as_register();
2536 Register tmp = op->tmp()->as_register();
2537 Register tmp_load_klass = rscratch1;
2538 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2539
2540 CodeStub* stub = op->stub();
2541 int flags = op->flags();
2542 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2543 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2544
2545 if (flags & LIR_OpArrayCopy::always_slow_path) {
2546 __ jmp(*stub->entry());
2547 __ bind(*stub->continuation());
2548 return;
2549 }
2550
2551 // if we don't know anything, just go through the generic arraycopy
2552 if (default_type == nullptr) {
2553 // save outgoing arguments on stack in case call to System.arraycopy is needed
2554 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2555 // for interpreter calling conventions. Now we have to do it in new style conventions.
2556 // For the moment until C1 gets the new register allocator I just force all the
2557 // args to the right place (except the register args) and then on the back side
2558 // reload the register args properly if we go slow path. Yuck
2559
2560 // These are proper for the calling convention
2561 store_parameter(length, 2);
2562 store_parameter(dst_pos, 1);
2563 store_parameter(dst, 0);
2564
2565 // these are just temporary placements until we need to reload
2566 store_parameter(src_pos, 3);
2567 store_parameter(src, 4);
2568
2569 address copyfunc_addr = StubRoutines::generic_arraycopy();
2570 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2571
2572 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2573 // The arguments are in java calling convention so we can trivially shift them to C
2574 // convention
2575 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2576 __ mov(c_rarg0, j_rarg0);
2577 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2578 __ mov(c_rarg1, j_rarg1);
2579 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2580 __ mov(c_rarg2, j_rarg2);
2581 assert_different_registers(c_rarg3, j_rarg4);
2582 __ mov(c_rarg3, j_rarg3);
2583 #ifdef _WIN64
2584 // Allocate abi space for args but be sure to keep stack aligned
2585 __ subptr(rsp, 6*wordSize);
2586 store_parameter(j_rarg4, 4);
2587 #ifndef PRODUCT
2588 if (PrintC1Statistics) {
2589 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2590 }
2591 #endif
2592 __ call(RuntimeAddress(copyfunc_addr));
2593 __ addptr(rsp, 6*wordSize);
2594 #else
2595 __ mov(c_rarg4, j_rarg4);
2596 #ifndef PRODUCT
2597 if (PrintC1Statistics) {
2598 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2599 }
2600 #endif
2601 __ call(RuntimeAddress(copyfunc_addr));
2602 #endif // _WIN64
2603
2604 __ testl(rax, rax);
2605 __ jcc(Assembler::equal, *stub->continuation());
2606
2607 __ mov(tmp, rax);
2608 __ xorl(tmp, -1);
2609
2610 // Reload values from the stack so they are where the stub
2611 // expects them.
2612 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2613 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2614 __ movptr (length, Address(rsp, 2*BytesPerWord));
2615 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2616 __ movptr (src, Address(rsp, 4*BytesPerWord));
2617
2618 __ subl(length, tmp);
2619 __ addl(src_pos, tmp);
2620 __ addl(dst_pos, tmp);
2621 __ jmp(*stub->entry());
2622
2623 __ bind(*stub->continuation());
2624 return;
2625 }
2626
2627 // Handle inline type arrays
2628 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2629 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2630 }
2631 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2632 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2633 }
2634
2635 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2636
2637 int elem_size = type2aelembytes(basic_type);
2638 Address::ScaleFactor scale;
2639
2640 switch (elem_size) {
2641 case 1 :
2642 scale = Address::times_1;
2643 break;
2644 case 2 :
2645 scale = Address::times_2;
2646 break;
2647 case 4 :
2648 scale = Address::times_4;
2649 break;
2650 case 8 :
2651 scale = Address::times_8;
2652 break;
2653 default:
2654 scale = Address::no_scale;
2655 ShouldNotReachHere();
2656 }
2657
2658 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2659 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2660
2661 // length and pos's are all sign extended at this point on 64bit
2662
2663 // test for null
2664 if (flags & LIR_OpArrayCopy::src_null_check) {
2665 __ testptr(src, src);
2666 __ jcc(Assembler::zero, *stub->entry());
2667 }
2668 if (flags & LIR_OpArrayCopy::dst_null_check) {
2669 __ testptr(dst, dst);
2670 __ jcc(Assembler::zero, *stub->entry());
2671 }
2672
2673 // If the compiler was not able to prove that exact type of the source or the destination
2674 // of the arraycopy is an array type, check at runtime if the source or the destination is
2675 // an instance type.
2676 if (flags & LIR_OpArrayCopy::type_check) {
2677 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2678 __ load_klass(tmp, dst, tmp_load_klass);
2679 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2680 __ jcc(Assembler::greaterEqual, *stub->entry());
2681 }
2682
2683 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2684 __ load_klass(tmp, src, tmp_load_klass);
2685 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2686 __ jcc(Assembler::greaterEqual, *stub->entry());
2687 }
2688 }
2689
2690 // check if negative
2691 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2692 __ testl(src_pos, src_pos);
2693 __ jcc(Assembler::less, *stub->entry());
2694 }
2695 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2696 __ testl(dst_pos, dst_pos);
2697 __ jcc(Assembler::less, *stub->entry());
2698 }
2699
2700 if (flags & LIR_OpArrayCopy::src_range_check) {
2701 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2702 __ cmpl(tmp, src_length_addr);
2703 __ jcc(Assembler::above, *stub->entry());
2704 }
2705 if (flags & LIR_OpArrayCopy::dst_range_check) {
2706 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2707 __ cmpl(tmp, dst_length_addr);
2708 __ jcc(Assembler::above, *stub->entry());
2709 }
2710
2711 if (flags & LIR_OpArrayCopy::length_positive_check) {
2712 __ testl(length, length);
2713 __ jcc(Assembler::less, *stub->entry());
2714 }
2715
2716 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2717 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2718
2719 if (flags & LIR_OpArrayCopy::type_check) {
2720 // We don't know the array types are compatible
2721 if (basic_type != T_OBJECT) {
2722 // Simple test for basic type arrays
2723 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2724 __ jcc(Assembler::notEqual, *stub->entry());
2725 } else {
2726 // For object arrays, if src is a sub class of dst then we can
2727 // safely do the copy.
2728 Label cont, slow;
2729
2730 __ push_ppx(src);
2731 __ push_ppx(dst);
2732
2733 __ load_klass(src, src, tmp_load_klass);
2734 __ load_klass(dst, dst, tmp_load_klass);
2735
2736 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2737
2738 __ push_ppx(src);
2739 __ push_ppx(dst);
2740 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2741 __ pop_ppx(dst);
2742 __ pop_ppx(src);
2743
2744 __ testl(src, src);
2745 __ jcc(Assembler::notEqual, cont);
2746
2747 __ bind(slow);
2748 __ pop_ppx(dst);
2749 __ pop_ppx(src);
2750
2751 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2752 if (copyfunc_addr != nullptr) { // use stub if available
2753 // src is not a sub class of dst so we have to do a
2754 // per-element check.
2755
2756 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2757 if ((flags & mask) != mask) {
2758 // Check that at least both of them object arrays.
2759 assert(flags & mask, "one of the two should be known to be an object array");
2760
2761 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2762 __ load_klass(tmp, src, tmp_load_klass);
2763 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2764 __ load_klass(tmp, dst, tmp_load_klass);
2765 }
2766 int lh_offset = in_bytes(Klass::layout_helper_offset());
2767 Address klass_lh_addr(tmp, lh_offset);
2768 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2769 __ cmpl(klass_lh_addr, objArray_lh);
2770 __ jcc(Assembler::notEqual, *stub->entry());
2771 }
2772
2773 // Spill because stubs can use any register they like and it's
2774 // easier to restore just those that we care about.
2775 store_parameter(dst, 0);
2776 store_parameter(dst_pos, 1);
2777 store_parameter(length, 2);
2778 store_parameter(src_pos, 3);
2779 store_parameter(src, 4);
2780
2781 __ movl2ptr(length, length); //higher 32bits must be null
2782
2783 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2784 assert_different_registers(c_rarg0, dst, dst_pos, length);
2785 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2786 assert_different_registers(c_rarg1, dst, length);
2787
2788 __ mov(c_rarg2, length);
2789 assert_different_registers(c_rarg2, dst);
2790
2791 #ifdef _WIN64
2792 // Allocate abi space for args but be sure to keep stack aligned
2793 __ subptr(rsp, 6*wordSize);
2794 __ load_klass(c_rarg3, dst, tmp_load_klass);
2795 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2796 store_parameter(c_rarg3, 4);
2797 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2798 __ call(RuntimeAddress(copyfunc_addr));
2799 __ addptr(rsp, 6*wordSize);
2800 #else
2801 __ load_klass(c_rarg4, dst, tmp_load_klass);
2802 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2803 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2804 __ call(RuntimeAddress(copyfunc_addr));
2805 #endif
2806
2807 #ifndef PRODUCT
2808 if (PrintC1Statistics) {
2809 Label failed;
2810 __ testl(rax, rax);
2811 __ jcc(Assembler::notZero, failed);
2812 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2813 __ bind(failed);
2814 }
2815 #endif
2816
2817 __ testl(rax, rax);
2818 __ jcc(Assembler::zero, *stub->continuation());
2819
2820 #ifndef PRODUCT
2821 if (PrintC1Statistics) {
2822 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2823 }
2824 #endif
2825
2826 __ mov(tmp, rax);
2827
2828 __ xorl(tmp, -1);
2829
2830 // Restore previously spilled arguments
2831 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2832 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2833 __ movptr (length, Address(rsp, 2*BytesPerWord));
2834 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2835 __ movptr (src, Address(rsp, 4*BytesPerWord));
2836
2837
2838 __ subl(length, tmp);
2839 __ addl(src_pos, tmp);
2840 __ addl(dst_pos, tmp);
2841 }
2842
2843 __ jmp(*stub->entry());
2844
2845 __ bind(cont);
2846 __ pop(dst);
2847 __ pop(src);
2848 }
2849 }
2850
2851 #ifdef ASSERT
2852 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2853 // Sanity check the known type with the incoming class. For the
2854 // primitive case the types must match exactly with src.klass and
2855 // dst.klass each exactly matching the default type. For the
2856 // object array case, if no type check is needed then either the
2857 // dst type is exactly the expected type and the src type is a
2858 // subtype which we can't check or src is the same array as dst
2859 // but not necessarily exactly of type default_type.
2860 Label known_ok, halt;
2861 __ mov_metadata(tmp, default_type->constant_encoding());
2862 if (UseCompressedClassPointers) {
2863 __ encode_klass_not_null(tmp, rscratch1);
2864 }
2865
2866 if (basic_type != T_OBJECT) {
2867 __ cmp_klass(tmp, dst, tmp2);
2868 __ jcc(Assembler::notEqual, halt);
2869 __ cmp_klass(tmp, src, tmp2);
2870 __ jcc(Assembler::equal, known_ok);
2871 } else {
2872 __ cmp_klass(tmp, dst, tmp2);
2873 __ jcc(Assembler::equal, known_ok);
2874 __ cmpptr(src, dst);
2875 __ jcc(Assembler::equal, known_ok);
2876 }
2877 __ bind(halt);
2878 __ stop("incorrect type information in arraycopy");
2879 __ bind(known_ok);
2880 }
2881 #endif
2882
2883 #ifndef PRODUCT
2884 if (PrintC1Statistics) {
2885 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2886 }
2887 #endif
2888
2889 assert_different_registers(c_rarg0, dst, dst_pos, length);
2890 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2891 assert_different_registers(c_rarg1, length);
2892 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2893 __ mov(c_rarg2, length);
2894
2895 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2896 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2897 const char *name;
2898 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2899 __ call_VM_leaf(entry, 0);
2900
2901 if (stub != nullptr) {
2902 __ bind(*stub->continuation());
2903 }
2904 }
2905
2906 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2907 assert(op->crc()->is_single_cpu(), "crc must be register");
2908 assert(op->val()->is_single_cpu(), "byte value must be register");
2909 assert(op->result_opr()->is_single_cpu(), "result must be register");
2910 Register crc = op->crc()->as_register();
2911 Register val = op->val()->as_register();
2912 Register res = op->result_opr()->as_register();
2913
2914 assert_different_registers(val, crc, res);
2915
2916 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2917 __ notl(crc); // ~crc
2918 __ update_byte_crc32(crc, val, res);
2919 __ notl(crc); // ~crc
2920 __ mov(res, crc);
2921 }
2922
2923 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2924 Register obj = op->obj_opr()->as_register(); // may not be an oop
2925 Register hdr = op->hdr_opr()->as_register();
2926 Register lock = op->lock_opr()->as_register();
2927 if (op->code() == lir_lock) {
2928 Register tmp = op->scratch_opr()->as_register();
2929 // add debug info for NullPointerException only if one is possible
2930 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2931 if (op->info() != nullptr) {
2932 add_debug_info_for_null_check(null_check_offset, op->info());
2933 }
2934 // done
2935 } else if (op->code() == lir_unlock) {
2936 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2937 } else {
2938 Unimplemented();
2939 }
2940 __ bind(*op->stub()->continuation());
2941 }
2942
2943 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2944 Register obj = op->obj()->as_pointer_register();
2945 Register result = op->result_opr()->as_pointer_register();
2946
2947 CodeEmitInfo* info = op->info();
2948 if (info != nullptr) {
2949 add_debug_info_for_null_check_here(info);
2950 }
2951
2952 __ load_klass(result, obj, rscratch1);
2953 }
2954
2955 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2956 ciMethod* method = op->profiled_method();
2957 int bci = op->profiled_bci();
2958 ciMethod* callee = op->profiled_callee();
2959 Register tmp_load_klass = rscratch1;
2960
2961 // Update counter for all call types
2962 ciMethodData* md = method->method_data_or_null();
2963 assert(md != nullptr, "Sanity");
2964 ciProfileData* data = md->bci_to_data(bci);
2965 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2966 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2967 Register mdo = op->mdo()->as_register();
2968 __ mov_metadata(mdo, md->constant_encoding());
2969 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2970 // Perform additional virtual call profiling for invokevirtual and
2971 // invokeinterface bytecodes
2972 if (op->should_profile_receiver_type()) {
2973 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2974 Register recv = op->recv()->as_register();
2975 assert_different_registers(mdo, recv);
2976 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2977 ciKlass* known_klass = op->known_holder();
2978 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2979 // We know the type that will be seen at this call site; we can
2980 // statically update the MethodData* rather than needing to do
2981 // dynamic tests on the receiver type.
2982 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2983 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2984 ciKlass* receiver = vc_data->receiver(i);
2985 if (known_klass->equals(receiver)) {
2986 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2987 __ addptr(data_addr, DataLayout::counter_increment);
2988 return;
2989 }
2990 }
2991 // Receiver type is not found in profile data.
2992 // Fall back to runtime helper to handle the rest at runtime.
2993 __ mov_metadata(recv, known_klass->constant_encoding());
2994 } else {
2995 __ load_klass(recv, recv, tmp_load_klass);
2996 }
2997 type_profile_helper(mdo, md, data, recv);
2998 } else {
2999 // Static call
3000 __ addptr(counter_addr, DataLayout::counter_increment);
3001 }
3002 }
3003
3004 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3005 Register obj = op->obj()->as_register();
3006 Register tmp = op->tmp()->as_pointer_register();
3007 Register tmp_load_klass = rscratch1;
3008 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3009 ciKlass* exact_klass = op->exact_klass();
3010 intptr_t current_klass = op->current_klass();
3011 bool not_null = op->not_null();
3012 bool no_conflict = op->no_conflict();
3013
3014 Label update, next, none;
3015
3016 bool do_null = !not_null;
3017 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3018 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3019
3020 assert(do_null || do_update, "why are we here?");
3021 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3022
3023 __ verify_oop(obj);
3024
3025 #ifdef ASSERT
3026 if (obj == tmp) {
3027 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3028 } else {
3029 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3030 }
3031 #endif
3032 if (do_null) {
3033 __ testptr(obj, obj);
3034 __ jccb(Assembler::notZero, update);
3035 if (!TypeEntries::was_null_seen(current_klass)) {
3036 __ testptr(mdo_addr, TypeEntries::null_seen);
3037 #ifndef ASSERT
3038 __ jccb(Assembler::notZero, next); // already set
3039 #else
3040 __ jcc(Assembler::notZero, next); // already set
3041 #endif
3042 // atomic update to prevent overwriting Klass* with 0
3043 __ lock();
3044 __ orptr(mdo_addr, TypeEntries::null_seen);
3045 }
3046 if (do_update) {
3047 #ifndef ASSERT
3048 __ jmpb(next);
3049 }
3050 #else
3051 __ jmp(next);
3052 }
3053 } else {
3054 __ testptr(obj, obj);
3055 __ jcc(Assembler::notZero, update);
3056 __ stop("unexpected null obj");
3057 #endif
3058 }
3059
3060 __ bind(update);
3061
3062 if (do_update) {
3063 #ifdef ASSERT
3064 if (exact_klass != nullptr) {
3065 Label ok;
3066 __ load_klass(tmp, obj, tmp_load_klass);
3067 __ push_ppx(tmp);
3068 __ mov_metadata(tmp, exact_klass->constant_encoding());
3069 __ cmpptr(tmp, Address(rsp, 0));
3070 __ jcc(Assembler::equal, ok);
3071 __ stop("exact klass and actual klass differ");
3072 __ bind(ok);
3073 __ pop_ppx(tmp);
3074 }
3075 #endif
3076 if (!no_conflict) {
3077 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3078 if (exact_klass != nullptr) {
3079 __ mov_metadata(tmp, exact_klass->constant_encoding());
3080 } else {
3081 __ load_klass(tmp, obj, tmp_load_klass);
3082 }
3083 __ mov(rscratch1, tmp); // save original value before XOR
3084 __ xorptr(tmp, mdo_addr);
3085 __ testptr(tmp, TypeEntries::type_klass_mask);
3086 // klass seen before, nothing to do. The unknown bit may have been
3087 // set already but no need to check.
3088 __ jccb(Assembler::zero, next);
3089
3090 __ testptr(tmp, TypeEntries::type_unknown);
3091 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3092
3093 if (TypeEntries::is_type_none(current_klass)) {
3094 __ testptr(mdo_addr, TypeEntries::type_mask);
3095 __ jccb(Assembler::zero, none);
3096 // There is a chance that the checks above (re-reading profiling
3097 // data from memory) fail if another thread has just set the
3098 // profiling to this obj's klass
3099 __ mov(tmp, rscratch1); // get back original value before XOR
3100 __ xorptr(tmp, mdo_addr);
3101 __ testptr(tmp, TypeEntries::type_klass_mask);
3102 __ jccb(Assembler::zero, next);
3103 }
3104 } else {
3105 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3106 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3107
3108 __ testptr(mdo_addr, TypeEntries::type_unknown);
3109 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3110 }
3111
3112 // different than before. Cannot keep accurate profile.
3113 __ orptr(mdo_addr, TypeEntries::type_unknown);
3114
3115 if (TypeEntries::is_type_none(current_klass)) {
3116 __ jmpb(next);
3117
3118 __ bind(none);
3119 // first time here. Set profile type.
3120 __ movptr(mdo_addr, tmp);
3121 #ifdef ASSERT
3122 __ andptr(tmp, TypeEntries::type_klass_mask);
3123 __ verify_klass_ptr(tmp);
3124 #endif
3125 }
3126 } else {
3127 // There's a single possible klass at this profile point
3128 assert(exact_klass != nullptr, "should be");
3129 if (TypeEntries::is_type_none(current_klass)) {
3130 __ mov_metadata(tmp, exact_klass->constant_encoding());
3131 __ xorptr(tmp, mdo_addr);
3132 __ testptr(tmp, TypeEntries::type_klass_mask);
3133 #ifdef ASSERT
3134 __ jcc(Assembler::zero, next);
3135
3136 {
3137 Label ok;
3138 __ push_ppx(tmp);
3139 __ testptr(mdo_addr, TypeEntries::type_mask);
3140 __ jcc(Assembler::zero, ok);
3141 // may have been set by another thread
3142 __ mov_metadata(tmp, exact_klass->constant_encoding());
3143 __ xorptr(tmp, mdo_addr);
3144 __ testptr(tmp, TypeEntries::type_mask);
3145 __ jcc(Assembler::zero, ok);
3146
3147 __ stop("unexpected profiling mismatch");
3148 __ bind(ok);
3149 __ pop_ppx(tmp);
3150 }
3151 #else
3152 __ jccb(Assembler::zero, next);
3153 #endif
3154 // first time here. Set profile type.
3155 __ movptr(mdo_addr, tmp);
3156 #ifdef ASSERT
3157 __ andptr(tmp, TypeEntries::type_klass_mask);
3158 __ verify_klass_ptr(tmp);
3159 #endif
3160 } else {
3161 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3162 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3163
3164 __ testptr(mdo_addr, TypeEntries::type_unknown);
3165 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3166
3167 __ orptr(mdo_addr, TypeEntries::type_unknown);
3168 }
3169 }
3170 }
3171 __ bind(next);
3172 }
3173
3174 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3175 Register obj = op->obj()->as_register();
3176 Register tmp = op->tmp()->as_pointer_register();
3177 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3178 bool not_null = op->not_null();
3179 int flag = op->flag();
3180
3181 Label not_inline_type;
3182 if (!not_null) {
3183 __ testptr(obj, obj);
3184 __ jccb(Assembler::zero, not_inline_type);
3185 }
3186
3187 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3188
3189 __ orb(mdo_addr, flag);
3190
3191 __ bind(not_inline_type);
3192 }
3193
3194
3195 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3196 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3197 }
3198
3199
3200 void LIR_Assembler::align_backward_branch_target() {
3201 __ align(BytesPerWord);
3202 }
3203
3204
3205 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3206 if (left->is_single_cpu()) {
3207 __ negl(left->as_register());
3208 move_regs(left->as_register(), dest->as_register());
3209
3210 } else if (left->is_double_cpu()) {
3211 Register lo = left->as_register_lo();
3212 Register dst = dest->as_register_lo();
3213 __ movptr(dst, lo);
3214 __ negptr(dst);
3215
3216 } else if (dest->is_single_xmm()) {
3217 assert(!tmp->is_valid(), "do not need temporary");
3218 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3219 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3220 }
3221 __ xorps(dest->as_xmm_float_reg(),
3222 ExternalAddress((address)float_signflip_pool),
3223 rscratch1);
3224 } else if (dest->is_double_xmm()) {
3225 assert(!tmp->is_valid(), "do not need temporary");
3226 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3227 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3228 }
3229 __ xorpd(dest->as_xmm_double_reg(),
3230 ExternalAddress((address)double_signflip_pool),
3231 rscratch1);
3232 } else {
3233 ShouldNotReachHere();
3234 }
3235 }
3236
3237
3238 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3239 assert(src->is_address(), "must be an address");
3240 assert(dest->is_register(), "must be a register");
3241
3242 PatchingStub* patch = nullptr;
3243 if (patch_code != lir_patch_none) {
3244 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3245 }
3246
3247 Register reg = dest->as_pointer_register();
3248 LIR_Address* addr = src->as_address_ptr();
3249 __ lea(reg, as_Address(addr));
3250
3251 if (patch != nullptr) {
3252 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3253 }
3254 }
3255
3256
3257
3258 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3259 assert(!tmp->is_valid(), "don't need temporary");
3260 __ call(RuntimeAddress(dest));
3261 if (info != nullptr) {
3262 add_call_info_here(info);
3263 }
3264 __ post_call_nop();
3265 }
3266
3267
3268 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3269 assert(type == T_LONG, "only for volatile long fields");
3270
3271 if (info != nullptr) {
3272 add_debug_info_for_null_check_here(info);
3273 }
3274
3275 if (src->is_double_xmm()) {
3276 if (dest->is_double_cpu()) {
3277 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3278 } else if (dest->is_double_stack()) {
3279 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3280 } else if (dest->is_address()) {
3281 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3282 } else {
3283 ShouldNotReachHere();
3284 }
3285
3286 } else if (dest->is_double_xmm()) {
3287 if (src->is_double_stack()) {
3288 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3289 } else if (src->is_address()) {
3290 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3291 } else {
3292 ShouldNotReachHere();
3293 }
3294
3295 } else {
3296 ShouldNotReachHere();
3297 }
3298 }
3299
3300 #ifdef ASSERT
3301 // emit run-time assertion
3302 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3303 assert(op->code() == lir_assert, "must be");
3304
3305 if (op->in_opr1()->is_valid()) {
3306 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3307 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3308 } else {
3309 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3310 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3311 }
3312
3313 Label ok;
3314 if (op->condition() != lir_cond_always) {
3315 Assembler::Condition acond = Assembler::zero;
3316 switch (op->condition()) {
3317 case lir_cond_equal: acond = Assembler::equal; break;
3318 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3319 case lir_cond_less: acond = Assembler::less; break;
3320 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3321 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3322 case lir_cond_greater: acond = Assembler::greater; break;
3323 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3324 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3325 default: ShouldNotReachHere();
3326 }
3327 __ jcc(acond, ok);
3328 }
3329 if (op->halt()) {
3330 const char* str = __ code_string(op->msg());
3331 __ stop(str);
3332 } else {
3333 breakpoint();
3334 }
3335 __ bind(ok);
3336 }
3337 #endif
3338
3339 void LIR_Assembler::membar() {
3340 // QQQ sparc TSO uses this,
3341 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3342 }
3343
3344 void LIR_Assembler::membar_acquire() {
3345 // No x86 machines currently require load fences
3346 }
3347
3348 void LIR_Assembler::membar_release() {
3349 // No x86 machines currently require store fences
3350 }
3351
3352 void LIR_Assembler::membar_loadload() {
3353 // no-op
3354 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3355 }
3356
3357 void LIR_Assembler::membar_storestore() {
3358 // no-op
3359 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3360 }
3361
3362 void LIR_Assembler::membar_loadstore() {
3363 // no-op
3364 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3365 }
3366
3367 void LIR_Assembler::membar_storeload() {
3368 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3369 }
3370
3371 void LIR_Assembler::on_spin_wait() {
3372 __ pause ();
3373 }
3374
3375 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3376 assert(result_reg->is_register(), "check");
3377 __ mov(result_reg->as_register(), r15_thread);
3378 }
3379
3380 void LIR_Assembler::check_orig_pc() {
3381 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3382 }
3383
3384 void LIR_Assembler::peephole(LIR_List*) {
3385 // do nothing for now
3386 }
3387
3388 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3389 assert(data == dest, "xchg/xadd uses only 2 operands");
3390
3391 if (data->type() == T_INT) {
3392 if (code == lir_xadd) {
3393 __ lock();
3394 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3395 } else {
3396 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3397 }
3398 } else if (data->is_oop()) {
3399 assert (code == lir_xchg, "xadd for oops");
3400 Register obj = data->as_register();
3401 if (UseCompressedOops) {
3402 __ encode_heap_oop(obj);
3403 __ xchgl(obj, as_Address(src->as_address_ptr()));
3404 __ decode_heap_oop(obj);
3405 } else {
3406 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3407 }
3408 } else if (data->type() == T_LONG) {
3409 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3410 if (code == lir_xadd) {
3411 __ lock();
3412 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3413 } else {
3414 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3415 }
3416 } else {
3417 ShouldNotReachHere();
3418 }
3419 }
3420
3421 #undef __