1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
61 operand[1] = hi;
62 return operand;
63 }
64
65 // Buffer for 128-bits masks used by SSE instructions.
66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
67
68 // Static initialization during VM startup.
69 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
70 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
71 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
72 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
73
74
75 NEEDS_CLEANUP // remove this definitions ?
76 const Register SYNC_header = rax; // synchronization header
77 const Register SHIFT_count = rcx; // where count for shift operations must be
78
79 #define __ _masm->
80
81
82 static void select_different_registers(Register preserve,
83 Register extra,
84 Register &tmp1,
85 Register &tmp2) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, extra);
91 tmp2 = extra;
92 }
93 assert_different_registers(preserve, tmp1, tmp2);
94 }
95
96
97
98 static void select_different_registers(Register preserve,
99 Register extra,
100 Register &tmp1,
101 Register &tmp2,
102 Register &tmp3) {
103 if (tmp1 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp1 = extra;
106 } else if (tmp2 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp2 = extra;
109 } else if (tmp3 == preserve) {
110 assert_different_registers(tmp1, tmp2, tmp3, extra);
111 tmp3 = extra;
112 }
113 assert_different_registers(preserve, tmp1, tmp2, tmp3);
114 }
115
116
117
118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
119 if (opr->is_constant()) {
120 LIR_Const* constant = opr->as_constant_ptr();
121 switch (constant->type()) {
122 case T_INT: {
123 return true;
124 }
125
126 default:
127 return false;
128 }
129 }
130 return false;
131 }
132
133
134 LIR_Opr LIR_Assembler::receiverOpr() {
135 return FrameMap::receiver_opr;
136 }
137
138 LIR_Opr LIR_Assembler::osrBufferPointer() {
139 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
140 }
141
142 //--------------fpu register translations-----------------------
143
144
145 address LIR_Assembler::float_constant(float f) {
146 address const_addr = __ float_constant(f);
147 if (const_addr == nullptr) {
148 bailout("const section overflow");
149 return __ code()->consts()->start();
150 } else {
151 return const_addr;
152 }
153 }
154
155
156 address LIR_Assembler::double_constant(double d) {
157 address const_addr = __ double_constant(d);
158 if (const_addr == nullptr) {
159 bailout("const section overflow");
160 return __ code()->consts()->start();
161 } else {
162 return const_addr;
163 }
164 }
165
166 void LIR_Assembler::breakpoint() {
167 __ int3();
168 }
169
170 void LIR_Assembler::push(LIR_Opr opr) {
171 if (opr->is_single_cpu()) {
172 __ push_reg(opr->as_register());
173 } else if (opr->is_double_cpu()) {
174 __ push_reg(opr->as_register_lo());
175 } else if (opr->is_stack()) {
176 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
177 } else if (opr->is_constant()) {
178 LIR_Const* const_opr = opr->as_constant_ptr();
179 if (const_opr->type() == T_OBJECT) {
180 __ push_oop(const_opr->as_jobject(), rscratch1);
181 } else if (const_opr->type() == T_INT) {
182 __ push_jint(const_opr->as_jint());
183 } else {
184 ShouldNotReachHere();
185 }
186
187 } else {
188 ShouldNotReachHere();
189 }
190 }
191
192 void LIR_Assembler::pop(LIR_Opr opr) {
193 if (opr->is_single_cpu()) {
194 __ pop_reg(opr->as_register());
195 } else {
196 ShouldNotReachHere();
197 }
198 }
199
200 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
201 return addr->base()->is_illegal() && addr->index()->is_illegal();
202 }
203
204 //-------------------------------------------
205
206 Address LIR_Assembler::as_Address(LIR_Address* addr) {
207 return as_Address(addr, rscratch1);
208 }
209
210 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
211 if (addr->base()->is_illegal()) {
212 assert(addr->index()->is_illegal(), "must be illegal too");
213 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
214 if (! __ reachable(laddr)) {
215 __ movptr(tmp, laddr.addr());
216 Address res(tmp, 0);
217 return res;
218 } else {
219 return __ as_Address(laddr);
220 }
221 }
222
223 Register base = addr->base()->as_pointer_register();
224
225 if (addr->index()->is_illegal()) {
226 return Address( base, addr->disp());
227 } else if (addr->index()->is_cpu_register()) {
228 Register index = addr->index()->as_pointer_register();
229 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
230 } else if (addr->index()->is_constant()) {
231 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
232 assert(Assembler::is_simm32(addr_offset), "must be");
233
234 return Address(base, addr_offset);
235 } else {
236 Unimplemented();
237 return Address();
238 }
239 }
240
241
242 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
243 Address base = as_Address(addr);
244 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
245 }
246
247
248 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
249 return as_Address(addr);
250 }
251
252
253 void LIR_Assembler::osr_entry() {
254 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
255 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
256 ValueStack* entry_state = osr_entry->state();
257 int number_of_locks = entry_state->locks_size();
258
259 // we jump here if osr happens with the interpreter
260 // state set up to continue at the beginning of the
261 // loop that triggered osr - in particular, we have
262 // the following registers setup:
263 //
264 // rcx: osr buffer
265 //
266
267 // build frame
268 ciMethod* m = compilation()->method();
269 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
270
271 // OSR buffer is
272 //
273 // locals[nlocals-1..0]
274 // monitors[0..number_of_locks]
275 //
276 // locals is a direct copy of the interpreter frame so in the osr buffer
277 // so first slot in the local array is the last local from the interpreter
278 // and last slot is local[0] (receiver) from the interpreter
279 //
280 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
281 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
282 // in the interpreter frame (the method lock if a sync method)
283
284 // Initialize monitors in the compiled activation.
285 // rcx: pointer to osr buffer
286 //
287 // All other registers are dead at this point and the locals will be
288 // copied into place by code emitted in the IR.
289
290 Register OSR_buf = osrBufferPointer()->as_pointer_register();
291 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
292 int monitor_offset = BytesPerWord * method()->max_locals() +
293 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
294 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
295 // the OSR buffer using 2 word entries: first the lock and then
296 // the oop.
297 for (int i = 0; i < number_of_locks; i++) {
298 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
299 #ifdef ASSERT
300 // verify the interpreter's monitor has a non-null object
301 {
302 Label L;
303 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
304 __ jcc(Assembler::notZero, L);
305 __ stop("locked object is null");
306 __ bind(L);
307 }
308 #endif
309 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
310 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
311 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
312 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
313 }
314 }
315 }
316
317
318 // inline cache check; done before the frame is built.
319 int LIR_Assembler::check_icache() {
320 return __ ic_check(CodeEntryAlignment);
321 }
322
323 void LIR_Assembler::clinit_barrier(ciMethod* method) {
324 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
325 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
326
327 Label L_skip_barrier;
328 Register klass = rscratch1;
329
330 __ mov_metadata(klass, method->holder()->constant_encoding());
331 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
332
333 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
334
335 __ bind(L_skip_barrier);
336 }
337
338 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
339 jobject o = nullptr;
340 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
341 __ movoop(reg, o);
342 patching_epilog(patch, lir_patch_normal, reg, info);
343 }
344
345 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
346 Metadata* o = nullptr;
347 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
348 __ mov_metadata(reg, o);
349 patching_epilog(patch, lir_patch_normal, reg, info);
350 }
351
352 // This specifies the rsp decrement needed to build the frame
353 int LIR_Assembler::initial_frame_size_in_bytes() const {
354 // if rounding, must let FrameMap know!
355
356 // The frame_map records size in slots (32bit word)
357
358 // subtract two words to account for return address and link
359 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
360 }
361
362
363 int LIR_Assembler::emit_exception_handler() {
364 // generate code for exception handler
365 address handler_base = __ start_a_stub(exception_handler_size());
366 if (handler_base == nullptr) {
367 // not enough space left for the handler
368 bailout("exception handler overflow");
369 return -1;
370 }
371
372 int offset = code_offset();
373
374 // the exception oop and pc are in rax, and rdx
375 // no other registers need to be preserved, so invalidate them
376 __ invalidate_registers(false, true, true, false, true, true);
377
378 // check that there is really an exception
379 __ verify_not_null_oop(rax);
380
381 // search an exception handler (rax: exception oop, rdx: throwing pc)
382 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id)));
383 __ should_not_reach_here();
384 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
385 __ end_a_stub();
386
387 return offset;
388 }
389
390
391 // Emit the code to remove the frame from the stack in the exception
392 // unwind path.
393 int LIR_Assembler::emit_unwind_handler() {
394 #ifndef PRODUCT
395 if (CommentedAssembly) {
396 _masm->block_comment("Unwind handler");
397 }
398 #endif
399
400 int offset = code_offset();
401
402 // Fetch the exception from TLS and clear out exception related thread state
403 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
404 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
405 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
406
407 __ bind(_unwind_handler_entry);
408 __ verify_not_null_oop(rax);
409 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
410 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
411 }
412
413 // Perform needed unlocking
414 MonitorExitStub* stub = nullptr;
415 if (method()->is_synchronized()) {
416 monitor_address(0, FrameMap::rax_opr);
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 if (LockingMode == LM_MONITOR) {
419 __ jmp(*stub->entry());
420 } else {
421 __ unlock_object(rdi, rsi, rax, *stub->entry());
422 }
423 __ bind(*stub->continuation());
424 }
425
426 if (compilation()->env()->dtrace_method_probes()) {
427 __ mov(rdi, r15_thread);
428 __ mov_metadata(rsi, method()->constant_encoding());
429 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
430 }
431
432 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
433 __ mov(rax, rbx); // Restore the exception
434 }
435
436 // remove the activation and dispatch to the unwind handler
437 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
438 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
439
440 // Emit the slow path assembly
441 if (stub != nullptr) {
442 stub->emit_code(this);
443 }
444
445 return offset;
446 }
447
448
449 int LIR_Assembler::emit_deopt_handler() {
450 // generate code for exception handler
451 address handler_base = __ start_a_stub(deopt_handler_size());
452 if (handler_base == nullptr) {
453 // not enough space left for the handler
454 bailout("deopt handler overflow");
455 return -1;
456 }
457
458 int offset = code_offset();
459 InternalAddress here(__ pc());
460
461 __ pushptr(here.addr(), rscratch1);
462 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 __ end_a_stub();
465
466 return offset;
467 }
468
469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
470 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
471 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
472 assert(result->fpu() == 0, "result must already be on TOS");
473 }
474 if (InlineTypeReturnedAsFields) {
475 #ifndef _LP64
476 Unimplemented();
477 #endif
478 // Check if we are returning an non-null inline type and load its fields into registers
479 ciType* return_type = compilation()->method()->return_type();
480 if (return_type->is_inlinetype()) {
481 ciInlineKlass* vk = return_type->as_inline_klass();
482 if (vk->can_be_returned_as_fields()) {
483 address unpack_handler = vk->unpack_handler();
484 assert(unpack_handler != nullptr, "must be");
485 __ call(RuntimeAddress(unpack_handler));
486 }
487 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
488 Label skip;
489 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
490
491 // Load fields from a buffered value with an inline class specific handler
492 __ load_klass(rdi, rax, rscratch1);
493 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
494 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
495 // Unpack handler can be null if inline type is not scalarizable in returns
496 __ testptr(rdi, rdi);
497 __ jcc(Assembler::zero, skip);
498 __ call(rdi);
499
500 __ bind(skip);
501 }
502 // At this point, rax points to the value object (for interpreter or C1 caller).
503 // The fields of the object are copied into registers (for C2 caller).
504 }
505
506 // Pop the stack before the safepoint code
507 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
508
509 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
510 __ reserved_stack_check();
511 }
512
513 // Note: we do not need to round double result; float result has the right precision
514 // the poll sets the condition code, but no data registers
515
516 code_stub->set_safepoint_offset(__ offset());
517 __ relocate(relocInfo::poll_return_type);
518 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
519 __ ret(0);
520 }
521
522
523 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
524 return (__ store_inline_type_fields_to_buf(vk, false));
525 }
526
527 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
528 guarantee(info != nullptr, "Shouldn't be null");
529 int offset = __ offset();
530 const Register poll_addr = rscratch1;
531 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
532 add_debug_info_for_branch(info);
533 __ relocate(relocInfo::poll_type);
534 address pre_pc = __ pc();
535 __ testl(rax, Address(poll_addr, 0));
536 address post_pc = __ pc();
537 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
538 return offset;
539 }
540
541
542 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
543 if (from_reg != to_reg) __ mov(to_reg, from_reg);
544 }
545
546 void LIR_Assembler::swap_reg(Register a, Register b) {
547 __ xchgptr(a, b);
548 }
549
550
551 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
552 assert(src->is_constant(), "should not call otherwise");
553 assert(dest->is_register(), "should not call otherwise");
554 LIR_Const* c = src->as_constant_ptr();
555
556 switch (c->type()) {
557 case T_INT: {
558 assert(patch_code == lir_patch_none, "no patching handled here");
559 __ movl(dest->as_register(), c->as_jint());
560 break;
561 }
562
563 case T_ADDRESS: {
564 assert(patch_code == lir_patch_none, "no patching handled here");
565 __ movptr(dest->as_register(), c->as_jint());
566 break;
567 }
568
569 case T_LONG: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
572 break;
573 }
574
575 case T_OBJECT: {
576 if (patch_code != lir_patch_none) {
577 jobject2reg_with_patching(dest->as_register(), info);
578 } else {
579 __ movoop(dest->as_register(), c->as_jobject());
580 }
581 break;
582 }
583
584 case T_METADATA: {
585 if (patch_code != lir_patch_none) {
586 klass2reg_with_patching(dest->as_register(), info);
587 } else {
588 __ mov_metadata(dest->as_register(), c->as_metadata());
589 }
590 break;
591 }
592
593 case T_FLOAT: {
594 if (dest->is_single_xmm()) {
595 if (UseAVX <= 2 && c->is_zero_float()) {
596 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
597 } else {
598 __ movflt(dest->as_xmm_float_reg(),
599 InternalAddress(float_constant(c->as_jfloat())));
600 }
601 } else {
602 ShouldNotReachHere();
603 }
604 break;
605 }
606
607 case T_DOUBLE: {
608 if (dest->is_double_xmm()) {
609 if (UseAVX <= 2 && c->is_zero_double()) {
610 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
611 } else {
612 __ movdbl(dest->as_xmm_double_reg(),
613 InternalAddress(double_constant(c->as_jdouble())));
614 }
615 } else {
616 ShouldNotReachHere();
617 }
618 break;
619 }
620
621 default:
622 ShouldNotReachHere();
623 }
624 }
625
626 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
627 assert(src->is_constant(), "should not call otherwise");
628 assert(dest->is_stack(), "should not call otherwise");
629 LIR_Const* c = src->as_constant_ptr();
630
631 switch (c->type()) {
632 case T_INT: // fall through
633 case T_FLOAT:
634 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
635 break;
636
637 case T_ADDRESS:
638 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
639 break;
640
641 case T_OBJECT:
642 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
643 break;
644
645 case T_LONG: // fall through
646 case T_DOUBLE:
647 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
648 lo_word_offset_in_bytes),
649 (intptr_t)c->as_jlong_bits(),
650 rscratch1);
651 break;
652
653 default:
654 ShouldNotReachHere();
655 }
656 }
657
658 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
659 assert(src->is_constant(), "should not call otherwise");
660 assert(dest->is_address(), "should not call otherwise");
661 LIR_Const* c = src->as_constant_ptr();
662 LIR_Address* addr = dest->as_address_ptr();
663
664 int null_check_here = code_offset();
665 switch (type) {
666 case T_INT: // fall through
667 case T_FLOAT:
668 __ movl(as_Address(addr), c->as_jint_bits());
669 break;
670
671 case T_ADDRESS:
672 __ movptr(as_Address(addr), c->as_jint_bits());
673 break;
674
675 case T_OBJECT: // fall through
676 case T_ARRAY:
677 if (c->as_jobject() == nullptr) {
678 if (UseCompressedOops && !wide) {
679 __ movl(as_Address(addr), NULL_WORD);
680 } else {
681 __ xorptr(rscratch1, rscratch1);
682 null_check_here = code_offset();
683 __ movptr(as_Address(addr), rscratch1);
684 }
685 } else {
686 if (is_literal_address(addr)) {
687 ShouldNotReachHere();
688 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
689 } else {
690 __ movoop(rscratch1, c->as_jobject());
691 if (UseCompressedOops && !wide) {
692 __ encode_heap_oop(rscratch1);
693 null_check_here = code_offset();
694 __ movl(as_Address_lo(addr), rscratch1);
695 } else {
696 null_check_here = code_offset();
697 __ movptr(as_Address_lo(addr), rscratch1);
698 }
699 }
700 }
701 break;
702
703 case T_LONG: // fall through
704 case T_DOUBLE:
705 if (is_literal_address(addr)) {
706 ShouldNotReachHere();
707 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
708 } else {
709 __ movptr(r10, (intptr_t)c->as_jlong_bits());
710 null_check_here = code_offset();
711 __ movptr(as_Address_lo(addr), r10);
712 }
713 break;
714
715 case T_BOOLEAN: // fall through
716 case T_BYTE:
717 __ movb(as_Address(addr), c->as_jint() & 0xFF);
718 break;
719
720 case T_CHAR: // fall through
721 case T_SHORT:
722 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
723 break;
724
725 default:
726 ShouldNotReachHere();
727 };
728
729 if (info != nullptr) {
730 add_debug_info_for_null_check(null_check_here, info);
731 }
732 }
733
734
735 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
736 assert(src->is_register(), "should not call otherwise");
737 assert(dest->is_register(), "should not call otherwise");
738
739 // move between cpu-registers
740 if (dest->is_single_cpu()) {
741 if (src->type() == T_LONG) {
742 // Can do LONG -> OBJECT
743 move_regs(src->as_register_lo(), dest->as_register());
744 return;
745 }
746 assert(src->is_single_cpu(), "must match");
747 if (src->type() == T_OBJECT) {
748 __ verify_oop(src->as_register());
749 }
750 move_regs(src->as_register(), dest->as_register());
751
752 } else if (dest->is_double_cpu()) {
753 if (is_reference_type(src->type())) {
754 // Surprising to me but we can see move of a long to t_object
755 __ verify_oop(src->as_register());
756 move_regs(src->as_register(), dest->as_register_lo());
757 return;
758 }
759 assert(src->is_double_cpu(), "must match");
760 Register f_lo = src->as_register_lo();
761 Register f_hi = src->as_register_hi();
762 Register t_lo = dest->as_register_lo();
763 Register t_hi = dest->as_register_hi();
764 assert(f_hi == f_lo, "must be same");
765 assert(t_hi == t_lo, "must be same");
766 move_regs(f_lo, t_lo);
767
768 // move between xmm-registers
769 } else if (dest->is_single_xmm()) {
770 assert(src->is_single_xmm(), "must match");
771 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
772 } else if (dest->is_double_xmm()) {
773 assert(src->is_double_xmm(), "must match");
774 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
775
776 } else {
777 ShouldNotReachHere();
778 }
779 }
780
781 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
782 assert(src->is_register(), "should not call otherwise");
783 assert(dest->is_stack(), "should not call otherwise");
784
785 if (src->is_single_cpu()) {
786 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
787 if (is_reference_type(type)) {
788 __ verify_oop(src->as_register());
789 __ movptr (dst, src->as_register());
790 } else if (type == T_METADATA || type == T_ADDRESS) {
791 __ movptr (dst, src->as_register());
792 } else {
793 __ movl (dst, src->as_register());
794 }
795
796 } else if (src->is_double_cpu()) {
797 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
798 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
799 __ movptr (dstLO, src->as_register_lo());
800
801 } else if (src->is_single_xmm()) {
802 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
803 __ movflt(dst_addr, src->as_xmm_float_reg());
804
805 } else if (src->is_double_xmm()) {
806 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
807 __ movdbl(dst_addr, src->as_xmm_double_reg());
808
809 } else {
810 ShouldNotReachHere();
811 }
812 }
813
814
815 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
816 LIR_Address* to_addr = dest->as_address_ptr();
817 PatchingStub* patch = nullptr;
818 Register compressed_src = rscratch1;
819
820 if (is_reference_type(type)) {
821 __ verify_oop(src->as_register());
822 if (UseCompressedOops && !wide) {
823 __ movptr(compressed_src, src->as_register());
824 __ encode_heap_oop(compressed_src);
825 if (patch_code != lir_patch_none) {
826 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
827 }
828 }
829 }
830
831 if (patch_code != lir_patch_none) {
832 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
833 Address toa = as_Address(to_addr);
834 assert(toa.disp() != 0, "must have");
835 }
836
837 int null_check_here = code_offset();
838 switch (type) {
839 case T_FLOAT: {
840 assert(src->is_single_xmm(), "not a float");
841 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
842 break;
843 }
844
845 case T_DOUBLE: {
846 assert(src->is_double_xmm(), "not a double");
847 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
848 break;
849 }
850
851 case T_ARRAY: // fall through
852 case T_OBJECT: // fall through
853 if (UseCompressedOops && !wide) {
854 __ movl(as_Address(to_addr), compressed_src);
855 } else {
856 __ movptr(as_Address(to_addr), src->as_register());
857 }
858 break;
859 case T_ADDRESS:
860 __ movptr(as_Address(to_addr), src->as_register());
861 break;
862 case T_INT:
863 __ movl(as_Address(to_addr), src->as_register());
864 break;
865
866 case T_LONG: {
867 Register from_lo = src->as_register_lo();
868 Register from_hi = src->as_register_hi();
869 __ movptr(as_Address_lo(to_addr), from_lo);
870 break;
871 }
872
873 case T_BYTE: // fall through
874 case T_BOOLEAN: {
875 Register src_reg = src->as_register();
876 Address dst_addr = as_Address(to_addr);
877 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
878 __ movb(dst_addr, src_reg);
879 break;
880 }
881
882 case T_CHAR: // fall through
883 case T_SHORT:
884 __ movw(as_Address(to_addr), src->as_register());
885 break;
886
887 default:
888 ShouldNotReachHere();
889 }
890 if (info != nullptr) {
891 add_debug_info_for_null_check(null_check_here, info);
892 }
893
894 if (patch_code != lir_patch_none) {
895 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
896 }
897 }
898
899
900 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
901 assert(src->is_stack(), "should not call otherwise");
902 assert(dest->is_register(), "should not call otherwise");
903
904 if (dest->is_single_cpu()) {
905 if (is_reference_type(type)) {
906 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
907 __ verify_oop(dest->as_register());
908 } else if (type == T_METADATA || type == T_ADDRESS) {
909 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
910 } else {
911 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
912 }
913
914 } else if (dest->is_double_cpu()) {
915 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
916 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
917 __ movptr(dest->as_register_lo(), src_addr_LO);
918
919 } else if (dest->is_single_xmm()) {
920 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
921 __ movflt(dest->as_xmm_float_reg(), src_addr);
922
923 } else if (dest->is_double_xmm()) {
924 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
925 __ movdbl(dest->as_xmm_double_reg(), src_addr);
926
927 } else {
928 ShouldNotReachHere();
929 }
930 }
931
932
933 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
934 if (src->is_single_stack()) {
935 if (is_reference_type(type)) {
936 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
937 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
938 } else {
939 //no pushl on 64bits
940 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
941 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
942 }
943
944 } else if (src->is_double_stack()) {
945 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
946 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
947
948 } else {
949 ShouldNotReachHere();
950 }
951 }
952
953
954 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
955 assert(src->is_address(), "should not call otherwise");
956 assert(dest->is_register(), "should not call otherwise");
957
958 LIR_Address* addr = src->as_address_ptr();
959 Address from_addr = as_Address(addr);
960
961 if (addr->base()->type() == T_OBJECT) {
962 __ verify_oop(addr->base()->as_pointer_register());
963 }
964
965 switch (type) {
966 case T_BOOLEAN: // fall through
967 case T_BYTE: // fall through
968 case T_CHAR: // fall through
969 case T_SHORT:
970 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
971 // on pre P6 processors we may get partial register stalls
972 // so blow away the value of to_rinfo before loading a
973 // partial word into it. Do it here so that it precedes
974 // the potential patch point below.
975 __ xorptr(dest->as_register(), dest->as_register());
976 }
977 break;
978 default:
979 break;
980 }
981
982 PatchingStub* patch = nullptr;
983 if (patch_code != lir_patch_none) {
984 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
985 assert(from_addr.disp() != 0, "must have");
986 }
987 if (info != nullptr) {
988 add_debug_info_for_null_check_here(info);
989 }
990
991 switch (type) {
992 case T_FLOAT: {
993 if (dest->is_single_xmm()) {
994 __ movflt(dest->as_xmm_float_reg(), from_addr);
995 } else {
996 ShouldNotReachHere();
997 }
998 break;
999 }
1000
1001 case T_DOUBLE: {
1002 if (dest->is_double_xmm()) {
1003 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1004 } else {
1005 ShouldNotReachHere();
1006 }
1007 break;
1008 }
1009
1010 case T_OBJECT: // fall through
1011 case T_ARRAY: // fall through
1012 if (UseCompressedOops && !wide) {
1013 __ movl(dest->as_register(), from_addr);
1014 } else {
1015 __ movptr(dest->as_register(), from_addr);
1016 }
1017 break;
1018
1019 case T_ADDRESS:
1020 __ movptr(dest->as_register(), from_addr);
1021 break;
1022 case T_INT:
1023 __ movl(dest->as_register(), from_addr);
1024 break;
1025
1026 case T_LONG: {
1027 Register to_lo = dest->as_register_lo();
1028 Register to_hi = dest->as_register_hi();
1029 __ movptr(to_lo, as_Address_lo(addr));
1030 break;
1031 }
1032
1033 case T_BOOLEAN: // fall through
1034 case T_BYTE: {
1035 Register dest_reg = dest->as_register();
1036 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1037 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1038 __ movsbl(dest_reg, from_addr);
1039 } else {
1040 __ movb(dest_reg, from_addr);
1041 __ shll(dest_reg, 24);
1042 __ sarl(dest_reg, 24);
1043 }
1044 break;
1045 }
1046
1047 case T_CHAR: {
1048 Register dest_reg = dest->as_register();
1049 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1050 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1051 __ movzwl(dest_reg, from_addr);
1052 } else {
1053 __ movw(dest_reg, from_addr);
1054 }
1055 break;
1056 }
1057
1058 case T_SHORT: {
1059 Register dest_reg = dest->as_register();
1060 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1061 __ movswl(dest_reg, from_addr);
1062 } else {
1063 __ movw(dest_reg, from_addr);
1064 __ shll(dest_reg, 16);
1065 __ sarl(dest_reg, 16);
1066 }
1067 break;
1068 }
1069
1070 default:
1071 ShouldNotReachHere();
1072 }
1073
1074 if (patch != nullptr) {
1075 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1076 }
1077
1078 if (is_reference_type(type)) {
1079 if (UseCompressedOops && !wide) {
1080 __ decode_heap_oop(dest->as_register());
1081 }
1082
1083 __ verify_oop(dest->as_register());
1084 }
1085 }
1086
1087
1088 NEEDS_CLEANUP; // This could be static?
1089 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1090 int elem_size = type2aelembytes(type);
1091 switch (elem_size) {
1092 case 1: return Address::times_1;
1093 case 2: return Address::times_2;
1094 case 4: return Address::times_4;
1095 case 8: return Address::times_8;
1096 }
1097 ShouldNotReachHere();
1098 return Address::no_scale;
1099 }
1100
1101
1102 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1103 switch (op->code()) {
1104 case lir_idiv:
1105 case lir_irem:
1106 arithmetic_idiv(op->code(),
1107 op->in_opr1(),
1108 op->in_opr2(),
1109 op->in_opr3(),
1110 op->result_opr(),
1111 op->info());
1112 break;
1113 case lir_fmad:
1114 __ fmad(op->result_opr()->as_xmm_double_reg(),
1115 op->in_opr1()->as_xmm_double_reg(),
1116 op->in_opr2()->as_xmm_double_reg(),
1117 op->in_opr3()->as_xmm_double_reg());
1118 break;
1119 case lir_fmaf:
1120 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1121 op->in_opr1()->as_xmm_float_reg(),
1122 op->in_opr2()->as_xmm_float_reg(),
1123 op->in_opr3()->as_xmm_float_reg());
1124 break;
1125 default: ShouldNotReachHere(); break;
1126 }
1127 }
1128
1129 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1130 #ifdef ASSERT
1131 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1132 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1133 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1134 #endif
1135
1136 if (op->cond() == lir_cond_always) {
1137 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1138 __ jmp (*(op->label()));
1139 } else {
1140 Assembler::Condition acond = Assembler::zero;
1141 if (op->code() == lir_cond_float_branch) {
1142 assert(op->ublock() != nullptr, "must have unordered successor");
1143 __ jcc(Assembler::parity, *(op->ublock()->label()));
1144 switch(op->cond()) {
1145 case lir_cond_equal: acond = Assembler::equal; break;
1146 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1147 case lir_cond_less: acond = Assembler::below; break;
1148 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1149 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1150 case lir_cond_greater: acond = Assembler::above; break;
1151 default: ShouldNotReachHere();
1152 }
1153 } else {
1154 switch (op->cond()) {
1155 case lir_cond_equal: acond = Assembler::equal; break;
1156 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1157 case lir_cond_less: acond = Assembler::less; break;
1158 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1159 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1160 case lir_cond_greater: acond = Assembler::greater; break;
1161 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1162 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1163 default: ShouldNotReachHere();
1164 }
1165 }
1166 __ jcc(acond,*(op->label()));
1167 }
1168 }
1169
1170 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1171 LIR_Opr src = op->in_opr();
1172 LIR_Opr dest = op->result_opr();
1173
1174 switch (op->bytecode()) {
1175 case Bytecodes::_i2l:
1176 __ movl2ptr(dest->as_register_lo(), src->as_register());
1177 break;
1178
1179 case Bytecodes::_l2i:
1180 __ movl(dest->as_register(), src->as_register_lo());
1181 break;
1182
1183 case Bytecodes::_i2b:
1184 move_regs(src->as_register(), dest->as_register());
1185 __ sign_extend_byte(dest->as_register());
1186 break;
1187
1188 case Bytecodes::_i2c:
1189 move_regs(src->as_register(), dest->as_register());
1190 __ andl(dest->as_register(), 0xFFFF);
1191 break;
1192
1193 case Bytecodes::_i2s:
1194 move_regs(src->as_register(), dest->as_register());
1195 __ sign_extend_short(dest->as_register());
1196 break;
1197
1198 case Bytecodes::_f2d:
1199 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1200 break;
1201
1202 case Bytecodes::_d2f:
1203 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1204 break;
1205
1206 case Bytecodes::_i2f:
1207 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1208 break;
1209
1210 case Bytecodes::_i2d:
1211 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1212 break;
1213
1214 case Bytecodes::_l2f:
1215 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1216 break;
1217
1218 case Bytecodes::_l2d:
1219 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1220 break;
1221
1222 case Bytecodes::_f2i:
1223 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1224 break;
1225
1226 case Bytecodes::_d2i:
1227 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1228 break;
1229
1230 case Bytecodes::_f2l:
1231 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1232 break;
1233
1234 case Bytecodes::_d2l:
1235 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1236 break;
1237
1238 default: ShouldNotReachHere();
1239 }
1240 }
1241
1242 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1243 if (op->init_check()) {
1244 add_debug_info_for_null_check_here(op->stub()->info());
1245 // init_state needs acquire, but x86 is TSO, and so we are already good.
1246 __ cmpb(Address(op->klass()->as_register(),
1247 InstanceKlass::init_state_offset()),
1248 InstanceKlass::fully_initialized);
1249 __ jcc(Assembler::notEqual, *op->stub()->entry());
1250 }
1251 __ allocate_object(op->obj()->as_register(),
1252 op->tmp1()->as_register(),
1253 op->tmp2()->as_register(),
1254 op->header_size(),
1255 op->object_size(),
1256 op->klass()->as_register(),
1257 *op->stub()->entry());
1258 __ bind(*op->stub()->continuation());
1259 }
1260
1261 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1262 Register len = op->len()->as_register();
1263 __ movslq(len, len);
1264
1265 if (UseSlowPath || op->is_null_free() ||
1266 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1267 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1268 __ jmp(*op->stub()->entry());
1269 } else {
1270 Register tmp1 = op->tmp1()->as_register();
1271 Register tmp2 = op->tmp2()->as_register();
1272 Register tmp3 = op->tmp3()->as_register();
1273 if (len == tmp1) {
1274 tmp1 = tmp3;
1275 } else if (len == tmp2) {
1276 tmp2 = tmp3;
1277 } else if (len == tmp3) {
1278 // everything is ok
1279 } else {
1280 __ mov(tmp3, len);
1281 }
1282 __ allocate_array(op->obj()->as_register(),
1283 len,
1284 tmp1,
1285 tmp2,
1286 arrayOopDesc::base_offset_in_bytes(op->type()),
1287 array_element_size(op->type()),
1288 op->klass()->as_register(),
1289 *op->stub()->entry(),
1290 op->zero_array());
1291 }
1292 __ bind(*op->stub()->continuation());
1293 }
1294
1295 void LIR_Assembler::type_profile_helper(Register mdo,
1296 ciMethodData *md, ciProfileData *data,
1297 Register recv, Label* update_done) {
1298 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1299 Label next_test;
1300 // See if the receiver is receiver[n].
1301 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1302 __ jccb(Assembler::notEqual, next_test);
1303 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1304 __ addptr(data_addr, DataLayout::counter_increment);
1305 __ jmp(*update_done);
1306 __ bind(next_test);
1307 }
1308
1309 // Didn't find receiver; find next empty slot and fill it in
1310 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1311 Label next_test;
1312 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1313 __ cmpptr(recv_addr, NULL_WORD);
1314 __ jccb(Assembler::notEqual, next_test);
1315 __ movptr(recv_addr, recv);
1316 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1317 __ jmp(*update_done);
1318 __ bind(next_test);
1319 }
1320 }
1321
1322 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1323 // we always need a stub for the failure case.
1324 CodeStub* stub = op->stub();
1325 Register obj = op->object()->as_register();
1326 Register k_RInfo = op->tmp1()->as_register();
1327 Register klass_RInfo = op->tmp2()->as_register();
1328 Register dst = op->result_opr()->as_register();
1329 ciKlass* k = op->klass();
1330 Register Rtmp1 = noreg;
1331 Register tmp_load_klass = rscratch1;
1332
1333 // check if it needs to be profiled
1334 ciMethodData* md = nullptr;
1335 ciProfileData* data = nullptr;
1336
1337 if (op->should_profile()) {
1338 ciMethod* method = op->profiled_method();
1339 assert(method != nullptr, "Should have method");
1340 int bci = op->profiled_bci();
1341 md = method->method_data_or_null();
1342 assert(md != nullptr, "Sanity");
1343 data = md->bci_to_data(bci);
1344 assert(data != nullptr, "need data for type check");
1345 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1346 }
1347 Label* success_target = success;
1348 Label* failure_target = failure;
1349
1350 if (obj == k_RInfo) {
1351 k_RInfo = dst;
1352 } else if (obj == klass_RInfo) {
1353 klass_RInfo = dst;
1354 }
1355 if (k->is_loaded() && !UseCompressedClassPointers) {
1356 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1357 } else {
1358 Rtmp1 = op->tmp3()->as_register();
1359 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1360 }
1361
1362 assert_different_registers(obj, k_RInfo, klass_RInfo);
1363
1364 if (op->need_null_check()) {
1365 __ testptr(obj, obj);
1366 if (op->should_profile()) {
1367 Label not_null;
1368 Register mdo = klass_RInfo;
1369 __ mov_metadata(mdo, md->constant_encoding());
1370 __ jccb(Assembler::notEqual, not_null);
1371 // Object is null; update MDO and exit
1372 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1373 int header_bits = BitData::null_seen_byte_constant();
1374 __ orb(data_addr, header_bits);
1375 __ jmp(*obj_is_null);
1376 __ bind(not_null);
1377
1378 Label update_done;
1379 Register recv = k_RInfo;
1380 __ load_klass(recv, obj, tmp_load_klass);
1381 type_profile_helper(mdo, md, data, recv, &update_done);
1382
1383 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1384 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1385
1386 __ bind(update_done);
1387 } else {
1388 __ jcc(Assembler::equal, *obj_is_null);
1389 }
1390 }
1391
1392 if (!k->is_loaded()) {
1393 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1394 } else {
1395 __ mov_metadata(k_RInfo, k->constant_encoding());
1396 }
1397 __ verify_oop(obj);
1398
1399 if (op->fast_check()) {
1400 // get object class
1401 // not a safepoint as obj null check happens earlier
1402 if (UseCompressedClassPointers) {
1403 __ load_klass(Rtmp1, obj, tmp_load_klass);
1404 __ cmpptr(k_RInfo, Rtmp1);
1405 } else {
1406 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1407 }
1408 __ jcc(Assembler::notEqual, *failure_target);
1409 // successful cast, fall through to profile or jump
1410 } else {
1411 // get object class
1412 // not a safepoint as obj null check happens earlier
1413 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1414 if (k->is_loaded()) {
1415 // See if we get an immediate positive hit
1416 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1417 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1418 __ jcc(Assembler::notEqual, *failure_target);
1419 // successful cast, fall through to profile or jump
1420 } else {
1421 // See if we get an immediate positive hit
1422 __ jcc(Assembler::equal, *success_target);
1423 // check for self
1424 __ cmpptr(klass_RInfo, k_RInfo);
1425 __ jcc(Assembler::equal, *success_target);
1426
1427 __ push(klass_RInfo);
1428 __ push(k_RInfo);
1429 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1430 __ pop(klass_RInfo);
1431 __ pop(klass_RInfo);
1432 // result is a boolean
1433 __ testl(klass_RInfo, klass_RInfo);
1434 __ jcc(Assembler::equal, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 }
1437 } else {
1438 // perform the fast part of the checking logic
1439 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1440 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1441 __ push(klass_RInfo);
1442 __ push(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1444 __ pop(klass_RInfo);
1445 __ pop(k_RInfo);
1446 // result is a boolean
1447 __ testl(k_RInfo, k_RInfo);
1448 __ jcc(Assembler::equal, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 }
1452 __ jmp(*success);
1453 }
1454
1455
1456 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1457 Register tmp_load_klass = rscratch1;
1458 LIR_Code code = op->code();
1459 if (code == lir_store_check) {
1460 Register value = op->object()->as_register();
1461 Register array = op->array()->as_register();
1462 Register k_RInfo = op->tmp1()->as_register();
1463 Register klass_RInfo = op->tmp2()->as_register();
1464 Register Rtmp1 = op->tmp3()->as_register();
1465
1466 CodeStub* stub = op->stub();
1467
1468 // check if it needs to be profiled
1469 ciMethodData* md = nullptr;
1470 ciProfileData* data = nullptr;
1471
1472 if (op->should_profile()) {
1473 ciMethod* method = op->profiled_method();
1474 assert(method != nullptr, "Should have method");
1475 int bci = op->profiled_bci();
1476 md = method->method_data_or_null();
1477 assert(md != nullptr, "Sanity");
1478 data = md->bci_to_data(bci);
1479 assert(data != nullptr, "need data for type check");
1480 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1481 }
1482 Label done;
1483 Label* success_target = &done;
1484 Label* failure_target = stub->entry();
1485
1486 __ testptr(value, value);
1487 if (op->should_profile()) {
1488 Label not_null;
1489 Register mdo = klass_RInfo;
1490 __ mov_metadata(mdo, md->constant_encoding());
1491 __ jccb(Assembler::notEqual, not_null);
1492 // Object is null; update MDO and exit
1493 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1494 int header_bits = BitData::null_seen_byte_constant();
1495 __ orb(data_addr, header_bits);
1496 __ jmp(done);
1497 __ bind(not_null);
1498
1499 Label update_done;
1500 Register recv = k_RInfo;
1501 __ load_klass(recv, value, tmp_load_klass);
1502 type_profile_helper(mdo, md, data, recv, &update_done);
1503
1504 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1505 __ addptr(counter_addr, DataLayout::counter_increment);
1506 __ bind(update_done);
1507 } else {
1508 __ jcc(Assembler::equal, done);
1509 }
1510
1511 add_debug_info_for_null_check_here(op->info_for_exception());
1512 __ load_klass(k_RInfo, array, tmp_load_klass);
1513 __ load_klass(klass_RInfo, value, tmp_load_klass);
1514
1515 // get instance klass (it's already uncompressed)
1516 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1517 // perform the fast part of the checking logic
1518 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1519 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1520 __ push(klass_RInfo);
1521 __ push(k_RInfo);
1522 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1523 __ pop(klass_RInfo);
1524 __ pop(k_RInfo);
1525 // result is a boolean
1526 __ testl(k_RInfo, k_RInfo);
1527 __ jcc(Assembler::equal, *failure_target);
1528 // fall through to the success case
1529
1530 __ bind(done);
1531 } else
1532 if (code == lir_checkcast) {
1533 Register obj = op->object()->as_register();
1534 Register dst = op->result_opr()->as_register();
1535 Label success;
1536 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1537 __ bind(success);
1538 if (dst != obj) {
1539 __ mov(dst, obj);
1540 }
1541 } else
1542 if (code == lir_instanceof) {
1543 Register obj = op->object()->as_register();
1544 Register dst = op->result_opr()->as_register();
1545 Label success, failure, done;
1546 emit_typecheck_helper(op, &success, &failure, &failure);
1547 __ bind(failure);
1548 __ xorptr(dst, dst);
1549 __ jmpb(done);
1550 __ bind(success);
1551 __ movptr(dst, 1);
1552 __ bind(done);
1553 } else {
1554 ShouldNotReachHere();
1555 }
1556
1557 }
1558
1559 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1560 // We are loading/storing from/to an array that *may* be a flat array (the
1561 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1562 // If this array is a flat array, take the slow path.
1563 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1564 if (!op->value()->is_illegal()) {
1565 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1566 // The array is not a flat array, but it might be null-free. If we are storing
1567 // a null into a null-free array, take the slow path (which will throw NPE).
1568 Label skip;
1569 __ cmpptr(op->value()->as_register(), NULL_WORD);
1570 __ jcc(Assembler::notEqual, skip);
1571 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572 __ bind(skip);
1573 }
1574 }
1575
1576 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1577 // We are storing into an array that *may* be null-free (the declared type is
1578 // Object[], abstract[], interface[] or VT.ref[]).
1579 Label test_mark_word;
1580 Register tmp = op->tmp()->as_register();
1581 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1582 __ testl(tmp, markWord::unlocked_value);
1583 __ jccb(Assembler::notZero, test_mark_word);
1584 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1585 __ bind(test_mark_word);
1586 __ testl(tmp, markWord::null_free_array_bit_in_place);
1587 }
1588
1589 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1590 Label L_oops_equal;
1591 Label L_oops_not_equal;
1592 Label L_end;
1593
1594 Register left = op->left()->as_register();
1595 Register right = op->right()->as_register();
1596
1597 __ cmpptr(left, right);
1598 __ jcc(Assembler::equal, L_oops_equal);
1599
1600 // (1) Null check -- if one of the operands is null, the other must not be null (because
1601 // the two references are not equal), so they are not substitutable,
1602 // FIXME: do null check only if the operand is nullable
1603 __ testptr(left, right);
1604 __ jcc(Assembler::zero, L_oops_not_equal);
1605
1606 ciKlass* left_klass = op->left_klass();
1607 ciKlass* right_klass = op->right_klass();
1608
1609 // (2) Inline type check -- if either of the operands is not a inline type,
1610 // they are not substitutable. We do this only if we are not sure that the
1611 // operands are inline type
1612 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1613 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1614 Register tmp1 = op->tmp1()->as_register();
1615 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1616 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1617 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1619 __ jcc(Assembler::notEqual, L_oops_not_equal);
1620 }
1621
1622 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1623 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1624 // No need to load klass -- the operands are statically known to be the same inline klass.
1625 __ jmp(*op->stub()->entry());
1626 } else {
1627 Register left_klass_op = op->left_klass_op()->as_register();
1628 Register right_klass_op = op->right_klass_op()->as_register();
1629
1630 if (UseCompressedClassPointers) {
1631 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1632 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1633 __ cmpl(left_klass_op, right_klass_op);
1634 } else {
1635 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1636 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1637 __ cmpptr(left_klass_op, right_klass_op);
1638 }
1639
1640 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1641 // fall through to L_oops_not_equal
1642 }
1643
1644 __ bind(L_oops_not_equal);
1645 move(op->not_equal_result(), op->result_opr());
1646 __ jmp(L_end);
1647
1648 __ bind(L_oops_equal);
1649 move(op->equal_result(), op->result_opr());
1650 __ jmp(L_end);
1651
1652 // We've returned from the stub. RAX contains 0x0 IFF the two
1653 // operands are not substitutable. (Don't compare against 0x1 in case the
1654 // C compiler is naughty)
1655 __ bind(*op->stub()->continuation());
1656 __ cmpl(rax, 0);
1657 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1658 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1659 // fall-through
1660 __ bind(L_end);
1661 }
1662
1663 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1664 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1665 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1666 Register newval = op->new_value()->as_register();
1667 Register cmpval = op->cmp_value()->as_register();
1668 assert(cmpval == rax, "wrong register");
1669 assert(newval != noreg, "new val must be register");
1670 assert(cmpval != newval, "cmp and new values must be in different registers");
1671 assert(cmpval != addr, "cmp and addr must be in different registers");
1672 assert(newval != addr, "new value and addr must be in different registers");
1673
1674 if (op->code() == lir_cas_obj) {
1675 if (UseCompressedOops) {
1676 __ encode_heap_oop(cmpval);
1677 __ mov(rscratch1, newval);
1678 __ encode_heap_oop(rscratch1);
1679 __ lock();
1680 // cmpval (rax) is implicitly used by this instruction
1681 __ cmpxchgl(rscratch1, Address(addr, 0));
1682 } else {
1683 __ lock();
1684 __ cmpxchgptr(newval, Address(addr, 0));
1685 }
1686 } else {
1687 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1688 __ lock();
1689 __ cmpxchgl(newval, Address(addr, 0));
1690 }
1691 } else if (op->code() == lir_cas_long) {
1692 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1693 Register newval = op->new_value()->as_register_lo();
1694 Register cmpval = op->cmp_value()->as_register_lo();
1695 assert(cmpval == rax, "wrong register");
1696 assert(newval != noreg, "new val must be register");
1697 assert(cmpval != newval, "cmp and new values must be in different registers");
1698 assert(cmpval != addr, "cmp and addr must be in different registers");
1699 assert(newval != addr, "new value and addr must be in different registers");
1700 __ lock();
1701 __ cmpxchgq(newval, Address(addr, 0));
1702 } else {
1703 Unimplemented();
1704 }
1705 }
1706
1707 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1708 assert(dst->is_cpu_register(), "must be");
1709 assert(dst->type() == src->type(), "must be");
1710
1711 if (src->is_cpu_register()) {
1712 reg2reg(src, dst);
1713 } else if (src->is_stack()) {
1714 stack2reg(src, dst, dst->type());
1715 } else if (src->is_constant()) {
1716 const2reg(src, dst, lir_patch_none, nullptr);
1717 } else {
1718 ShouldNotReachHere();
1719 }
1720 }
1721
1722 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1723 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1724 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1725
1726 Assembler::Condition acond, ncond;
1727 switch (condition) {
1728 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1729 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1730 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1731 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1732 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1733 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1734 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1735 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1736 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1737 ShouldNotReachHere();
1738 }
1739
1740 if (opr1->is_cpu_register()) {
1741 reg2reg(opr1, result);
1742 } else if (opr1->is_stack()) {
1743 stack2reg(opr1, result, result->type());
1744 } else if (opr1->is_constant()) {
1745 const2reg(opr1, result, lir_patch_none, nullptr);
1746 } else {
1747 ShouldNotReachHere();
1748 }
1749
1750 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1751 // optimized version that does not require a branch
1752 if (opr2->is_single_cpu()) {
1753 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1754 __ cmov(ncond, result->as_register(), opr2->as_register());
1755 } else if (opr2->is_double_cpu()) {
1756 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1757 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1758 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1759 } else if (opr2->is_single_stack()) {
1760 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1761 } else if (opr2->is_double_stack()) {
1762 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1763 } else {
1764 ShouldNotReachHere();
1765 }
1766
1767 } else {
1768 Label skip;
1769 __ jccb(acond, skip);
1770 if (opr2->is_cpu_register()) {
1771 reg2reg(opr2, result);
1772 } else if (opr2->is_stack()) {
1773 stack2reg(opr2, result, result->type());
1774 } else if (opr2->is_constant()) {
1775 const2reg(opr2, result, lir_patch_none, nullptr);
1776 } else {
1777 ShouldNotReachHere();
1778 }
1779 __ bind(skip);
1780 }
1781 }
1782
1783
1784 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1785 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1786
1787 if (left->is_single_cpu()) {
1788 assert(left == dest, "left and dest must be equal");
1789 Register lreg = left->as_register();
1790
1791 if (right->is_single_cpu()) {
1792 // cpu register - cpu register
1793 Register rreg = right->as_register();
1794 switch (code) {
1795 case lir_add: __ addl (lreg, rreg); break;
1796 case lir_sub: __ subl (lreg, rreg); break;
1797 case lir_mul: __ imull(lreg, rreg); break;
1798 default: ShouldNotReachHere();
1799 }
1800
1801 } else if (right->is_stack()) {
1802 // cpu register - stack
1803 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1804 switch (code) {
1805 case lir_add: __ addl(lreg, raddr); break;
1806 case lir_sub: __ subl(lreg, raddr); break;
1807 default: ShouldNotReachHere();
1808 }
1809
1810 } else if (right->is_constant()) {
1811 // cpu register - constant
1812 jint c = right->as_constant_ptr()->as_jint();
1813 switch (code) {
1814 case lir_add: {
1815 __ incrementl(lreg, c);
1816 break;
1817 }
1818 case lir_sub: {
1819 __ decrementl(lreg, c);
1820 break;
1821 }
1822 default: ShouldNotReachHere();
1823 }
1824
1825 } else {
1826 ShouldNotReachHere();
1827 }
1828
1829 } else if (left->is_double_cpu()) {
1830 assert(left == dest, "left and dest must be equal");
1831 Register lreg_lo = left->as_register_lo();
1832 Register lreg_hi = left->as_register_hi();
1833
1834 if (right->is_double_cpu()) {
1835 // cpu register - cpu register
1836 Register rreg_lo = right->as_register_lo();
1837 Register rreg_hi = right->as_register_hi();
1838 assert_different_registers(lreg_lo, rreg_lo);
1839 switch (code) {
1840 case lir_add:
1841 __ addptr(lreg_lo, rreg_lo);
1842 break;
1843 case lir_sub:
1844 __ subptr(lreg_lo, rreg_lo);
1845 break;
1846 case lir_mul:
1847 __ imulq(lreg_lo, rreg_lo);
1848 break;
1849 default:
1850 ShouldNotReachHere();
1851 }
1852
1853 } else if (right->is_constant()) {
1854 // cpu register - constant
1855 jlong c = right->as_constant_ptr()->as_jlong_bits();
1856 __ movptr(r10, (intptr_t) c);
1857 switch (code) {
1858 case lir_add:
1859 __ addptr(lreg_lo, r10);
1860 break;
1861 case lir_sub:
1862 __ subptr(lreg_lo, r10);
1863 break;
1864 default:
1865 ShouldNotReachHere();
1866 }
1867
1868 } else {
1869 ShouldNotReachHere();
1870 }
1871
1872 } else if (left->is_single_xmm()) {
1873 assert(left == dest, "left and dest must be equal");
1874 XMMRegister lreg = left->as_xmm_float_reg();
1875
1876 if (right->is_single_xmm()) {
1877 XMMRegister rreg = right->as_xmm_float_reg();
1878 switch (code) {
1879 case lir_add: __ addss(lreg, rreg); break;
1880 case lir_sub: __ subss(lreg, rreg); break;
1881 case lir_mul: __ mulss(lreg, rreg); break;
1882 case lir_div: __ divss(lreg, rreg); break;
1883 default: ShouldNotReachHere();
1884 }
1885 } else {
1886 Address raddr;
1887 if (right->is_single_stack()) {
1888 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1889 } else if (right->is_constant()) {
1890 // hack for now
1891 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1892 } else {
1893 ShouldNotReachHere();
1894 }
1895 switch (code) {
1896 case lir_add: __ addss(lreg, raddr); break;
1897 case lir_sub: __ subss(lreg, raddr); break;
1898 case lir_mul: __ mulss(lreg, raddr); break;
1899 case lir_div: __ divss(lreg, raddr); break;
1900 default: ShouldNotReachHere();
1901 }
1902 }
1903
1904 } else if (left->is_double_xmm()) {
1905 assert(left == dest, "left and dest must be equal");
1906
1907 XMMRegister lreg = left->as_xmm_double_reg();
1908 if (right->is_double_xmm()) {
1909 XMMRegister rreg = right->as_xmm_double_reg();
1910 switch (code) {
1911 case lir_add: __ addsd(lreg, rreg); break;
1912 case lir_sub: __ subsd(lreg, rreg); break;
1913 case lir_mul: __ mulsd(lreg, rreg); break;
1914 case lir_div: __ divsd(lreg, rreg); break;
1915 default: ShouldNotReachHere();
1916 }
1917 } else {
1918 Address raddr;
1919 if (right->is_double_stack()) {
1920 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1921 } else if (right->is_constant()) {
1922 // hack for now
1923 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1924 } else {
1925 ShouldNotReachHere();
1926 }
1927 switch (code) {
1928 case lir_add: __ addsd(lreg, raddr); break;
1929 case lir_sub: __ subsd(lreg, raddr); break;
1930 case lir_mul: __ mulsd(lreg, raddr); break;
1931 case lir_div: __ divsd(lreg, raddr); break;
1932 default: ShouldNotReachHere();
1933 }
1934 }
1935
1936 } else if (left->is_single_stack() || left->is_address()) {
1937 assert(left == dest, "left and dest must be equal");
1938
1939 Address laddr;
1940 if (left->is_single_stack()) {
1941 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1942 } else if (left->is_address()) {
1943 laddr = as_Address(left->as_address_ptr());
1944 } else {
1945 ShouldNotReachHere();
1946 }
1947
1948 if (right->is_single_cpu()) {
1949 Register rreg = right->as_register();
1950 switch (code) {
1951 case lir_add: __ addl(laddr, rreg); break;
1952 case lir_sub: __ subl(laddr, rreg); break;
1953 default: ShouldNotReachHere();
1954 }
1955 } else if (right->is_constant()) {
1956 jint c = right->as_constant_ptr()->as_jint();
1957 switch (code) {
1958 case lir_add: {
1959 __ incrementl(laddr, c);
1960 break;
1961 }
1962 case lir_sub: {
1963 __ decrementl(laddr, c);
1964 break;
1965 }
1966 default: ShouldNotReachHere();
1967 }
1968 } else {
1969 ShouldNotReachHere();
1970 }
1971
1972 } else {
1973 ShouldNotReachHere();
1974 }
1975 }
1976
1977
1978 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1979 if (value->is_double_xmm()) {
1980 switch(code) {
1981 case lir_abs :
1982 {
1983 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1984 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1985 }
1986 assert(!tmp->is_valid(), "do not need temporary");
1987 __ andpd(dest->as_xmm_double_reg(),
1988 ExternalAddress((address)double_signmask_pool),
1989 rscratch1);
1990 }
1991 break;
1992
1993 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1994 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1995 default : ShouldNotReachHere();
1996 }
1997
1998 } else if (code == lir_f2hf) {
1999 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2000 } else if (code == lir_hf2f) {
2001 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2002 } else {
2003 Unimplemented();
2004 }
2005 }
2006
2007 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2008 // assert(left->destroys_register(), "check");
2009 if (left->is_single_cpu()) {
2010 Register reg = left->as_register();
2011 if (right->is_constant()) {
2012 int val = right->as_constant_ptr()->as_jint();
2013 switch (code) {
2014 case lir_logic_and: __ andl (reg, val); break;
2015 case lir_logic_or: __ orl (reg, val); break;
2016 case lir_logic_xor: __ xorl (reg, val); break;
2017 default: ShouldNotReachHere();
2018 }
2019 } else if (right->is_stack()) {
2020 // added support for stack operands
2021 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2022 switch (code) {
2023 case lir_logic_and: __ andl (reg, raddr); break;
2024 case lir_logic_or: __ orl (reg, raddr); break;
2025 case lir_logic_xor: __ xorl (reg, raddr); break;
2026 default: ShouldNotReachHere();
2027 }
2028 } else {
2029 Register rright = right->as_register();
2030 switch (code) {
2031 case lir_logic_and: __ andptr (reg, rright); break;
2032 case lir_logic_or : __ orptr (reg, rright); break;
2033 case lir_logic_xor: __ xorptr (reg, rright); break;
2034 default: ShouldNotReachHere();
2035 }
2036 }
2037 move_regs(reg, dst->as_register());
2038 } else {
2039 Register l_lo = left->as_register_lo();
2040 Register l_hi = left->as_register_hi();
2041 if (right->is_constant()) {
2042 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2043 switch (code) {
2044 case lir_logic_and:
2045 __ andq(l_lo, rscratch1);
2046 break;
2047 case lir_logic_or:
2048 __ orq(l_lo, rscratch1);
2049 break;
2050 case lir_logic_xor:
2051 __ xorq(l_lo, rscratch1);
2052 break;
2053 default: ShouldNotReachHere();
2054 }
2055 } else {
2056 Register r_lo;
2057 if (is_reference_type(right->type())) {
2058 r_lo = right->as_register();
2059 } else {
2060 r_lo = right->as_register_lo();
2061 }
2062 switch (code) {
2063 case lir_logic_and:
2064 __ andptr(l_lo, r_lo);
2065 break;
2066 case lir_logic_or:
2067 __ orptr(l_lo, r_lo);
2068 break;
2069 case lir_logic_xor:
2070 __ xorptr(l_lo, r_lo);
2071 break;
2072 default: ShouldNotReachHere();
2073 }
2074 }
2075
2076 Register dst_lo = dst->as_register_lo();
2077 Register dst_hi = dst->as_register_hi();
2078
2079 move_regs(l_lo, dst_lo);
2080 }
2081 }
2082
2083
2084 // we assume that rax, and rdx can be overwritten
2085 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2086
2087 assert(left->is_single_cpu(), "left must be register");
2088 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2089 assert(result->is_single_cpu(), "result must be register");
2090
2091 // assert(left->destroys_register(), "check");
2092 // assert(right->destroys_register(), "check");
2093
2094 Register lreg = left->as_register();
2095 Register dreg = result->as_register();
2096
2097 if (right->is_constant()) {
2098 jint divisor = right->as_constant_ptr()->as_jint();
2099 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2100 if (code == lir_idiv) {
2101 assert(lreg == rax, "must be rax,");
2102 assert(temp->as_register() == rdx, "tmp register must be rdx");
2103 __ cdql(); // sign extend into rdx:rax
2104 if (divisor == 2) {
2105 __ subl(lreg, rdx);
2106 } else {
2107 __ andl(rdx, divisor - 1);
2108 __ addl(lreg, rdx);
2109 }
2110 __ sarl(lreg, log2i_exact(divisor));
2111 move_regs(lreg, dreg);
2112 } else if (code == lir_irem) {
2113 Label done;
2114 __ mov(dreg, lreg);
2115 __ andl(dreg, 0x80000000 | (divisor - 1));
2116 __ jcc(Assembler::positive, done);
2117 __ decrement(dreg);
2118 __ orl(dreg, ~(divisor - 1));
2119 __ increment(dreg);
2120 __ bind(done);
2121 } else {
2122 ShouldNotReachHere();
2123 }
2124 } else {
2125 Register rreg = right->as_register();
2126 assert(lreg == rax, "left register must be rax,");
2127 assert(rreg != rdx, "right register must not be rdx");
2128 assert(temp->as_register() == rdx, "tmp register must be rdx");
2129
2130 move_regs(lreg, rax);
2131
2132 int idivl_offset = __ corrected_idivl(rreg);
2133 if (ImplicitDiv0Checks) {
2134 add_debug_info_for_div0(idivl_offset, info);
2135 }
2136 if (code == lir_irem) {
2137 move_regs(rdx, dreg); // result is in rdx
2138 } else {
2139 move_regs(rax, dreg);
2140 }
2141 }
2142 }
2143
2144
2145 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2146 if (opr1->is_single_cpu()) {
2147 Register reg1 = opr1->as_register();
2148 if (opr2->is_single_cpu()) {
2149 // cpu register - cpu register
2150 if (is_reference_type(opr1->type())) {
2151 __ cmpoop(reg1, opr2->as_register());
2152 } else {
2153 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2154 __ cmpl(reg1, opr2->as_register());
2155 }
2156 } else if (opr2->is_stack()) {
2157 // cpu register - stack
2158 if (is_reference_type(opr1->type())) {
2159 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2160 } else {
2161 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2162 }
2163 } else if (opr2->is_constant()) {
2164 // cpu register - constant
2165 LIR_Const* c = opr2->as_constant_ptr();
2166 if (c->type() == T_INT) {
2167 jint i = c->as_jint();
2168 if (i == 0) {
2169 __ testl(reg1, reg1);
2170 } else {
2171 __ cmpl(reg1, i);
2172 }
2173 } else if (c->type() == T_METADATA) {
2174 // All we need for now is a comparison with null for equality.
2175 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2176 Metadata* m = c->as_metadata();
2177 if (m == nullptr) {
2178 __ testptr(reg1, reg1);
2179 } else {
2180 ShouldNotReachHere();
2181 }
2182 } else if (is_reference_type(c->type())) {
2183 // In 64bit oops are single register
2184 jobject o = c->as_jobject();
2185 if (o == nullptr) {
2186 __ testptr(reg1, reg1);
2187 } else {
2188 __ cmpoop(reg1, o, rscratch1);
2189 }
2190 } else {
2191 fatal("unexpected type: %s", basictype_to_str(c->type()));
2192 }
2193 // cpu register - address
2194 } else if (opr2->is_address()) {
2195 if (op->info() != nullptr) {
2196 add_debug_info_for_null_check_here(op->info());
2197 }
2198 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2199 } else {
2200 ShouldNotReachHere();
2201 }
2202
2203 } else if(opr1->is_double_cpu()) {
2204 Register xlo = opr1->as_register_lo();
2205 Register xhi = opr1->as_register_hi();
2206 if (opr2->is_double_cpu()) {
2207 __ cmpptr(xlo, opr2->as_register_lo());
2208 } else if (opr2->is_constant()) {
2209 // cpu register - constant 0
2210 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2211 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2212 } else {
2213 ShouldNotReachHere();
2214 }
2215
2216 } else if (opr1->is_single_xmm()) {
2217 XMMRegister reg1 = opr1->as_xmm_float_reg();
2218 if (opr2->is_single_xmm()) {
2219 // xmm register - xmm register
2220 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2221 } else if (opr2->is_stack()) {
2222 // xmm register - stack
2223 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2224 } else if (opr2->is_constant()) {
2225 // xmm register - constant
2226 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2227 } else if (opr2->is_address()) {
2228 // xmm register - address
2229 if (op->info() != nullptr) {
2230 add_debug_info_for_null_check_here(op->info());
2231 }
2232 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2233 } else {
2234 ShouldNotReachHere();
2235 }
2236
2237 } else if (opr1->is_double_xmm()) {
2238 XMMRegister reg1 = opr1->as_xmm_double_reg();
2239 if (opr2->is_double_xmm()) {
2240 // xmm register - xmm register
2241 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2242 } else if (opr2->is_stack()) {
2243 // xmm register - stack
2244 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2245 } else if (opr2->is_constant()) {
2246 // xmm register - constant
2247 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2248 } else if (opr2->is_address()) {
2249 // xmm register - address
2250 if (op->info() != nullptr) {
2251 add_debug_info_for_null_check_here(op->info());
2252 }
2253 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2254 } else {
2255 ShouldNotReachHere();
2256 }
2257
2258 } else if (opr1->is_address() && opr2->is_constant()) {
2259 LIR_Const* c = opr2->as_constant_ptr();
2260 if (is_reference_type(c->type())) {
2261 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2262 __ movoop(rscratch1, c->as_jobject());
2263 }
2264 if (op->info() != nullptr) {
2265 add_debug_info_for_null_check_here(op->info());
2266 }
2267 // special case: address - constant
2268 LIR_Address* addr = opr1->as_address_ptr();
2269 if (c->type() == T_INT) {
2270 __ cmpl(as_Address(addr), c->as_jint());
2271 } else if (is_reference_type(c->type())) {
2272 // %%% Make this explode if addr isn't reachable until we figure out a
2273 // better strategy by giving noreg as the temp for as_Address
2274 __ cmpoop(rscratch1, as_Address(addr, noreg));
2275 } else {
2276 ShouldNotReachHere();
2277 }
2278
2279 } else {
2280 ShouldNotReachHere();
2281 }
2282 }
2283
2284 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2285 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2286 if (left->is_single_xmm()) {
2287 assert(right->is_single_xmm(), "must match");
2288 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2289 } else if (left->is_double_xmm()) {
2290 assert(right->is_double_xmm(), "must match");
2291 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2292
2293 } else {
2294 ShouldNotReachHere();
2295 }
2296 } else {
2297 assert(code == lir_cmp_l2i, "check");
2298 Label done;
2299 Register dest = dst->as_register();
2300 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2301 __ movl(dest, -1);
2302 __ jccb(Assembler::less, done);
2303 __ setb(Assembler::notZero, dest);
2304 __ movzbl(dest, dest);
2305 __ bind(done);
2306 }
2307 }
2308
2309
2310 void LIR_Assembler::align_call(LIR_Code code) {
2311 // make sure that the displacement word of the call ends up word aligned
2312 int offset = __ offset();
2313 switch (code) {
2314 case lir_static_call:
2315 case lir_optvirtual_call:
2316 case lir_dynamic_call:
2317 offset += NativeCall::displacement_offset;
2318 break;
2319 case lir_icvirtual_call:
2320 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2321 break;
2322 default: ShouldNotReachHere();
2323 }
2324 __ align(BytesPerWord, offset);
2325 }
2326
2327
2328 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2329 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2330 "must be aligned");
2331 __ call(AddressLiteral(op->addr(), rtype));
2332 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2333 __ post_call_nop();
2334 }
2335
2336
2337 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2338 __ ic_call(op->addr());
2339 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2340 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2341 "must be aligned");
2342 __ post_call_nop();
2343 }
2344
2345
2346 void LIR_Assembler::emit_static_call_stub() {
2347 address call_pc = __ pc();
2348 address stub = __ start_a_stub(call_stub_size());
2349 if (stub == nullptr) {
2350 bailout("static call stub overflow");
2351 return;
2352 }
2353
2354 int start = __ offset();
2355
2356 // make sure that the displacement word of the call ends up word aligned
2357 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2358 __ relocate(static_stub_Relocation::spec(call_pc));
2359 __ mov_metadata(rbx, (Metadata*)nullptr);
2360 // must be set to -1 at code generation time
2361 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2362 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2363 __ jump(RuntimeAddress(__ pc()));
2364
2365 assert(__ offset() - start <= call_stub_size(), "stub too big");
2366 __ end_a_stub();
2367 }
2368
2369
2370 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2371 assert(exceptionOop->as_register() == rax, "must match");
2372 assert(exceptionPC->as_register() == rdx, "must match");
2373
2374 // exception object is not added to oop map by LinearScan
2375 // (LinearScan assumes that no oops are in fixed registers)
2376 info->add_register_oop(exceptionOop);
2377 C1StubId unwind_id;
2378
2379 // get current pc information
2380 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2381 int pc_for_athrow_offset = __ offset();
2382 InternalAddress pc_for_athrow(__ pc());
2383 __ lea(exceptionPC->as_register(), pc_for_athrow);
2384 add_call_info(pc_for_athrow_offset, info); // for exception handler
2385
2386 __ verify_not_null_oop(rax);
2387 // search an exception handler (rax: exception oop, rdx: throwing pc)
2388 if (compilation()->has_fpu_code()) {
2389 unwind_id = C1StubId::handle_exception_id;
2390 } else {
2391 unwind_id = C1StubId::handle_exception_nofpu_id;
2392 }
2393 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2394
2395 // enough room for two byte trap
2396 __ nop();
2397 }
2398
2399
2400 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2401 assert(exceptionOop->as_register() == rax, "must match");
2402
2403 __ jmp(_unwind_handler_entry);
2404 }
2405
2406
2407 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2408
2409 // optimized version for linear scan:
2410 // * count must be already in ECX (guaranteed by LinearScan)
2411 // * left and dest must be equal
2412 // * tmp must be unused
2413 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2414 assert(left == dest, "left and dest must be equal");
2415 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2416
2417 if (left->is_single_cpu()) {
2418 Register value = left->as_register();
2419 assert(value != SHIFT_count, "left cannot be ECX");
2420
2421 switch (code) {
2422 case lir_shl: __ shll(value); break;
2423 case lir_shr: __ sarl(value); break;
2424 case lir_ushr: __ shrl(value); break;
2425 default: ShouldNotReachHere();
2426 }
2427 } else if (left->is_double_cpu()) {
2428 Register lo = left->as_register_lo();
2429 Register hi = left->as_register_hi();
2430 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2431 switch (code) {
2432 case lir_shl: __ shlptr(lo); break;
2433 case lir_shr: __ sarptr(lo); break;
2434 case lir_ushr: __ shrptr(lo); break;
2435 default: ShouldNotReachHere();
2436 }
2437 } else {
2438 ShouldNotReachHere();
2439 }
2440 }
2441
2442
2443 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2444 if (dest->is_single_cpu()) {
2445 // first move left into dest so that left is not destroyed by the shift
2446 Register value = dest->as_register();
2447 count = count & 0x1F; // Java spec
2448
2449 move_regs(left->as_register(), value);
2450 switch (code) {
2451 case lir_shl: __ shll(value, count); break;
2452 case lir_shr: __ sarl(value, count); break;
2453 case lir_ushr: __ shrl(value, count); break;
2454 default: ShouldNotReachHere();
2455 }
2456 } else if (dest->is_double_cpu()) {
2457 // first move left into dest so that left is not destroyed by the shift
2458 Register value = dest->as_register_lo();
2459 count = count & 0x1F; // Java spec
2460
2461 move_regs(left->as_register_lo(), value);
2462 switch (code) {
2463 case lir_shl: __ shlptr(value, count); break;
2464 case lir_shr: __ sarptr(value, count); break;
2465 case lir_ushr: __ shrptr(value, count); break;
2466 default: ShouldNotReachHere();
2467 }
2468 } else {
2469 ShouldNotReachHere();
2470 }
2471 }
2472
2473
2474 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2475 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2476 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2477 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2478 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2479 }
2480
2481
2482 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2483 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2484 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2485 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2486 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2487 }
2488
2489
2490 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2491 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2492 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2493 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2494 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2495 }
2496
2497
2498 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2499 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2500 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2501 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2502 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2503 }
2504
2505
2506 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2507 if (null_check) {
2508 __ testptr(obj, obj);
2509 __ jcc(Assembler::zero, *slow_path->entry());
2510 }
2511 if (is_dest) {
2512 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2513 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2514 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2515 } else {
2516 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2517 }
2518 }
2519
2520
2521 // This code replaces a call to arraycopy; no exception may
2522 // be thrown in this code, they must be thrown in the System.arraycopy
2523 // activation frame; we could save some checks if this would not be the case
2524 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2525 ciArrayKlass* default_type = op->expected_type();
2526 Register src = op->src()->as_register();
2527 Register dst = op->dst()->as_register();
2528 Register src_pos = op->src_pos()->as_register();
2529 Register dst_pos = op->dst_pos()->as_register();
2530 Register length = op->length()->as_register();
2531 Register tmp = op->tmp()->as_register();
2532 Register tmp_load_klass = rscratch1;
2533 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2534
2535 CodeStub* stub = op->stub();
2536 int flags = op->flags();
2537 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2538 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2539
2540 if (flags & LIR_OpArrayCopy::always_slow_path) {
2541 __ jmp(*stub->entry());
2542 __ bind(*stub->continuation());
2543 return;
2544 }
2545
2546 // if we don't know anything, just go through the generic arraycopy
2547 if (default_type == nullptr) {
2548 // save outgoing arguments on stack in case call to System.arraycopy is needed
2549 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2550 // for interpreter calling conventions. Now we have to do it in new style conventions.
2551 // For the moment until C1 gets the new register allocator I just force all the
2552 // args to the right place (except the register args) and then on the back side
2553 // reload the register args properly if we go slow path. Yuck
2554
2555 // These are proper for the calling convention
2556 store_parameter(length, 2);
2557 store_parameter(dst_pos, 1);
2558 store_parameter(dst, 0);
2559
2560 // these are just temporary placements until we need to reload
2561 store_parameter(src_pos, 3);
2562 store_parameter(src, 4);
2563
2564 address copyfunc_addr = StubRoutines::generic_arraycopy();
2565 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2566
2567 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2568 // The arguments are in java calling convention so we can trivially shift them to C
2569 // convention
2570 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2571 __ mov(c_rarg0, j_rarg0);
2572 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2573 __ mov(c_rarg1, j_rarg1);
2574 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2575 __ mov(c_rarg2, j_rarg2);
2576 assert_different_registers(c_rarg3, j_rarg4);
2577 __ mov(c_rarg3, j_rarg3);
2578 #ifdef _WIN64
2579 // Allocate abi space for args but be sure to keep stack aligned
2580 __ subptr(rsp, 6*wordSize);
2581 store_parameter(j_rarg4, 4);
2582 #ifndef PRODUCT
2583 if (PrintC1Statistics) {
2584 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2585 }
2586 #endif
2587 __ call(RuntimeAddress(copyfunc_addr));
2588 __ addptr(rsp, 6*wordSize);
2589 #else
2590 __ mov(c_rarg4, j_rarg4);
2591 #ifndef PRODUCT
2592 if (PrintC1Statistics) {
2593 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2594 }
2595 #endif
2596 __ call(RuntimeAddress(copyfunc_addr));
2597 #endif // _WIN64
2598
2599 __ testl(rax, rax);
2600 __ jcc(Assembler::equal, *stub->continuation());
2601
2602 __ mov(tmp, rax);
2603 __ xorl(tmp, -1);
2604
2605 // Reload values from the stack so they are where the stub
2606 // expects them.
2607 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2608 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2609 __ movptr (length, Address(rsp, 2*BytesPerWord));
2610 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2611 __ movptr (src, Address(rsp, 4*BytesPerWord));
2612
2613 __ subl(length, tmp);
2614 __ addl(src_pos, tmp);
2615 __ addl(dst_pos, tmp);
2616 __ jmp(*stub->entry());
2617
2618 __ bind(*stub->continuation());
2619 return;
2620 }
2621
2622 // Handle inline type arrays
2623 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2624 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2625 }
2626 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2627 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2628 }
2629
2630 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2631
2632 int elem_size = type2aelembytes(basic_type);
2633 Address::ScaleFactor scale;
2634
2635 switch (elem_size) {
2636 case 1 :
2637 scale = Address::times_1;
2638 break;
2639 case 2 :
2640 scale = Address::times_2;
2641 break;
2642 case 4 :
2643 scale = Address::times_4;
2644 break;
2645 case 8 :
2646 scale = Address::times_8;
2647 break;
2648 default:
2649 scale = Address::no_scale;
2650 ShouldNotReachHere();
2651 }
2652
2653 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2654 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2655
2656 // length and pos's are all sign extended at this point on 64bit
2657
2658 // test for null
2659 if (flags & LIR_OpArrayCopy::src_null_check) {
2660 __ testptr(src, src);
2661 __ jcc(Assembler::zero, *stub->entry());
2662 }
2663 if (flags & LIR_OpArrayCopy::dst_null_check) {
2664 __ testptr(dst, dst);
2665 __ jcc(Assembler::zero, *stub->entry());
2666 }
2667
2668 // If the compiler was not able to prove that exact type of the source or the destination
2669 // of the arraycopy is an array type, check at runtime if the source or the destination is
2670 // an instance type.
2671 if (flags & LIR_OpArrayCopy::type_check) {
2672 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2673 __ load_klass(tmp, dst, tmp_load_klass);
2674 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2675 __ jcc(Assembler::greaterEqual, *stub->entry());
2676 }
2677
2678 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2679 __ load_klass(tmp, src, tmp_load_klass);
2680 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2681 __ jcc(Assembler::greaterEqual, *stub->entry());
2682 }
2683 }
2684
2685 // check if negative
2686 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2687 __ testl(src_pos, src_pos);
2688 __ jcc(Assembler::less, *stub->entry());
2689 }
2690 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2691 __ testl(dst_pos, dst_pos);
2692 __ jcc(Assembler::less, *stub->entry());
2693 }
2694
2695 if (flags & LIR_OpArrayCopy::src_range_check) {
2696 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2697 __ cmpl(tmp, src_length_addr);
2698 __ jcc(Assembler::above, *stub->entry());
2699 }
2700 if (flags & LIR_OpArrayCopy::dst_range_check) {
2701 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2702 __ cmpl(tmp, dst_length_addr);
2703 __ jcc(Assembler::above, *stub->entry());
2704 }
2705
2706 if (flags & LIR_OpArrayCopy::length_positive_check) {
2707 __ testl(length, length);
2708 __ jcc(Assembler::less, *stub->entry());
2709 }
2710
2711 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2712 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2713
2714 if (flags & LIR_OpArrayCopy::type_check) {
2715 // We don't know the array types are compatible
2716 if (basic_type != T_OBJECT) {
2717 // Simple test for basic type arrays
2718 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2719 __ jcc(Assembler::notEqual, *stub->entry());
2720 } else {
2721 // For object arrays, if src is a sub class of dst then we can
2722 // safely do the copy.
2723 Label cont, slow;
2724
2725 __ push(src);
2726 __ push(dst);
2727
2728 __ load_klass(src, src, tmp_load_klass);
2729 __ load_klass(dst, dst, tmp_load_klass);
2730
2731 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2732
2733 __ push(src);
2734 __ push(dst);
2735 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
2736 __ pop(dst);
2737 __ pop(src);
2738
2739 __ testl(src, src);
2740 __ jcc(Assembler::notEqual, cont);
2741
2742 __ bind(slow);
2743 __ pop(dst);
2744 __ pop(src);
2745
2746 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2747 if (copyfunc_addr != nullptr) { // use stub if available
2748 // src is not a sub class of dst so we have to do a
2749 // per-element check.
2750
2751 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2752 if ((flags & mask) != mask) {
2753 // Check that at least both of them object arrays.
2754 assert(flags & mask, "one of the two should be known to be an object array");
2755
2756 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2757 __ load_klass(tmp, src, tmp_load_klass);
2758 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2759 __ load_klass(tmp, dst, tmp_load_klass);
2760 }
2761 int lh_offset = in_bytes(Klass::layout_helper_offset());
2762 Address klass_lh_addr(tmp, lh_offset);
2763 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2764 __ cmpl(klass_lh_addr, objArray_lh);
2765 __ jcc(Assembler::notEqual, *stub->entry());
2766 }
2767
2768 // Spill because stubs can use any register they like and it's
2769 // easier to restore just those that we care about.
2770 store_parameter(dst, 0);
2771 store_parameter(dst_pos, 1);
2772 store_parameter(length, 2);
2773 store_parameter(src_pos, 3);
2774 store_parameter(src, 4);
2775
2776 __ movl2ptr(length, length); //higher 32bits must be null
2777
2778 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2779 assert_different_registers(c_rarg0, dst, dst_pos, length);
2780 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2781 assert_different_registers(c_rarg1, dst, length);
2782
2783 __ mov(c_rarg2, length);
2784 assert_different_registers(c_rarg2, dst);
2785
2786 #ifdef _WIN64
2787 // Allocate abi space for args but be sure to keep stack aligned
2788 __ subptr(rsp, 6*wordSize);
2789 __ load_klass(c_rarg3, dst, tmp_load_klass);
2790 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2791 store_parameter(c_rarg3, 4);
2792 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2793 __ call(RuntimeAddress(copyfunc_addr));
2794 __ addptr(rsp, 6*wordSize);
2795 #else
2796 __ load_klass(c_rarg4, dst, tmp_load_klass);
2797 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2798 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2799 __ call(RuntimeAddress(copyfunc_addr));
2800 #endif
2801
2802 #ifndef PRODUCT
2803 if (PrintC1Statistics) {
2804 Label failed;
2805 __ testl(rax, rax);
2806 __ jcc(Assembler::notZero, failed);
2807 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2808 __ bind(failed);
2809 }
2810 #endif
2811
2812 __ testl(rax, rax);
2813 __ jcc(Assembler::zero, *stub->continuation());
2814
2815 #ifndef PRODUCT
2816 if (PrintC1Statistics) {
2817 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2818 }
2819 #endif
2820
2821 __ mov(tmp, rax);
2822
2823 __ xorl(tmp, -1);
2824
2825 // Restore previously spilled arguments
2826 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2827 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2828 __ movptr (length, Address(rsp, 2*BytesPerWord));
2829 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2830 __ movptr (src, Address(rsp, 4*BytesPerWord));
2831
2832
2833 __ subl(length, tmp);
2834 __ addl(src_pos, tmp);
2835 __ addl(dst_pos, tmp);
2836 }
2837
2838 __ jmp(*stub->entry());
2839
2840 __ bind(cont);
2841 __ pop(dst);
2842 __ pop(src);
2843 }
2844 }
2845
2846 #ifdef ASSERT
2847 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2848 // Sanity check the known type with the incoming class. For the
2849 // primitive case the types must match exactly with src.klass and
2850 // dst.klass each exactly matching the default type. For the
2851 // object array case, if no type check is needed then either the
2852 // dst type is exactly the expected type and the src type is a
2853 // subtype which we can't check or src is the same array as dst
2854 // but not necessarily exactly of type default_type.
2855 Label known_ok, halt;
2856 __ mov_metadata(tmp, default_type->constant_encoding());
2857 if (UseCompressedClassPointers) {
2858 __ encode_klass_not_null(tmp, rscratch1);
2859 }
2860
2861 if (basic_type != T_OBJECT) {
2862 __ cmp_klass(tmp, dst, tmp2);
2863 __ jcc(Assembler::notEqual, halt);
2864 __ cmp_klass(tmp, src, tmp2);
2865 __ jcc(Assembler::equal, known_ok);
2866 } else {
2867 __ cmp_klass(tmp, dst, tmp2);
2868 __ jcc(Assembler::equal, known_ok);
2869 __ cmpptr(src, dst);
2870 __ jcc(Assembler::equal, known_ok);
2871 }
2872 __ bind(halt);
2873 __ stop("incorrect type information in arraycopy");
2874 __ bind(known_ok);
2875 }
2876 #endif
2877
2878 #ifndef PRODUCT
2879 if (PrintC1Statistics) {
2880 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2881 }
2882 #endif
2883
2884 assert_different_registers(c_rarg0, dst, dst_pos, length);
2885 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2886 assert_different_registers(c_rarg1, length);
2887 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2888 __ mov(c_rarg2, length);
2889
2890 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2891 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2892 const char *name;
2893 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2894 __ call_VM_leaf(entry, 0);
2895
2896 if (stub != nullptr) {
2897 __ bind(*stub->continuation());
2898 }
2899 }
2900
2901 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2902 assert(op->crc()->is_single_cpu(), "crc must be register");
2903 assert(op->val()->is_single_cpu(), "byte value must be register");
2904 assert(op->result_opr()->is_single_cpu(), "result must be register");
2905 Register crc = op->crc()->as_register();
2906 Register val = op->val()->as_register();
2907 Register res = op->result_opr()->as_register();
2908
2909 assert_different_registers(val, crc, res);
2910
2911 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2912 __ notl(crc); // ~crc
2913 __ update_byte_crc32(crc, val, res);
2914 __ notl(crc); // ~crc
2915 __ mov(res, crc);
2916 }
2917
2918 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2919 Register obj = op->obj_opr()->as_register(); // may not be an oop
2920 Register hdr = op->hdr_opr()->as_register();
2921 Register lock = op->lock_opr()->as_register();
2922 if (LockingMode == LM_MONITOR) {
2923 if (op->info() != nullptr) {
2924 add_debug_info_for_null_check_here(op->info());
2925 __ null_check(obj);
2926 }
2927 __ jmp(*op->stub()->entry());
2928 } else if (op->code() == lir_lock) {
2929 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2930 Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
2931 // add debug info for NullPointerException only if one is possible
2932 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2933 if (op->info() != nullptr) {
2934 add_debug_info_for_null_check(null_check_offset, op->info());
2935 }
2936 // done
2937 } else if (op->code() == lir_unlock) {
2938 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2939 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2940 } else {
2941 Unimplemented();
2942 }
2943 __ bind(*op->stub()->continuation());
2944 }
2945
2946 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2947 Register obj = op->obj()->as_pointer_register();
2948 Register result = op->result_opr()->as_pointer_register();
2949
2950 CodeEmitInfo* info = op->info();
2951 if (info != nullptr) {
2952 add_debug_info_for_null_check_here(info);
2953 }
2954
2955 __ load_klass(result, obj, rscratch1);
2956 }
2957
2958 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2959 ciMethod* method = op->profiled_method();
2960 int bci = op->profiled_bci();
2961 ciMethod* callee = op->profiled_callee();
2962 Register tmp_load_klass = rscratch1;
2963
2964 // Update counter for all call types
2965 ciMethodData* md = method->method_data_or_null();
2966 assert(md != nullptr, "Sanity");
2967 ciProfileData* data = md->bci_to_data(bci);
2968 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2969 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2970 Register mdo = op->mdo()->as_register();
2971 __ mov_metadata(mdo, md->constant_encoding());
2972 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2973 // Perform additional virtual call profiling for invokevirtual and
2974 // invokeinterface bytecodes
2975 if (op->should_profile_receiver_type()) {
2976 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2977 Register recv = op->recv()->as_register();
2978 assert_different_registers(mdo, recv);
2979 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2980 ciKlass* known_klass = op->known_holder();
2981 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2982 // We know the type that will be seen at this call site; we can
2983 // statically update the MethodData* rather than needing to do
2984 // dynamic tests on the receiver type
2985
2986 // NOTE: we should probably put a lock around this search to
2987 // avoid collisions by concurrent compilations
2988 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2989 uint i;
2990 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2991 ciKlass* receiver = vc_data->receiver(i);
2992 if (known_klass->equals(receiver)) {
2993 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2994 __ addptr(data_addr, DataLayout::counter_increment);
2995 return;
2996 }
2997 }
2998
2999 // Receiver type not found in profile data; select an empty slot
3000
3001 // Note that this is less efficient than it should be because it
3002 // always does a write to the receiver part of the
3003 // VirtualCallData rather than just the first time
3004 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3005 ciKlass* receiver = vc_data->receiver(i);
3006 if (receiver == nullptr) {
3007 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3008 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3009 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3010 __ addptr(data_addr, DataLayout::counter_increment);
3011 return;
3012 }
3013 }
3014 } else {
3015 __ load_klass(recv, recv, tmp_load_klass);
3016 Label update_done;
3017 type_profile_helper(mdo, md, data, recv, &update_done);
3018 // Receiver did not match any saved receiver and there is no empty row for it.
3019 // Increment total counter to indicate polymorphic case.
3020 __ addptr(counter_addr, DataLayout::counter_increment);
3021
3022 __ bind(update_done);
3023 }
3024 } else {
3025 // Static call
3026 __ addptr(counter_addr, DataLayout::counter_increment);
3027 }
3028 }
3029
3030 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3031 Register obj = op->obj()->as_register();
3032 Register tmp = op->tmp()->as_pointer_register();
3033 Register tmp_load_klass = rscratch1;
3034 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3035 ciKlass* exact_klass = op->exact_klass();
3036 intptr_t current_klass = op->current_klass();
3037 bool not_null = op->not_null();
3038 bool no_conflict = op->no_conflict();
3039
3040 Label update, next, none;
3041
3042 bool do_null = !not_null;
3043 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3044 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3045
3046 assert(do_null || do_update, "why are we here?");
3047 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3048
3049 __ verify_oop(obj);
3050
3051 #ifdef ASSERT
3052 if (obj == tmp) {
3053 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3054 } else {
3055 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3056 }
3057 #endif
3058 if (do_null) {
3059 __ testptr(obj, obj);
3060 __ jccb(Assembler::notZero, update);
3061 if (!TypeEntries::was_null_seen(current_klass)) {
3062 __ testptr(mdo_addr, TypeEntries::null_seen);
3063 #ifndef ASSERT
3064 __ jccb(Assembler::notZero, next); // already set
3065 #else
3066 __ jcc(Assembler::notZero, next); // already set
3067 #endif
3068 // atomic update to prevent overwriting Klass* with 0
3069 __ lock();
3070 __ orptr(mdo_addr, TypeEntries::null_seen);
3071 }
3072 if (do_update) {
3073 #ifndef ASSERT
3074 __ jmpb(next);
3075 }
3076 #else
3077 __ jmp(next);
3078 }
3079 } else {
3080 __ testptr(obj, obj);
3081 __ jcc(Assembler::notZero, update);
3082 __ stop("unexpected null obj");
3083 #endif
3084 }
3085
3086 __ bind(update);
3087
3088 if (do_update) {
3089 #ifdef ASSERT
3090 if (exact_klass != nullptr) {
3091 Label ok;
3092 __ load_klass(tmp, obj, tmp_load_klass);
3093 __ push(tmp);
3094 __ mov_metadata(tmp, exact_klass->constant_encoding());
3095 __ cmpptr(tmp, Address(rsp, 0));
3096 __ jcc(Assembler::equal, ok);
3097 __ stop("exact klass and actual klass differ");
3098 __ bind(ok);
3099 __ pop(tmp);
3100 }
3101 #endif
3102 if (!no_conflict) {
3103 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3104 if (exact_klass != nullptr) {
3105 __ mov_metadata(tmp, exact_klass->constant_encoding());
3106 } else {
3107 __ load_klass(tmp, obj, tmp_load_klass);
3108 }
3109 __ mov(rscratch1, tmp); // save original value before XOR
3110 __ xorptr(tmp, mdo_addr);
3111 __ testptr(tmp, TypeEntries::type_klass_mask);
3112 // klass seen before, nothing to do. The unknown bit may have been
3113 // set already but no need to check.
3114 __ jccb(Assembler::zero, next);
3115
3116 __ testptr(tmp, TypeEntries::type_unknown);
3117 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3118
3119 if (TypeEntries::is_type_none(current_klass)) {
3120 __ testptr(mdo_addr, TypeEntries::type_mask);
3121 __ jccb(Assembler::zero, none);
3122 // There is a chance that the checks above (re-reading profiling
3123 // data from memory) fail if another thread has just set the
3124 // profiling to this obj's klass
3125 __ mov(tmp, rscratch1); // get back original value before XOR
3126 __ xorptr(tmp, mdo_addr);
3127 __ testptr(tmp, TypeEntries::type_klass_mask);
3128 __ jccb(Assembler::zero, next);
3129 }
3130 } else {
3131 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3132 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3133
3134 __ testptr(mdo_addr, TypeEntries::type_unknown);
3135 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3136 }
3137
3138 // different than before. Cannot keep accurate profile.
3139 __ orptr(mdo_addr, TypeEntries::type_unknown);
3140
3141 if (TypeEntries::is_type_none(current_klass)) {
3142 __ jmpb(next);
3143
3144 __ bind(none);
3145 // first time here. Set profile type.
3146 __ movptr(mdo_addr, tmp);
3147 #ifdef ASSERT
3148 __ andptr(tmp, TypeEntries::type_klass_mask);
3149 __ verify_klass_ptr(tmp);
3150 #endif
3151 }
3152 } else {
3153 // There's a single possible klass at this profile point
3154 assert(exact_klass != nullptr, "should be");
3155 if (TypeEntries::is_type_none(current_klass)) {
3156 __ mov_metadata(tmp, exact_klass->constant_encoding());
3157 __ xorptr(tmp, mdo_addr);
3158 __ testptr(tmp, TypeEntries::type_klass_mask);
3159 #ifdef ASSERT
3160 __ jcc(Assembler::zero, next);
3161
3162 {
3163 Label ok;
3164 __ push(tmp);
3165 __ testptr(mdo_addr, TypeEntries::type_mask);
3166 __ jcc(Assembler::zero, ok);
3167 // may have been set by another thread
3168 __ mov_metadata(tmp, exact_klass->constant_encoding());
3169 __ xorptr(tmp, mdo_addr);
3170 __ testptr(tmp, TypeEntries::type_mask);
3171 __ jcc(Assembler::zero, ok);
3172
3173 __ stop("unexpected profiling mismatch");
3174 __ bind(ok);
3175 __ pop(tmp);
3176 }
3177 #else
3178 __ jccb(Assembler::zero, next);
3179 #endif
3180 // first time here. Set profile type.
3181 __ movptr(mdo_addr, tmp);
3182 #ifdef ASSERT
3183 __ andptr(tmp, TypeEntries::type_klass_mask);
3184 __ verify_klass_ptr(tmp);
3185 #endif
3186 } else {
3187 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3188 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3189
3190 __ testptr(mdo_addr, TypeEntries::type_unknown);
3191 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3192
3193 __ orptr(mdo_addr, TypeEntries::type_unknown);
3194 }
3195 }
3196 }
3197 __ bind(next);
3198 }
3199
3200 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3201 Register obj = op->obj()->as_register();
3202 Register tmp = op->tmp()->as_pointer_register();
3203 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3204 bool not_null = op->not_null();
3205 int flag = op->flag();
3206
3207 Label not_inline_type;
3208 if (!not_null) {
3209 __ testptr(obj, obj);
3210 __ jccb(Assembler::zero, not_inline_type);
3211 }
3212
3213 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3214
3215 __ orb(mdo_addr, flag);
3216
3217 __ bind(not_inline_type);
3218 }
3219
3220 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3221 Unimplemented();
3222 }
3223
3224
3225 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3226 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3227 }
3228
3229
3230 void LIR_Assembler::align_backward_branch_target() {
3231 __ align(BytesPerWord);
3232 }
3233
3234
3235 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3236 if (left->is_single_cpu()) {
3237 __ negl(left->as_register());
3238 move_regs(left->as_register(), dest->as_register());
3239
3240 } else if (left->is_double_cpu()) {
3241 Register lo = left->as_register_lo();
3242 Register dst = dest->as_register_lo();
3243 __ movptr(dst, lo);
3244 __ negptr(dst);
3245
3246 } else if (dest->is_single_xmm()) {
3247 assert(!tmp->is_valid(), "do not need temporary");
3248 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3249 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3250 }
3251 __ xorps(dest->as_xmm_float_reg(),
3252 ExternalAddress((address)float_signflip_pool),
3253 rscratch1);
3254 } else if (dest->is_double_xmm()) {
3255 assert(!tmp->is_valid(), "do not need temporary");
3256 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3257 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3258 }
3259 __ xorpd(dest->as_xmm_double_reg(),
3260 ExternalAddress((address)double_signflip_pool),
3261 rscratch1);
3262 } else {
3263 ShouldNotReachHere();
3264 }
3265 }
3266
3267
3268 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3269 assert(src->is_address(), "must be an address");
3270 assert(dest->is_register(), "must be a register");
3271
3272 PatchingStub* patch = nullptr;
3273 if (patch_code != lir_patch_none) {
3274 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3275 }
3276
3277 Register reg = dest->as_pointer_register();
3278 LIR_Address* addr = src->as_address_ptr();
3279 __ lea(reg, as_Address(addr));
3280
3281 if (patch != nullptr) {
3282 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3283 }
3284 }
3285
3286
3287
3288 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3289 assert(!tmp->is_valid(), "don't need temporary");
3290 __ call(RuntimeAddress(dest));
3291 if (info != nullptr) {
3292 add_call_info_here(info);
3293 }
3294 __ post_call_nop();
3295 }
3296
3297
3298 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3299 assert(type == T_LONG, "only for volatile long fields");
3300
3301 if (info != nullptr) {
3302 add_debug_info_for_null_check_here(info);
3303 }
3304
3305 if (src->is_double_xmm()) {
3306 if (dest->is_double_cpu()) {
3307 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3308 } else if (dest->is_double_stack()) {
3309 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3310 } else if (dest->is_address()) {
3311 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3312 } else {
3313 ShouldNotReachHere();
3314 }
3315
3316 } else if (dest->is_double_xmm()) {
3317 if (src->is_double_stack()) {
3318 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3319 } else if (src->is_address()) {
3320 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3321 } else {
3322 ShouldNotReachHere();
3323 }
3324
3325 } else {
3326 ShouldNotReachHere();
3327 }
3328 }
3329
3330 #ifdef ASSERT
3331 // emit run-time assertion
3332 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3333 assert(op->code() == lir_assert, "must be");
3334
3335 if (op->in_opr1()->is_valid()) {
3336 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3337 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3338 } else {
3339 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3340 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3341 }
3342
3343 Label ok;
3344 if (op->condition() != lir_cond_always) {
3345 Assembler::Condition acond = Assembler::zero;
3346 switch (op->condition()) {
3347 case lir_cond_equal: acond = Assembler::equal; break;
3348 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3349 case lir_cond_less: acond = Assembler::less; break;
3350 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3351 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3352 case lir_cond_greater: acond = Assembler::greater; break;
3353 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3354 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3355 default: ShouldNotReachHere();
3356 }
3357 __ jcc(acond, ok);
3358 }
3359 if (op->halt()) {
3360 const char* str = __ code_string(op->msg());
3361 __ stop(str);
3362 } else {
3363 breakpoint();
3364 }
3365 __ bind(ok);
3366 }
3367 #endif
3368
3369 void LIR_Assembler::membar() {
3370 // QQQ sparc TSO uses this,
3371 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3372 }
3373
3374 void LIR_Assembler::membar_acquire() {
3375 // No x86 machines currently require load fences
3376 }
3377
3378 void LIR_Assembler::membar_release() {
3379 // No x86 machines currently require store fences
3380 }
3381
3382 void LIR_Assembler::membar_loadload() {
3383 // no-op
3384 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3385 }
3386
3387 void LIR_Assembler::membar_storestore() {
3388 // no-op
3389 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3390 }
3391
3392 void LIR_Assembler::membar_loadstore() {
3393 // no-op
3394 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3395 }
3396
3397 void LIR_Assembler::membar_storeload() {
3398 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3399 }
3400
3401 void LIR_Assembler::on_spin_wait() {
3402 __ pause ();
3403 }
3404
3405 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3406 assert(result_reg->is_register(), "check");
3407 __ mov(result_reg->as_register(), r15_thread);
3408 }
3409
3410 void LIR_Assembler::check_orig_pc() {
3411 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3412 }
3413
3414 void LIR_Assembler::peephole(LIR_List*) {
3415 // do nothing for now
3416 }
3417
3418 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3419 assert(data == dest, "xchg/xadd uses only 2 operands");
3420
3421 if (data->type() == T_INT) {
3422 if (code == lir_xadd) {
3423 __ lock();
3424 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3425 } else {
3426 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3427 }
3428 } else if (data->is_oop()) {
3429 assert (code == lir_xchg, "xadd for oops");
3430 Register obj = data->as_register();
3431 if (UseCompressedOops) {
3432 __ encode_heap_oop(obj);
3433 __ xchgl(obj, as_Address(src->as_address_ptr()));
3434 __ decode_heap_oop(obj);
3435 } else {
3436 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3437 }
3438 } else if (data->type() == T_LONG) {
3439 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3440 if (code == lir_xadd) {
3441 __ lock();
3442 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3443 } else {
3444 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3445 }
3446 } else {
3447 ShouldNotReachHere();
3448 }
3449 }
3450
3451 #undef __