1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/powerOfTwo.hpp"
49 #include "vmreg_x86.inline.hpp"
50
51
52 // These masks are used to provide 128-bit aligned bitmasks to the XMM
53 // instructions, to allow sign-masking or sign-bit flipping. They allow
54 // fast versions of NegF/NegD and AbsF/AbsD.
55
56 // Note: 'double' and 'long long' have 32-bits alignment on x86.
57 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
58 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
59 // of 128-bits operands for SSE instructions.
60 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
61 // Store the value to a 128-bits operand.
62 operand[0] = lo;
63 operand[1] = hi;
64 return operand;
65 }
66
67 // Buffer for 128-bits masks used by SSE instructions.
68 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
69
70 // Static initialization during VM startup.
71 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
72 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
73 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
74 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
75
76
77 NEEDS_CLEANUP // remove this definitions ?
78 const Register SYNC_header = rax; // synchronization header
79 const Register SHIFT_count = rcx; // where count for shift operations must be
80
81 #define __ _masm->
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2,
87 Register &tmp3) {
88 if (tmp1 == preserve) {
89 assert_different_registers(tmp1, tmp2, tmp3, extra);
90 tmp1 = extra;
91 } else if (tmp2 == preserve) {
92 assert_different_registers(tmp1, tmp2, tmp3, extra);
93 tmp2 = extra;
94 } else if (tmp3 == preserve) {
95 assert_different_registers(tmp1, tmp2, tmp3, extra);
96 tmp3 = extra;
97 }
98 assert_different_registers(preserve, tmp1, tmp2, tmp3);
99 }
100
101
102
103 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
104 if (opr->is_constant()) {
105 LIR_Const* constant = opr->as_constant_ptr();
106 switch (constant->type()) {
107 case T_INT: {
108 return true;
109 }
110
111 default:
112 return false;
113 }
114 }
115 return false;
116 }
117
118
119 LIR_Opr LIR_Assembler::receiverOpr() {
120 return FrameMap::receiver_opr;
121 }
122
123 LIR_Opr LIR_Assembler::osrBufferPointer() {
124 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
125 }
126
127 //--------------fpu register translations-----------------------
128
129
130 address LIR_Assembler::float_constant(float f) {
131 address const_addr = __ float_constant(f);
132 if (const_addr == nullptr) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140
141 address LIR_Assembler::double_constant(double d) {
142 address const_addr = __ double_constant(d);
143 if (const_addr == nullptr) {
144 bailout("const section overflow");
145 return __ code()->consts()->start();
146 } else {
147 return const_addr;
148 }
149 }
150
151 void LIR_Assembler::breakpoint() {
152 __ int3();
153 }
154
155 void LIR_Assembler::push(LIR_Opr opr) {
156 if (opr->is_single_cpu()) {
157 __ push_reg(opr->as_register());
158 } else if (opr->is_double_cpu()) {
159 __ push_reg(opr->as_register_lo());
160 } else if (opr->is_stack()) {
161 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
162 } else if (opr->is_constant()) {
163 LIR_Const* const_opr = opr->as_constant_ptr();
164 if (const_opr->type() == T_OBJECT) {
165 __ push_oop(const_opr->as_jobject(), rscratch1);
166 } else if (const_opr->type() == T_INT) {
167 __ push_jint(const_opr->as_jint());
168 } else {
169 ShouldNotReachHere();
170 }
171
172 } else {
173 ShouldNotReachHere();
174 }
175 }
176
177 void LIR_Assembler::pop(LIR_Opr opr) {
178 if (opr->is_single_cpu()) {
179 __ pop_reg(opr->as_register());
180 } else {
181 ShouldNotReachHere();
182 }
183 }
184
185 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
186 return addr->base()->is_illegal() && addr->index()->is_illegal();
187 }
188
189 //-------------------------------------------
190
191 Address LIR_Assembler::as_Address(LIR_Address* addr) {
192 return as_Address(addr, rscratch1);
193 }
194
195 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
196 if (addr->base()->is_illegal()) {
197 assert(addr->index()->is_illegal(), "must be illegal too");
198 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
199 if (! __ reachable(laddr)) {
200 __ movptr(tmp, laddr.addr());
201 Address res(tmp, 0);
202 return res;
203 } else {
204 return __ as_Address(laddr);
205 }
206 }
207
208 Register base = addr->base()->as_pointer_register();
209
210 if (addr->index()->is_illegal()) {
211 return Address( base, addr->disp());
212 } else if (addr->index()->is_cpu_register()) {
213 Register index = addr->index()->as_pointer_register();
214 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
215 } else if (addr->index()->is_constant()) {
216 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
217 assert(Assembler::is_simm32(addr_offset), "must be");
218
219 return Address(base, addr_offset);
220 } else {
221 Unimplemented();
222 return Address();
223 }
224 }
225
226
227 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
228 Address base = as_Address(addr);
229 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
230 }
231
232
233 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
234 return as_Address(addr);
235 }
236
237
238 void LIR_Assembler::osr_entry() {
239 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
240 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
241 ValueStack* entry_state = osr_entry->state();
242 int number_of_locks = entry_state->locks_size();
243
244 // we jump here if osr happens with the interpreter
245 // state set up to continue at the beginning of the
246 // loop that triggered osr - in particular, we have
247 // the following registers setup:
248 //
249 // rcx: osr buffer
250 //
251
252 // build frame
253 ciMethod* m = compilation()->method();
254 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
255
256 // OSR buffer is
257 //
258 // locals[nlocals-1..0]
259 // monitors[0..number_of_locks]
260 //
261 // locals is a direct copy of the interpreter frame so in the osr buffer
262 // so first slot in the local array is the last local from the interpreter
263 // and last slot is local[0] (receiver) from the interpreter
264 //
265 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
266 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
267 // in the interpreter frame (the method lock if a sync method)
268
269 // Initialize monitors in the compiled activation.
270 // rcx: pointer to osr buffer
271 //
272 // All other registers are dead at this point and the locals will be
273 // copied into place by code emitted in the IR.
274
275 Register OSR_buf = osrBufferPointer()->as_pointer_register();
276 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
277 int monitor_offset = BytesPerWord * method()->max_locals() +
278 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
279 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
280 // the OSR buffer using 2 word entries: first the lock and then
281 // the oop.
282 for (int i = 0; i < number_of_locks; i++) {
283 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
284 #ifdef ASSERT
285 // verify the interpreter's monitor has a non-null object
286 {
287 Label L;
288 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
289 __ jcc(Assembler::notZero, L);
290 __ stop("locked object is null");
291 __ bind(L);
292 }
293 #endif
294 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
295 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
296 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
297 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
298 }
299 }
300 }
301
302
303 // inline cache check; done before the frame is built.
304 int LIR_Assembler::check_icache() {
305 return __ ic_check(CodeEntryAlignment);
306 }
307
308 void LIR_Assembler::clinit_barrier(ciMethod* method) {
309 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
310 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
311
312 Label L_skip_barrier;
313 Register klass = rscratch1;
314
315 __ mov_metadata(klass, method->holder()->constant_encoding());
316 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
317
318 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
319
320 __ bind(L_skip_barrier);
321 }
322
323 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
324 jobject o = nullptr;
325 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
326 __ movoop(reg, o);
327 patching_epilog(patch, lir_patch_normal, reg, info);
328 }
329
330 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
331 Metadata* o = nullptr;
332 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
333 __ mov_metadata(reg, o);
334 patching_epilog(patch, lir_patch_normal, reg, info);
335 }
336
337 // This specifies the rsp decrement needed to build the frame
338 int LIR_Assembler::initial_frame_size_in_bytes() const {
339 // if rounding, must let FrameMap know!
340
341 // The frame_map records size in slots (32bit word)
342
343 // subtract two words to account for return address and link
344 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
345 }
346
347
348 int LIR_Assembler::emit_exception_handler() {
349 // generate code for exception handler
350 address handler_base = __ start_a_stub(exception_handler_size());
351 if (handler_base == nullptr) {
352 // not enough space left for the handler
353 bailout("exception handler overflow");
354 return -1;
355 }
356
357 int offset = code_offset();
358
359 // the exception oop and pc are in rax, and rdx
360 // no other registers need to be preserved, so invalidate them
361 __ invalidate_registers(false, true, true, false, true, true);
362
363 // check that there is really an exception
364 __ verify_not_null_oop(rax);
365
366 // search an exception handler (rax: exception oop, rdx: throwing pc)
367 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
368 __ should_not_reach_here();
369 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
370 __ end_a_stub();
371
372 return offset;
373 }
374
375
376 // Emit the code to remove the frame from the stack in the exception
377 // unwind path.
378 int LIR_Assembler::emit_unwind_handler() {
379 #ifndef PRODUCT
380 if (CommentedAssembly) {
381 _masm->block_comment("Unwind handler");
382 }
383 #endif
384
385 int offset = code_offset();
386
387 // Fetch the exception from TLS and clear out exception related thread state
388 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
389 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
390 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
391
392 __ bind(_unwind_handler_entry);
393 __ verify_not_null_oop(rax);
394 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
395 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
396 }
397
398 // Perform needed unlocking
399 MonitorExitStub* stub = nullptr;
400 if (method()->is_synchronized()) {
401 monitor_address(0, FrameMap::rax_opr);
402 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
403 __ unlock_object(rdi, rsi, rax, *stub->entry());
404 __ bind(*stub->continuation());
405 }
406
407 if (compilation()->env()->dtrace_method_probes()) {
408 __ mov(rdi, r15_thread);
409 __ mov_metadata(rsi, method()->constant_encoding());
410 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
411 }
412
413 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
414 __ mov(rax, rbx); // Restore the exception
415 }
416
417 // remove the activation and dispatch to the unwind handler
418 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
419 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
420
421 // Emit the slow path assembly
422 if (stub != nullptr) {
423 stub->emit_code(this);
424 }
425
426 return offset;
427 }
428
429
430 int LIR_Assembler::emit_deopt_handler() {
431 // generate code for exception handler
432 address handler_base = __ start_a_stub(deopt_handler_size());
433 if (handler_base == nullptr) {
434 // not enough space left for the handler
435 bailout("deopt handler overflow");
436 return -1;
437 }
438
439 int offset = code_offset();
440
441 Label start;
442 __ bind(start);
443
444 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
445
446 int entry_offset = __ offset();
447
448 __ jmp(start);
449
450 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
451 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
452 "out of bounds read in post-call NOP check");
453 __ end_a_stub();
454
455 return entry_offset;
456 }
457
458 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
459 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
460 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
461 assert(result->fpu() == 0, "result must already be on TOS");
462 }
463 if (InlineTypeReturnedAsFields) {
464 #ifndef _LP64
465 Unimplemented();
466 #endif
467 // Check if we are returning an non-null inline type and load its fields into registers
468 ciType* return_type = compilation()->method()->return_type();
469 if (return_type->is_inlinetype()) {
470 ciInlineKlass* vk = return_type->as_inline_klass();
471 if (vk->can_be_returned_as_fields()) {
472 address unpack_handler = vk->unpack_handler();
473 assert(unpack_handler != nullptr, "must be");
474 __ call(RuntimeAddress(unpack_handler));
475 }
476 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
477 Label skip;
478 Label not_null;
479 __ testptr(rax, rax);
480 __ jcc(Assembler::notZero, not_null);
481 // Returned value is null, zero all return registers because they may belong to oop fields
482 __ xorq(j_rarg1, j_rarg1);
483 __ xorq(j_rarg2, j_rarg2);
484 __ xorq(j_rarg3, j_rarg3);
485 __ xorq(j_rarg4, j_rarg4);
486 __ xorq(j_rarg5, j_rarg5);
487 __ jmp(skip);
488 __ bind(not_null);
489
490 // Check if we are returning an non-null inline type and load its fields into registers
491 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
492
493 // Load fields from a buffered value with an inline class specific handler
494 __ load_klass(rdi, rax, rscratch1);
495 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
496 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
497 // Unpack handler can be null if inline type is not scalarizable in returns
498 __ testptr(rdi, rdi);
499 __ jcc(Assembler::zero, skip);
500 __ call(rdi);
501
502 __ bind(skip);
503 }
504 // At this point, rax points to the value object (for interpreter or C1 caller).
505 // The fields of the object are copied into registers (for C2 caller).
506 }
507
508 // Pop the stack before the safepoint code
509 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
510
511 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
512 __ reserved_stack_check();
513 }
514
515 // Note: we do not need to round double result; float result has the right precision
516 // the poll sets the condition code, but no data registers
517
518 code_stub->set_safepoint_offset(__ offset());
519 __ relocate(relocInfo::poll_return_type);
520 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
521 __ ret(0);
522 }
523
524
525 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
526 return (__ store_inline_type_fields_to_buf(vk, false));
527 }
528
529 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
530 guarantee(info != nullptr, "Shouldn't be null");
531 int offset = __ offset();
532 const Register poll_addr = rscratch1;
533 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
534 add_debug_info_for_branch(info);
535 __ relocate(relocInfo::poll_type);
536 address pre_pc = __ pc();
537 __ testl(rax, Address(poll_addr, 0));
538 address post_pc = __ pc();
539 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
540 return offset;
541 }
542
543
544 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
545 if (from_reg != to_reg) __ mov(to_reg, from_reg);
546 }
547
548 void LIR_Assembler::swap_reg(Register a, Register b) {
549 __ xchgptr(a, b);
550 }
551
552
553 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
554 assert(src->is_constant(), "should not call otherwise");
555 assert(dest->is_register(), "should not call otherwise");
556 LIR_Const* c = src->as_constant_ptr();
557
558 switch (c->type()) {
559 case T_INT: {
560 assert(patch_code == lir_patch_none, "no patching handled here");
561 __ movl(dest->as_register(), c->as_jint());
562 break;
563 }
564
565 case T_ADDRESS: {
566 assert(patch_code == lir_patch_none, "no patching handled here");
567 __ movptr(dest->as_register(), c->as_jint());
568 break;
569 }
570
571 case T_LONG: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 #if INCLUDE_CDS
574 if (AOTCodeCache::is_on_for_dump()) {
575 address b = c->as_pointer();
576 if (AOTRuntimeConstants::contains(b)) {
577 __ load_aotrc_address(dest->as_register_lo(), b);
578 break;
579 }
580 }
581 #endif
582 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
583 break;
584 }
585
586 case T_OBJECT: {
587 if (patch_code != lir_patch_none) {
588 jobject2reg_with_patching(dest->as_register(), info);
589 } else {
590 __ movoop(dest->as_register(), c->as_jobject());
591 }
592 break;
593 }
594
595 case T_METADATA: {
596 if (patch_code != lir_patch_none) {
597 klass2reg_with_patching(dest->as_register(), info);
598 } else {
599 __ mov_metadata(dest->as_register(), c->as_metadata());
600 }
601 break;
602 }
603
604 case T_FLOAT: {
605 if (dest->is_single_xmm()) {
606 if (UseAVX <= 2 && c->is_zero_float()) {
607 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
608 } else {
609 __ movflt(dest->as_xmm_float_reg(),
610 InternalAddress(float_constant(c->as_jfloat())));
611 }
612 } else {
613 ShouldNotReachHere();
614 }
615 break;
616 }
617
618 case T_DOUBLE: {
619 if (dest->is_double_xmm()) {
620 if (UseAVX <= 2 && c->is_zero_double()) {
621 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
622 } else {
623 __ movdbl(dest->as_xmm_double_reg(),
624 InternalAddress(double_constant(c->as_jdouble())));
625 }
626 } else {
627 ShouldNotReachHere();
628 }
629 break;
630 }
631
632 default:
633 ShouldNotReachHere();
634 }
635 }
636
637 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
638 assert(src->is_constant(), "should not call otherwise");
639 assert(dest->is_stack(), "should not call otherwise");
640 LIR_Const* c = src->as_constant_ptr();
641
642 switch (c->type()) {
643 case T_INT: // fall through
644 case T_FLOAT:
645 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
646 break;
647
648 case T_ADDRESS:
649 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
650 break;
651
652 case T_OBJECT:
653 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
654 break;
655
656 case T_LONG: // fall through
657 case T_DOUBLE:
658 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
659 lo_word_offset_in_bytes),
660 (intptr_t)c->as_jlong_bits(),
661 rscratch1);
662 break;
663
664 default:
665 ShouldNotReachHere();
666 }
667 }
668
669 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
670 assert(src->is_constant(), "should not call otherwise");
671 assert(dest->is_address(), "should not call otherwise");
672 LIR_Const* c = src->as_constant_ptr();
673 LIR_Address* addr = dest->as_address_ptr();
674
675 int null_check_here = code_offset();
676 switch (type) {
677 case T_INT: // fall through
678 case T_FLOAT:
679 __ movl(as_Address(addr), c->as_jint_bits());
680 break;
681
682 case T_ADDRESS:
683 __ movptr(as_Address(addr), c->as_jint_bits());
684 break;
685
686 case T_OBJECT: // fall through
687 case T_ARRAY:
688 if (c->as_jobject() == nullptr) {
689 if (UseCompressedOops && !wide) {
690 __ movl(as_Address(addr), NULL_WORD);
691 } else {
692 __ xorptr(rscratch1, rscratch1);
693 null_check_here = code_offset();
694 __ movptr(as_Address(addr), rscratch1);
695 }
696 } else {
697 if (is_literal_address(addr)) {
698 ShouldNotReachHere();
699 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
700 } else {
701 __ movoop(rscratch1, c->as_jobject());
702 if (UseCompressedOops && !wide) {
703 __ encode_heap_oop(rscratch1);
704 null_check_here = code_offset();
705 __ movl(as_Address_lo(addr), rscratch1);
706 } else {
707 null_check_here = code_offset();
708 __ movptr(as_Address_lo(addr), rscratch1);
709 }
710 }
711 }
712 break;
713
714 case T_LONG: // fall through
715 case T_DOUBLE:
716 if (is_literal_address(addr)) {
717 ShouldNotReachHere();
718 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
719 } else {
720 __ movptr(r10, (intptr_t)c->as_jlong_bits());
721 null_check_here = code_offset();
722 __ movptr(as_Address_lo(addr), r10);
723 }
724 break;
725
726 case T_BOOLEAN: // fall through
727 case T_BYTE:
728 __ movb(as_Address(addr), c->as_jint() & 0xFF);
729 break;
730
731 case T_CHAR: // fall through
732 case T_SHORT:
733 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
734 break;
735
736 default:
737 ShouldNotReachHere();
738 };
739
740 if (info != nullptr) {
741 add_debug_info_for_null_check(null_check_here, info);
742 }
743 }
744
745
746 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
747 assert(src->is_register(), "should not call otherwise");
748 assert(dest->is_register(), "should not call otherwise");
749
750 // move between cpu-registers
751 if (dest->is_single_cpu()) {
752 if (src->type() == T_LONG) {
753 // Can do LONG -> OBJECT
754 move_regs(src->as_register_lo(), dest->as_register());
755 return;
756 }
757 assert(src->is_single_cpu(), "must match");
758 if (src->type() == T_OBJECT) {
759 __ verify_oop(src->as_register());
760 }
761 move_regs(src->as_register(), dest->as_register());
762
763 } else if (dest->is_double_cpu()) {
764 if (is_reference_type(src->type())) {
765 // Surprising to me but we can see move of a long to t_object
766 __ verify_oop(src->as_register());
767 move_regs(src->as_register(), dest->as_register_lo());
768 return;
769 }
770 assert(src->is_double_cpu(), "must match");
771 Register f_lo = src->as_register_lo();
772 Register f_hi = src->as_register_hi();
773 Register t_lo = dest->as_register_lo();
774 Register t_hi = dest->as_register_hi();
775 assert(f_hi == f_lo, "must be same");
776 assert(t_hi == t_lo, "must be same");
777 move_regs(f_lo, t_lo);
778
779 // move between xmm-registers
780 } else if (dest->is_single_xmm()) {
781 assert(src->is_single_xmm(), "must match");
782 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
783 } else if (dest->is_double_xmm()) {
784 assert(src->is_double_xmm(), "must match");
785 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
786
787 } else {
788 ShouldNotReachHere();
789 }
790 }
791
792 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
793 assert(src->is_register(), "should not call otherwise");
794 assert(dest->is_stack(), "should not call otherwise");
795
796 if (src->is_single_cpu()) {
797 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
798 if (is_reference_type(type)) {
799 __ verify_oop(src->as_register());
800 __ movptr (dst, src->as_register());
801 } else if (type == T_METADATA || type == T_ADDRESS) {
802 __ movptr (dst, src->as_register());
803 } else {
804 __ movl (dst, src->as_register());
805 }
806
807 } else if (src->is_double_cpu()) {
808 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
809 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
810 __ movptr (dstLO, src->as_register_lo());
811
812 } else if (src->is_single_xmm()) {
813 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
814 __ movflt(dst_addr, src->as_xmm_float_reg());
815
816 } else if (src->is_double_xmm()) {
817 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
818 __ movdbl(dst_addr, src->as_xmm_double_reg());
819
820 } else {
821 ShouldNotReachHere();
822 }
823 }
824
825
826 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
827 LIR_Address* to_addr = dest->as_address_ptr();
828 PatchingStub* patch = nullptr;
829 Register compressed_src = rscratch1;
830
831 if (is_reference_type(type)) {
832 __ verify_oop(src->as_register());
833 if (UseCompressedOops && !wide) {
834 __ movptr(compressed_src, src->as_register());
835 __ encode_heap_oop(compressed_src);
836 if (patch_code != lir_patch_none) {
837 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
838 }
839 }
840 }
841
842 if (patch_code != lir_patch_none) {
843 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
844 Address toa = as_Address(to_addr);
845 assert(toa.disp() != 0, "must have");
846 }
847
848 int null_check_here = code_offset();
849 switch (type) {
850 case T_FLOAT: {
851 assert(src->is_single_xmm(), "not a float");
852 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
853 break;
854 }
855
856 case T_DOUBLE: {
857 assert(src->is_double_xmm(), "not a double");
858 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
859 break;
860 }
861
862 case T_ARRAY: // fall through
863 case T_OBJECT: // fall through
864 if (UseCompressedOops && !wide) {
865 __ movl(as_Address(to_addr), compressed_src);
866 } else {
867 __ movptr(as_Address(to_addr), src->as_register());
868 }
869 break;
870 case T_ADDRESS:
871 __ movptr(as_Address(to_addr), src->as_register());
872 break;
873 case T_INT:
874 __ movl(as_Address(to_addr), src->as_register());
875 break;
876
877 case T_LONG: {
878 Register from_lo = src->as_register_lo();
879 Register from_hi = src->as_register_hi();
880 __ movptr(as_Address_lo(to_addr), from_lo);
881 break;
882 }
883
884 case T_BYTE: // fall through
885 case T_BOOLEAN: {
886 Register src_reg = src->as_register();
887 Address dst_addr = as_Address(to_addr);
888 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
889 __ movb(dst_addr, src_reg);
890 break;
891 }
892
893 case T_CHAR: // fall through
894 case T_SHORT:
895 __ movw(as_Address(to_addr), src->as_register());
896 break;
897
898 default:
899 ShouldNotReachHere();
900 }
901 if (info != nullptr) {
902 add_debug_info_for_null_check(null_check_here, info);
903 }
904
905 if (patch_code != lir_patch_none) {
906 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
907 }
908 }
909
910
911 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
912 assert(src->is_stack(), "should not call otherwise");
913 assert(dest->is_register(), "should not call otherwise");
914
915 if (dest->is_single_cpu()) {
916 if (is_reference_type(type)) {
917 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
918 __ verify_oop(dest->as_register());
919 } else if (type == T_METADATA || type == T_ADDRESS) {
920 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
921 } else {
922 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
923 }
924
925 } else if (dest->is_double_cpu()) {
926 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
927 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
928 __ movptr(dest->as_register_lo(), src_addr_LO);
929
930 } else if (dest->is_single_xmm()) {
931 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
932 __ movflt(dest->as_xmm_float_reg(), src_addr);
933
934 } else if (dest->is_double_xmm()) {
935 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
936 __ movdbl(dest->as_xmm_double_reg(), src_addr);
937
938 } else {
939 ShouldNotReachHere();
940 }
941 }
942
943
944 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
945 if (src->is_single_stack()) {
946 if (is_reference_type(type)) {
947 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
948 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
949 } else {
950 //no pushl on 64bits
951 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
952 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
953 }
954
955 } else if (src->is_double_stack()) {
956 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
957 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
958
959 } else {
960 ShouldNotReachHere();
961 }
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
966 assert(src->is_address(), "should not call otherwise");
967 assert(dest->is_register(), "should not call otherwise");
968
969 LIR_Address* addr = src->as_address_ptr();
970 Address from_addr = as_Address(addr);
971
972 if (addr->base()->type() == T_OBJECT) {
973 __ verify_oop(addr->base()->as_pointer_register());
974 }
975
976 switch (type) {
977 case T_BOOLEAN: // fall through
978 case T_BYTE: // fall through
979 case T_CHAR: // fall through
980 case T_SHORT:
981 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
982 // on pre P6 processors we may get partial register stalls
983 // so blow away the value of to_rinfo before loading a
984 // partial word into it. Do it here so that it precedes
985 // the potential patch point below.
986 __ xorptr(dest->as_register(), dest->as_register());
987 }
988 break;
989 default:
990 break;
991 }
992
993 PatchingStub* patch = nullptr;
994 if (patch_code != lir_patch_none) {
995 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
996 assert(from_addr.disp() != 0, "must have");
997 }
998 if (info != nullptr) {
999 add_debug_info_for_null_check_here(info);
1000 }
1001
1002 switch (type) {
1003 case T_FLOAT: {
1004 if (dest->is_single_xmm()) {
1005 __ movflt(dest->as_xmm_float_reg(), from_addr);
1006 } else {
1007 ShouldNotReachHere();
1008 }
1009 break;
1010 }
1011
1012 case T_DOUBLE: {
1013 if (dest->is_double_xmm()) {
1014 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1015 } else {
1016 ShouldNotReachHere();
1017 }
1018 break;
1019 }
1020
1021 case T_OBJECT: // fall through
1022 case T_ARRAY: // fall through
1023 if (UseCompressedOops && !wide) {
1024 __ movl(dest->as_register(), from_addr);
1025 } else {
1026 __ movptr(dest->as_register(), from_addr);
1027 }
1028 break;
1029
1030 case T_ADDRESS:
1031 __ movptr(dest->as_register(), from_addr);
1032 break;
1033 case T_INT:
1034 __ movl(dest->as_register(), from_addr);
1035 break;
1036
1037 case T_LONG: {
1038 Register to_lo = dest->as_register_lo();
1039 Register to_hi = dest->as_register_hi();
1040 __ movptr(to_lo, as_Address_lo(addr));
1041 break;
1042 }
1043
1044 case T_BOOLEAN: // fall through
1045 case T_BYTE: {
1046 Register dest_reg = dest->as_register();
1047 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1048 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1049 __ movsbl(dest_reg, from_addr);
1050 } else {
1051 __ movb(dest_reg, from_addr);
1052 __ shll(dest_reg, 24);
1053 __ sarl(dest_reg, 24);
1054 }
1055 break;
1056 }
1057
1058 case T_CHAR: {
1059 Register dest_reg = dest->as_register();
1060 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1061 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1062 __ movzwl(dest_reg, from_addr);
1063 } else {
1064 __ movw(dest_reg, from_addr);
1065 }
1066 break;
1067 }
1068
1069 case T_SHORT: {
1070 Register dest_reg = dest->as_register();
1071 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1072 __ movswl(dest_reg, from_addr);
1073 } else {
1074 __ movw(dest_reg, from_addr);
1075 __ shll(dest_reg, 16);
1076 __ sarl(dest_reg, 16);
1077 }
1078 break;
1079 }
1080
1081 default:
1082 ShouldNotReachHere();
1083 }
1084
1085 if (patch != nullptr) {
1086 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1087 }
1088
1089 if (is_reference_type(type)) {
1090 if (UseCompressedOops && !wide) {
1091 __ decode_heap_oop(dest->as_register());
1092 }
1093
1094 __ verify_oop(dest->as_register());
1095 }
1096 }
1097
1098
1099 NEEDS_CLEANUP; // This could be static?
1100 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1101 int elem_size = type2aelembytes(type);
1102 switch (elem_size) {
1103 case 1: return Address::times_1;
1104 case 2: return Address::times_2;
1105 case 4: return Address::times_4;
1106 case 8: return Address::times_8;
1107 }
1108 ShouldNotReachHere();
1109 return Address::no_scale;
1110 }
1111
1112
1113 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1114 switch (op->code()) {
1115 case lir_idiv:
1116 case lir_irem:
1117 arithmetic_idiv(op->code(),
1118 op->in_opr1(),
1119 op->in_opr2(),
1120 op->in_opr3(),
1121 op->result_opr(),
1122 op->info());
1123 break;
1124 case lir_fmad:
1125 __ fmad(op->result_opr()->as_xmm_double_reg(),
1126 op->in_opr1()->as_xmm_double_reg(),
1127 op->in_opr2()->as_xmm_double_reg(),
1128 op->in_opr3()->as_xmm_double_reg());
1129 break;
1130 case lir_fmaf:
1131 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1132 op->in_opr1()->as_xmm_float_reg(),
1133 op->in_opr2()->as_xmm_float_reg(),
1134 op->in_opr3()->as_xmm_float_reg());
1135 break;
1136 default: ShouldNotReachHere(); break;
1137 }
1138 }
1139
1140 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1141 #ifdef ASSERT
1142 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1143 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1144 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1145 #endif
1146
1147 if (op->cond() == lir_cond_always) {
1148 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1149 __ jmp (*(op->label()));
1150 } else {
1151 Assembler::Condition acond = Assembler::zero;
1152 if (op->code() == lir_cond_float_branch) {
1153 assert(op->ublock() != nullptr, "must have unordered successor");
1154 __ jcc(Assembler::parity, *(op->ublock()->label()));
1155 switch(op->cond()) {
1156 case lir_cond_equal: acond = Assembler::equal; break;
1157 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1158 case lir_cond_less: acond = Assembler::below; break;
1159 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1160 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1161 case lir_cond_greater: acond = Assembler::above; break;
1162 default: ShouldNotReachHere();
1163 }
1164 } else {
1165 switch (op->cond()) {
1166 case lir_cond_equal: acond = Assembler::equal; break;
1167 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1168 case lir_cond_less: acond = Assembler::less; break;
1169 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1170 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1171 case lir_cond_greater: acond = Assembler::greater; break;
1172 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1173 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1174 default: ShouldNotReachHere();
1175 }
1176 }
1177 __ jcc(acond,*(op->label()));
1178 }
1179 }
1180
1181 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1182 LIR_Opr src = op->in_opr();
1183 LIR_Opr dest = op->result_opr();
1184
1185 switch (op->bytecode()) {
1186 case Bytecodes::_i2l:
1187 __ movl2ptr(dest->as_register_lo(), src->as_register());
1188 break;
1189
1190 case Bytecodes::_l2i:
1191 __ movl(dest->as_register(), src->as_register_lo());
1192 break;
1193
1194 case Bytecodes::_i2b:
1195 move_regs(src->as_register(), dest->as_register());
1196 __ sign_extend_byte(dest->as_register());
1197 break;
1198
1199 case Bytecodes::_i2c:
1200 move_regs(src->as_register(), dest->as_register());
1201 __ andl(dest->as_register(), 0xFFFF);
1202 break;
1203
1204 case Bytecodes::_i2s:
1205 move_regs(src->as_register(), dest->as_register());
1206 __ sign_extend_short(dest->as_register());
1207 break;
1208
1209 case Bytecodes::_f2d:
1210 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1211 break;
1212
1213 case Bytecodes::_d2f:
1214 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1215 break;
1216
1217 case Bytecodes::_i2f:
1218 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1219 break;
1220
1221 case Bytecodes::_i2d:
1222 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1223 break;
1224
1225 case Bytecodes::_l2f:
1226 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1227 break;
1228
1229 case Bytecodes::_l2d:
1230 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1231 break;
1232
1233 case Bytecodes::_f2i:
1234 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1235 break;
1236
1237 case Bytecodes::_d2i:
1238 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1239 break;
1240
1241 case Bytecodes::_f2l:
1242 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1243 break;
1244
1245 case Bytecodes::_d2l:
1246 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1247 break;
1248
1249 default: ShouldNotReachHere();
1250 }
1251 }
1252
1253 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1254 if (op->init_check()) {
1255 add_debug_info_for_null_check_here(op->stub()->info());
1256 // init_state needs acquire, but x86 is TSO, and so we are already good.
1257 __ cmpb(Address(op->klass()->as_register(),
1258 InstanceKlass::init_state_offset()),
1259 InstanceKlass::fully_initialized);
1260 __ jcc(Assembler::notEqual, *op->stub()->entry());
1261 }
1262 __ allocate_object(op->obj()->as_register(),
1263 op->tmp1()->as_register(),
1264 op->tmp2()->as_register(),
1265 op->header_size(),
1266 op->object_size(),
1267 op->klass()->as_register(),
1268 *op->stub()->entry());
1269 __ bind(*op->stub()->continuation());
1270 }
1271
1272 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1273 Register len = op->len()->as_register();
1274 __ movslq(len, len);
1275
1276 if (UseSlowPath || op->always_slow_path() ||
1277 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1278 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1279 __ jmp(*op->stub()->entry());
1280 } else {
1281 Register tmp1 = op->tmp1()->as_register();
1282 Register tmp2 = op->tmp2()->as_register();
1283 Register tmp3 = op->tmp3()->as_register();
1284 if (len == tmp1) {
1285 tmp1 = tmp3;
1286 } else if (len == tmp2) {
1287 tmp2 = tmp3;
1288 } else if (len == tmp3) {
1289 // everything is ok
1290 } else {
1291 __ mov(tmp3, len);
1292 }
1293 __ allocate_array(op->obj()->as_register(),
1294 len,
1295 tmp1,
1296 tmp2,
1297 arrayOopDesc::base_offset_in_bytes(op->type()),
1298 array_element_size(op->type()),
1299 op->klass()->as_register(),
1300 *op->stub()->entry(),
1301 op->zero_array());
1302 }
1303 __ bind(*op->stub()->continuation());
1304 }
1305
1306 void LIR_Assembler::type_profile_helper(Register mdo,
1307 ciMethodData *md, ciProfileData *data,
1308 Register recv) {
1309 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1310 __ profile_receiver_type(recv, mdo, mdp_offset);
1311 }
1312
1313 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1314 // we always need a stub for the failure case.
1315 CodeStub* stub = op->stub();
1316 Register obj = op->object()->as_register();
1317 Register k_RInfo = op->tmp1()->as_register();
1318 Register klass_RInfo = op->tmp2()->as_register();
1319 Register dst = op->result_opr()->as_register();
1320 ciKlass* k = op->klass();
1321 Register Rtmp1 = noreg;
1322 Register tmp_load_klass = rscratch1;
1323
1324 // check if it needs to be profiled
1325 ciMethodData* md = nullptr;
1326 ciProfileData* data = nullptr;
1327
1328 if (op->should_profile()) {
1329 ciMethod* method = op->profiled_method();
1330 assert(method != nullptr, "Should have method");
1331 int bci = op->profiled_bci();
1332 md = method->method_data_or_null();
1333 assert(md != nullptr, "Sanity");
1334 data = md->bci_to_data(bci);
1335 assert(data != nullptr, "need data for type check");
1336 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1337 }
1338 Label* success_target = success;
1339 Label* failure_target = failure;
1340
1341 if (obj == k_RInfo) {
1342 k_RInfo = dst;
1343 } else if (obj == klass_RInfo) {
1344 klass_RInfo = dst;
1345 }
1346 Rtmp1 = op->tmp3()->as_register();
1347 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1348
1349 assert_different_registers(obj, k_RInfo, klass_RInfo);
1350
1351 if (op->need_null_check()) {
1352 __ testptr(obj, obj);
1353 if (op->should_profile()) {
1354 Label not_null;
1355 Register mdo = klass_RInfo;
1356 __ mov_metadata(mdo, md->constant_encoding());
1357 __ jccb(Assembler::notEqual, not_null);
1358 // Object is null; update MDO and exit
1359 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1360 int header_bits = BitData::null_seen_byte_constant();
1361 __ orb(data_addr, header_bits);
1362 __ jmp(*obj_is_null);
1363 __ bind(not_null);
1364
1365 Register recv = k_RInfo;
1366 __ load_klass(recv, obj, tmp_load_klass);
1367 type_profile_helper(mdo, md, data, recv);
1368 } else {
1369 __ jcc(Assembler::equal, *obj_is_null);
1370 }
1371 }
1372
1373 if (!k->is_loaded()) {
1374 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1375 } else {
1376 __ mov_metadata(k_RInfo, k->constant_encoding());
1377 }
1378 __ verify_oop(obj);
1379
1380 if (op->fast_check()) {
1381 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1382 // get object class
1383 // not a safepoint as obj null check happens earlier
1384 __ load_klass(Rtmp1, obj, tmp_load_klass);
1385 __ cmpptr(k_RInfo, Rtmp1);
1386 __ jcc(Assembler::notEqual, *failure_target);
1387 // successful cast, fall through to profile or jump
1388 } else {
1389 // get object class
1390 // not a safepoint as obj null check happens earlier
1391 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1392 if (k->is_loaded()) {
1393 // See if we get an immediate positive hit
1394 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1395 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1396 __ jcc(Assembler::notEqual, *failure_target);
1397 // successful cast, fall through to profile or jump
1398 } else {
1399 // See if we get an immediate positive hit
1400 __ jcc(Assembler::equal, *success_target);
1401 // check for self
1402 if (k->is_loaded() && k->is_obj_array_klass()) {
1403 // For a direct pointer comparison, we need the refined array klass pointer
1404 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1405 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1406 __ cmpptr(klass_RInfo, tmp_load_klass);
1407 } else {
1408 __ cmpptr(klass_RInfo, k_RInfo);
1409 }
1410 __ jcc(Assembler::equal, *success_target);
1411
1412 __ push_ppx(klass_RInfo);
1413 __ push_ppx(k_RInfo);
1414 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1415 __ pop_ppx(klass_RInfo);
1416 __ pop_ppx(klass_RInfo);
1417 // result is a boolean
1418 __ testl(klass_RInfo, klass_RInfo);
1419 __ jcc(Assembler::equal, *failure_target);
1420 // successful cast, fall through to profile or jump
1421 }
1422 } else {
1423 // perform the fast part of the checking logic
1424 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1425 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1426 __ push_ppx(klass_RInfo);
1427 __ push_ppx(k_RInfo);
1428 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1429 __ pop_ppx(klass_RInfo);
1430 __ pop_ppx(k_RInfo);
1431 // result is a boolean
1432 __ testl(k_RInfo, k_RInfo);
1433 __ jcc(Assembler::equal, *failure_target);
1434 // successful cast, fall through to profile or jump
1435 }
1436 }
1437 __ jmp(*success);
1438 }
1439
1440
1441 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1442 Register tmp_load_klass = rscratch1;
1443 LIR_Code code = op->code();
1444 if (code == lir_store_check) {
1445 Register value = op->object()->as_register();
1446 Register array = op->array()->as_register();
1447 Register k_RInfo = op->tmp1()->as_register();
1448 Register klass_RInfo = op->tmp2()->as_register();
1449 Register Rtmp1 = op->tmp3()->as_register();
1450
1451 CodeStub* stub = op->stub();
1452
1453 // check if it needs to be profiled
1454 ciMethodData* md = nullptr;
1455 ciProfileData* data = nullptr;
1456
1457 if (op->should_profile()) {
1458 ciMethod* method = op->profiled_method();
1459 assert(method != nullptr, "Should have method");
1460 int bci = op->profiled_bci();
1461 md = method->method_data_or_null();
1462 assert(md != nullptr, "Sanity");
1463 data = md->bci_to_data(bci);
1464 assert(data != nullptr, "need data for type check");
1465 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1466 }
1467 Label done;
1468 Label* success_target = &done;
1469 Label* failure_target = stub->entry();
1470
1471 __ testptr(value, value);
1472 if (op->should_profile()) {
1473 Label not_null;
1474 Register mdo = klass_RInfo;
1475 __ mov_metadata(mdo, md->constant_encoding());
1476 __ jccb(Assembler::notEqual, not_null);
1477 // Object is null; update MDO and exit
1478 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1479 int header_bits = BitData::null_seen_byte_constant();
1480 __ orb(data_addr, header_bits);
1481 __ jmp(done);
1482 __ bind(not_null);
1483
1484 Register recv = k_RInfo;
1485 __ load_klass(recv, value, tmp_load_klass);
1486 type_profile_helper(mdo, md, data, recv);
1487 } else {
1488 __ jcc(Assembler::equal, done);
1489 }
1490
1491 add_debug_info_for_null_check_here(op->info_for_exception());
1492 __ load_klass(k_RInfo, array, tmp_load_klass);
1493 __ load_klass(klass_RInfo, value, tmp_load_klass);
1494
1495 // get instance klass (it's already uncompressed)
1496 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1497 // perform the fast part of the checking logic
1498 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1499 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1500 __ push_ppx(klass_RInfo);
1501 __ push_ppx(k_RInfo);
1502 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1503 __ pop_ppx(klass_RInfo);
1504 __ pop_ppx(k_RInfo);
1505 // result is a boolean
1506 __ testl(k_RInfo, k_RInfo);
1507 __ jcc(Assembler::equal, *failure_target);
1508 // fall through to the success case
1509
1510 __ bind(done);
1511 } else
1512 if (code == lir_checkcast) {
1513 Register obj = op->object()->as_register();
1514 Register dst = op->result_opr()->as_register();
1515 Label success;
1516 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1517 __ bind(success);
1518 if (dst != obj) {
1519 __ mov(dst, obj);
1520 }
1521 } else
1522 if (code == lir_instanceof) {
1523 Register obj = op->object()->as_register();
1524 Register dst = op->result_opr()->as_register();
1525 Label success, failure, done;
1526 emit_typecheck_helper(op, &success, &failure, &failure);
1527 __ bind(failure);
1528 __ xorptr(dst, dst);
1529 __ jmpb(done);
1530 __ bind(success);
1531 __ movptr(dst, 1);
1532 __ bind(done);
1533 } else {
1534 ShouldNotReachHere();
1535 }
1536
1537 }
1538
1539 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1540 // We are loading/storing from/to an array that *may* be a flat array (the
1541 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1542 // If this array is a flat array, take the slow path.
1543 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1544 if (!op->value()->is_illegal()) {
1545 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1546 // The array is not a flat array, but it might be null-free. If we are storing
1547 // a null into a null-free array, take the slow path (which will throw NPE).
1548 Label skip;
1549 __ cmpptr(op->value()->as_register(), NULL_WORD);
1550 __ jcc(Assembler::notEqual, skip);
1551 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1552 __ bind(skip);
1553 }
1554 }
1555
1556 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1557 // We are storing into an array that *may* be null-free (the declared type is
1558 // Object[], abstract[], interface[] or VT.ref[]).
1559 Label test_mark_word;
1560 Register tmp = op->tmp()->as_register();
1561 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1562 __ testl(tmp, markWord::unlocked_value);
1563 __ jccb(Assembler::notZero, test_mark_word);
1564 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1565 __ bind(test_mark_word);
1566 __ testl(tmp, markWord::null_free_array_bit_in_place);
1567 }
1568
1569 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1570 Label L_oops_equal;
1571 Label L_oops_not_equal;
1572 Label L_end;
1573
1574 Register left = op->left()->as_register();
1575 Register right = op->right()->as_register();
1576
1577 __ cmpptr(left, right);
1578 __ jcc(Assembler::equal, L_oops_equal);
1579
1580 // (1) Null check -- if one of the operands is null, the other must not be null (because
1581 // the two references are not equal), so they are not substitutable,
1582 // FIXME: do null check only if the operand is nullable
1583 __ testptr(left, right);
1584 __ jcc(Assembler::zero, L_oops_not_equal);
1585
1586 ciKlass* left_klass = op->left_klass();
1587 ciKlass* right_klass = op->right_klass();
1588
1589 // (2) Inline type check -- if either of the operands is not a inline type,
1590 // they are not substitutable. We do this only if we are not sure that the
1591 // operands are inline type
1592 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1593 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1594 Register tmp = op->tmp1()->as_register();
1595 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1596 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1597 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1598 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1599 __ jcc(Assembler::notEqual, L_oops_not_equal);
1600 }
1601
1602 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1603 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1604 // No need to load klass -- the operands are statically known to be the same inline klass.
1605 __ jmp(*op->stub()->entry());
1606 } else {
1607 Register tmp1 = op->tmp1()->as_register();
1608 Register tmp2 = op->tmp2()->as_register();
1609 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1610 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1611 // fall through to L_oops_not_equal
1612 }
1613
1614 __ bind(L_oops_not_equal);
1615 move(op->not_equal_result(), op->result_opr());
1616 __ jmp(L_end);
1617
1618 __ bind(L_oops_equal);
1619 move(op->equal_result(), op->result_opr());
1620 __ jmp(L_end);
1621
1622 // We've returned from the stub. RAX contains 0x0 IFF the two
1623 // operands are not substitutable. (Don't compare against 0x1 in case the
1624 // C compiler is naughty)
1625 __ bind(*op->stub()->continuation());
1626 __ cmpl(rax, 0);
1627 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1628 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1629 // fall-through
1630 __ bind(L_end);
1631 }
1632
1633 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1634 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1635 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1636 Register newval = op->new_value()->as_register();
1637 Register cmpval = op->cmp_value()->as_register();
1638 assert(cmpval == rax, "wrong register");
1639 assert(newval != noreg, "new val must be register");
1640 assert(cmpval != newval, "cmp and new values must be in different registers");
1641 assert(cmpval != addr, "cmp and addr must be in different registers");
1642 assert(newval != addr, "new value and addr must be in different registers");
1643
1644 if (op->code() == lir_cas_obj) {
1645 if (UseCompressedOops) {
1646 __ encode_heap_oop(cmpval);
1647 __ mov(rscratch1, newval);
1648 __ encode_heap_oop(rscratch1);
1649 __ lock();
1650 // cmpval (rax) is implicitly used by this instruction
1651 __ cmpxchgl(rscratch1, Address(addr, 0));
1652 } else {
1653 __ lock();
1654 __ cmpxchgptr(newval, Address(addr, 0));
1655 }
1656 } else {
1657 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1658 __ lock();
1659 __ cmpxchgl(newval, Address(addr, 0));
1660 }
1661 } else if (op->code() == lir_cas_long) {
1662 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1663 Register newval = op->new_value()->as_register_lo();
1664 Register cmpval = op->cmp_value()->as_register_lo();
1665 assert(cmpval == rax, "wrong register");
1666 assert(newval != noreg, "new val must be register");
1667 assert(cmpval != newval, "cmp and new values must be in different registers");
1668 assert(cmpval != addr, "cmp and addr must be in different registers");
1669 assert(newval != addr, "new value and addr must be in different registers");
1670 __ lock();
1671 __ cmpxchgq(newval, Address(addr, 0));
1672 } else {
1673 Unimplemented();
1674 }
1675 }
1676
1677 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1678 assert(dst->is_cpu_register(), "must be");
1679 assert(dst->type() == src->type(), "must be");
1680
1681 if (src->is_cpu_register()) {
1682 reg2reg(src, dst);
1683 } else if (src->is_stack()) {
1684 stack2reg(src, dst, dst->type());
1685 } else if (src->is_constant()) {
1686 const2reg(src, dst, lir_patch_none, nullptr);
1687 } else {
1688 ShouldNotReachHere();
1689 }
1690 }
1691
1692 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1693 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1694 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1695
1696 Assembler::Condition acond, ncond;
1697 switch (condition) {
1698 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1699 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1700 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1701 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1702 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1703 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1704 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1705 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1706 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1707 ShouldNotReachHere();
1708 }
1709
1710 if (opr1->is_cpu_register()) {
1711 reg2reg(opr1, result);
1712 } else if (opr1->is_stack()) {
1713 stack2reg(opr1, result, result->type());
1714 } else if (opr1->is_constant()) {
1715 const2reg(opr1, result, lir_patch_none, nullptr);
1716 } else {
1717 ShouldNotReachHere();
1718 }
1719
1720 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1721 // optimized version that does not require a branch
1722 if (opr2->is_single_cpu()) {
1723 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1724 __ cmov(ncond, result->as_register(), opr2->as_register());
1725 } else if (opr2->is_double_cpu()) {
1726 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1727 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1728 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1729 } else if (opr2->is_single_stack()) {
1730 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1731 } else if (opr2->is_double_stack()) {
1732 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1733 } else {
1734 ShouldNotReachHere();
1735 }
1736
1737 } else {
1738 Label skip;
1739 __ jccb(acond, skip);
1740 if (opr2->is_cpu_register()) {
1741 reg2reg(opr2, result);
1742 } else if (opr2->is_stack()) {
1743 stack2reg(opr2, result, result->type());
1744 } else if (opr2->is_constant()) {
1745 const2reg(opr2, result, lir_patch_none, nullptr);
1746 } else {
1747 ShouldNotReachHere();
1748 }
1749 __ bind(skip);
1750 }
1751 }
1752
1753
1754 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1755 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1756
1757 if (left->is_single_cpu()) {
1758 assert(left == dest, "left and dest must be equal");
1759 Register lreg = left->as_register();
1760
1761 if (right->is_single_cpu()) {
1762 // cpu register - cpu register
1763 Register rreg = right->as_register();
1764 switch (code) {
1765 case lir_add: __ addl (lreg, rreg); break;
1766 case lir_sub: __ subl (lreg, rreg); break;
1767 case lir_mul: __ imull(lreg, rreg); break;
1768 default: ShouldNotReachHere();
1769 }
1770
1771 } else if (right->is_stack()) {
1772 // cpu register - stack
1773 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1774 switch (code) {
1775 case lir_add: __ addl(lreg, raddr); break;
1776 case lir_sub: __ subl(lreg, raddr); break;
1777 default: ShouldNotReachHere();
1778 }
1779
1780 } else if (right->is_constant()) {
1781 // cpu register - constant
1782 jint c = right->as_constant_ptr()->as_jint();
1783 switch (code) {
1784 case lir_add: {
1785 __ incrementl(lreg, c);
1786 break;
1787 }
1788 case lir_sub: {
1789 __ decrementl(lreg, c);
1790 break;
1791 }
1792 default: ShouldNotReachHere();
1793 }
1794
1795 } else {
1796 ShouldNotReachHere();
1797 }
1798
1799 } else if (left->is_double_cpu()) {
1800 assert(left == dest, "left and dest must be equal");
1801 Register lreg_lo = left->as_register_lo();
1802 Register lreg_hi = left->as_register_hi();
1803
1804 if (right->is_double_cpu()) {
1805 // cpu register - cpu register
1806 Register rreg_lo = right->as_register_lo();
1807 Register rreg_hi = right->as_register_hi();
1808 assert_different_registers(lreg_lo, rreg_lo);
1809 switch (code) {
1810 case lir_add:
1811 __ addptr(lreg_lo, rreg_lo);
1812 break;
1813 case lir_sub:
1814 __ subptr(lreg_lo, rreg_lo);
1815 break;
1816 case lir_mul:
1817 __ imulq(lreg_lo, rreg_lo);
1818 break;
1819 default:
1820 ShouldNotReachHere();
1821 }
1822
1823 } else if (right->is_constant()) {
1824 // cpu register - constant
1825 jlong c = right->as_constant_ptr()->as_jlong_bits();
1826 __ movptr(r10, (intptr_t) c);
1827 switch (code) {
1828 case lir_add:
1829 __ addptr(lreg_lo, r10);
1830 break;
1831 case lir_sub:
1832 __ subptr(lreg_lo, r10);
1833 break;
1834 default:
1835 ShouldNotReachHere();
1836 }
1837
1838 } else {
1839 ShouldNotReachHere();
1840 }
1841
1842 } else if (left->is_single_xmm()) {
1843 assert(left == dest, "left and dest must be equal");
1844 XMMRegister lreg = left->as_xmm_float_reg();
1845
1846 if (right->is_single_xmm()) {
1847 XMMRegister rreg = right->as_xmm_float_reg();
1848 switch (code) {
1849 case lir_add: __ addss(lreg, rreg); break;
1850 case lir_sub: __ subss(lreg, rreg); break;
1851 case lir_mul: __ mulss(lreg, rreg); break;
1852 case lir_div: __ divss(lreg, rreg); break;
1853 default: ShouldNotReachHere();
1854 }
1855 } else {
1856 Address raddr;
1857 if (right->is_single_stack()) {
1858 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1859 } else if (right->is_constant()) {
1860 // hack for now
1861 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1862 } else {
1863 ShouldNotReachHere();
1864 }
1865 switch (code) {
1866 case lir_add: __ addss(lreg, raddr); break;
1867 case lir_sub: __ subss(lreg, raddr); break;
1868 case lir_mul: __ mulss(lreg, raddr); break;
1869 case lir_div: __ divss(lreg, raddr); break;
1870 default: ShouldNotReachHere();
1871 }
1872 }
1873
1874 } else if (left->is_double_xmm()) {
1875 assert(left == dest, "left and dest must be equal");
1876
1877 XMMRegister lreg = left->as_xmm_double_reg();
1878 if (right->is_double_xmm()) {
1879 XMMRegister rreg = right->as_xmm_double_reg();
1880 switch (code) {
1881 case lir_add: __ addsd(lreg, rreg); break;
1882 case lir_sub: __ subsd(lreg, rreg); break;
1883 case lir_mul: __ mulsd(lreg, rreg); break;
1884 case lir_div: __ divsd(lreg, rreg); break;
1885 default: ShouldNotReachHere();
1886 }
1887 } else {
1888 Address raddr;
1889 if (right->is_double_stack()) {
1890 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1891 } else if (right->is_constant()) {
1892 // hack for now
1893 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1894 } else {
1895 ShouldNotReachHere();
1896 }
1897 switch (code) {
1898 case lir_add: __ addsd(lreg, raddr); break;
1899 case lir_sub: __ subsd(lreg, raddr); break;
1900 case lir_mul: __ mulsd(lreg, raddr); break;
1901 case lir_div: __ divsd(lreg, raddr); break;
1902 default: ShouldNotReachHere();
1903 }
1904 }
1905
1906 } else if (left->is_single_stack() || left->is_address()) {
1907 assert(left == dest, "left and dest must be equal");
1908
1909 Address laddr;
1910 if (left->is_single_stack()) {
1911 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1912 } else if (left->is_address()) {
1913 laddr = as_Address(left->as_address_ptr());
1914 } else {
1915 ShouldNotReachHere();
1916 }
1917
1918 if (right->is_single_cpu()) {
1919 Register rreg = right->as_register();
1920 switch (code) {
1921 case lir_add: __ addl(laddr, rreg); break;
1922 case lir_sub: __ subl(laddr, rreg); break;
1923 default: ShouldNotReachHere();
1924 }
1925 } else if (right->is_constant()) {
1926 jint c = right->as_constant_ptr()->as_jint();
1927 switch (code) {
1928 case lir_add: {
1929 __ incrementl(laddr, c);
1930 break;
1931 }
1932 case lir_sub: {
1933 __ decrementl(laddr, c);
1934 break;
1935 }
1936 default: ShouldNotReachHere();
1937 }
1938 } else {
1939 ShouldNotReachHere();
1940 }
1941
1942 } else {
1943 ShouldNotReachHere();
1944 }
1945 }
1946
1947
1948 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1949 if (value->is_double_xmm()) {
1950 switch(code) {
1951 case lir_abs :
1952 {
1953 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1954 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1955 }
1956 assert(!tmp->is_valid(), "do not need temporary");
1957 __ andpd(dest->as_xmm_double_reg(),
1958 ExternalAddress((address)double_signmask_pool),
1959 rscratch1);
1960 }
1961 break;
1962
1963 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1964 // all other intrinsics are not available in the SSE instruction set, so FPU is used
1965 default : ShouldNotReachHere();
1966 }
1967
1968 } else if (code == lir_f2hf) {
1969 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1970 } else if (code == lir_hf2f) {
1971 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1972 } else {
1973 Unimplemented();
1974 }
1975 }
1976
1977 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1978 // assert(left->destroys_register(), "check");
1979 if (left->is_single_cpu()) {
1980 Register reg = left->as_register();
1981 if (right->is_constant()) {
1982 int val = right->as_constant_ptr()->as_jint();
1983 switch (code) {
1984 case lir_logic_and: __ andl (reg, val); break;
1985 case lir_logic_or: __ orl (reg, val); break;
1986 case lir_logic_xor: __ xorl (reg, val); break;
1987 default: ShouldNotReachHere();
1988 }
1989 } else if (right->is_stack()) {
1990 // added support for stack operands
1991 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1992 switch (code) {
1993 case lir_logic_and: __ andl (reg, raddr); break;
1994 case lir_logic_or: __ orl (reg, raddr); break;
1995 case lir_logic_xor: __ xorl (reg, raddr); break;
1996 default: ShouldNotReachHere();
1997 }
1998 } else {
1999 Register rright = right->as_register();
2000 switch (code) {
2001 case lir_logic_and: __ andptr (reg, rright); break;
2002 case lir_logic_or : __ orptr (reg, rright); break;
2003 case lir_logic_xor: __ xorptr (reg, rright); break;
2004 default: ShouldNotReachHere();
2005 }
2006 }
2007 move_regs(reg, dst->as_register());
2008 } else {
2009 Register l_lo = left->as_register_lo();
2010 Register l_hi = left->as_register_hi();
2011 if (right->is_constant()) {
2012 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2013 switch (code) {
2014 case lir_logic_and:
2015 __ andq(l_lo, rscratch1);
2016 break;
2017 case lir_logic_or:
2018 __ orq(l_lo, rscratch1);
2019 break;
2020 case lir_logic_xor:
2021 __ xorq(l_lo, rscratch1);
2022 break;
2023 default: ShouldNotReachHere();
2024 }
2025 } else {
2026 Register r_lo;
2027 if (is_reference_type(right->type())) {
2028 r_lo = right->as_register();
2029 } else {
2030 r_lo = right->as_register_lo();
2031 }
2032 switch (code) {
2033 case lir_logic_and:
2034 __ andptr(l_lo, r_lo);
2035 break;
2036 case lir_logic_or:
2037 __ orptr(l_lo, r_lo);
2038 break;
2039 case lir_logic_xor:
2040 __ xorptr(l_lo, r_lo);
2041 break;
2042 default: ShouldNotReachHere();
2043 }
2044 }
2045
2046 Register dst_lo = dst->as_register_lo();
2047 Register dst_hi = dst->as_register_hi();
2048
2049 move_regs(l_lo, dst_lo);
2050 }
2051 }
2052
2053
2054 // we assume that rax, and rdx can be overwritten
2055 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2056
2057 assert(left->is_single_cpu(), "left must be register");
2058 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2059 assert(result->is_single_cpu(), "result must be register");
2060
2061 // assert(left->destroys_register(), "check");
2062 // assert(right->destroys_register(), "check");
2063
2064 Register lreg = left->as_register();
2065 Register dreg = result->as_register();
2066
2067 if (right->is_constant()) {
2068 jint divisor = right->as_constant_ptr()->as_jint();
2069 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2070 if (code == lir_idiv) {
2071 assert(lreg == rax, "must be rax,");
2072 assert(temp->as_register() == rdx, "tmp register must be rdx");
2073 __ cdql(); // sign extend into rdx:rax
2074 if (divisor == 2) {
2075 __ subl(lreg, rdx);
2076 } else {
2077 __ andl(rdx, divisor - 1);
2078 __ addl(lreg, rdx);
2079 }
2080 __ sarl(lreg, log2i_exact(divisor));
2081 move_regs(lreg, dreg);
2082 } else if (code == lir_irem) {
2083 Label done;
2084 __ mov(dreg, lreg);
2085 __ andl(dreg, 0x80000000 | (divisor - 1));
2086 __ jcc(Assembler::positive, done);
2087 __ decrement(dreg);
2088 __ orl(dreg, ~(divisor - 1));
2089 __ increment(dreg);
2090 __ bind(done);
2091 } else {
2092 ShouldNotReachHere();
2093 }
2094 } else {
2095 Register rreg = right->as_register();
2096 assert(lreg == rax, "left register must be rax,");
2097 assert(rreg != rdx, "right register must not be rdx");
2098 assert(temp->as_register() == rdx, "tmp register must be rdx");
2099
2100 move_regs(lreg, rax);
2101
2102 int idivl_offset = __ corrected_idivl(rreg);
2103 if (ImplicitDiv0Checks) {
2104 add_debug_info_for_div0(idivl_offset, info);
2105 }
2106 if (code == lir_irem) {
2107 move_regs(rdx, dreg); // result is in rdx
2108 } else {
2109 move_regs(rax, dreg);
2110 }
2111 }
2112 }
2113
2114
2115 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2116 if (opr1->is_single_cpu()) {
2117 Register reg1 = opr1->as_register();
2118 if (opr2->is_single_cpu()) {
2119 // cpu register - cpu register
2120 if (is_reference_type(opr1->type())) {
2121 __ cmpoop(reg1, opr2->as_register());
2122 } else {
2123 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2124 __ cmpl(reg1, opr2->as_register());
2125 }
2126 } else if (opr2->is_stack()) {
2127 // cpu register - stack
2128 if (is_reference_type(opr1->type())) {
2129 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2130 } else {
2131 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2132 }
2133 } else if (opr2->is_constant()) {
2134 // cpu register - constant
2135 LIR_Const* c = opr2->as_constant_ptr();
2136 if (c->type() == T_INT) {
2137 jint i = c->as_jint();
2138 if (i == 0) {
2139 __ testl(reg1, reg1);
2140 } else {
2141 __ cmpl(reg1, i);
2142 }
2143 } else if (c->type() == T_METADATA) {
2144 // All we need for now is a comparison with null for equality.
2145 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2146 Metadata* m = c->as_metadata();
2147 if (m == nullptr) {
2148 __ testptr(reg1, reg1);
2149 } else {
2150 ShouldNotReachHere();
2151 }
2152 } else if (is_reference_type(c->type())) {
2153 // In 64bit oops are single register
2154 jobject o = c->as_jobject();
2155 if (o == nullptr) {
2156 __ testptr(reg1, reg1);
2157 } else {
2158 __ cmpoop(reg1, o, rscratch1);
2159 }
2160 } else {
2161 fatal("unexpected type: %s", basictype_to_str(c->type()));
2162 }
2163 // cpu register - address
2164 } else if (opr2->is_address()) {
2165 if (op->info() != nullptr) {
2166 add_debug_info_for_null_check_here(op->info());
2167 }
2168 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2169 } else {
2170 ShouldNotReachHere();
2171 }
2172
2173 } else if(opr1->is_double_cpu()) {
2174 Register xlo = opr1->as_register_lo();
2175 Register xhi = opr1->as_register_hi();
2176 if (opr2->is_double_cpu()) {
2177 __ cmpptr(xlo, opr2->as_register_lo());
2178 } else if (opr2->is_constant()) {
2179 // cpu register - constant 0
2180 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2181 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2182 } else {
2183 ShouldNotReachHere();
2184 }
2185
2186 } else if (opr1->is_single_xmm()) {
2187 XMMRegister reg1 = opr1->as_xmm_float_reg();
2188 if (opr2->is_single_xmm()) {
2189 // xmm register - xmm register
2190 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2191 } else if (opr2->is_stack()) {
2192 // xmm register - stack
2193 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2194 } else if (opr2->is_constant()) {
2195 // xmm register - constant
2196 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2197 } else if (opr2->is_address()) {
2198 // xmm register - address
2199 if (op->info() != nullptr) {
2200 add_debug_info_for_null_check_here(op->info());
2201 }
2202 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2203 } else {
2204 ShouldNotReachHere();
2205 }
2206
2207 } else if (opr1->is_double_xmm()) {
2208 XMMRegister reg1 = opr1->as_xmm_double_reg();
2209 if (opr2->is_double_xmm()) {
2210 // xmm register - xmm register
2211 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2212 } else if (opr2->is_stack()) {
2213 // xmm register - stack
2214 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2215 } else if (opr2->is_constant()) {
2216 // xmm register - constant
2217 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2218 } else if (opr2->is_address()) {
2219 // xmm register - address
2220 if (op->info() != nullptr) {
2221 add_debug_info_for_null_check_here(op->info());
2222 }
2223 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2224 } else {
2225 ShouldNotReachHere();
2226 }
2227
2228 } else if (opr1->is_address() && opr2->is_constant()) {
2229 LIR_Const* c = opr2->as_constant_ptr();
2230 if (is_reference_type(c->type())) {
2231 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2232 __ movoop(rscratch1, c->as_jobject());
2233 }
2234 if (op->info() != nullptr) {
2235 add_debug_info_for_null_check_here(op->info());
2236 }
2237 // special case: address - constant
2238 LIR_Address* addr = opr1->as_address_ptr();
2239 if (c->type() == T_INT) {
2240 __ cmpl(as_Address(addr), c->as_jint());
2241 } else if (is_reference_type(c->type())) {
2242 // %%% Make this explode if addr isn't reachable until we figure out a
2243 // better strategy by giving noreg as the temp for as_Address
2244 __ cmpoop(rscratch1, as_Address(addr, noreg));
2245 } else {
2246 ShouldNotReachHere();
2247 }
2248
2249 } else {
2250 ShouldNotReachHere();
2251 }
2252 }
2253
2254 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2255 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2256 if (left->is_single_xmm()) {
2257 assert(right->is_single_xmm(), "must match");
2258 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2259 } else if (left->is_double_xmm()) {
2260 assert(right->is_double_xmm(), "must match");
2261 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2262
2263 } else {
2264 ShouldNotReachHere();
2265 }
2266 } else {
2267 assert(code == lir_cmp_l2i, "check");
2268 Label done;
2269 Register dest = dst->as_register();
2270 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2271 __ movl(dest, -1);
2272 __ jccb(Assembler::less, done);
2273 __ setb(Assembler::notZero, dest);
2274 __ movzbl(dest, dest);
2275 __ bind(done);
2276 }
2277 }
2278
2279
2280 void LIR_Assembler::align_call(LIR_Code code) {
2281 // make sure that the displacement word of the call ends up word aligned
2282 int offset = __ offset();
2283 switch (code) {
2284 case lir_static_call:
2285 case lir_optvirtual_call:
2286 case lir_dynamic_call:
2287 offset += NativeCall::displacement_offset;
2288 break;
2289 case lir_icvirtual_call:
2290 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2291 break;
2292 default: ShouldNotReachHere();
2293 }
2294 __ align(BytesPerWord, offset);
2295 }
2296
2297
2298 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2299 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2300 "must be aligned");
2301 __ call(AddressLiteral(op->addr(), rtype));
2302 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2303 __ post_call_nop();
2304 }
2305
2306
2307 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2308 __ ic_call(op->addr());
2309 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2310 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2311 "must be aligned");
2312 __ post_call_nop();
2313 }
2314
2315
2316 void LIR_Assembler::emit_static_call_stub() {
2317 address call_pc = __ pc();
2318 address stub = __ start_a_stub(call_stub_size());
2319 if (stub == nullptr) {
2320 bailout("static call stub overflow");
2321 return;
2322 }
2323
2324 int start = __ offset();
2325
2326 // make sure that the displacement word of the call ends up word aligned
2327 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2328 __ relocate(static_stub_Relocation::spec(call_pc));
2329 __ mov_metadata(rbx, (Metadata*)nullptr);
2330 // must be set to -1 at code generation time
2331 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2332 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2333 __ jump(RuntimeAddress(__ pc()));
2334
2335 assert(__ offset() - start <= call_stub_size(), "stub too big");
2336 __ end_a_stub();
2337 }
2338
2339
2340 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2341 assert(exceptionOop->as_register() == rax, "must match");
2342 assert(exceptionPC->as_register() == rdx, "must match");
2343
2344 // exception object is not added to oop map by LinearScan
2345 // (LinearScan assumes that no oops are in fixed registers)
2346 info->add_register_oop(exceptionOop);
2347 StubId unwind_id;
2348
2349 // get current pc information
2350 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2351 int pc_for_athrow_offset = __ offset();
2352 InternalAddress pc_for_athrow(__ pc());
2353 __ lea(exceptionPC->as_register(), pc_for_athrow);
2354 add_call_info(pc_for_athrow_offset, info); // for exception handler
2355
2356 __ verify_not_null_oop(rax);
2357 // search an exception handler (rax: exception oop, rdx: throwing pc)
2358 if (compilation()->has_fpu_code()) {
2359 unwind_id = StubId::c1_handle_exception_id;
2360 } else {
2361 unwind_id = StubId::c1_handle_exception_nofpu_id;
2362 }
2363 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2364
2365 // enough room for two byte trap
2366 __ nop();
2367 }
2368
2369
2370 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2371 assert(exceptionOop->as_register() == rax, "must match");
2372
2373 __ jmp(_unwind_handler_entry);
2374 }
2375
2376
2377 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2378
2379 // optimized version for linear scan:
2380 // * count must be already in ECX (guaranteed by LinearScan)
2381 // * left and dest must be equal
2382 // * tmp must be unused
2383 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2384 assert(left == dest, "left and dest must be equal");
2385 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2386
2387 if (left->is_single_cpu()) {
2388 Register value = left->as_register();
2389 assert(value != SHIFT_count, "left cannot be ECX");
2390
2391 switch (code) {
2392 case lir_shl: __ shll(value); break;
2393 case lir_shr: __ sarl(value); break;
2394 case lir_ushr: __ shrl(value); break;
2395 default: ShouldNotReachHere();
2396 }
2397 } else if (left->is_double_cpu()) {
2398 Register lo = left->as_register_lo();
2399 Register hi = left->as_register_hi();
2400 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2401 switch (code) {
2402 case lir_shl: __ shlptr(lo); break;
2403 case lir_shr: __ sarptr(lo); break;
2404 case lir_ushr: __ shrptr(lo); break;
2405 default: ShouldNotReachHere();
2406 }
2407 } else {
2408 ShouldNotReachHere();
2409 }
2410 }
2411
2412
2413 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2414 if (dest->is_single_cpu()) {
2415 // first move left into dest so that left is not destroyed by the shift
2416 Register value = dest->as_register();
2417 count = count & 0x1F; // Java spec
2418
2419 move_regs(left->as_register(), value);
2420 switch (code) {
2421 case lir_shl: __ shll(value, count); break;
2422 case lir_shr: __ sarl(value, count); break;
2423 case lir_ushr: __ shrl(value, count); break;
2424 default: ShouldNotReachHere();
2425 }
2426 } else if (dest->is_double_cpu()) {
2427 // first move left into dest so that left is not destroyed by the shift
2428 Register value = dest->as_register_lo();
2429 count = count & 0x1F; // Java spec
2430
2431 move_regs(left->as_register_lo(), value);
2432 switch (code) {
2433 case lir_shl: __ shlptr(value, count); break;
2434 case lir_shr: __ sarptr(value, count); break;
2435 case lir_ushr: __ shrptr(value, count); break;
2436 default: ShouldNotReachHere();
2437 }
2438 } else {
2439 ShouldNotReachHere();
2440 }
2441 }
2442
2443
2444 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2445 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2446 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2447 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2448 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2449 }
2450
2451
2452 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2453 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2454 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2455 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2456 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2457 }
2458
2459
2460 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2461 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2462 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2463 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2464 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2465 }
2466
2467
2468 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2469 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2470 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2471 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2472 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2473 }
2474
2475
2476 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2477 if (null_check) {
2478 __ testptr(obj, obj);
2479 __ jcc(Assembler::zero, *slow_path->entry());
2480 }
2481 if (is_dest) {
2482 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2483 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2484 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2485 } else {
2486 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2487 }
2488 }
2489
2490
2491 // This code replaces a call to arraycopy; no exception may
2492 // be thrown in this code, they must be thrown in the System.arraycopy
2493 // activation frame; we could save some checks if this would not be the case
2494 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2495 ciArrayKlass* default_type = op->expected_type();
2496 Register src = op->src()->as_register();
2497 Register dst = op->dst()->as_register();
2498 Register src_pos = op->src_pos()->as_register();
2499 Register dst_pos = op->dst_pos()->as_register();
2500 Register length = op->length()->as_register();
2501 Register tmp = op->tmp()->as_register();
2502 Register tmp_load_klass = rscratch1;
2503 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2504
2505 CodeStub* stub = op->stub();
2506 int flags = op->flags();
2507 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2508 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2509
2510 if (flags & LIR_OpArrayCopy::always_slow_path) {
2511 __ jmp(*stub->entry());
2512 __ bind(*stub->continuation());
2513 return;
2514 }
2515
2516 // if we don't know anything, just go through the generic arraycopy
2517 if (default_type == nullptr) {
2518 // save outgoing arguments on stack in case call to System.arraycopy is needed
2519 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2520 // for interpreter calling conventions. Now we have to do it in new style conventions.
2521 // For the moment until C1 gets the new register allocator I just force all the
2522 // args to the right place (except the register args) and then on the back side
2523 // reload the register args properly if we go slow path. Yuck
2524
2525 // These are proper for the calling convention
2526 store_parameter(length, 2);
2527 store_parameter(dst_pos, 1);
2528 store_parameter(dst, 0);
2529
2530 // these are just temporary placements until we need to reload
2531 store_parameter(src_pos, 3);
2532 store_parameter(src, 4);
2533
2534 address copyfunc_addr = StubRoutines::generic_arraycopy();
2535 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2536
2537 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2538 // The arguments are in java calling convention so we can trivially shift them to C
2539 // convention
2540 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2541 __ mov(c_rarg0, j_rarg0);
2542 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2543 __ mov(c_rarg1, j_rarg1);
2544 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2545 __ mov(c_rarg2, j_rarg2);
2546 assert_different_registers(c_rarg3, j_rarg4);
2547 __ mov(c_rarg3, j_rarg3);
2548 #ifdef _WIN64
2549 // Allocate abi space for args but be sure to keep stack aligned
2550 __ subptr(rsp, 6*wordSize);
2551 store_parameter(j_rarg4, 4);
2552 #ifndef PRODUCT
2553 if (PrintC1Statistics) {
2554 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2555 }
2556 #endif
2557 __ call(RuntimeAddress(copyfunc_addr));
2558 __ addptr(rsp, 6*wordSize);
2559 #else
2560 __ mov(c_rarg4, j_rarg4);
2561 #ifndef PRODUCT
2562 if (PrintC1Statistics) {
2563 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2564 }
2565 #endif
2566 __ call(RuntimeAddress(copyfunc_addr));
2567 #endif // _WIN64
2568
2569 __ testl(rax, rax);
2570 __ jcc(Assembler::equal, *stub->continuation());
2571
2572 __ mov(tmp, rax);
2573 __ xorl(tmp, -1);
2574
2575 // Reload values from the stack so they are where the stub
2576 // expects them.
2577 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2578 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2579 __ movptr (length, Address(rsp, 2*BytesPerWord));
2580 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2581 __ movptr (src, Address(rsp, 4*BytesPerWord));
2582
2583 __ subl(length, tmp);
2584 __ addl(src_pos, tmp);
2585 __ addl(dst_pos, tmp);
2586 __ jmp(*stub->entry());
2587
2588 __ bind(*stub->continuation());
2589 return;
2590 }
2591
2592 // Handle inline type arrays
2593 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2594 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2595 }
2596 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2597 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2598 }
2599
2600 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2601
2602 int elem_size = type2aelembytes(basic_type);
2603 Address::ScaleFactor scale;
2604
2605 switch (elem_size) {
2606 case 1 :
2607 scale = Address::times_1;
2608 break;
2609 case 2 :
2610 scale = Address::times_2;
2611 break;
2612 case 4 :
2613 scale = Address::times_4;
2614 break;
2615 case 8 :
2616 scale = Address::times_8;
2617 break;
2618 default:
2619 scale = Address::no_scale;
2620 ShouldNotReachHere();
2621 }
2622
2623 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2624 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2625
2626 // length and pos's are all sign extended at this point on 64bit
2627
2628 // test for null
2629 if (flags & LIR_OpArrayCopy::src_null_check) {
2630 __ testptr(src, src);
2631 __ jcc(Assembler::zero, *stub->entry());
2632 }
2633 if (flags & LIR_OpArrayCopy::dst_null_check) {
2634 __ testptr(dst, dst);
2635 __ jcc(Assembler::zero, *stub->entry());
2636 }
2637
2638 // If the compiler was not able to prove that exact type of the source or the destination
2639 // of the arraycopy is an array type, check at runtime if the source or the destination is
2640 // an instance type.
2641 if (flags & LIR_OpArrayCopy::type_check) {
2642 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2643 __ load_klass(tmp, dst, tmp_load_klass);
2644 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2645 __ jcc(Assembler::greaterEqual, *stub->entry());
2646 }
2647
2648 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2649 __ load_klass(tmp, src, tmp_load_klass);
2650 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2651 __ jcc(Assembler::greaterEqual, *stub->entry());
2652 }
2653 }
2654
2655 // check if negative
2656 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2657 __ testl(src_pos, src_pos);
2658 __ jcc(Assembler::less, *stub->entry());
2659 }
2660 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2661 __ testl(dst_pos, dst_pos);
2662 __ jcc(Assembler::less, *stub->entry());
2663 }
2664
2665 if (flags & LIR_OpArrayCopy::src_range_check) {
2666 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2667 __ cmpl(tmp, src_length_addr);
2668 __ jcc(Assembler::above, *stub->entry());
2669 }
2670 if (flags & LIR_OpArrayCopy::dst_range_check) {
2671 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2672 __ cmpl(tmp, dst_length_addr);
2673 __ jcc(Assembler::above, *stub->entry());
2674 }
2675
2676 if (flags & LIR_OpArrayCopy::length_positive_check) {
2677 __ testl(length, length);
2678 __ jcc(Assembler::less, *stub->entry());
2679 }
2680
2681 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2682 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2683
2684 if (flags & LIR_OpArrayCopy::type_check) {
2685 // We don't know the array types are compatible
2686 if (basic_type != T_OBJECT) {
2687 // Simple test for basic type arrays
2688 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2689 __ jcc(Assembler::notEqual, *stub->entry());
2690 } else {
2691 // For object arrays, if src is a sub class of dst then we can
2692 // safely do the copy.
2693 Label cont, slow;
2694
2695 __ push_ppx(src);
2696 __ push_ppx(dst);
2697
2698 __ load_klass(src, src, tmp_load_klass);
2699 __ load_klass(dst, dst, tmp_load_klass);
2700
2701 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2702
2703 __ push_ppx(src);
2704 __ push_ppx(dst);
2705 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2706 __ pop_ppx(dst);
2707 __ pop_ppx(src);
2708
2709 __ testl(src, src);
2710 __ jcc(Assembler::notEqual, cont);
2711
2712 __ bind(slow);
2713 __ pop_ppx(dst);
2714 __ pop_ppx(src);
2715
2716 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2717 if (copyfunc_addr != nullptr) { // use stub if available
2718 // src is not a sub class of dst so we have to do a
2719 // per-element check.
2720
2721 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2722 if ((flags & mask) != mask) {
2723 // Check that at least both of them object arrays.
2724 assert(flags & mask, "one of the two should be known to be an object array");
2725
2726 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2727 __ load_klass(tmp, src, tmp_load_klass);
2728 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2729 __ load_klass(tmp, dst, tmp_load_klass);
2730 }
2731 int lh_offset = in_bytes(Klass::layout_helper_offset());
2732 Address klass_lh_addr(tmp, lh_offset);
2733 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2734 __ cmpl(klass_lh_addr, objArray_lh);
2735 __ jcc(Assembler::notEqual, *stub->entry());
2736 }
2737
2738 // Spill because stubs can use any register they like and it's
2739 // easier to restore just those that we care about.
2740 store_parameter(dst, 0);
2741 store_parameter(dst_pos, 1);
2742 store_parameter(length, 2);
2743 store_parameter(src_pos, 3);
2744 store_parameter(src, 4);
2745
2746 __ movl2ptr(length, length); //higher 32bits must be null
2747
2748 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2749 assert_different_registers(c_rarg0, dst, dst_pos, length);
2750 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2751 assert_different_registers(c_rarg1, dst, length);
2752
2753 __ mov(c_rarg2, length);
2754 assert_different_registers(c_rarg2, dst);
2755
2756 #ifdef _WIN64
2757 // Allocate abi space for args but be sure to keep stack aligned
2758 __ subptr(rsp, 6*wordSize);
2759 __ load_klass(c_rarg3, dst, tmp_load_klass);
2760 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2761 store_parameter(c_rarg3, 4);
2762 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2763 __ call(RuntimeAddress(copyfunc_addr));
2764 __ addptr(rsp, 6*wordSize);
2765 #else
2766 __ load_klass(c_rarg4, dst, tmp_load_klass);
2767 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2768 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2769 __ call(RuntimeAddress(copyfunc_addr));
2770 #endif
2771
2772 #ifndef PRODUCT
2773 if (PrintC1Statistics) {
2774 Label failed;
2775 __ testl(rax, rax);
2776 __ jcc(Assembler::notZero, failed);
2777 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2778 __ bind(failed);
2779 }
2780 #endif
2781
2782 __ testl(rax, rax);
2783 __ jcc(Assembler::zero, *stub->continuation());
2784
2785 #ifndef PRODUCT
2786 if (PrintC1Statistics) {
2787 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2788 }
2789 #endif
2790
2791 __ mov(tmp, rax);
2792
2793 __ xorl(tmp, -1);
2794
2795 // Restore previously spilled arguments
2796 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2797 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2798 __ movptr (length, Address(rsp, 2*BytesPerWord));
2799 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2800 __ movptr (src, Address(rsp, 4*BytesPerWord));
2801
2802
2803 __ subl(length, tmp);
2804 __ addl(src_pos, tmp);
2805 __ addl(dst_pos, tmp);
2806 }
2807
2808 __ jmp(*stub->entry());
2809
2810 __ bind(cont);
2811 __ pop(dst);
2812 __ pop(src);
2813 }
2814 }
2815
2816 #ifdef ASSERT
2817 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2818 // Sanity check the known type with the incoming class. For the
2819 // primitive case the types must match exactly with src.klass and
2820 // dst.klass each exactly matching the default type. For the
2821 // object array case, if no type check is needed then either the
2822 // dst type is exactly the expected type and the src type is a
2823 // subtype which we can't check or src is the same array as dst
2824 // but not necessarily exactly of type default_type.
2825 Label known_ok, halt;
2826 __ mov_metadata(tmp, default_type->constant_encoding());
2827 __ encode_klass_not_null(tmp, rscratch1);
2828
2829 if (basic_type != T_OBJECT) {
2830 __ cmp_klass(tmp, dst, tmp2);
2831 __ jcc(Assembler::notEqual, halt);
2832 __ cmp_klass(tmp, src, tmp2);
2833 __ jcc(Assembler::equal, known_ok);
2834 } else {
2835 __ cmp_klass(tmp, dst, tmp2);
2836 __ jcc(Assembler::equal, known_ok);
2837 __ cmpptr(src, dst);
2838 __ jcc(Assembler::equal, known_ok);
2839 }
2840 __ bind(halt);
2841 __ stop("incorrect type information in arraycopy");
2842 __ bind(known_ok);
2843 }
2844 #endif
2845
2846 #ifndef PRODUCT
2847 if (PrintC1Statistics) {
2848 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2849 }
2850 #endif
2851
2852 assert_different_registers(c_rarg0, dst, dst_pos, length);
2853 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2854 assert_different_registers(c_rarg1, length);
2855 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2856 __ mov(c_rarg2, length);
2857
2858 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2859 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2860 const char *name;
2861 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2862 __ call_VM_leaf(entry, 0);
2863
2864 if (stub != nullptr) {
2865 __ bind(*stub->continuation());
2866 }
2867 }
2868
2869 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2870 assert(op->crc()->is_single_cpu(), "crc must be register");
2871 assert(op->val()->is_single_cpu(), "byte value must be register");
2872 assert(op->result_opr()->is_single_cpu(), "result must be register");
2873 Register crc = op->crc()->as_register();
2874 Register val = op->val()->as_register();
2875 Register res = op->result_opr()->as_register();
2876
2877 assert_different_registers(val, crc, res);
2878
2879 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2880 __ notl(crc); // ~crc
2881 __ update_byte_crc32(crc, val, res);
2882 __ notl(crc); // ~crc
2883 __ mov(res, crc);
2884 }
2885
2886 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2887 Register obj = op->obj_opr()->as_register(); // may not be an oop
2888 Register hdr = op->hdr_opr()->as_register();
2889 Register lock = op->lock_opr()->as_register();
2890 if (op->code() == lir_lock) {
2891 Register tmp = op->scratch_opr()->as_register();
2892 // add debug info for NullPointerException only if one is possible
2893 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2894 if (op->info() != nullptr) {
2895 add_debug_info_for_null_check(null_check_offset, op->info());
2896 }
2897 // done
2898 } else if (op->code() == lir_unlock) {
2899 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2900 } else {
2901 Unimplemented();
2902 }
2903 __ bind(*op->stub()->continuation());
2904 }
2905
2906 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2907 Register obj = op->obj()->as_pointer_register();
2908 Register result = op->result_opr()->as_pointer_register();
2909
2910 CodeEmitInfo* info = op->info();
2911 if (info != nullptr) {
2912 add_debug_info_for_null_check_here(info);
2913 }
2914
2915 __ load_klass(result, obj, rscratch1);
2916 }
2917
2918 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2919 ciMethod* method = op->profiled_method();
2920 int bci = op->profiled_bci();
2921 ciMethod* callee = op->profiled_callee();
2922 Register tmp_load_klass = rscratch1;
2923
2924 // Update counter for all call types
2925 ciMethodData* md = method->method_data_or_null();
2926 assert(md != nullptr, "Sanity");
2927 ciProfileData* data = md->bci_to_data(bci);
2928 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2929 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2930 Register mdo = op->mdo()->as_register();
2931 __ mov_metadata(mdo, md->constant_encoding());
2932 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2933 // Perform additional virtual call profiling for invokevirtual and
2934 // invokeinterface bytecodes
2935 if (op->should_profile_receiver_type()) {
2936 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2937 Register recv = op->recv()->as_register();
2938 assert_different_registers(mdo, recv);
2939 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2940 ciKlass* known_klass = op->known_holder();
2941 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2942 // We know the type that will be seen at this call site; we can
2943 // statically update the MethodData* rather than needing to do
2944 // dynamic tests on the receiver type.
2945 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2946 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2947 ciKlass* receiver = vc_data->receiver(i);
2948 if (known_klass->equals(receiver)) {
2949 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2950 __ addptr(data_addr, DataLayout::counter_increment);
2951 return;
2952 }
2953 }
2954 // Receiver type is not found in profile data.
2955 // Fall back to runtime helper to handle the rest at runtime.
2956 __ mov_metadata(recv, known_klass->constant_encoding());
2957 } else {
2958 __ load_klass(recv, recv, tmp_load_klass);
2959 }
2960 type_profile_helper(mdo, md, data, recv);
2961 } else {
2962 // Static call
2963 __ addptr(counter_addr, DataLayout::counter_increment);
2964 }
2965 }
2966
2967 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2968 Register obj = op->obj()->as_register();
2969 Register tmp = op->tmp()->as_pointer_register();
2970 Register tmp_load_klass = rscratch1;
2971 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2972 ciKlass* exact_klass = op->exact_klass();
2973 intptr_t current_klass = op->current_klass();
2974 bool not_null = op->not_null();
2975 bool no_conflict = op->no_conflict();
2976
2977 Label update, next, none;
2978
2979 bool do_null = !not_null;
2980 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2981 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2982
2983 assert(do_null || do_update, "why are we here?");
2984 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2985
2986 __ verify_oop(obj);
2987
2988 #ifdef ASSERT
2989 if (obj == tmp) {
2990 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2991 } else {
2992 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2993 }
2994 #endif
2995 if (do_null) {
2996 __ testptr(obj, obj);
2997 __ jccb(Assembler::notZero, update);
2998 if (!TypeEntries::was_null_seen(current_klass)) {
2999 __ testptr(mdo_addr, TypeEntries::null_seen);
3000 #ifndef ASSERT
3001 __ jccb(Assembler::notZero, next); // already set
3002 #else
3003 __ jcc(Assembler::notZero, next); // already set
3004 #endif
3005 // atomic update to prevent overwriting Klass* with 0
3006 __ lock();
3007 __ orptr(mdo_addr, TypeEntries::null_seen);
3008 }
3009 if (do_update) {
3010 #ifndef ASSERT
3011 __ jmpb(next);
3012 }
3013 #else
3014 __ jmp(next);
3015 }
3016 } else {
3017 __ testptr(obj, obj);
3018 __ jcc(Assembler::notZero, update);
3019 __ stop("unexpected null obj");
3020 #endif
3021 }
3022
3023 __ bind(update);
3024
3025 if (do_update) {
3026 #ifdef ASSERT
3027 if (exact_klass != nullptr) {
3028 Label ok;
3029 __ load_klass(tmp, obj, tmp_load_klass);
3030 __ push_ppx(tmp);
3031 __ mov_metadata(tmp, exact_klass->constant_encoding());
3032 __ cmpptr(tmp, Address(rsp, 0));
3033 __ jcc(Assembler::equal, ok);
3034 __ stop("exact klass and actual klass differ");
3035 __ bind(ok);
3036 __ pop_ppx(tmp);
3037 }
3038 #endif
3039 if (!no_conflict) {
3040 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3041 if (exact_klass != nullptr) {
3042 __ mov_metadata(tmp, exact_klass->constant_encoding());
3043 } else {
3044 __ load_klass(tmp, obj, tmp_load_klass);
3045 }
3046 __ mov(rscratch1, tmp); // save original value before XOR
3047 __ xorptr(tmp, mdo_addr);
3048 __ testptr(tmp, TypeEntries::type_klass_mask);
3049 // klass seen before, nothing to do. The unknown bit may have been
3050 // set already but no need to check.
3051 __ jccb(Assembler::zero, next);
3052
3053 __ testptr(tmp, TypeEntries::type_unknown);
3054 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3055
3056 if (TypeEntries::is_type_none(current_klass)) {
3057 __ testptr(mdo_addr, TypeEntries::type_mask);
3058 __ jccb(Assembler::zero, none);
3059 // There is a chance that the checks above (re-reading profiling
3060 // data from memory) fail if another thread has just set the
3061 // profiling to this obj's klass
3062 __ mov(tmp, rscratch1); // get back original value before XOR
3063 __ xorptr(tmp, mdo_addr);
3064 __ testptr(tmp, TypeEntries::type_klass_mask);
3065 __ jccb(Assembler::zero, next);
3066 }
3067 } else {
3068 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3069 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3070
3071 __ testptr(mdo_addr, TypeEntries::type_unknown);
3072 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3073 }
3074
3075 // different than before. Cannot keep accurate profile.
3076 __ orptr(mdo_addr, TypeEntries::type_unknown);
3077
3078 if (TypeEntries::is_type_none(current_klass)) {
3079 __ jmpb(next);
3080
3081 __ bind(none);
3082 // first time here. Set profile type.
3083 __ movptr(mdo_addr, tmp);
3084 #ifdef ASSERT
3085 __ andptr(tmp, TypeEntries::type_klass_mask);
3086 __ verify_klass_ptr(tmp);
3087 #endif
3088 }
3089 } else {
3090 // There's a single possible klass at this profile point
3091 assert(exact_klass != nullptr, "should be");
3092 if (TypeEntries::is_type_none(current_klass)) {
3093 __ mov_metadata(tmp, exact_klass->constant_encoding());
3094 __ xorptr(tmp, mdo_addr);
3095 __ testptr(tmp, TypeEntries::type_klass_mask);
3096 #ifdef ASSERT
3097 __ jcc(Assembler::zero, next);
3098
3099 {
3100 Label ok;
3101 __ push_ppx(tmp);
3102 __ testptr(mdo_addr, TypeEntries::type_mask);
3103 __ jcc(Assembler::zero, ok);
3104 // may have been set by another thread
3105 __ mov_metadata(tmp, exact_klass->constant_encoding());
3106 __ xorptr(tmp, mdo_addr);
3107 __ testptr(tmp, TypeEntries::type_mask);
3108 __ jcc(Assembler::zero, ok);
3109
3110 __ stop("unexpected profiling mismatch");
3111 __ bind(ok);
3112 __ pop_ppx(tmp);
3113 }
3114 #else
3115 __ jccb(Assembler::zero, next);
3116 #endif
3117 // first time here. Set profile type.
3118 __ movptr(mdo_addr, tmp);
3119 #ifdef ASSERT
3120 __ andptr(tmp, TypeEntries::type_klass_mask);
3121 __ verify_klass_ptr(tmp);
3122 #endif
3123 } else {
3124 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3125 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3126
3127 __ testptr(mdo_addr, TypeEntries::type_unknown);
3128 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3129
3130 __ orptr(mdo_addr, TypeEntries::type_unknown);
3131 }
3132 }
3133 }
3134 __ bind(next);
3135 }
3136
3137 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3138 Register obj = op->obj()->as_register();
3139 Register tmp = op->tmp()->as_pointer_register();
3140 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3141 bool not_null = op->not_null();
3142 int flag = op->flag();
3143
3144 Label not_inline_type;
3145 if (!not_null) {
3146 __ testptr(obj, obj);
3147 __ jccb(Assembler::zero, not_inline_type);
3148 }
3149
3150 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3151
3152 __ orb(mdo_addr, flag);
3153
3154 __ bind(not_inline_type);
3155 }
3156
3157
3158 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3159 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3160 }
3161
3162
3163 void LIR_Assembler::align_backward_branch_target() {
3164 __ align(BytesPerWord);
3165 }
3166
3167
3168 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3169 if (left->is_single_cpu()) {
3170 __ negl(left->as_register());
3171 move_regs(left->as_register(), dest->as_register());
3172
3173 } else if (left->is_double_cpu()) {
3174 Register lo = left->as_register_lo();
3175 Register dst = dest->as_register_lo();
3176 __ movptr(dst, lo);
3177 __ negptr(dst);
3178
3179 } else if (dest->is_single_xmm()) {
3180 assert(!tmp->is_valid(), "do not need temporary");
3181 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3182 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3183 }
3184 __ xorps(dest->as_xmm_float_reg(),
3185 ExternalAddress((address)float_signflip_pool),
3186 rscratch1);
3187 } else if (dest->is_double_xmm()) {
3188 assert(!tmp->is_valid(), "do not need temporary");
3189 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3190 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3191 }
3192 __ xorpd(dest->as_xmm_double_reg(),
3193 ExternalAddress((address)double_signflip_pool),
3194 rscratch1);
3195 } else {
3196 ShouldNotReachHere();
3197 }
3198 }
3199
3200
3201 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3202 assert(src->is_address(), "must be an address");
3203 assert(dest->is_register(), "must be a register");
3204
3205 PatchingStub* patch = nullptr;
3206 if (patch_code != lir_patch_none) {
3207 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3208 }
3209
3210 Register reg = dest->as_pointer_register();
3211 LIR_Address* addr = src->as_address_ptr();
3212 __ lea(reg, as_Address(addr));
3213
3214 if (patch != nullptr) {
3215 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3216 }
3217 }
3218
3219
3220
3221 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3222 assert(!tmp->is_valid(), "don't need temporary");
3223 __ call(RuntimeAddress(dest));
3224 if (info != nullptr) {
3225 add_call_info_here(info);
3226 }
3227 __ post_call_nop();
3228 }
3229
3230
3231 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3232 assert(type == T_LONG, "only for volatile long fields");
3233
3234 if (info != nullptr) {
3235 add_debug_info_for_null_check_here(info);
3236 }
3237
3238 if (src->is_double_xmm()) {
3239 if (dest->is_double_cpu()) {
3240 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3241 } else if (dest->is_double_stack()) {
3242 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3243 } else if (dest->is_address()) {
3244 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3245 } else {
3246 ShouldNotReachHere();
3247 }
3248
3249 } else if (dest->is_double_xmm()) {
3250 if (src->is_double_stack()) {
3251 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3252 } else if (src->is_address()) {
3253 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3254 } else {
3255 ShouldNotReachHere();
3256 }
3257
3258 } else {
3259 ShouldNotReachHere();
3260 }
3261 }
3262
3263 #ifdef ASSERT
3264 // emit run-time assertion
3265 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3266 assert(op->code() == lir_assert, "must be");
3267
3268 if (op->in_opr1()->is_valid()) {
3269 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3270 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3271 } else {
3272 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3273 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3274 }
3275
3276 Label ok;
3277 if (op->condition() != lir_cond_always) {
3278 Assembler::Condition acond = Assembler::zero;
3279 switch (op->condition()) {
3280 case lir_cond_equal: acond = Assembler::equal; break;
3281 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3282 case lir_cond_less: acond = Assembler::less; break;
3283 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3284 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3285 case lir_cond_greater: acond = Assembler::greater; break;
3286 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3287 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3288 default: ShouldNotReachHere();
3289 }
3290 __ jcc(acond, ok);
3291 }
3292 if (op->halt()) {
3293 const char* str = __ code_string(op->msg());
3294 __ stop(str);
3295 } else {
3296 breakpoint();
3297 }
3298 __ bind(ok);
3299 }
3300 #endif
3301
3302 void LIR_Assembler::membar() {
3303 // QQQ sparc TSO uses this,
3304 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3305 }
3306
3307 void LIR_Assembler::membar_acquire() {
3308 // No x86 machines currently require load fences
3309 }
3310
3311 void LIR_Assembler::membar_release() {
3312 // No x86 machines currently require store fences
3313 }
3314
3315 void LIR_Assembler::membar_loadload() {
3316 // no-op
3317 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3318 }
3319
3320 void LIR_Assembler::membar_storestore() {
3321 // no-op
3322 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3323 }
3324
3325 void LIR_Assembler::membar_loadstore() {
3326 // no-op
3327 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3328 }
3329
3330 void LIR_Assembler::membar_storeload() {
3331 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3332 }
3333
3334 void LIR_Assembler::on_spin_wait() {
3335 __ pause ();
3336 }
3337
3338 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3339 assert(result_reg->is_register(), "check");
3340 __ mov(result_reg->as_register(), r15_thread);
3341 }
3342
3343 void LIR_Assembler::check_orig_pc() {
3344 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3345 }
3346
3347 void LIR_Assembler::peephole(LIR_List*) {
3348 // do nothing for now
3349 }
3350
3351 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3352 assert(data == dest, "xchg/xadd uses only 2 operands");
3353
3354 if (data->type() == T_INT) {
3355 if (code == lir_xadd) {
3356 __ lock();
3357 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3358 } else {
3359 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3360 }
3361 } else if (data->is_oop()) {
3362 assert (code == lir_xchg, "xadd for oops");
3363 Register obj = data->as_register();
3364 if (UseCompressedOops) {
3365 __ encode_heap_oop(obj);
3366 __ xchgl(obj, as_Address(src->as_address_ptr()));
3367 __ decode_heap_oop(obj);
3368 } else {
3369 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3370 }
3371 } else if (data->type() == T_LONG) {
3372 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3373 if (code == lir_xadd) {
3374 __ lock();
3375 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3376 } else {
3377 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3378 }
3379 } else {
3380 ShouldNotReachHere();
3381 }
3382 }
3383
3384 #undef __