1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456 InternalAddress here(__ pc());
457
458 __ pushptr(here.addr(), rscratch1);
459 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
460 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
461 __ end_a_stub();
462
463 return offset;
464 }
465
466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
467 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
468 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
469 assert(result->fpu() == 0, "result must already be on TOS");
470 }
471 if (InlineTypeReturnedAsFields) {
472 #ifndef _LP64
473 Unimplemented();
474 #endif
475 // Check if we are returning an non-null inline type and load its fields into registers
476 ciType* return_type = compilation()->method()->return_type();
477 if (return_type->is_inlinetype()) {
478 ciInlineKlass* vk = return_type->as_inline_klass();
479 if (vk->can_be_returned_as_fields()) {
480 address unpack_handler = vk->unpack_handler();
481 assert(unpack_handler != nullptr, "must be");
482 __ call(RuntimeAddress(unpack_handler));
483 }
484 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
485 Label skip;
486 Label not_null;
487 __ testptr(rax, rax);
488 __ jcc(Assembler::notZero, not_null);
489 // Returned value is null, zero all return registers because they may belong to oop fields
490 __ xorq(j_rarg1, j_rarg1);
491 __ xorq(j_rarg2, j_rarg2);
492 __ xorq(j_rarg3, j_rarg3);
493 __ xorq(j_rarg4, j_rarg4);
494 __ xorq(j_rarg5, j_rarg5);
495 __ jmp(skip);
496 __ bind(not_null);
497
498 // Check if we are returning an non-null inline type and load its fields into registers
499 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
500
501 // Load fields from a buffered value with an inline class specific handler
502 __ load_klass(rdi, rax, rscratch1);
503 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
504 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
505 // Unpack handler can be null if inline type is not scalarizable in returns
506 __ testptr(rdi, rdi);
507 __ jcc(Assembler::zero, skip);
508 __ call(rdi);
509
510 __ bind(skip);
511 }
512 // At this point, rax points to the value object (for interpreter or C1 caller).
513 // The fields of the object are copied into registers (for C2 caller).
514 }
515
516 // Pop the stack before the safepoint code
517 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
518
519 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
520 __ reserved_stack_check();
521 }
522
523 // Note: we do not need to round double result; float result has the right precision
524 // the poll sets the condition code, but no data registers
525
526 code_stub->set_safepoint_offset(__ offset());
527 __ relocate(relocInfo::poll_return_type);
528 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
529 __ ret(0);
530 }
531
532
533 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
534 return (__ store_inline_type_fields_to_buf(vk, false));
535 }
536
537 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
538 guarantee(info != nullptr, "Shouldn't be null");
539 int offset = __ offset();
540 const Register poll_addr = rscratch1;
541 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
542 add_debug_info_for_branch(info);
543 __ relocate(relocInfo::poll_type);
544 address pre_pc = __ pc();
545 __ testl(rax, Address(poll_addr, 0));
546 address post_pc = __ pc();
547 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
548 return offset;
549 }
550
551
552 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
553 if (from_reg != to_reg) __ mov(to_reg, from_reg);
554 }
555
556 void LIR_Assembler::swap_reg(Register a, Register b) {
557 __ xchgptr(a, b);
558 }
559
560
561 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
562 assert(src->is_constant(), "should not call otherwise");
563 assert(dest->is_register(), "should not call otherwise");
564 LIR_Const* c = src->as_constant_ptr();
565
566 switch (c->type()) {
567 case T_INT: {
568 assert(patch_code == lir_patch_none, "no patching handled here");
569 __ movl(dest->as_register(), c->as_jint());
570 break;
571 }
572
573 case T_ADDRESS: {
574 assert(patch_code == lir_patch_none, "no patching handled here");
575 __ movptr(dest->as_register(), c->as_jint());
576 break;
577 }
578
579 case T_LONG: {
580 assert(patch_code == lir_patch_none, "no patching handled here");
581 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
582 break;
583 }
584
585 case T_OBJECT: {
586 if (patch_code != lir_patch_none) {
587 jobject2reg_with_patching(dest->as_register(), info);
588 } else {
589 __ movoop(dest->as_register(), c->as_jobject());
590 }
591 break;
592 }
593
594 case T_METADATA: {
595 if (patch_code != lir_patch_none) {
596 klass2reg_with_patching(dest->as_register(), info);
597 } else {
598 __ mov_metadata(dest->as_register(), c->as_metadata());
599 }
600 break;
601 }
602
603 case T_FLOAT: {
604 if (dest->is_single_xmm()) {
605 if (UseAVX <= 2 && c->is_zero_float()) {
606 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
607 } else {
608 __ movflt(dest->as_xmm_float_reg(),
609 InternalAddress(float_constant(c->as_jfloat())));
610 }
611 } else {
612 ShouldNotReachHere();
613 }
614 break;
615 }
616
617 case T_DOUBLE: {
618 if (dest->is_double_xmm()) {
619 if (UseAVX <= 2 && c->is_zero_double()) {
620 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
621 } else {
622 __ movdbl(dest->as_xmm_double_reg(),
623 InternalAddress(double_constant(c->as_jdouble())));
624 }
625 } else {
626 ShouldNotReachHere();
627 }
628 break;
629 }
630
631 default:
632 ShouldNotReachHere();
633 }
634 }
635
636 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
637 assert(src->is_constant(), "should not call otherwise");
638 assert(dest->is_stack(), "should not call otherwise");
639 LIR_Const* c = src->as_constant_ptr();
640
641 switch (c->type()) {
642 case T_INT: // fall through
643 case T_FLOAT:
644 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
645 break;
646
647 case T_ADDRESS:
648 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
649 break;
650
651 case T_OBJECT:
652 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
653 break;
654
655 case T_LONG: // fall through
656 case T_DOUBLE:
657 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
658 lo_word_offset_in_bytes),
659 (intptr_t)c->as_jlong_bits(),
660 rscratch1);
661 break;
662
663 default:
664 ShouldNotReachHere();
665 }
666 }
667
668 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
669 assert(src->is_constant(), "should not call otherwise");
670 assert(dest->is_address(), "should not call otherwise");
671 LIR_Const* c = src->as_constant_ptr();
672 LIR_Address* addr = dest->as_address_ptr();
673
674 int null_check_here = code_offset();
675 switch (type) {
676 case T_INT: // fall through
677 case T_FLOAT:
678 __ movl(as_Address(addr), c->as_jint_bits());
679 break;
680
681 case T_ADDRESS:
682 __ movptr(as_Address(addr), c->as_jint_bits());
683 break;
684
685 case T_OBJECT: // fall through
686 case T_ARRAY:
687 if (c->as_jobject() == nullptr) {
688 if (UseCompressedOops && !wide) {
689 __ movl(as_Address(addr), NULL_WORD);
690 } else {
691 __ xorptr(rscratch1, rscratch1);
692 null_check_here = code_offset();
693 __ movptr(as_Address(addr), rscratch1);
694 }
695 } else {
696 if (is_literal_address(addr)) {
697 ShouldNotReachHere();
698 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
699 } else {
700 __ movoop(rscratch1, c->as_jobject());
701 if (UseCompressedOops && !wide) {
702 __ encode_heap_oop(rscratch1);
703 null_check_here = code_offset();
704 __ movl(as_Address_lo(addr), rscratch1);
705 } else {
706 null_check_here = code_offset();
707 __ movptr(as_Address_lo(addr), rscratch1);
708 }
709 }
710 }
711 break;
712
713 case T_LONG: // fall through
714 case T_DOUBLE:
715 if (is_literal_address(addr)) {
716 ShouldNotReachHere();
717 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
718 } else {
719 __ movptr(r10, (intptr_t)c->as_jlong_bits());
720 null_check_here = code_offset();
721 __ movptr(as_Address_lo(addr), r10);
722 }
723 break;
724
725 case T_BOOLEAN: // fall through
726 case T_BYTE:
727 __ movb(as_Address(addr), c->as_jint() & 0xFF);
728 break;
729
730 case T_CHAR: // fall through
731 case T_SHORT:
732 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
733 break;
734
735 default:
736 ShouldNotReachHere();
737 };
738
739 if (info != nullptr) {
740 add_debug_info_for_null_check(null_check_here, info);
741 }
742 }
743
744
745 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
746 assert(src->is_register(), "should not call otherwise");
747 assert(dest->is_register(), "should not call otherwise");
748
749 // move between cpu-registers
750 if (dest->is_single_cpu()) {
751 if (src->type() == T_LONG) {
752 // Can do LONG -> OBJECT
753 move_regs(src->as_register_lo(), dest->as_register());
754 return;
755 }
756 assert(src->is_single_cpu(), "must match");
757 if (src->type() == T_OBJECT) {
758 __ verify_oop(src->as_register());
759 }
760 move_regs(src->as_register(), dest->as_register());
761
762 } else if (dest->is_double_cpu()) {
763 if (is_reference_type(src->type())) {
764 // Surprising to me but we can see move of a long to t_object
765 __ verify_oop(src->as_register());
766 move_regs(src->as_register(), dest->as_register_lo());
767 return;
768 }
769 assert(src->is_double_cpu(), "must match");
770 Register f_lo = src->as_register_lo();
771 Register f_hi = src->as_register_hi();
772 Register t_lo = dest->as_register_lo();
773 Register t_hi = dest->as_register_hi();
774 assert(f_hi == f_lo, "must be same");
775 assert(t_hi == t_lo, "must be same");
776 move_regs(f_lo, t_lo);
777
778 // move between xmm-registers
779 } else if (dest->is_single_xmm()) {
780 assert(src->is_single_xmm(), "must match");
781 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
782 } else if (dest->is_double_xmm()) {
783 assert(src->is_double_xmm(), "must match");
784 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
785
786 } else {
787 ShouldNotReachHere();
788 }
789 }
790
791 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
792 assert(src->is_register(), "should not call otherwise");
793 assert(dest->is_stack(), "should not call otherwise");
794
795 if (src->is_single_cpu()) {
796 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
797 if (is_reference_type(type)) {
798 __ verify_oop(src->as_register());
799 __ movptr (dst, src->as_register());
800 } else if (type == T_METADATA || type == T_ADDRESS) {
801 __ movptr (dst, src->as_register());
802 } else {
803 __ movl (dst, src->as_register());
804 }
805
806 } else if (src->is_double_cpu()) {
807 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
808 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
809 __ movptr (dstLO, src->as_register_lo());
810
811 } else if (src->is_single_xmm()) {
812 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
813 __ movflt(dst_addr, src->as_xmm_float_reg());
814
815 } else if (src->is_double_xmm()) {
816 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
817 __ movdbl(dst_addr, src->as_xmm_double_reg());
818
819 } else {
820 ShouldNotReachHere();
821 }
822 }
823
824
825 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
826 LIR_Address* to_addr = dest->as_address_ptr();
827 PatchingStub* patch = nullptr;
828 Register compressed_src = rscratch1;
829
830 if (is_reference_type(type)) {
831 __ verify_oop(src->as_register());
832 if (UseCompressedOops && !wide) {
833 __ movptr(compressed_src, src->as_register());
834 __ encode_heap_oop(compressed_src);
835 if (patch_code != lir_patch_none) {
836 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
837 }
838 }
839 }
840
841 if (patch_code != lir_patch_none) {
842 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
843 Address toa = as_Address(to_addr);
844 assert(toa.disp() != 0, "must have");
845 }
846
847 int null_check_here = code_offset();
848 switch (type) {
849 case T_FLOAT: {
850 assert(src->is_single_xmm(), "not a float");
851 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
852 break;
853 }
854
855 case T_DOUBLE: {
856 assert(src->is_double_xmm(), "not a double");
857 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
858 break;
859 }
860
861 case T_ARRAY: // fall through
862 case T_OBJECT: // fall through
863 if (UseCompressedOops && !wide) {
864 __ movl(as_Address(to_addr), compressed_src);
865 } else {
866 __ movptr(as_Address(to_addr), src->as_register());
867 }
868 break;
869 case T_ADDRESS:
870 __ movptr(as_Address(to_addr), src->as_register());
871 break;
872 case T_INT:
873 __ movl(as_Address(to_addr), src->as_register());
874 break;
875
876 case T_LONG: {
877 Register from_lo = src->as_register_lo();
878 Register from_hi = src->as_register_hi();
879 __ movptr(as_Address_lo(to_addr), from_lo);
880 break;
881 }
882
883 case T_BYTE: // fall through
884 case T_BOOLEAN: {
885 Register src_reg = src->as_register();
886 Address dst_addr = as_Address(to_addr);
887 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
888 __ movb(dst_addr, src_reg);
889 break;
890 }
891
892 case T_CHAR: // fall through
893 case T_SHORT:
894 __ movw(as_Address(to_addr), src->as_register());
895 break;
896
897 default:
898 ShouldNotReachHere();
899 }
900 if (info != nullptr) {
901 add_debug_info_for_null_check(null_check_here, info);
902 }
903
904 if (patch_code != lir_patch_none) {
905 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
906 }
907 }
908
909
910 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
911 assert(src->is_stack(), "should not call otherwise");
912 assert(dest->is_register(), "should not call otherwise");
913
914 if (dest->is_single_cpu()) {
915 if (is_reference_type(type)) {
916 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
917 __ verify_oop(dest->as_register());
918 } else if (type == T_METADATA || type == T_ADDRESS) {
919 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
920 } else {
921 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
922 }
923
924 } else if (dest->is_double_cpu()) {
925 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
926 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
927 __ movptr(dest->as_register_lo(), src_addr_LO);
928
929 } else if (dest->is_single_xmm()) {
930 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
931 __ movflt(dest->as_xmm_float_reg(), src_addr);
932
933 } else if (dest->is_double_xmm()) {
934 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
935 __ movdbl(dest->as_xmm_double_reg(), src_addr);
936
937 } else {
938 ShouldNotReachHere();
939 }
940 }
941
942
943 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
944 if (src->is_single_stack()) {
945 if (is_reference_type(type)) {
946 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
947 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
948 } else {
949 //no pushl on 64bits
950 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
951 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
952 }
953
954 } else if (src->is_double_stack()) {
955 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
956 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
957
958 } else {
959 ShouldNotReachHere();
960 }
961 }
962
963
964 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
965 assert(src->is_address(), "should not call otherwise");
966 assert(dest->is_register(), "should not call otherwise");
967
968 LIR_Address* addr = src->as_address_ptr();
969 Address from_addr = as_Address(addr);
970
971 if (addr->base()->type() == T_OBJECT) {
972 __ verify_oop(addr->base()->as_pointer_register());
973 }
974
975 switch (type) {
976 case T_BOOLEAN: // fall through
977 case T_BYTE: // fall through
978 case T_CHAR: // fall through
979 case T_SHORT:
980 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
981 // on pre P6 processors we may get partial register stalls
982 // so blow away the value of to_rinfo before loading a
983 // partial word into it. Do it here so that it precedes
984 // the potential patch point below.
985 __ xorptr(dest->as_register(), dest->as_register());
986 }
987 break;
988 default:
989 break;
990 }
991
992 PatchingStub* patch = nullptr;
993 if (patch_code != lir_patch_none) {
994 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
995 assert(from_addr.disp() != 0, "must have");
996 }
997 if (info != nullptr) {
998 add_debug_info_for_null_check_here(info);
999 }
1000
1001 switch (type) {
1002 case T_FLOAT: {
1003 if (dest->is_single_xmm()) {
1004 __ movflt(dest->as_xmm_float_reg(), from_addr);
1005 } else {
1006 ShouldNotReachHere();
1007 }
1008 break;
1009 }
1010
1011 case T_DOUBLE: {
1012 if (dest->is_double_xmm()) {
1013 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1014 } else {
1015 ShouldNotReachHere();
1016 }
1017 break;
1018 }
1019
1020 case T_OBJECT: // fall through
1021 case T_ARRAY: // fall through
1022 if (UseCompressedOops && !wide) {
1023 __ movl(dest->as_register(), from_addr);
1024 } else {
1025 __ movptr(dest->as_register(), from_addr);
1026 }
1027 break;
1028
1029 case T_ADDRESS:
1030 __ movptr(dest->as_register(), from_addr);
1031 break;
1032 case T_INT:
1033 __ movl(dest->as_register(), from_addr);
1034 break;
1035
1036 case T_LONG: {
1037 Register to_lo = dest->as_register_lo();
1038 Register to_hi = dest->as_register_hi();
1039 __ movptr(to_lo, as_Address_lo(addr));
1040 break;
1041 }
1042
1043 case T_BOOLEAN: // fall through
1044 case T_BYTE: {
1045 Register dest_reg = dest->as_register();
1046 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1047 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1048 __ movsbl(dest_reg, from_addr);
1049 } else {
1050 __ movb(dest_reg, from_addr);
1051 __ shll(dest_reg, 24);
1052 __ sarl(dest_reg, 24);
1053 }
1054 break;
1055 }
1056
1057 case T_CHAR: {
1058 Register dest_reg = dest->as_register();
1059 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1060 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1061 __ movzwl(dest_reg, from_addr);
1062 } else {
1063 __ movw(dest_reg, from_addr);
1064 }
1065 break;
1066 }
1067
1068 case T_SHORT: {
1069 Register dest_reg = dest->as_register();
1070 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1071 __ movswl(dest_reg, from_addr);
1072 } else {
1073 __ movw(dest_reg, from_addr);
1074 __ shll(dest_reg, 16);
1075 __ sarl(dest_reg, 16);
1076 }
1077 break;
1078 }
1079
1080 default:
1081 ShouldNotReachHere();
1082 }
1083
1084 if (patch != nullptr) {
1085 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1086 }
1087
1088 if (is_reference_type(type)) {
1089 if (UseCompressedOops && !wide) {
1090 __ decode_heap_oop(dest->as_register());
1091 }
1092
1093 __ verify_oop(dest->as_register());
1094 }
1095 }
1096
1097
1098 NEEDS_CLEANUP; // This could be static?
1099 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1100 int elem_size = type2aelembytes(type);
1101 switch (elem_size) {
1102 case 1: return Address::times_1;
1103 case 2: return Address::times_2;
1104 case 4: return Address::times_4;
1105 case 8: return Address::times_8;
1106 }
1107 ShouldNotReachHere();
1108 return Address::no_scale;
1109 }
1110
1111
1112 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1113 switch (op->code()) {
1114 case lir_idiv:
1115 case lir_irem:
1116 arithmetic_idiv(op->code(),
1117 op->in_opr1(),
1118 op->in_opr2(),
1119 op->in_opr3(),
1120 op->result_opr(),
1121 op->info());
1122 break;
1123 case lir_fmad:
1124 __ fmad(op->result_opr()->as_xmm_double_reg(),
1125 op->in_opr1()->as_xmm_double_reg(),
1126 op->in_opr2()->as_xmm_double_reg(),
1127 op->in_opr3()->as_xmm_double_reg());
1128 break;
1129 case lir_fmaf:
1130 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1131 op->in_opr1()->as_xmm_float_reg(),
1132 op->in_opr2()->as_xmm_float_reg(),
1133 op->in_opr3()->as_xmm_float_reg());
1134 break;
1135 default: ShouldNotReachHere(); break;
1136 }
1137 }
1138
1139 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1140 #ifdef ASSERT
1141 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1142 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1143 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1144 #endif
1145
1146 if (op->cond() == lir_cond_always) {
1147 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1148 __ jmp (*(op->label()));
1149 } else {
1150 Assembler::Condition acond = Assembler::zero;
1151 if (op->code() == lir_cond_float_branch) {
1152 assert(op->ublock() != nullptr, "must have unordered successor");
1153 __ jcc(Assembler::parity, *(op->ublock()->label()));
1154 switch(op->cond()) {
1155 case lir_cond_equal: acond = Assembler::equal; break;
1156 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1157 case lir_cond_less: acond = Assembler::below; break;
1158 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1159 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1160 case lir_cond_greater: acond = Assembler::above; break;
1161 default: ShouldNotReachHere();
1162 }
1163 } else {
1164 switch (op->cond()) {
1165 case lir_cond_equal: acond = Assembler::equal; break;
1166 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1167 case lir_cond_less: acond = Assembler::less; break;
1168 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1169 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1170 case lir_cond_greater: acond = Assembler::greater; break;
1171 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1172 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1173 default: ShouldNotReachHere();
1174 }
1175 }
1176 __ jcc(acond,*(op->label()));
1177 }
1178 }
1179
1180 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1181 LIR_Opr src = op->in_opr();
1182 LIR_Opr dest = op->result_opr();
1183
1184 switch (op->bytecode()) {
1185 case Bytecodes::_i2l:
1186 __ movl2ptr(dest->as_register_lo(), src->as_register());
1187 break;
1188
1189 case Bytecodes::_l2i:
1190 __ movl(dest->as_register(), src->as_register_lo());
1191 break;
1192
1193 case Bytecodes::_i2b:
1194 move_regs(src->as_register(), dest->as_register());
1195 __ sign_extend_byte(dest->as_register());
1196 break;
1197
1198 case Bytecodes::_i2c:
1199 move_regs(src->as_register(), dest->as_register());
1200 __ andl(dest->as_register(), 0xFFFF);
1201 break;
1202
1203 case Bytecodes::_i2s:
1204 move_regs(src->as_register(), dest->as_register());
1205 __ sign_extend_short(dest->as_register());
1206 break;
1207
1208 case Bytecodes::_f2d:
1209 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1210 break;
1211
1212 case Bytecodes::_d2f:
1213 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1214 break;
1215
1216 case Bytecodes::_i2f:
1217 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1218 break;
1219
1220 case Bytecodes::_i2d:
1221 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1222 break;
1223
1224 case Bytecodes::_l2f:
1225 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1226 break;
1227
1228 case Bytecodes::_l2d:
1229 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1230 break;
1231
1232 case Bytecodes::_f2i:
1233 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1234 break;
1235
1236 case Bytecodes::_d2i:
1237 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1238 break;
1239
1240 case Bytecodes::_f2l:
1241 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1242 break;
1243
1244 case Bytecodes::_d2l:
1245 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1246 break;
1247
1248 default: ShouldNotReachHere();
1249 }
1250 }
1251
1252 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1253 if (op->init_check()) {
1254 add_debug_info_for_null_check_here(op->stub()->info());
1255 // init_state needs acquire, but x86 is TSO, and so we are already good.
1256 __ cmpb(Address(op->klass()->as_register(),
1257 InstanceKlass::init_state_offset()),
1258 InstanceKlass::fully_initialized);
1259 __ jcc(Assembler::notEqual, *op->stub()->entry());
1260 }
1261 __ allocate_object(op->obj()->as_register(),
1262 op->tmp1()->as_register(),
1263 op->tmp2()->as_register(),
1264 op->header_size(),
1265 op->object_size(),
1266 op->klass()->as_register(),
1267 *op->stub()->entry());
1268 __ bind(*op->stub()->continuation());
1269 }
1270
1271 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1272 Register len = op->len()->as_register();
1273 __ movslq(len, len);
1274
1275 if (UseSlowPath || op->always_slow_path() ||
1276 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1277 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1278 __ jmp(*op->stub()->entry());
1279 } else {
1280 Register tmp1 = op->tmp1()->as_register();
1281 Register tmp2 = op->tmp2()->as_register();
1282 Register tmp3 = op->tmp3()->as_register();
1283 if (len == tmp1) {
1284 tmp1 = tmp3;
1285 } else if (len == tmp2) {
1286 tmp2 = tmp3;
1287 } else if (len == tmp3) {
1288 // everything is ok
1289 } else {
1290 __ mov(tmp3, len);
1291 }
1292 __ allocate_array(op->obj()->as_register(),
1293 len,
1294 tmp1,
1295 tmp2,
1296 arrayOopDesc::base_offset_in_bytes(op->type()),
1297 array_element_size(op->type()),
1298 op->klass()->as_register(),
1299 *op->stub()->entry(),
1300 op->zero_array());
1301 }
1302 __ bind(*op->stub()->continuation());
1303 }
1304
1305 void LIR_Assembler::type_profile_helper(Register mdo,
1306 ciMethodData *md, ciProfileData *data,
1307 Register recv, Label* update_done) {
1308 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1309 Label next_test;
1310 // See if the receiver is receiver[n].
1311 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1312 __ jccb(Assembler::notEqual, next_test);
1313 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1314 __ addptr(data_addr, DataLayout::counter_increment);
1315 __ jmp(*update_done);
1316 __ bind(next_test);
1317 }
1318
1319 // Didn't find receiver; find next empty slot and fill it in
1320 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1321 Label next_test;
1322 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1323 __ cmpptr(recv_addr, NULL_WORD);
1324 __ jccb(Assembler::notEqual, next_test);
1325 __ movptr(recv_addr, recv);
1326 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1327 __ jmp(*update_done);
1328 __ bind(next_test);
1329 }
1330 }
1331
1332 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1333 // we always need a stub for the failure case.
1334 CodeStub* stub = op->stub();
1335 Register obj = op->object()->as_register();
1336 Register k_RInfo = op->tmp1()->as_register();
1337 Register klass_RInfo = op->tmp2()->as_register();
1338 Register dst = op->result_opr()->as_register();
1339 ciKlass* k = op->klass();
1340 Register Rtmp1 = noreg;
1341 Register tmp_load_klass = rscratch1;
1342
1343 // check if it needs to be profiled
1344 ciMethodData* md = nullptr;
1345 ciProfileData* data = nullptr;
1346
1347 if (op->should_profile()) {
1348 ciMethod* method = op->profiled_method();
1349 assert(method != nullptr, "Should have method");
1350 int bci = op->profiled_bci();
1351 md = method->method_data_or_null();
1352 assert(md != nullptr, "Sanity");
1353 data = md->bci_to_data(bci);
1354 assert(data != nullptr, "need data for type check");
1355 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1356 }
1357 Label* success_target = success;
1358 Label* failure_target = failure;
1359
1360 if (obj == k_RInfo) {
1361 k_RInfo = dst;
1362 } else if (obj == klass_RInfo) {
1363 klass_RInfo = dst;
1364 }
1365 if (k->is_loaded() && !UseCompressedClassPointers) {
1366 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1367 } else {
1368 Rtmp1 = op->tmp3()->as_register();
1369 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1370 }
1371
1372 assert_different_registers(obj, k_RInfo, klass_RInfo);
1373
1374 if (op->need_null_check()) {
1375 __ testptr(obj, obj);
1376 if (op->should_profile()) {
1377 Label not_null;
1378 Register mdo = klass_RInfo;
1379 __ mov_metadata(mdo, md->constant_encoding());
1380 __ jccb(Assembler::notEqual, not_null);
1381 // Object is null; update MDO and exit
1382 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1383 int header_bits = BitData::null_seen_byte_constant();
1384 __ orb(data_addr, header_bits);
1385 __ jmp(*obj_is_null);
1386 __ bind(not_null);
1387
1388 Label update_done;
1389 Register recv = k_RInfo;
1390 __ load_klass(recv, obj, tmp_load_klass);
1391 type_profile_helper(mdo, md, data, recv, &update_done);
1392
1393 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1394 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1395
1396 __ bind(update_done);
1397 } else {
1398 __ jcc(Assembler::equal, *obj_is_null);
1399 }
1400 }
1401
1402 if (!k->is_loaded()) {
1403 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1404 } else {
1405 __ mov_metadata(k_RInfo, k->constant_encoding());
1406 }
1407 __ verify_oop(obj);
1408
1409 if (op->fast_check()) {
1410 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1411 // get object class
1412 // not a safepoint as obj null check happens earlier
1413 if (UseCompressedClassPointers) {
1414 __ load_klass(Rtmp1, obj, tmp_load_klass);
1415 __ cmpptr(k_RInfo, Rtmp1);
1416 } else {
1417 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1418 }
1419 __ jcc(Assembler::notEqual, *failure_target);
1420 // successful cast, fall through to profile or jump
1421 } else {
1422 // get object class
1423 // not a safepoint as obj null check happens earlier
1424 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1425 if (k->is_loaded()) {
1426 // See if we get an immediate positive hit
1427 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1428 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1429 __ jcc(Assembler::notEqual, *failure_target);
1430 // successful cast, fall through to profile or jump
1431 } else {
1432 // See if we get an immediate positive hit
1433 __ jcc(Assembler::equal, *success_target);
1434 // check for self
1435 if (k->is_loaded() && k->is_obj_array_klass()) {
1436 // For a direct pointer comparison, we need the refined array klass pointer
1437 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1438 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1439 __ cmpptr(klass_RInfo, tmp_load_klass);
1440 } else {
1441 __ cmpptr(klass_RInfo, k_RInfo);
1442 }
1443 __ jcc(Assembler::equal, *success_target);
1444
1445 __ push_ppx(klass_RInfo);
1446 __ push_ppx(k_RInfo);
1447 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1448 __ pop_ppx(klass_RInfo);
1449 __ pop_ppx(klass_RInfo);
1450 // result is a boolean
1451 __ testl(klass_RInfo, klass_RInfo);
1452 __ jcc(Assembler::equal, *failure_target);
1453 // successful cast, fall through to profile or jump
1454 }
1455 } else {
1456 // perform the fast part of the checking logic
1457 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1458 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1459 __ push_ppx(klass_RInfo);
1460 __ push_ppx(k_RInfo);
1461 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1462 __ pop_ppx(klass_RInfo);
1463 __ pop_ppx(k_RInfo);
1464 // result is a boolean
1465 __ testl(k_RInfo, k_RInfo);
1466 __ jcc(Assembler::equal, *failure_target);
1467 // successful cast, fall through to profile or jump
1468 }
1469 }
1470 __ jmp(*success);
1471 }
1472
1473
1474 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1475 Register tmp_load_klass = rscratch1;
1476 LIR_Code code = op->code();
1477 if (code == lir_store_check) {
1478 Register value = op->object()->as_register();
1479 Register array = op->array()->as_register();
1480 Register k_RInfo = op->tmp1()->as_register();
1481 Register klass_RInfo = op->tmp2()->as_register();
1482 Register Rtmp1 = op->tmp3()->as_register();
1483
1484 CodeStub* stub = op->stub();
1485
1486 // check if it needs to be profiled
1487 ciMethodData* md = nullptr;
1488 ciProfileData* data = nullptr;
1489
1490 if (op->should_profile()) {
1491 ciMethod* method = op->profiled_method();
1492 assert(method != nullptr, "Should have method");
1493 int bci = op->profiled_bci();
1494 md = method->method_data_or_null();
1495 assert(md != nullptr, "Sanity");
1496 data = md->bci_to_data(bci);
1497 assert(data != nullptr, "need data for type check");
1498 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1499 }
1500 Label done;
1501 Label* success_target = &done;
1502 Label* failure_target = stub->entry();
1503
1504 __ testptr(value, value);
1505 if (op->should_profile()) {
1506 Label not_null;
1507 Register mdo = klass_RInfo;
1508 __ mov_metadata(mdo, md->constant_encoding());
1509 __ jccb(Assembler::notEqual, not_null);
1510 // Object is null; update MDO and exit
1511 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1512 int header_bits = BitData::null_seen_byte_constant();
1513 __ orb(data_addr, header_bits);
1514 __ jmp(done);
1515 __ bind(not_null);
1516
1517 Label update_done;
1518 Register recv = k_RInfo;
1519 __ load_klass(recv, value, tmp_load_klass);
1520 type_profile_helper(mdo, md, data, recv, &update_done);
1521
1522 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1523 __ addptr(counter_addr, DataLayout::counter_increment);
1524 __ bind(update_done);
1525 } else {
1526 __ jcc(Assembler::equal, done);
1527 }
1528
1529 add_debug_info_for_null_check_here(op->info_for_exception());
1530 __ load_klass(k_RInfo, array, tmp_load_klass);
1531 __ load_klass(klass_RInfo, value, tmp_load_klass);
1532
1533 // get instance klass (it's already uncompressed)
1534 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1535 // perform the fast part of the checking logic
1536 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1537 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1538 __ push_ppx(klass_RInfo);
1539 __ push_ppx(k_RInfo);
1540 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1541 __ pop_ppx(klass_RInfo);
1542 __ pop_ppx(k_RInfo);
1543 // result is a boolean
1544 __ testl(k_RInfo, k_RInfo);
1545 __ jcc(Assembler::equal, *failure_target);
1546 // fall through to the success case
1547
1548 __ bind(done);
1549 } else
1550 if (code == lir_checkcast) {
1551 Register obj = op->object()->as_register();
1552 Register dst = op->result_opr()->as_register();
1553 Label success;
1554 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1555 __ bind(success);
1556 if (dst != obj) {
1557 __ mov(dst, obj);
1558 }
1559 } else
1560 if (code == lir_instanceof) {
1561 Register obj = op->object()->as_register();
1562 Register dst = op->result_opr()->as_register();
1563 Label success, failure, done;
1564 emit_typecheck_helper(op, &success, &failure, &failure);
1565 __ bind(failure);
1566 __ xorptr(dst, dst);
1567 __ jmpb(done);
1568 __ bind(success);
1569 __ movptr(dst, 1);
1570 __ bind(done);
1571 } else {
1572 ShouldNotReachHere();
1573 }
1574
1575 }
1576
1577 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1578 // We are loading/storing from/to an array that *may* be a flat array (the
1579 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1580 // If this array is a flat array, take the slow path.
1581 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582 if (!op->value()->is_illegal()) {
1583 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1584 // The array is not a flat array, but it might be null-free. If we are storing
1585 // a null into a null-free array, take the slow path (which will throw NPE).
1586 Label skip;
1587 __ cmpptr(op->value()->as_register(), NULL_WORD);
1588 __ jcc(Assembler::notEqual, skip);
1589 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590 __ bind(skip);
1591 }
1592 }
1593
1594 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1595 // We are storing into an array that *may* be null-free (the declared type is
1596 // Object[], abstract[], interface[] or VT.ref[]).
1597 Label test_mark_word;
1598 Register tmp = op->tmp()->as_register();
1599 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600 __ testl(tmp, markWord::unlocked_value);
1601 __ jccb(Assembler::notZero, test_mark_word);
1602 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1603 __ bind(test_mark_word);
1604 __ testl(tmp, markWord::null_free_array_bit_in_place);
1605 }
1606
1607 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1608 Label L_oops_equal;
1609 Label L_oops_not_equal;
1610 Label L_end;
1611
1612 Register left = op->left()->as_register();
1613 Register right = op->right()->as_register();
1614
1615 __ cmpptr(left, right);
1616 __ jcc(Assembler::equal, L_oops_equal);
1617
1618 // (1) Null check -- if one of the operands is null, the other must not be null (because
1619 // the two references are not equal), so they are not substitutable,
1620 // FIXME: do null check only if the operand is nullable
1621 __ testptr(left, right);
1622 __ jcc(Assembler::zero, L_oops_not_equal);
1623
1624 ciKlass* left_klass = op->left_klass();
1625 ciKlass* right_klass = op->right_klass();
1626
1627 // (2) Inline type check -- if either of the operands is not a inline type,
1628 // they are not substitutable. We do this only if we are not sure that the
1629 // operands are inline type
1630 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1631 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1632 Register tmp1 = op->tmp1()->as_register();
1633 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1634 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1635 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1636 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1637 __ jcc(Assembler::notEqual, L_oops_not_equal);
1638 }
1639
1640 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1641 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1642 // No need to load klass -- the operands are statically known to be the same inline klass.
1643 __ jmp(*op->stub()->entry());
1644 } else {
1645 Register left_klass_op = op->left_klass_op()->as_register();
1646 Register right_klass_op = op->right_klass_op()->as_register();
1647
1648 if (UseCompressedClassPointers) {
1649 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1650 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651 __ cmpl(left_klass_op, right_klass_op);
1652 } else {
1653 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1654 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1655 __ cmpptr(left_klass_op, right_klass_op);
1656 }
1657
1658 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1659 // fall through to L_oops_not_equal
1660 }
1661
1662 __ bind(L_oops_not_equal);
1663 move(op->not_equal_result(), op->result_opr());
1664 __ jmp(L_end);
1665
1666 __ bind(L_oops_equal);
1667 move(op->equal_result(), op->result_opr());
1668 __ jmp(L_end);
1669
1670 // We've returned from the stub. RAX contains 0x0 IFF the two
1671 // operands are not substitutable. (Don't compare against 0x1 in case the
1672 // C compiler is naughty)
1673 __ bind(*op->stub()->continuation());
1674 __ cmpl(rax, 0);
1675 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1676 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1677 // fall-through
1678 __ bind(L_end);
1679 }
1680
1681 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1682 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1683 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1684 Register newval = op->new_value()->as_register();
1685 Register cmpval = op->cmp_value()->as_register();
1686 assert(cmpval == rax, "wrong register");
1687 assert(newval != noreg, "new val must be register");
1688 assert(cmpval != newval, "cmp and new values must be in different registers");
1689 assert(cmpval != addr, "cmp and addr must be in different registers");
1690 assert(newval != addr, "new value and addr must be in different registers");
1691
1692 if (op->code() == lir_cas_obj) {
1693 if (UseCompressedOops) {
1694 __ encode_heap_oop(cmpval);
1695 __ mov(rscratch1, newval);
1696 __ encode_heap_oop(rscratch1);
1697 __ lock();
1698 // cmpval (rax) is implicitly used by this instruction
1699 __ cmpxchgl(rscratch1, Address(addr, 0));
1700 } else {
1701 __ lock();
1702 __ cmpxchgptr(newval, Address(addr, 0));
1703 }
1704 } else {
1705 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1706 __ lock();
1707 __ cmpxchgl(newval, Address(addr, 0));
1708 }
1709 } else if (op->code() == lir_cas_long) {
1710 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1711 Register newval = op->new_value()->as_register_lo();
1712 Register cmpval = op->cmp_value()->as_register_lo();
1713 assert(cmpval == rax, "wrong register");
1714 assert(newval != noreg, "new val must be register");
1715 assert(cmpval != newval, "cmp and new values must be in different registers");
1716 assert(cmpval != addr, "cmp and addr must be in different registers");
1717 assert(newval != addr, "new value and addr must be in different registers");
1718 __ lock();
1719 __ cmpxchgq(newval, Address(addr, 0));
1720 } else {
1721 Unimplemented();
1722 }
1723 }
1724
1725 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1726 assert(dst->is_cpu_register(), "must be");
1727 assert(dst->type() == src->type(), "must be");
1728
1729 if (src->is_cpu_register()) {
1730 reg2reg(src, dst);
1731 } else if (src->is_stack()) {
1732 stack2reg(src, dst, dst->type());
1733 } else if (src->is_constant()) {
1734 const2reg(src, dst, lir_patch_none, nullptr);
1735 } else {
1736 ShouldNotReachHere();
1737 }
1738 }
1739
1740 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1741 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1742 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1743
1744 Assembler::Condition acond, ncond;
1745 switch (condition) {
1746 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1747 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1748 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1749 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1750 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1751 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1752 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1753 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1754 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1755 ShouldNotReachHere();
1756 }
1757
1758 if (opr1->is_cpu_register()) {
1759 reg2reg(opr1, result);
1760 } else if (opr1->is_stack()) {
1761 stack2reg(opr1, result, result->type());
1762 } else if (opr1->is_constant()) {
1763 const2reg(opr1, result, lir_patch_none, nullptr);
1764 } else {
1765 ShouldNotReachHere();
1766 }
1767
1768 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1769 // optimized version that does not require a branch
1770 if (opr2->is_single_cpu()) {
1771 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1772 __ cmov(ncond, result->as_register(), opr2->as_register());
1773 } else if (opr2->is_double_cpu()) {
1774 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1775 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1776 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1777 } else if (opr2->is_single_stack()) {
1778 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1779 } else if (opr2->is_double_stack()) {
1780 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1781 } else {
1782 ShouldNotReachHere();
1783 }
1784
1785 } else {
1786 Label skip;
1787 __ jccb(acond, skip);
1788 if (opr2->is_cpu_register()) {
1789 reg2reg(opr2, result);
1790 } else if (opr2->is_stack()) {
1791 stack2reg(opr2, result, result->type());
1792 } else if (opr2->is_constant()) {
1793 const2reg(opr2, result, lir_patch_none, nullptr);
1794 } else {
1795 ShouldNotReachHere();
1796 }
1797 __ bind(skip);
1798 }
1799 }
1800
1801
1802 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1803 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1804
1805 if (left->is_single_cpu()) {
1806 assert(left == dest, "left and dest must be equal");
1807 Register lreg = left->as_register();
1808
1809 if (right->is_single_cpu()) {
1810 // cpu register - cpu register
1811 Register rreg = right->as_register();
1812 switch (code) {
1813 case lir_add: __ addl (lreg, rreg); break;
1814 case lir_sub: __ subl (lreg, rreg); break;
1815 case lir_mul: __ imull(lreg, rreg); break;
1816 default: ShouldNotReachHere();
1817 }
1818
1819 } else if (right->is_stack()) {
1820 // cpu register - stack
1821 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1822 switch (code) {
1823 case lir_add: __ addl(lreg, raddr); break;
1824 case lir_sub: __ subl(lreg, raddr); break;
1825 default: ShouldNotReachHere();
1826 }
1827
1828 } else if (right->is_constant()) {
1829 // cpu register - constant
1830 jint c = right->as_constant_ptr()->as_jint();
1831 switch (code) {
1832 case lir_add: {
1833 __ incrementl(lreg, c);
1834 break;
1835 }
1836 case lir_sub: {
1837 __ decrementl(lreg, c);
1838 break;
1839 }
1840 default: ShouldNotReachHere();
1841 }
1842
1843 } else {
1844 ShouldNotReachHere();
1845 }
1846
1847 } else if (left->is_double_cpu()) {
1848 assert(left == dest, "left and dest must be equal");
1849 Register lreg_lo = left->as_register_lo();
1850 Register lreg_hi = left->as_register_hi();
1851
1852 if (right->is_double_cpu()) {
1853 // cpu register - cpu register
1854 Register rreg_lo = right->as_register_lo();
1855 Register rreg_hi = right->as_register_hi();
1856 assert_different_registers(lreg_lo, rreg_lo);
1857 switch (code) {
1858 case lir_add:
1859 __ addptr(lreg_lo, rreg_lo);
1860 break;
1861 case lir_sub:
1862 __ subptr(lreg_lo, rreg_lo);
1863 break;
1864 case lir_mul:
1865 __ imulq(lreg_lo, rreg_lo);
1866 break;
1867 default:
1868 ShouldNotReachHere();
1869 }
1870
1871 } else if (right->is_constant()) {
1872 // cpu register - constant
1873 jlong c = right->as_constant_ptr()->as_jlong_bits();
1874 __ movptr(r10, (intptr_t) c);
1875 switch (code) {
1876 case lir_add:
1877 __ addptr(lreg_lo, r10);
1878 break;
1879 case lir_sub:
1880 __ subptr(lreg_lo, r10);
1881 break;
1882 default:
1883 ShouldNotReachHere();
1884 }
1885
1886 } else {
1887 ShouldNotReachHere();
1888 }
1889
1890 } else if (left->is_single_xmm()) {
1891 assert(left == dest, "left and dest must be equal");
1892 XMMRegister lreg = left->as_xmm_float_reg();
1893
1894 if (right->is_single_xmm()) {
1895 XMMRegister rreg = right->as_xmm_float_reg();
1896 switch (code) {
1897 case lir_add: __ addss(lreg, rreg); break;
1898 case lir_sub: __ subss(lreg, rreg); break;
1899 case lir_mul: __ mulss(lreg, rreg); break;
1900 case lir_div: __ divss(lreg, rreg); break;
1901 default: ShouldNotReachHere();
1902 }
1903 } else {
1904 Address raddr;
1905 if (right->is_single_stack()) {
1906 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1907 } else if (right->is_constant()) {
1908 // hack for now
1909 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1910 } else {
1911 ShouldNotReachHere();
1912 }
1913 switch (code) {
1914 case lir_add: __ addss(lreg, raddr); break;
1915 case lir_sub: __ subss(lreg, raddr); break;
1916 case lir_mul: __ mulss(lreg, raddr); break;
1917 case lir_div: __ divss(lreg, raddr); break;
1918 default: ShouldNotReachHere();
1919 }
1920 }
1921
1922 } else if (left->is_double_xmm()) {
1923 assert(left == dest, "left and dest must be equal");
1924
1925 XMMRegister lreg = left->as_xmm_double_reg();
1926 if (right->is_double_xmm()) {
1927 XMMRegister rreg = right->as_xmm_double_reg();
1928 switch (code) {
1929 case lir_add: __ addsd(lreg, rreg); break;
1930 case lir_sub: __ subsd(lreg, rreg); break;
1931 case lir_mul: __ mulsd(lreg, rreg); break;
1932 case lir_div: __ divsd(lreg, rreg); break;
1933 default: ShouldNotReachHere();
1934 }
1935 } else {
1936 Address raddr;
1937 if (right->is_double_stack()) {
1938 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1939 } else if (right->is_constant()) {
1940 // hack for now
1941 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1942 } else {
1943 ShouldNotReachHere();
1944 }
1945 switch (code) {
1946 case lir_add: __ addsd(lreg, raddr); break;
1947 case lir_sub: __ subsd(lreg, raddr); break;
1948 case lir_mul: __ mulsd(lreg, raddr); break;
1949 case lir_div: __ divsd(lreg, raddr); break;
1950 default: ShouldNotReachHere();
1951 }
1952 }
1953
1954 } else if (left->is_single_stack() || left->is_address()) {
1955 assert(left == dest, "left and dest must be equal");
1956
1957 Address laddr;
1958 if (left->is_single_stack()) {
1959 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1960 } else if (left->is_address()) {
1961 laddr = as_Address(left->as_address_ptr());
1962 } else {
1963 ShouldNotReachHere();
1964 }
1965
1966 if (right->is_single_cpu()) {
1967 Register rreg = right->as_register();
1968 switch (code) {
1969 case lir_add: __ addl(laddr, rreg); break;
1970 case lir_sub: __ subl(laddr, rreg); break;
1971 default: ShouldNotReachHere();
1972 }
1973 } else if (right->is_constant()) {
1974 jint c = right->as_constant_ptr()->as_jint();
1975 switch (code) {
1976 case lir_add: {
1977 __ incrementl(laddr, c);
1978 break;
1979 }
1980 case lir_sub: {
1981 __ decrementl(laddr, c);
1982 break;
1983 }
1984 default: ShouldNotReachHere();
1985 }
1986 } else {
1987 ShouldNotReachHere();
1988 }
1989
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 }
1994
1995
1996 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1997 if (value->is_double_xmm()) {
1998 switch(code) {
1999 case lir_abs :
2000 {
2001 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2002 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2003 }
2004 assert(!tmp->is_valid(), "do not need temporary");
2005 __ andpd(dest->as_xmm_double_reg(),
2006 ExternalAddress((address)double_signmask_pool),
2007 rscratch1);
2008 }
2009 break;
2010
2011 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2012 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2013 default : ShouldNotReachHere();
2014 }
2015
2016 } else if (code == lir_f2hf) {
2017 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2018 } else if (code == lir_hf2f) {
2019 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2020 } else {
2021 Unimplemented();
2022 }
2023 }
2024
2025 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2026 // assert(left->destroys_register(), "check");
2027 if (left->is_single_cpu()) {
2028 Register reg = left->as_register();
2029 if (right->is_constant()) {
2030 int val = right->as_constant_ptr()->as_jint();
2031 switch (code) {
2032 case lir_logic_and: __ andl (reg, val); break;
2033 case lir_logic_or: __ orl (reg, val); break;
2034 case lir_logic_xor: __ xorl (reg, val); break;
2035 default: ShouldNotReachHere();
2036 }
2037 } else if (right->is_stack()) {
2038 // added support for stack operands
2039 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2040 switch (code) {
2041 case lir_logic_and: __ andl (reg, raddr); break;
2042 case lir_logic_or: __ orl (reg, raddr); break;
2043 case lir_logic_xor: __ xorl (reg, raddr); break;
2044 default: ShouldNotReachHere();
2045 }
2046 } else {
2047 Register rright = right->as_register();
2048 switch (code) {
2049 case lir_logic_and: __ andptr (reg, rright); break;
2050 case lir_logic_or : __ orptr (reg, rright); break;
2051 case lir_logic_xor: __ xorptr (reg, rright); break;
2052 default: ShouldNotReachHere();
2053 }
2054 }
2055 move_regs(reg, dst->as_register());
2056 } else {
2057 Register l_lo = left->as_register_lo();
2058 Register l_hi = left->as_register_hi();
2059 if (right->is_constant()) {
2060 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2061 switch (code) {
2062 case lir_logic_and:
2063 __ andq(l_lo, rscratch1);
2064 break;
2065 case lir_logic_or:
2066 __ orq(l_lo, rscratch1);
2067 break;
2068 case lir_logic_xor:
2069 __ xorq(l_lo, rscratch1);
2070 break;
2071 default: ShouldNotReachHere();
2072 }
2073 } else {
2074 Register r_lo;
2075 if (is_reference_type(right->type())) {
2076 r_lo = right->as_register();
2077 } else {
2078 r_lo = right->as_register_lo();
2079 }
2080 switch (code) {
2081 case lir_logic_and:
2082 __ andptr(l_lo, r_lo);
2083 break;
2084 case lir_logic_or:
2085 __ orptr(l_lo, r_lo);
2086 break;
2087 case lir_logic_xor:
2088 __ xorptr(l_lo, r_lo);
2089 break;
2090 default: ShouldNotReachHere();
2091 }
2092 }
2093
2094 Register dst_lo = dst->as_register_lo();
2095 Register dst_hi = dst->as_register_hi();
2096
2097 move_regs(l_lo, dst_lo);
2098 }
2099 }
2100
2101
2102 // we assume that rax, and rdx can be overwritten
2103 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2104
2105 assert(left->is_single_cpu(), "left must be register");
2106 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2107 assert(result->is_single_cpu(), "result must be register");
2108
2109 // assert(left->destroys_register(), "check");
2110 // assert(right->destroys_register(), "check");
2111
2112 Register lreg = left->as_register();
2113 Register dreg = result->as_register();
2114
2115 if (right->is_constant()) {
2116 jint divisor = right->as_constant_ptr()->as_jint();
2117 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2118 if (code == lir_idiv) {
2119 assert(lreg == rax, "must be rax,");
2120 assert(temp->as_register() == rdx, "tmp register must be rdx");
2121 __ cdql(); // sign extend into rdx:rax
2122 if (divisor == 2) {
2123 __ subl(lreg, rdx);
2124 } else {
2125 __ andl(rdx, divisor - 1);
2126 __ addl(lreg, rdx);
2127 }
2128 __ sarl(lreg, log2i_exact(divisor));
2129 move_regs(lreg, dreg);
2130 } else if (code == lir_irem) {
2131 Label done;
2132 __ mov(dreg, lreg);
2133 __ andl(dreg, 0x80000000 | (divisor - 1));
2134 __ jcc(Assembler::positive, done);
2135 __ decrement(dreg);
2136 __ orl(dreg, ~(divisor - 1));
2137 __ increment(dreg);
2138 __ bind(done);
2139 } else {
2140 ShouldNotReachHere();
2141 }
2142 } else {
2143 Register rreg = right->as_register();
2144 assert(lreg == rax, "left register must be rax,");
2145 assert(rreg != rdx, "right register must not be rdx");
2146 assert(temp->as_register() == rdx, "tmp register must be rdx");
2147
2148 move_regs(lreg, rax);
2149
2150 int idivl_offset = __ corrected_idivl(rreg);
2151 if (ImplicitDiv0Checks) {
2152 add_debug_info_for_div0(idivl_offset, info);
2153 }
2154 if (code == lir_irem) {
2155 move_regs(rdx, dreg); // result is in rdx
2156 } else {
2157 move_regs(rax, dreg);
2158 }
2159 }
2160 }
2161
2162
2163 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2164 if (opr1->is_single_cpu()) {
2165 Register reg1 = opr1->as_register();
2166 if (opr2->is_single_cpu()) {
2167 // cpu register - cpu register
2168 if (is_reference_type(opr1->type())) {
2169 __ cmpoop(reg1, opr2->as_register());
2170 } else {
2171 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2172 __ cmpl(reg1, opr2->as_register());
2173 }
2174 } else if (opr2->is_stack()) {
2175 // cpu register - stack
2176 if (is_reference_type(opr1->type())) {
2177 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2178 } else {
2179 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2180 }
2181 } else if (opr2->is_constant()) {
2182 // cpu register - constant
2183 LIR_Const* c = opr2->as_constant_ptr();
2184 if (c->type() == T_INT) {
2185 jint i = c->as_jint();
2186 if (i == 0) {
2187 __ testl(reg1, reg1);
2188 } else {
2189 __ cmpl(reg1, i);
2190 }
2191 } else if (c->type() == T_METADATA) {
2192 // All we need for now is a comparison with null for equality.
2193 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2194 Metadata* m = c->as_metadata();
2195 if (m == nullptr) {
2196 __ testptr(reg1, reg1);
2197 } else {
2198 ShouldNotReachHere();
2199 }
2200 } else if (is_reference_type(c->type())) {
2201 // In 64bit oops are single register
2202 jobject o = c->as_jobject();
2203 if (o == nullptr) {
2204 __ testptr(reg1, reg1);
2205 } else {
2206 __ cmpoop(reg1, o, rscratch1);
2207 }
2208 } else {
2209 fatal("unexpected type: %s", basictype_to_str(c->type()));
2210 }
2211 // cpu register - address
2212 } else if (opr2->is_address()) {
2213 if (op->info() != nullptr) {
2214 add_debug_info_for_null_check_here(op->info());
2215 }
2216 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2217 } else {
2218 ShouldNotReachHere();
2219 }
2220
2221 } else if(opr1->is_double_cpu()) {
2222 Register xlo = opr1->as_register_lo();
2223 Register xhi = opr1->as_register_hi();
2224 if (opr2->is_double_cpu()) {
2225 __ cmpptr(xlo, opr2->as_register_lo());
2226 } else if (opr2->is_constant()) {
2227 // cpu register - constant 0
2228 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2229 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2230 } else {
2231 ShouldNotReachHere();
2232 }
2233
2234 } else if (opr1->is_single_xmm()) {
2235 XMMRegister reg1 = opr1->as_xmm_float_reg();
2236 if (opr2->is_single_xmm()) {
2237 // xmm register - xmm register
2238 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2239 } else if (opr2->is_stack()) {
2240 // xmm register - stack
2241 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2242 } else if (opr2->is_constant()) {
2243 // xmm register - constant
2244 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2245 } else if (opr2->is_address()) {
2246 // xmm register - address
2247 if (op->info() != nullptr) {
2248 add_debug_info_for_null_check_here(op->info());
2249 }
2250 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2251 } else {
2252 ShouldNotReachHere();
2253 }
2254
2255 } else if (opr1->is_double_xmm()) {
2256 XMMRegister reg1 = opr1->as_xmm_double_reg();
2257 if (opr2->is_double_xmm()) {
2258 // xmm register - xmm register
2259 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2260 } else if (opr2->is_stack()) {
2261 // xmm register - stack
2262 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2263 } else if (opr2->is_constant()) {
2264 // xmm register - constant
2265 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2266 } else if (opr2->is_address()) {
2267 // xmm register - address
2268 if (op->info() != nullptr) {
2269 add_debug_info_for_null_check_here(op->info());
2270 }
2271 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2272 } else {
2273 ShouldNotReachHere();
2274 }
2275
2276 } else if (opr1->is_address() && opr2->is_constant()) {
2277 LIR_Const* c = opr2->as_constant_ptr();
2278 if (is_reference_type(c->type())) {
2279 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2280 __ movoop(rscratch1, c->as_jobject());
2281 }
2282 if (op->info() != nullptr) {
2283 add_debug_info_for_null_check_here(op->info());
2284 }
2285 // special case: address - constant
2286 LIR_Address* addr = opr1->as_address_ptr();
2287 if (c->type() == T_INT) {
2288 __ cmpl(as_Address(addr), c->as_jint());
2289 } else if (is_reference_type(c->type())) {
2290 // %%% Make this explode if addr isn't reachable until we figure out a
2291 // better strategy by giving noreg as the temp for as_Address
2292 __ cmpoop(rscratch1, as_Address(addr, noreg));
2293 } else {
2294 ShouldNotReachHere();
2295 }
2296
2297 } else {
2298 ShouldNotReachHere();
2299 }
2300 }
2301
2302 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2303 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2304 if (left->is_single_xmm()) {
2305 assert(right->is_single_xmm(), "must match");
2306 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2307 } else if (left->is_double_xmm()) {
2308 assert(right->is_double_xmm(), "must match");
2309 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2310
2311 } else {
2312 ShouldNotReachHere();
2313 }
2314 } else {
2315 assert(code == lir_cmp_l2i, "check");
2316 Label done;
2317 Register dest = dst->as_register();
2318 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2319 __ movl(dest, -1);
2320 __ jccb(Assembler::less, done);
2321 __ setb(Assembler::notZero, dest);
2322 __ movzbl(dest, dest);
2323 __ bind(done);
2324 }
2325 }
2326
2327
2328 void LIR_Assembler::align_call(LIR_Code code) {
2329 // make sure that the displacement word of the call ends up word aligned
2330 int offset = __ offset();
2331 switch (code) {
2332 case lir_static_call:
2333 case lir_optvirtual_call:
2334 case lir_dynamic_call:
2335 offset += NativeCall::displacement_offset;
2336 break;
2337 case lir_icvirtual_call:
2338 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2339 break;
2340 default: ShouldNotReachHere();
2341 }
2342 __ align(BytesPerWord, offset);
2343 }
2344
2345
2346 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2347 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2348 "must be aligned");
2349 __ call(AddressLiteral(op->addr(), rtype));
2350 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2351 __ post_call_nop();
2352 }
2353
2354
2355 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2356 __ ic_call(op->addr());
2357 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2358 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2359 "must be aligned");
2360 __ post_call_nop();
2361 }
2362
2363
2364 void LIR_Assembler::emit_static_call_stub() {
2365 address call_pc = __ pc();
2366 address stub = __ start_a_stub(call_stub_size());
2367 if (stub == nullptr) {
2368 bailout("static call stub overflow");
2369 return;
2370 }
2371
2372 int start = __ offset();
2373
2374 // make sure that the displacement word of the call ends up word aligned
2375 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2376 __ relocate(static_stub_Relocation::spec(call_pc));
2377 __ mov_metadata(rbx, (Metadata*)nullptr);
2378 // must be set to -1 at code generation time
2379 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2380 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2381 __ jump(RuntimeAddress(__ pc()));
2382
2383 assert(__ offset() - start <= call_stub_size(), "stub too big");
2384 __ end_a_stub();
2385 }
2386
2387
2388 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2389 assert(exceptionOop->as_register() == rax, "must match");
2390 assert(exceptionPC->as_register() == rdx, "must match");
2391
2392 // exception object is not added to oop map by LinearScan
2393 // (LinearScan assumes that no oops are in fixed registers)
2394 info->add_register_oop(exceptionOop);
2395 StubId unwind_id;
2396
2397 // get current pc information
2398 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2399 int pc_for_athrow_offset = __ offset();
2400 InternalAddress pc_for_athrow(__ pc());
2401 __ lea(exceptionPC->as_register(), pc_for_athrow);
2402 add_call_info(pc_for_athrow_offset, info); // for exception handler
2403
2404 __ verify_not_null_oop(rax);
2405 // search an exception handler (rax: exception oop, rdx: throwing pc)
2406 if (compilation()->has_fpu_code()) {
2407 unwind_id = StubId::c1_handle_exception_id;
2408 } else {
2409 unwind_id = StubId::c1_handle_exception_nofpu_id;
2410 }
2411 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2412
2413 // enough room for two byte trap
2414 __ nop();
2415 }
2416
2417
2418 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2419 assert(exceptionOop->as_register() == rax, "must match");
2420
2421 __ jmp(_unwind_handler_entry);
2422 }
2423
2424
2425 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2426
2427 // optimized version for linear scan:
2428 // * count must be already in ECX (guaranteed by LinearScan)
2429 // * left and dest must be equal
2430 // * tmp must be unused
2431 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2432 assert(left == dest, "left and dest must be equal");
2433 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2434
2435 if (left->is_single_cpu()) {
2436 Register value = left->as_register();
2437 assert(value != SHIFT_count, "left cannot be ECX");
2438
2439 switch (code) {
2440 case lir_shl: __ shll(value); break;
2441 case lir_shr: __ sarl(value); break;
2442 case lir_ushr: __ shrl(value); break;
2443 default: ShouldNotReachHere();
2444 }
2445 } else if (left->is_double_cpu()) {
2446 Register lo = left->as_register_lo();
2447 Register hi = left->as_register_hi();
2448 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2449 switch (code) {
2450 case lir_shl: __ shlptr(lo); break;
2451 case lir_shr: __ sarptr(lo); break;
2452 case lir_ushr: __ shrptr(lo); break;
2453 default: ShouldNotReachHere();
2454 }
2455 } else {
2456 ShouldNotReachHere();
2457 }
2458 }
2459
2460
2461 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2462 if (dest->is_single_cpu()) {
2463 // first move left into dest so that left is not destroyed by the shift
2464 Register value = dest->as_register();
2465 count = count & 0x1F; // Java spec
2466
2467 move_regs(left->as_register(), value);
2468 switch (code) {
2469 case lir_shl: __ shll(value, count); break;
2470 case lir_shr: __ sarl(value, count); break;
2471 case lir_ushr: __ shrl(value, count); break;
2472 default: ShouldNotReachHere();
2473 }
2474 } else if (dest->is_double_cpu()) {
2475 // first move left into dest so that left is not destroyed by the shift
2476 Register value = dest->as_register_lo();
2477 count = count & 0x1F; // Java spec
2478
2479 move_regs(left->as_register_lo(), value);
2480 switch (code) {
2481 case lir_shl: __ shlptr(value, count); break;
2482 case lir_shr: __ sarptr(value, count); break;
2483 case lir_ushr: __ shrptr(value, count); break;
2484 default: ShouldNotReachHere();
2485 }
2486 } else {
2487 ShouldNotReachHere();
2488 }
2489 }
2490
2491
2492 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2493 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2497 }
2498
2499
2500 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2501 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2505 }
2506
2507
2508 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2509 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2513 }
2514
2515
2516 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2517 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2521 }
2522
2523
2524 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2525 if (null_check) {
2526 __ testptr(obj, obj);
2527 __ jcc(Assembler::zero, *slow_path->entry());
2528 }
2529 if (is_dest) {
2530 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2531 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2532 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2533 } else {
2534 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2535 }
2536 }
2537
2538
2539 // This code replaces a call to arraycopy; no exception may
2540 // be thrown in this code, they must be thrown in the System.arraycopy
2541 // activation frame; we could save some checks if this would not be the case
2542 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2543 ciArrayKlass* default_type = op->expected_type();
2544 Register src = op->src()->as_register();
2545 Register dst = op->dst()->as_register();
2546 Register src_pos = op->src_pos()->as_register();
2547 Register dst_pos = op->dst_pos()->as_register();
2548 Register length = op->length()->as_register();
2549 Register tmp = op->tmp()->as_register();
2550 Register tmp_load_klass = rscratch1;
2551 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2552
2553 CodeStub* stub = op->stub();
2554 int flags = op->flags();
2555 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2556 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2557
2558 if (flags & LIR_OpArrayCopy::always_slow_path) {
2559 __ jmp(*stub->entry());
2560 __ bind(*stub->continuation());
2561 return;
2562 }
2563
2564 // if we don't know anything, just go through the generic arraycopy
2565 if (default_type == nullptr) {
2566 // save outgoing arguments on stack in case call to System.arraycopy is needed
2567 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2568 // for interpreter calling conventions. Now we have to do it in new style conventions.
2569 // For the moment until C1 gets the new register allocator I just force all the
2570 // args to the right place (except the register args) and then on the back side
2571 // reload the register args properly if we go slow path. Yuck
2572
2573 // These are proper for the calling convention
2574 store_parameter(length, 2);
2575 store_parameter(dst_pos, 1);
2576 store_parameter(dst, 0);
2577
2578 // these are just temporary placements until we need to reload
2579 store_parameter(src_pos, 3);
2580 store_parameter(src, 4);
2581
2582 address copyfunc_addr = StubRoutines::generic_arraycopy();
2583 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2584
2585 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2586 // The arguments are in java calling convention so we can trivially shift them to C
2587 // convention
2588 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2589 __ mov(c_rarg0, j_rarg0);
2590 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2591 __ mov(c_rarg1, j_rarg1);
2592 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2593 __ mov(c_rarg2, j_rarg2);
2594 assert_different_registers(c_rarg3, j_rarg4);
2595 __ mov(c_rarg3, j_rarg3);
2596 #ifdef _WIN64
2597 // Allocate abi space for args but be sure to keep stack aligned
2598 __ subptr(rsp, 6*wordSize);
2599 store_parameter(j_rarg4, 4);
2600 #ifndef PRODUCT
2601 if (PrintC1Statistics) {
2602 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2603 }
2604 #endif
2605 __ call(RuntimeAddress(copyfunc_addr));
2606 __ addptr(rsp, 6*wordSize);
2607 #else
2608 __ mov(c_rarg4, j_rarg4);
2609 #ifndef PRODUCT
2610 if (PrintC1Statistics) {
2611 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2612 }
2613 #endif
2614 __ call(RuntimeAddress(copyfunc_addr));
2615 #endif // _WIN64
2616
2617 __ testl(rax, rax);
2618 __ jcc(Assembler::equal, *stub->continuation());
2619
2620 __ mov(tmp, rax);
2621 __ xorl(tmp, -1);
2622
2623 // Reload values from the stack so they are where the stub
2624 // expects them.
2625 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2626 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2627 __ movptr (length, Address(rsp, 2*BytesPerWord));
2628 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2629 __ movptr (src, Address(rsp, 4*BytesPerWord));
2630
2631 __ subl(length, tmp);
2632 __ addl(src_pos, tmp);
2633 __ addl(dst_pos, tmp);
2634 __ jmp(*stub->entry());
2635
2636 __ bind(*stub->continuation());
2637 return;
2638 }
2639
2640 // Handle inline type arrays
2641 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2642 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2643 }
2644 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2645 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2646 }
2647
2648 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2649
2650 int elem_size = type2aelembytes(basic_type);
2651 Address::ScaleFactor scale;
2652
2653 switch (elem_size) {
2654 case 1 :
2655 scale = Address::times_1;
2656 break;
2657 case 2 :
2658 scale = Address::times_2;
2659 break;
2660 case 4 :
2661 scale = Address::times_4;
2662 break;
2663 case 8 :
2664 scale = Address::times_8;
2665 break;
2666 default:
2667 scale = Address::no_scale;
2668 ShouldNotReachHere();
2669 }
2670
2671 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2672 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2673
2674 // length and pos's are all sign extended at this point on 64bit
2675
2676 // test for null
2677 if (flags & LIR_OpArrayCopy::src_null_check) {
2678 __ testptr(src, src);
2679 __ jcc(Assembler::zero, *stub->entry());
2680 }
2681 if (flags & LIR_OpArrayCopy::dst_null_check) {
2682 __ testptr(dst, dst);
2683 __ jcc(Assembler::zero, *stub->entry());
2684 }
2685
2686 // If the compiler was not able to prove that exact type of the source or the destination
2687 // of the arraycopy is an array type, check at runtime if the source or the destination is
2688 // an instance type.
2689 if (flags & LIR_OpArrayCopy::type_check) {
2690 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2691 __ load_klass(tmp, dst, tmp_load_klass);
2692 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2693 __ jcc(Assembler::greaterEqual, *stub->entry());
2694 }
2695
2696 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2697 __ load_klass(tmp, src, tmp_load_klass);
2698 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2699 __ jcc(Assembler::greaterEqual, *stub->entry());
2700 }
2701 }
2702
2703 // check if negative
2704 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2705 __ testl(src_pos, src_pos);
2706 __ jcc(Assembler::less, *stub->entry());
2707 }
2708 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2709 __ testl(dst_pos, dst_pos);
2710 __ jcc(Assembler::less, *stub->entry());
2711 }
2712
2713 if (flags & LIR_OpArrayCopy::src_range_check) {
2714 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2715 __ cmpl(tmp, src_length_addr);
2716 __ jcc(Assembler::above, *stub->entry());
2717 }
2718 if (flags & LIR_OpArrayCopy::dst_range_check) {
2719 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2720 __ cmpl(tmp, dst_length_addr);
2721 __ jcc(Assembler::above, *stub->entry());
2722 }
2723
2724 if (flags & LIR_OpArrayCopy::length_positive_check) {
2725 __ testl(length, length);
2726 __ jcc(Assembler::less, *stub->entry());
2727 }
2728
2729 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2730 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2731
2732 if (flags & LIR_OpArrayCopy::type_check) {
2733 // We don't know the array types are compatible
2734 if (basic_type != T_OBJECT) {
2735 // Simple test for basic type arrays
2736 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2737 __ jcc(Assembler::notEqual, *stub->entry());
2738 } else {
2739 // For object arrays, if src is a sub class of dst then we can
2740 // safely do the copy.
2741 Label cont, slow;
2742
2743 __ push_ppx(src);
2744 __ push_ppx(dst);
2745
2746 __ load_klass(src, src, tmp_load_klass);
2747 __ load_klass(dst, dst, tmp_load_klass);
2748
2749 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2750
2751 __ push_ppx(src);
2752 __ push_ppx(dst);
2753 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2754 __ pop_ppx(dst);
2755 __ pop_ppx(src);
2756
2757 __ testl(src, src);
2758 __ jcc(Assembler::notEqual, cont);
2759
2760 __ bind(slow);
2761 __ pop_ppx(dst);
2762 __ pop_ppx(src);
2763
2764 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2765 if (copyfunc_addr != nullptr) { // use stub if available
2766 // src is not a sub class of dst so we have to do a
2767 // per-element check.
2768
2769 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2770 if ((flags & mask) != mask) {
2771 // Check that at least both of them object arrays.
2772 assert(flags & mask, "one of the two should be known to be an object array");
2773
2774 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2775 __ load_klass(tmp, src, tmp_load_klass);
2776 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2777 __ load_klass(tmp, dst, tmp_load_klass);
2778 }
2779 int lh_offset = in_bytes(Klass::layout_helper_offset());
2780 Address klass_lh_addr(tmp, lh_offset);
2781 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2782 __ cmpl(klass_lh_addr, objArray_lh);
2783 __ jcc(Assembler::notEqual, *stub->entry());
2784 }
2785
2786 // Spill because stubs can use any register they like and it's
2787 // easier to restore just those that we care about.
2788 store_parameter(dst, 0);
2789 store_parameter(dst_pos, 1);
2790 store_parameter(length, 2);
2791 store_parameter(src_pos, 3);
2792 store_parameter(src, 4);
2793
2794 __ movl2ptr(length, length); //higher 32bits must be null
2795
2796 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2797 assert_different_registers(c_rarg0, dst, dst_pos, length);
2798 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2799 assert_different_registers(c_rarg1, dst, length);
2800
2801 __ mov(c_rarg2, length);
2802 assert_different_registers(c_rarg2, dst);
2803
2804 #ifdef _WIN64
2805 // Allocate abi space for args but be sure to keep stack aligned
2806 __ subptr(rsp, 6*wordSize);
2807 __ load_klass(c_rarg3, dst, tmp_load_klass);
2808 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2809 store_parameter(c_rarg3, 4);
2810 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2811 __ call(RuntimeAddress(copyfunc_addr));
2812 __ addptr(rsp, 6*wordSize);
2813 #else
2814 __ load_klass(c_rarg4, dst, tmp_load_klass);
2815 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2816 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2817 __ call(RuntimeAddress(copyfunc_addr));
2818 #endif
2819
2820 #ifndef PRODUCT
2821 if (PrintC1Statistics) {
2822 Label failed;
2823 __ testl(rax, rax);
2824 __ jcc(Assembler::notZero, failed);
2825 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2826 __ bind(failed);
2827 }
2828 #endif
2829
2830 __ testl(rax, rax);
2831 __ jcc(Assembler::zero, *stub->continuation());
2832
2833 #ifndef PRODUCT
2834 if (PrintC1Statistics) {
2835 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2836 }
2837 #endif
2838
2839 __ mov(tmp, rax);
2840
2841 __ xorl(tmp, -1);
2842
2843 // Restore previously spilled arguments
2844 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2845 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2846 __ movptr (length, Address(rsp, 2*BytesPerWord));
2847 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2848 __ movptr (src, Address(rsp, 4*BytesPerWord));
2849
2850
2851 __ subl(length, tmp);
2852 __ addl(src_pos, tmp);
2853 __ addl(dst_pos, tmp);
2854 }
2855
2856 __ jmp(*stub->entry());
2857
2858 __ bind(cont);
2859 __ pop(dst);
2860 __ pop(src);
2861 }
2862 }
2863
2864 #ifdef ASSERT
2865 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2866 // Sanity check the known type with the incoming class. For the
2867 // primitive case the types must match exactly with src.klass and
2868 // dst.klass each exactly matching the default type. For the
2869 // object array case, if no type check is needed then either the
2870 // dst type is exactly the expected type and the src type is a
2871 // subtype which we can't check or src is the same array as dst
2872 // but not necessarily exactly of type default_type.
2873 Label known_ok, halt;
2874
2875 __ mov_metadata(tmp, default_type->constant_encoding());
2876 if (UseCompressedClassPointers) {
2877 __ encode_klass_not_null(tmp, rscratch1);
2878 }
2879
2880 if (basic_type != T_OBJECT) {
2881 __ cmp_klass(tmp, dst, tmp2);
2882 __ jcc(Assembler::notEqual, halt);
2883 __ cmp_klass(tmp, src, tmp2);
2884 __ jcc(Assembler::equal, known_ok);
2885 } else {
2886 __ cmp_klass(tmp, dst, tmp2);
2887 __ jcc(Assembler::equal, known_ok);
2888 __ cmpptr(src, dst);
2889 __ jcc(Assembler::equal, known_ok);
2890 }
2891 __ bind(halt);
2892 __ stop("incorrect type information in arraycopy");
2893 __ bind(known_ok);
2894 }
2895 #endif
2896
2897 #ifndef PRODUCT
2898 if (PrintC1Statistics) {
2899 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2900 }
2901 #endif
2902
2903 assert_different_registers(c_rarg0, dst, dst_pos, length);
2904 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2905 assert_different_registers(c_rarg1, length);
2906 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2907 __ mov(c_rarg2, length);
2908
2909 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2910 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2911 const char *name;
2912 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2913 __ call_VM_leaf(entry, 0);
2914
2915 if (stub != nullptr) {
2916 __ bind(*stub->continuation());
2917 }
2918 }
2919
2920 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2921 assert(op->crc()->is_single_cpu(), "crc must be register");
2922 assert(op->val()->is_single_cpu(), "byte value must be register");
2923 assert(op->result_opr()->is_single_cpu(), "result must be register");
2924 Register crc = op->crc()->as_register();
2925 Register val = op->val()->as_register();
2926 Register res = op->result_opr()->as_register();
2927
2928 assert_different_registers(val, crc, res);
2929
2930 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2931 __ notl(crc); // ~crc
2932 __ update_byte_crc32(crc, val, res);
2933 __ notl(crc); // ~crc
2934 __ mov(res, crc);
2935 }
2936
2937 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2938 Register obj = op->obj_opr()->as_register(); // may not be an oop
2939 Register hdr = op->hdr_opr()->as_register();
2940 Register lock = op->lock_opr()->as_register();
2941 if (op->code() == lir_lock) {
2942 Register tmp = op->scratch_opr()->as_register();
2943 // add debug info for NullPointerException only if one is possible
2944 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2945 if (op->info() != nullptr) {
2946 add_debug_info_for_null_check(null_check_offset, op->info());
2947 }
2948 // done
2949 } else if (op->code() == lir_unlock) {
2950 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2951 } else {
2952 Unimplemented();
2953 }
2954 __ bind(*op->stub()->continuation());
2955 }
2956
2957 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2958 Register obj = op->obj()->as_pointer_register();
2959 Register result = op->result_opr()->as_pointer_register();
2960
2961 CodeEmitInfo* info = op->info();
2962 if (info != nullptr) {
2963 add_debug_info_for_null_check_here(info);
2964 }
2965
2966 __ load_klass(result, obj, rscratch1);
2967 }
2968
2969 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2970 ciMethod* method = op->profiled_method();
2971 int bci = op->profiled_bci();
2972 ciMethod* callee = op->profiled_callee();
2973 Register tmp_load_klass = rscratch1;
2974
2975 // Update counter for all call types
2976 ciMethodData* md = method->method_data_or_null();
2977 assert(md != nullptr, "Sanity");
2978 ciProfileData* data = md->bci_to_data(bci);
2979 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2980 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2981 Register mdo = op->mdo()->as_register();
2982 __ mov_metadata(mdo, md->constant_encoding());
2983 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2984 // Perform additional virtual call profiling for invokevirtual and
2985 // invokeinterface bytecodes
2986 if (op->should_profile_receiver_type()) {
2987 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2988 Register recv = op->recv()->as_register();
2989 assert_different_registers(mdo, recv);
2990 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2991 ciKlass* known_klass = op->known_holder();
2992 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2993 // We know the type that will be seen at this call site; we can
2994 // statically update the MethodData* rather than needing to do
2995 // dynamic tests on the receiver type
2996
2997 // NOTE: we should probably put a lock around this search to
2998 // avoid collisions by concurrent compilations
2999 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3000 uint i;
3001 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3002 ciKlass* receiver = vc_data->receiver(i);
3003 if (known_klass->equals(receiver)) {
3004 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3005 __ addptr(data_addr, DataLayout::counter_increment);
3006 return;
3007 }
3008 }
3009
3010 // Receiver type not found in profile data; select an empty slot
3011
3012 // Note that this is less efficient than it should be because it
3013 // always does a write to the receiver part of the
3014 // VirtualCallData rather than just the first time
3015 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3016 ciKlass* receiver = vc_data->receiver(i);
3017 if (receiver == nullptr) {
3018 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3019 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3020 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3021 __ addptr(data_addr, DataLayout::counter_increment);
3022 return;
3023 }
3024 }
3025 } else {
3026 __ load_klass(recv, recv, tmp_load_klass);
3027 Label update_done;
3028 type_profile_helper(mdo, md, data, recv, &update_done);
3029 // Receiver did not match any saved receiver and there is no empty row for it.
3030 // Increment total counter to indicate polymorphic case.
3031 __ addptr(counter_addr, DataLayout::counter_increment);
3032
3033 __ bind(update_done);
3034 }
3035 } else {
3036 // Static call
3037 __ addptr(counter_addr, DataLayout::counter_increment);
3038 }
3039 }
3040
3041 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3042 Register obj = op->obj()->as_register();
3043 Register tmp = op->tmp()->as_pointer_register();
3044 Register tmp_load_klass = rscratch1;
3045 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3046 ciKlass* exact_klass = op->exact_klass();
3047 intptr_t current_klass = op->current_klass();
3048 bool not_null = op->not_null();
3049 bool no_conflict = op->no_conflict();
3050
3051 Label update, next, none;
3052
3053 bool do_null = !not_null;
3054 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3055 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3056
3057 assert(do_null || do_update, "why are we here?");
3058 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3059
3060 __ verify_oop(obj);
3061
3062 #ifdef ASSERT
3063 if (obj == tmp) {
3064 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3065 } else {
3066 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3067 }
3068 #endif
3069 if (do_null) {
3070 __ testptr(obj, obj);
3071 __ jccb(Assembler::notZero, update);
3072 if (!TypeEntries::was_null_seen(current_klass)) {
3073 __ testptr(mdo_addr, TypeEntries::null_seen);
3074 #ifndef ASSERT
3075 __ jccb(Assembler::notZero, next); // already set
3076 #else
3077 __ jcc(Assembler::notZero, next); // already set
3078 #endif
3079 // atomic update to prevent overwriting Klass* with 0
3080 __ lock();
3081 __ orptr(mdo_addr, TypeEntries::null_seen);
3082 }
3083 if (do_update) {
3084 #ifndef ASSERT
3085 __ jmpb(next);
3086 }
3087 #else
3088 __ jmp(next);
3089 }
3090 } else {
3091 __ testptr(obj, obj);
3092 __ jcc(Assembler::notZero, update);
3093 __ stop("unexpected null obj");
3094 #endif
3095 }
3096
3097 __ bind(update);
3098
3099 if (do_update) {
3100 #ifdef ASSERT
3101 if (exact_klass != nullptr) {
3102 Label ok;
3103 __ load_klass(tmp, obj, tmp_load_klass);
3104 __ push_ppx(tmp);
3105 __ mov_metadata(tmp, exact_klass->constant_encoding());
3106 __ cmpptr(tmp, Address(rsp, 0));
3107 __ jcc(Assembler::equal, ok);
3108 __ stop("exact klass and actual klass differ");
3109 __ bind(ok);
3110 __ pop_ppx(tmp);
3111 }
3112 #endif
3113 if (!no_conflict) {
3114 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3115 if (exact_klass != nullptr) {
3116 __ mov_metadata(tmp, exact_klass->constant_encoding());
3117 } else {
3118 __ load_klass(tmp, obj, tmp_load_klass);
3119 }
3120 __ mov(rscratch1, tmp); // save original value before XOR
3121 __ xorptr(tmp, mdo_addr);
3122 __ testptr(tmp, TypeEntries::type_klass_mask);
3123 // klass seen before, nothing to do. The unknown bit may have been
3124 // set already but no need to check.
3125 __ jccb(Assembler::zero, next);
3126
3127 __ testptr(tmp, TypeEntries::type_unknown);
3128 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3129
3130 if (TypeEntries::is_type_none(current_klass)) {
3131 __ testptr(mdo_addr, TypeEntries::type_mask);
3132 __ jccb(Assembler::zero, none);
3133 // There is a chance that the checks above (re-reading profiling
3134 // data from memory) fail if another thread has just set the
3135 // profiling to this obj's klass
3136 __ mov(tmp, rscratch1); // get back original value before XOR
3137 __ xorptr(tmp, mdo_addr);
3138 __ testptr(tmp, TypeEntries::type_klass_mask);
3139 __ jccb(Assembler::zero, next);
3140 }
3141 } else {
3142 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3143 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3144
3145 __ testptr(mdo_addr, TypeEntries::type_unknown);
3146 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3147 }
3148
3149 // different than before. Cannot keep accurate profile.
3150 __ orptr(mdo_addr, TypeEntries::type_unknown);
3151
3152 if (TypeEntries::is_type_none(current_klass)) {
3153 __ jmpb(next);
3154
3155 __ bind(none);
3156 // first time here. Set profile type.
3157 __ movptr(mdo_addr, tmp);
3158 #ifdef ASSERT
3159 __ andptr(tmp, TypeEntries::type_klass_mask);
3160 __ verify_klass_ptr(tmp);
3161 #endif
3162 }
3163 } else {
3164 // There's a single possible klass at this profile point
3165 assert(exact_klass != nullptr, "should be");
3166 if (TypeEntries::is_type_none(current_klass)) {
3167 __ mov_metadata(tmp, exact_klass->constant_encoding());
3168 __ xorptr(tmp, mdo_addr);
3169 __ testptr(tmp, TypeEntries::type_klass_mask);
3170 #ifdef ASSERT
3171 __ jcc(Assembler::zero, next);
3172
3173 {
3174 Label ok;
3175 __ push_ppx(tmp);
3176 __ testptr(mdo_addr, TypeEntries::type_mask);
3177 __ jcc(Assembler::zero, ok);
3178 // may have been set by another thread
3179 __ mov_metadata(tmp, exact_klass->constant_encoding());
3180 __ xorptr(tmp, mdo_addr);
3181 __ testptr(tmp, TypeEntries::type_mask);
3182 __ jcc(Assembler::zero, ok);
3183
3184 __ stop("unexpected profiling mismatch");
3185 __ bind(ok);
3186 __ pop_ppx(tmp);
3187 }
3188 #else
3189 __ jccb(Assembler::zero, next);
3190 #endif
3191 // first time here. Set profile type.
3192 __ movptr(mdo_addr, tmp);
3193 #ifdef ASSERT
3194 __ andptr(tmp, TypeEntries::type_klass_mask);
3195 __ verify_klass_ptr(tmp);
3196 #endif
3197 } else {
3198 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3199 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3200
3201 __ testptr(mdo_addr, TypeEntries::type_unknown);
3202 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3203
3204 __ orptr(mdo_addr, TypeEntries::type_unknown);
3205 }
3206 }
3207 }
3208 __ bind(next);
3209 }
3210
3211 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3212 Register obj = op->obj()->as_register();
3213 Register tmp = op->tmp()->as_pointer_register();
3214 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3215 bool not_null = op->not_null();
3216 int flag = op->flag();
3217
3218 Label not_inline_type;
3219 if (!not_null) {
3220 __ testptr(obj, obj);
3221 __ jccb(Assembler::zero, not_inline_type);
3222 }
3223
3224 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3225
3226 __ orb(mdo_addr, flag);
3227
3228 __ bind(not_inline_type);
3229 }
3230
3231
3232 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3233 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3234 }
3235
3236
3237 void LIR_Assembler::align_backward_branch_target() {
3238 __ align(BytesPerWord);
3239 }
3240
3241
3242 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3243 if (left->is_single_cpu()) {
3244 __ negl(left->as_register());
3245 move_regs(left->as_register(), dest->as_register());
3246
3247 } else if (left->is_double_cpu()) {
3248 Register lo = left->as_register_lo();
3249 Register dst = dest->as_register_lo();
3250 __ movptr(dst, lo);
3251 __ negptr(dst);
3252
3253 } else if (dest->is_single_xmm()) {
3254 assert(!tmp->is_valid(), "do not need temporary");
3255 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3256 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3257 }
3258 __ xorps(dest->as_xmm_float_reg(),
3259 ExternalAddress((address)float_signflip_pool),
3260 rscratch1);
3261 } else if (dest->is_double_xmm()) {
3262 assert(!tmp->is_valid(), "do not need temporary");
3263 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3264 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3265 }
3266 __ xorpd(dest->as_xmm_double_reg(),
3267 ExternalAddress((address)double_signflip_pool),
3268 rscratch1);
3269 } else {
3270 ShouldNotReachHere();
3271 }
3272 }
3273
3274
3275 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3276 assert(src->is_address(), "must be an address");
3277 assert(dest->is_register(), "must be a register");
3278
3279 PatchingStub* patch = nullptr;
3280 if (patch_code != lir_patch_none) {
3281 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3282 }
3283
3284 Register reg = dest->as_pointer_register();
3285 LIR_Address* addr = src->as_address_ptr();
3286 __ lea(reg, as_Address(addr));
3287
3288 if (patch != nullptr) {
3289 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3290 }
3291 }
3292
3293
3294
3295 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3296 assert(!tmp->is_valid(), "don't need temporary");
3297 __ call(RuntimeAddress(dest));
3298 if (info != nullptr) {
3299 add_call_info_here(info);
3300 }
3301 __ post_call_nop();
3302 }
3303
3304
3305 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3306 assert(type == T_LONG, "only for volatile long fields");
3307
3308 if (info != nullptr) {
3309 add_debug_info_for_null_check_here(info);
3310 }
3311
3312 if (src->is_double_xmm()) {
3313 if (dest->is_double_cpu()) {
3314 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3315 } else if (dest->is_double_stack()) {
3316 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3317 } else if (dest->is_address()) {
3318 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3319 } else {
3320 ShouldNotReachHere();
3321 }
3322
3323 } else if (dest->is_double_xmm()) {
3324 if (src->is_double_stack()) {
3325 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3326 } else if (src->is_address()) {
3327 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3328 } else {
3329 ShouldNotReachHere();
3330 }
3331
3332 } else {
3333 ShouldNotReachHere();
3334 }
3335 }
3336
3337 #ifdef ASSERT
3338 // emit run-time assertion
3339 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3340 assert(op->code() == lir_assert, "must be");
3341
3342 if (op->in_opr1()->is_valid()) {
3343 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3344 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3345 } else {
3346 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3347 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3348 }
3349
3350 Label ok;
3351 if (op->condition() != lir_cond_always) {
3352 Assembler::Condition acond = Assembler::zero;
3353 switch (op->condition()) {
3354 case lir_cond_equal: acond = Assembler::equal; break;
3355 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3356 case lir_cond_less: acond = Assembler::less; break;
3357 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3358 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3359 case lir_cond_greater: acond = Assembler::greater; break;
3360 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3361 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3362 default: ShouldNotReachHere();
3363 }
3364 __ jcc(acond, ok);
3365 }
3366 if (op->halt()) {
3367 const char* str = __ code_string(op->msg());
3368 __ stop(str);
3369 } else {
3370 breakpoint();
3371 }
3372 __ bind(ok);
3373 }
3374 #endif
3375
3376 void LIR_Assembler::membar() {
3377 // QQQ sparc TSO uses this,
3378 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3379 }
3380
3381 void LIR_Assembler::membar_acquire() {
3382 // No x86 machines currently require load fences
3383 }
3384
3385 void LIR_Assembler::membar_release() {
3386 // No x86 machines currently require store fences
3387 }
3388
3389 void LIR_Assembler::membar_loadload() {
3390 // no-op
3391 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3392 }
3393
3394 void LIR_Assembler::membar_storestore() {
3395 // no-op
3396 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3397 }
3398
3399 void LIR_Assembler::membar_loadstore() {
3400 // no-op
3401 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3402 }
3403
3404 void LIR_Assembler::membar_storeload() {
3405 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3406 }
3407
3408 void LIR_Assembler::on_spin_wait() {
3409 __ pause ();
3410 }
3411
3412 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3413 assert(result_reg->is_register(), "check");
3414 __ mov(result_reg->as_register(), r15_thread);
3415 }
3416
3417 void LIR_Assembler::check_orig_pc() {
3418 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3419 }
3420
3421 void LIR_Assembler::peephole(LIR_List*) {
3422 // do nothing for now
3423 }
3424
3425 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3426 assert(data == dest, "xchg/xadd uses only 2 operands");
3427
3428 if (data->type() == T_INT) {
3429 if (code == lir_xadd) {
3430 __ lock();
3431 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3432 } else {
3433 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3434 }
3435 } else if (data->is_oop()) {
3436 assert (code == lir_xchg, "xadd for oops");
3437 Register obj = data->as_register();
3438 if (UseCompressedOops) {
3439 __ encode_heap_oop(obj);
3440 __ xchgl(obj, as_Address(src->as_address_ptr()));
3441 __ decode_heap_oop(obj);
3442 } else {
3443 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3444 }
3445 } else if (data->type() == T_LONG) {
3446 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3447 if (code == lir_xadd) {
3448 __ lock();
3449 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3450 } else {
3451 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3452 }
3453 } else {
3454 ShouldNotReachHere();
3455 }
3456 }
3457
3458 #undef __