1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2);
95 }
96
97
98
99 static void select_different_registers(Register preserve,
100 Register extra,
101 Register &tmp1,
102 Register &tmp2,
103 Register &tmp3) {
104 if (tmp1 == preserve) {
105 assert_different_registers(tmp1, tmp2, tmp3, extra);
106 tmp1 = extra;
107 } else if (tmp2 == preserve) {
108 assert_different_registers(tmp1, tmp2, tmp3, extra);
109 tmp2 = extra;
110 } else if (tmp3 == preserve) {
111 assert_different_registers(tmp1, tmp2, tmp3, extra);
112 tmp3 = extra;
113 }
114 assert_different_registers(preserve, tmp1, tmp2, tmp3);
115 }
116
117
118
119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
120 if (opr->is_constant()) {
121 LIR_Const* constant = opr->as_constant_ptr();
122 switch (constant->type()) {
123 case T_INT: {
124 return true;
125 }
126
127 default:
128 return false;
129 }
130 }
131 return false;
132 }
133
134
135 LIR_Opr LIR_Assembler::receiverOpr() {
136 return FrameMap::receiver_opr;
137 }
138
139 LIR_Opr LIR_Assembler::osrBufferPointer() {
140 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
141 }
142
143 //--------------fpu register translations-----------------------
144
145
146 address LIR_Assembler::float_constant(float f) {
147 address const_addr = __ float_constant(f);
148 if (const_addr == nullptr) {
149 bailout("const section overflow");
150 return __ code()->consts()->start();
151 } else {
152 return const_addr;
153 }
154 }
155
156
157 address LIR_Assembler::double_constant(double d) {
158 address const_addr = __ double_constant(d);
159 if (const_addr == nullptr) {
160 bailout("const section overflow");
161 return __ code()->consts()->start();
162 } else {
163 return const_addr;
164 }
165 }
166
167 void LIR_Assembler::breakpoint() {
168 __ int3();
169 }
170
171 void LIR_Assembler::push(LIR_Opr opr) {
172 if (opr->is_single_cpu()) {
173 __ push_reg(opr->as_register());
174 } else if (opr->is_double_cpu()) {
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330
331 __ mov_metadata(klass, method->holder()->constant_encoding());
332 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
333
334 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
335
336 __ bind(L_skip_barrier);
337 }
338
339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
340 jobject o = nullptr;
341 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
342 __ movoop(reg, o);
343 patching_epilog(patch, lir_patch_normal, reg, info);
344 }
345
346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
347 Metadata* o = nullptr;
348 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
349 __ mov_metadata(reg, o);
350 patching_epilog(patch, lir_patch_normal, reg, info);
351 }
352
353 // This specifies the rsp decrement needed to build the frame
354 int LIR_Assembler::initial_frame_size_in_bytes() const {
355 // if rounding, must let FrameMap know!
356
357 // The frame_map records size in slots (32bit word)
358
359 // subtract two words to account for return address and link
360 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
361 }
362
363
364 int LIR_Assembler::emit_exception_handler() {
365 // generate code for exception handler
366 address handler_base = __ start_a_stub(exception_handler_size());
367 if (handler_base == nullptr) {
368 // not enough space left for the handler
369 bailout("exception handler overflow");
370 return -1;
371 }
372
373 int offset = code_offset();
374
375 // the exception oop and pc are in rax, and rdx
376 // no other registers need to be preserved, so invalidate them
377 __ invalidate_registers(false, true, true, false, true, true);
378
379 // check that there is really an exception
380 __ verify_not_null_oop(rax);
381
382 // search an exception handler (rax: exception oop, rdx: throwing pc)
383 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
384 __ should_not_reach_here();
385 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
386 __ end_a_stub();
387
388 return offset;
389 }
390
391
392 // Emit the code to remove the frame from the stack in the exception
393 // unwind path.
394 int LIR_Assembler::emit_unwind_handler() {
395 #ifndef PRODUCT
396 if (CommentedAssembly) {
397 _masm->block_comment("Unwind handler");
398 }
399 #endif
400
401 int offset = code_offset();
402
403 // Fetch the exception from TLS and clear out exception related thread state
404 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
405 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
406 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
407
408 __ bind(_unwind_handler_entry);
409 __ verify_not_null_oop(rax);
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
412 }
413
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456
457 Label start;
458 __ bind(start);
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
468 "out of bounds read in post-call NOP check");
469 __ end_a_stub();
470
471 return entry_offset;
472 }
473
474 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
475 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
476 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
477 assert(result->fpu() == 0, "result must already be on TOS");
478 }
479 if (InlineTypeReturnedAsFields) {
480 #ifndef _LP64
481 Unimplemented();
482 #endif
483 // Check if we are returning an non-null inline type and load its fields into registers
484 ciType* return_type = compilation()->method()->return_type();
485 if (return_type->is_inlinetype()) {
486 ciInlineKlass* vk = return_type->as_inline_klass();
487 if (vk->can_be_returned_as_fields()) {
488 address unpack_handler = vk->unpack_handler();
489 assert(unpack_handler != nullptr, "must be");
490 __ call(RuntimeAddress(unpack_handler));
491 }
492 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
493 Label skip;
494 Label not_null;
495 __ testptr(rax, rax);
496 __ jcc(Assembler::notZero, not_null);
497 // Returned value is null, zero all return registers because they may belong to oop fields
498 __ xorq(j_rarg1, j_rarg1);
499 __ xorq(j_rarg2, j_rarg2);
500 __ xorq(j_rarg3, j_rarg3);
501 __ xorq(j_rarg4, j_rarg4);
502 __ xorq(j_rarg5, j_rarg5);
503 __ jmp(skip);
504 __ bind(not_null);
505
506 // Check if we are returning an non-null inline type and load its fields into registers
507 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
508
509 // Load fields from a buffered value with an inline class specific handler
510 __ load_klass(rdi, rax, rscratch1);
511 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
512 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
513 // Unpack handler can be null if inline type is not scalarizable in returns
514 __ testptr(rdi, rdi);
515 __ jcc(Assembler::zero, skip);
516 __ call(rdi);
517
518 __ bind(skip);
519 }
520 // At this point, rax points to the value object (for interpreter or C1 caller).
521 // The fields of the object are copied into registers (for C2 caller).
522 }
523
524 // Pop the stack before the safepoint code
525 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
526
527 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
528 __ reserved_stack_check();
529 }
530
531 // Note: we do not need to round double result; float result has the right precision
532 // the poll sets the condition code, but no data registers
533
534 code_stub->set_safepoint_offset(__ offset());
535 __ relocate(relocInfo::poll_return_type);
536 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
537 __ ret(0);
538 }
539
540
541 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
542 return (__ store_inline_type_fields_to_buf(vk, false));
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 guarantee(info != nullptr, "Shouldn't be null");
547 int offset = __ offset();
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 add_debug_info_for_branch(info);
551 __ relocate(relocInfo::poll_type);
552 address pre_pc = __ pc();
553 __ testl(rax, Address(poll_addr, 0));
554 address post_pc = __ pc();
555 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
556 return offset;
557 }
558
559
560 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
561 if (from_reg != to_reg) __ mov(to_reg, from_reg);
562 }
563
564 void LIR_Assembler::swap_reg(Register a, Register b) {
565 __ xchgptr(a, b);
566 }
567
568
569 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
570 assert(src->is_constant(), "should not call otherwise");
571 assert(dest->is_register(), "should not call otherwise");
572 LIR_Const* c = src->as_constant_ptr();
573
574 switch (c->type()) {
575 case T_INT: {
576 assert(patch_code == lir_patch_none, "no patching handled here");
577 __ movl(dest->as_register(), c->as_jint());
578 break;
579 }
580
581 case T_ADDRESS: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movptr(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_LONG: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
590 break;
591 }
592
593 case T_OBJECT: {
594 if (patch_code != lir_patch_none) {
595 jobject2reg_with_patching(dest->as_register(), info);
596 } else {
597 __ movoop(dest->as_register(), c->as_jobject());
598 }
599 break;
600 }
601
602 case T_METADATA: {
603 if (patch_code != lir_patch_none) {
604 klass2reg_with_patching(dest->as_register(), info);
605 } else {
606 __ mov_metadata(dest->as_register(), c->as_metadata());
607 }
608 break;
609 }
610
611 case T_FLOAT: {
612 if (dest->is_single_xmm()) {
613 if (UseAVX <= 2 && c->is_zero_float()) {
614 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
615 } else {
616 __ movflt(dest->as_xmm_float_reg(),
617 InternalAddress(float_constant(c->as_jfloat())));
618 }
619 } else {
620 ShouldNotReachHere();
621 }
622 break;
623 }
624
625 case T_DOUBLE: {
626 if (dest->is_double_xmm()) {
627 if (UseAVX <= 2 && c->is_zero_double()) {
628 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
629 } else {
630 __ movdbl(dest->as_xmm_double_reg(),
631 InternalAddress(double_constant(c->as_jdouble())));
632 }
633 } else {
634 ShouldNotReachHere();
635 }
636 break;
637 }
638
639 default:
640 ShouldNotReachHere();
641 }
642 }
643
644 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
645 assert(src->is_constant(), "should not call otherwise");
646 assert(dest->is_stack(), "should not call otherwise");
647 LIR_Const* c = src->as_constant_ptr();
648
649 switch (c->type()) {
650 case T_INT: // fall through
651 case T_FLOAT:
652 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
653 break;
654
655 case T_ADDRESS:
656 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
657 break;
658
659 case T_OBJECT:
660 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
661 break;
662
663 case T_LONG: // fall through
664 case T_DOUBLE:
665 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
666 lo_word_offset_in_bytes),
667 (intptr_t)c->as_jlong_bits(),
668 rscratch1);
669 break;
670
671 default:
672 ShouldNotReachHere();
673 }
674 }
675
676 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
677 assert(src->is_constant(), "should not call otherwise");
678 assert(dest->is_address(), "should not call otherwise");
679 LIR_Const* c = src->as_constant_ptr();
680 LIR_Address* addr = dest->as_address_ptr();
681
682 int null_check_here = code_offset();
683 switch (type) {
684 case T_INT: // fall through
685 case T_FLOAT:
686 __ movl(as_Address(addr), c->as_jint_bits());
687 break;
688
689 case T_ADDRESS:
690 __ movptr(as_Address(addr), c->as_jint_bits());
691 break;
692
693 case T_OBJECT: // fall through
694 case T_ARRAY:
695 if (c->as_jobject() == nullptr) {
696 if (UseCompressedOops && !wide) {
697 __ movl(as_Address(addr), NULL_WORD);
698 } else {
699 __ xorptr(rscratch1, rscratch1);
700 null_check_here = code_offset();
701 __ movptr(as_Address(addr), rscratch1);
702 }
703 } else {
704 if (is_literal_address(addr)) {
705 ShouldNotReachHere();
706 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
707 } else {
708 __ movoop(rscratch1, c->as_jobject());
709 if (UseCompressedOops && !wide) {
710 __ encode_heap_oop(rscratch1);
711 null_check_here = code_offset();
712 __ movl(as_Address_lo(addr), rscratch1);
713 } else {
714 null_check_here = code_offset();
715 __ movptr(as_Address_lo(addr), rscratch1);
716 }
717 }
718 }
719 break;
720
721 case T_LONG: // fall through
722 case T_DOUBLE:
723 if (is_literal_address(addr)) {
724 ShouldNotReachHere();
725 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
726 } else {
727 __ movptr(r10, (intptr_t)c->as_jlong_bits());
728 null_check_here = code_offset();
729 __ movptr(as_Address_lo(addr), r10);
730 }
731 break;
732
733 case T_BOOLEAN: // fall through
734 case T_BYTE:
735 __ movb(as_Address(addr), c->as_jint() & 0xFF);
736 break;
737
738 case T_CHAR: // fall through
739 case T_SHORT:
740 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
741 break;
742
743 default:
744 ShouldNotReachHere();
745 };
746
747 if (info != nullptr) {
748 add_debug_info_for_null_check(null_check_here, info);
749 }
750 }
751
752
753 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
754 assert(src->is_register(), "should not call otherwise");
755 assert(dest->is_register(), "should not call otherwise");
756
757 // move between cpu-registers
758 if (dest->is_single_cpu()) {
759 if (src->type() == T_LONG) {
760 // Can do LONG -> OBJECT
761 move_regs(src->as_register_lo(), dest->as_register());
762 return;
763 }
764 assert(src->is_single_cpu(), "must match");
765 if (src->type() == T_OBJECT) {
766 __ verify_oop(src->as_register());
767 }
768 move_regs(src->as_register(), dest->as_register());
769
770 } else if (dest->is_double_cpu()) {
771 if (is_reference_type(src->type())) {
772 // Surprising to me but we can see move of a long to t_object
773 __ verify_oop(src->as_register());
774 move_regs(src->as_register(), dest->as_register_lo());
775 return;
776 }
777 assert(src->is_double_cpu(), "must match");
778 Register f_lo = src->as_register_lo();
779 Register f_hi = src->as_register_hi();
780 Register t_lo = dest->as_register_lo();
781 Register t_hi = dest->as_register_hi();
782 assert(f_hi == f_lo, "must be same");
783 assert(t_hi == t_lo, "must be same");
784 move_regs(f_lo, t_lo);
785
786 // move between xmm-registers
787 } else if (dest->is_single_xmm()) {
788 assert(src->is_single_xmm(), "must match");
789 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
790 } else if (dest->is_double_xmm()) {
791 assert(src->is_double_xmm(), "must match");
792 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
793
794 } else {
795 ShouldNotReachHere();
796 }
797 }
798
799 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
800 assert(src->is_register(), "should not call otherwise");
801 assert(dest->is_stack(), "should not call otherwise");
802
803 if (src->is_single_cpu()) {
804 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
805 if (is_reference_type(type)) {
806 __ verify_oop(src->as_register());
807 __ movptr (dst, src->as_register());
808 } else if (type == T_METADATA || type == T_ADDRESS) {
809 __ movptr (dst, src->as_register());
810 } else {
811 __ movl (dst, src->as_register());
812 }
813
814 } else if (src->is_double_cpu()) {
815 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
816 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
817 __ movptr (dstLO, src->as_register_lo());
818
819 } else if (src->is_single_xmm()) {
820 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
821 __ movflt(dst_addr, src->as_xmm_float_reg());
822
823 } else if (src->is_double_xmm()) {
824 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
825 __ movdbl(dst_addr, src->as_xmm_double_reg());
826
827 } else {
828 ShouldNotReachHere();
829 }
830 }
831
832
833 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
834 LIR_Address* to_addr = dest->as_address_ptr();
835 PatchingStub* patch = nullptr;
836 Register compressed_src = rscratch1;
837
838 if (is_reference_type(type)) {
839 __ verify_oop(src->as_register());
840 if (UseCompressedOops && !wide) {
841 __ movptr(compressed_src, src->as_register());
842 __ encode_heap_oop(compressed_src);
843 if (patch_code != lir_patch_none) {
844 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
845 }
846 }
847 }
848
849 if (patch_code != lir_patch_none) {
850 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
851 Address toa = as_Address(to_addr);
852 assert(toa.disp() != 0, "must have");
853 }
854
855 int null_check_here = code_offset();
856 switch (type) {
857 case T_FLOAT: {
858 assert(src->is_single_xmm(), "not a float");
859 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
860 break;
861 }
862
863 case T_DOUBLE: {
864 assert(src->is_double_xmm(), "not a double");
865 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
866 break;
867 }
868
869 case T_ARRAY: // fall through
870 case T_OBJECT: // fall through
871 if (UseCompressedOops && !wide) {
872 __ movl(as_Address(to_addr), compressed_src);
873 } else {
874 __ movptr(as_Address(to_addr), src->as_register());
875 }
876 break;
877 case T_ADDRESS:
878 __ movptr(as_Address(to_addr), src->as_register());
879 break;
880 case T_INT:
881 __ movl(as_Address(to_addr), src->as_register());
882 break;
883
884 case T_LONG: {
885 Register from_lo = src->as_register_lo();
886 Register from_hi = src->as_register_hi();
887 __ movptr(as_Address_lo(to_addr), from_lo);
888 break;
889 }
890
891 case T_BYTE: // fall through
892 case T_BOOLEAN: {
893 Register src_reg = src->as_register();
894 Address dst_addr = as_Address(to_addr);
895 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
896 __ movb(dst_addr, src_reg);
897 break;
898 }
899
900 case T_CHAR: // fall through
901 case T_SHORT:
902 __ movw(as_Address(to_addr), src->as_register());
903 break;
904
905 default:
906 ShouldNotReachHere();
907 }
908 if (info != nullptr) {
909 add_debug_info_for_null_check(null_check_here, info);
910 }
911
912 if (patch_code != lir_patch_none) {
913 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
914 }
915 }
916
917
918 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
919 assert(src->is_stack(), "should not call otherwise");
920 assert(dest->is_register(), "should not call otherwise");
921
922 if (dest->is_single_cpu()) {
923 if (is_reference_type(type)) {
924 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
925 __ verify_oop(dest->as_register());
926 } else if (type == T_METADATA || type == T_ADDRESS) {
927 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
928 } else {
929 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
930 }
931
932 } else if (dest->is_double_cpu()) {
933 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
934 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
935 __ movptr(dest->as_register_lo(), src_addr_LO);
936
937 } else if (dest->is_single_xmm()) {
938 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
939 __ movflt(dest->as_xmm_float_reg(), src_addr);
940
941 } else if (dest->is_double_xmm()) {
942 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
943 __ movdbl(dest->as_xmm_double_reg(), src_addr);
944
945 } else {
946 ShouldNotReachHere();
947 }
948 }
949
950
951 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
952 if (src->is_single_stack()) {
953 if (is_reference_type(type)) {
954 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
955 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
956 } else {
957 //no pushl on 64bits
958 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
959 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
960 }
961
962 } else if (src->is_double_stack()) {
963 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
964 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
965
966 } else {
967 ShouldNotReachHere();
968 }
969 }
970
971
972 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
973 assert(src->is_address(), "should not call otherwise");
974 assert(dest->is_register(), "should not call otherwise");
975
976 LIR_Address* addr = src->as_address_ptr();
977 Address from_addr = as_Address(addr);
978
979 if (addr->base()->type() == T_OBJECT) {
980 __ verify_oop(addr->base()->as_pointer_register());
981 }
982
983 switch (type) {
984 case T_BOOLEAN: // fall through
985 case T_BYTE: // fall through
986 case T_CHAR: // fall through
987 case T_SHORT:
988 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
989 // on pre P6 processors we may get partial register stalls
990 // so blow away the value of to_rinfo before loading a
991 // partial word into it. Do it here so that it precedes
992 // the potential patch point below.
993 __ xorptr(dest->as_register(), dest->as_register());
994 }
995 break;
996 default:
997 break;
998 }
999
1000 PatchingStub* patch = nullptr;
1001 if (patch_code != lir_patch_none) {
1002 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1003 assert(from_addr.disp() != 0, "must have");
1004 }
1005 if (info != nullptr) {
1006 add_debug_info_for_null_check_here(info);
1007 }
1008
1009 switch (type) {
1010 case T_FLOAT: {
1011 if (dest->is_single_xmm()) {
1012 __ movflt(dest->as_xmm_float_reg(), from_addr);
1013 } else {
1014 ShouldNotReachHere();
1015 }
1016 break;
1017 }
1018
1019 case T_DOUBLE: {
1020 if (dest->is_double_xmm()) {
1021 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1022 } else {
1023 ShouldNotReachHere();
1024 }
1025 break;
1026 }
1027
1028 case T_OBJECT: // fall through
1029 case T_ARRAY: // fall through
1030 if (UseCompressedOops && !wide) {
1031 __ movl(dest->as_register(), from_addr);
1032 } else {
1033 __ movptr(dest->as_register(), from_addr);
1034 }
1035 break;
1036
1037 case T_ADDRESS:
1038 __ movptr(dest->as_register(), from_addr);
1039 break;
1040 case T_INT:
1041 __ movl(dest->as_register(), from_addr);
1042 break;
1043
1044 case T_LONG: {
1045 Register to_lo = dest->as_register_lo();
1046 Register to_hi = dest->as_register_hi();
1047 __ movptr(to_lo, as_Address_lo(addr));
1048 break;
1049 }
1050
1051 case T_BOOLEAN: // fall through
1052 case T_BYTE: {
1053 Register dest_reg = dest->as_register();
1054 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1055 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1056 __ movsbl(dest_reg, from_addr);
1057 } else {
1058 __ movb(dest_reg, from_addr);
1059 __ shll(dest_reg, 24);
1060 __ sarl(dest_reg, 24);
1061 }
1062 break;
1063 }
1064
1065 case T_CHAR: {
1066 Register dest_reg = dest->as_register();
1067 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1068 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1069 __ movzwl(dest_reg, from_addr);
1070 } else {
1071 __ movw(dest_reg, from_addr);
1072 }
1073 break;
1074 }
1075
1076 case T_SHORT: {
1077 Register dest_reg = dest->as_register();
1078 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1079 __ movswl(dest_reg, from_addr);
1080 } else {
1081 __ movw(dest_reg, from_addr);
1082 __ shll(dest_reg, 16);
1083 __ sarl(dest_reg, 16);
1084 }
1085 break;
1086 }
1087
1088 default:
1089 ShouldNotReachHere();
1090 }
1091
1092 if (patch != nullptr) {
1093 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1094 }
1095
1096 if (is_reference_type(type)) {
1097 if (UseCompressedOops && !wide) {
1098 __ decode_heap_oop(dest->as_register());
1099 }
1100
1101 __ verify_oop(dest->as_register());
1102 }
1103 }
1104
1105
1106 NEEDS_CLEANUP; // This could be static?
1107 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1108 int elem_size = type2aelembytes(type);
1109 switch (elem_size) {
1110 case 1: return Address::times_1;
1111 case 2: return Address::times_2;
1112 case 4: return Address::times_4;
1113 case 8: return Address::times_8;
1114 }
1115 ShouldNotReachHere();
1116 return Address::no_scale;
1117 }
1118
1119
1120 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1121 switch (op->code()) {
1122 case lir_idiv:
1123 case lir_irem:
1124 arithmetic_idiv(op->code(),
1125 op->in_opr1(),
1126 op->in_opr2(),
1127 op->in_opr3(),
1128 op->result_opr(),
1129 op->info());
1130 break;
1131 case lir_fmad:
1132 __ fmad(op->result_opr()->as_xmm_double_reg(),
1133 op->in_opr1()->as_xmm_double_reg(),
1134 op->in_opr2()->as_xmm_double_reg(),
1135 op->in_opr3()->as_xmm_double_reg());
1136 break;
1137 case lir_fmaf:
1138 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1139 op->in_opr1()->as_xmm_float_reg(),
1140 op->in_opr2()->as_xmm_float_reg(),
1141 op->in_opr3()->as_xmm_float_reg());
1142 break;
1143 default: ShouldNotReachHere(); break;
1144 }
1145 }
1146
1147 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1148 #ifdef ASSERT
1149 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1150 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1151 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1152 #endif
1153
1154 if (op->cond() == lir_cond_always) {
1155 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1156 __ jmp (*(op->label()));
1157 } else {
1158 Assembler::Condition acond = Assembler::zero;
1159 if (op->code() == lir_cond_float_branch) {
1160 assert(op->ublock() != nullptr, "must have unordered successor");
1161 __ jcc(Assembler::parity, *(op->ublock()->label()));
1162 switch(op->cond()) {
1163 case lir_cond_equal: acond = Assembler::equal; break;
1164 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1165 case lir_cond_less: acond = Assembler::below; break;
1166 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1167 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1168 case lir_cond_greater: acond = Assembler::above; break;
1169 default: ShouldNotReachHere();
1170 }
1171 } else {
1172 switch (op->cond()) {
1173 case lir_cond_equal: acond = Assembler::equal; break;
1174 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1175 case lir_cond_less: acond = Assembler::less; break;
1176 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1177 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1178 case lir_cond_greater: acond = Assembler::greater; break;
1179 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1180 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1181 default: ShouldNotReachHere();
1182 }
1183 }
1184 __ jcc(acond,*(op->label()));
1185 }
1186 }
1187
1188 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1189 LIR_Opr src = op->in_opr();
1190 LIR_Opr dest = op->result_opr();
1191
1192 switch (op->bytecode()) {
1193 case Bytecodes::_i2l:
1194 __ movl2ptr(dest->as_register_lo(), src->as_register());
1195 break;
1196
1197 case Bytecodes::_l2i:
1198 __ movl(dest->as_register(), src->as_register_lo());
1199 break;
1200
1201 case Bytecodes::_i2b:
1202 move_regs(src->as_register(), dest->as_register());
1203 __ sign_extend_byte(dest->as_register());
1204 break;
1205
1206 case Bytecodes::_i2c:
1207 move_regs(src->as_register(), dest->as_register());
1208 __ andl(dest->as_register(), 0xFFFF);
1209 break;
1210
1211 case Bytecodes::_i2s:
1212 move_regs(src->as_register(), dest->as_register());
1213 __ sign_extend_short(dest->as_register());
1214 break;
1215
1216 case Bytecodes::_f2d:
1217 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1218 break;
1219
1220 case Bytecodes::_d2f:
1221 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1222 break;
1223
1224 case Bytecodes::_i2f:
1225 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1226 break;
1227
1228 case Bytecodes::_i2d:
1229 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1230 break;
1231
1232 case Bytecodes::_l2f:
1233 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1234 break;
1235
1236 case Bytecodes::_l2d:
1237 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1238 break;
1239
1240 case Bytecodes::_f2i:
1241 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1242 break;
1243
1244 case Bytecodes::_d2i:
1245 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1246 break;
1247
1248 case Bytecodes::_f2l:
1249 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1250 break;
1251
1252 case Bytecodes::_d2l:
1253 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1254 break;
1255
1256 default: ShouldNotReachHere();
1257 }
1258 }
1259
1260 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1261 if (op->init_check()) {
1262 add_debug_info_for_null_check_here(op->stub()->info());
1263 // init_state needs acquire, but x86 is TSO, and so we are already good.
1264 __ cmpb(Address(op->klass()->as_register(),
1265 InstanceKlass::init_state_offset()),
1266 InstanceKlass::fully_initialized);
1267 __ jcc(Assembler::notEqual, *op->stub()->entry());
1268 }
1269 __ allocate_object(op->obj()->as_register(),
1270 op->tmp1()->as_register(),
1271 op->tmp2()->as_register(),
1272 op->header_size(),
1273 op->object_size(),
1274 op->klass()->as_register(),
1275 *op->stub()->entry());
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1280 Register len = op->len()->as_register();
1281 __ movslq(len, len);
1282
1283 if (UseSlowPath || op->always_slow_path() ||
1284 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1285 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1286 __ jmp(*op->stub()->entry());
1287 } else {
1288 Register tmp1 = op->tmp1()->as_register();
1289 Register tmp2 = op->tmp2()->as_register();
1290 Register tmp3 = op->tmp3()->as_register();
1291 if (len == tmp1) {
1292 tmp1 = tmp3;
1293 } else if (len == tmp2) {
1294 tmp2 = tmp3;
1295 } else if (len == tmp3) {
1296 // everything is ok
1297 } else {
1298 __ mov(tmp3, len);
1299 }
1300 __ allocate_array(op->obj()->as_register(),
1301 len,
1302 tmp1,
1303 tmp2,
1304 arrayOopDesc::base_offset_in_bytes(op->type()),
1305 array_element_size(op->type()),
1306 op->klass()->as_register(),
1307 *op->stub()->entry(),
1308 op->zero_array());
1309 }
1310 __ bind(*op->stub()->continuation());
1311 }
1312
1313 void LIR_Assembler::type_profile_helper(Register mdo,
1314 ciMethodData *md, ciProfileData *data,
1315 Register recv, Label* update_done) {
1316 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1317 Label next_test;
1318 // See if the receiver is receiver[n].
1319 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1320 __ jccb(Assembler::notEqual, next_test);
1321 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1322 __ addptr(data_addr, DataLayout::counter_increment);
1323 __ jmp(*update_done);
1324 __ bind(next_test);
1325 }
1326
1327 // Didn't find receiver; find next empty slot and fill it in
1328 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1329 Label next_test;
1330 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1331 __ cmpptr(recv_addr, NULL_WORD);
1332 __ jccb(Assembler::notEqual, next_test);
1333 __ movptr(recv_addr, recv);
1334 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1335 __ jmp(*update_done);
1336 __ bind(next_test);
1337 }
1338 }
1339
1340 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1341 // we always need a stub for the failure case.
1342 CodeStub* stub = op->stub();
1343 Register obj = op->object()->as_register();
1344 Register k_RInfo = op->tmp1()->as_register();
1345 Register klass_RInfo = op->tmp2()->as_register();
1346 Register dst = op->result_opr()->as_register();
1347 ciKlass* k = op->klass();
1348 Register Rtmp1 = noreg;
1349 Register tmp_load_klass = rscratch1;
1350
1351 // check if it needs to be profiled
1352 ciMethodData* md = nullptr;
1353 ciProfileData* data = nullptr;
1354
1355 if (op->should_profile()) {
1356 ciMethod* method = op->profiled_method();
1357 assert(method != nullptr, "Should have method");
1358 int bci = op->profiled_bci();
1359 md = method->method_data_or_null();
1360 assert(md != nullptr, "Sanity");
1361 data = md->bci_to_data(bci);
1362 assert(data != nullptr, "need data for type check");
1363 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1364 }
1365 Label* success_target = success;
1366 Label* failure_target = failure;
1367
1368 if (obj == k_RInfo) {
1369 k_RInfo = dst;
1370 } else if (obj == klass_RInfo) {
1371 klass_RInfo = dst;
1372 }
1373 if (k->is_loaded() && !UseCompressedClassPointers) {
1374 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1375 } else {
1376 Rtmp1 = op->tmp3()->as_register();
1377 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1378 }
1379
1380 assert_different_registers(obj, k_RInfo, klass_RInfo);
1381
1382 if (op->need_null_check()) {
1383 __ testptr(obj, obj);
1384 if (op->should_profile()) {
1385 Label not_null;
1386 Register mdo = klass_RInfo;
1387 __ mov_metadata(mdo, md->constant_encoding());
1388 __ jccb(Assembler::notEqual, not_null);
1389 // Object is null; update MDO and exit
1390 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1391 int header_bits = BitData::null_seen_byte_constant();
1392 __ orb(data_addr, header_bits);
1393 __ jmp(*obj_is_null);
1394 __ bind(not_null);
1395
1396 Label update_done;
1397 Register recv = k_RInfo;
1398 __ load_klass(recv, obj, tmp_load_klass);
1399 type_profile_helper(mdo, md, data, recv, &update_done);
1400
1401 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1402 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1403
1404 __ bind(update_done);
1405 } else {
1406 __ jcc(Assembler::equal, *obj_is_null);
1407 }
1408 }
1409
1410 if (!k->is_loaded()) {
1411 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1412 } else {
1413 __ mov_metadata(k_RInfo, k->constant_encoding());
1414 }
1415 __ verify_oop(obj);
1416
1417 if (op->fast_check()) {
1418 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1419 // get object class
1420 // not a safepoint as obj null check happens earlier
1421 if (UseCompressedClassPointers) {
1422 __ load_klass(Rtmp1, obj, tmp_load_klass);
1423 __ cmpptr(k_RInfo, Rtmp1);
1424 } else {
1425 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1426 }
1427 __ jcc(Assembler::notEqual, *failure_target);
1428 // successful cast, fall through to profile or jump
1429 } else {
1430 // get object class
1431 // not a safepoint as obj null check happens earlier
1432 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1433 if (k->is_loaded()) {
1434 // See if we get an immediate positive hit
1435 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1436 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1437 __ jcc(Assembler::notEqual, *failure_target);
1438 // successful cast, fall through to profile or jump
1439 } else {
1440 // See if we get an immediate positive hit
1441 __ jcc(Assembler::equal, *success_target);
1442 // check for self
1443 if (k->is_loaded() && k->is_obj_array_klass()) {
1444 // For a direct pointer comparison, we need the refined array klass pointer
1445 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1446 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1447 __ cmpptr(klass_RInfo, tmp_load_klass);
1448 } else {
1449 __ cmpptr(klass_RInfo, k_RInfo);
1450 }
1451 __ jcc(Assembler::equal, *success_target);
1452
1453 __ push_ppx(klass_RInfo);
1454 __ push_ppx(k_RInfo);
1455 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1456 __ pop_ppx(klass_RInfo);
1457 __ pop_ppx(klass_RInfo);
1458 // result is a boolean
1459 __ testl(klass_RInfo, klass_RInfo);
1460 __ jcc(Assembler::equal, *failure_target);
1461 // successful cast, fall through to profile or jump
1462 }
1463 } else {
1464 // perform the fast part of the checking logic
1465 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1466 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1467 __ push_ppx(klass_RInfo);
1468 __ push_ppx(k_RInfo);
1469 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1470 __ pop_ppx(klass_RInfo);
1471 __ pop_ppx(k_RInfo);
1472 // result is a boolean
1473 __ testl(k_RInfo, k_RInfo);
1474 __ jcc(Assembler::equal, *failure_target);
1475 // successful cast, fall through to profile or jump
1476 }
1477 }
1478 __ jmp(*success);
1479 }
1480
1481
1482 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1483 Register tmp_load_klass = rscratch1;
1484 LIR_Code code = op->code();
1485 if (code == lir_store_check) {
1486 Register value = op->object()->as_register();
1487 Register array = op->array()->as_register();
1488 Register k_RInfo = op->tmp1()->as_register();
1489 Register klass_RInfo = op->tmp2()->as_register();
1490 Register Rtmp1 = op->tmp3()->as_register();
1491
1492 CodeStub* stub = op->stub();
1493
1494 // check if it needs to be profiled
1495 ciMethodData* md = nullptr;
1496 ciProfileData* data = nullptr;
1497
1498 if (op->should_profile()) {
1499 ciMethod* method = op->profiled_method();
1500 assert(method != nullptr, "Should have method");
1501 int bci = op->profiled_bci();
1502 md = method->method_data_or_null();
1503 assert(md != nullptr, "Sanity");
1504 data = md->bci_to_data(bci);
1505 assert(data != nullptr, "need data for type check");
1506 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1507 }
1508 Label done;
1509 Label* success_target = &done;
1510 Label* failure_target = stub->entry();
1511
1512 __ testptr(value, value);
1513 if (op->should_profile()) {
1514 Label not_null;
1515 Register mdo = klass_RInfo;
1516 __ mov_metadata(mdo, md->constant_encoding());
1517 __ jccb(Assembler::notEqual, not_null);
1518 // Object is null; update MDO and exit
1519 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1520 int header_bits = BitData::null_seen_byte_constant();
1521 __ orb(data_addr, header_bits);
1522 __ jmp(done);
1523 __ bind(not_null);
1524
1525 Label update_done;
1526 Register recv = k_RInfo;
1527 __ load_klass(recv, value, tmp_load_klass);
1528 type_profile_helper(mdo, md, data, recv, &update_done);
1529
1530 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1531 __ addptr(counter_addr, DataLayout::counter_increment);
1532 __ bind(update_done);
1533 } else {
1534 __ jcc(Assembler::equal, done);
1535 }
1536
1537 add_debug_info_for_null_check_here(op->info_for_exception());
1538 __ load_klass(k_RInfo, array, tmp_load_klass);
1539 __ load_klass(klass_RInfo, value, tmp_load_klass);
1540
1541 // get instance klass (it's already uncompressed)
1542 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1543 // perform the fast part of the checking logic
1544 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1545 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1546 __ push_ppx(klass_RInfo);
1547 __ push_ppx(k_RInfo);
1548 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1549 __ pop_ppx(klass_RInfo);
1550 __ pop_ppx(k_RInfo);
1551 // result is a boolean
1552 __ testl(k_RInfo, k_RInfo);
1553 __ jcc(Assembler::equal, *failure_target);
1554 // fall through to the success case
1555
1556 __ bind(done);
1557 } else
1558 if (code == lir_checkcast) {
1559 Register obj = op->object()->as_register();
1560 Register dst = op->result_opr()->as_register();
1561 Label success;
1562 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1563 __ bind(success);
1564 if (dst != obj) {
1565 __ mov(dst, obj);
1566 }
1567 } else
1568 if (code == lir_instanceof) {
1569 Register obj = op->object()->as_register();
1570 Register dst = op->result_opr()->as_register();
1571 Label success, failure, done;
1572 emit_typecheck_helper(op, &success, &failure, &failure);
1573 __ bind(failure);
1574 __ xorptr(dst, dst);
1575 __ jmpb(done);
1576 __ bind(success);
1577 __ movptr(dst, 1);
1578 __ bind(done);
1579 } else {
1580 ShouldNotReachHere();
1581 }
1582
1583 }
1584
1585 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1586 // We are loading/storing from/to an array that *may* be a flat array (the
1587 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1588 // If this array is a flat array, take the slow path.
1589 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590 if (!op->value()->is_illegal()) {
1591 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1592 // The array is not a flat array, but it might be null-free. If we are storing
1593 // a null into a null-free array, take the slow path (which will throw NPE).
1594 Label skip;
1595 __ cmpptr(op->value()->as_register(), NULL_WORD);
1596 __ jcc(Assembler::notEqual, skip);
1597 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1598 __ bind(skip);
1599 }
1600 }
1601
1602 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1603 // We are storing into an array that *may* be null-free (the declared type is
1604 // Object[], abstract[], interface[] or VT.ref[]).
1605 Label test_mark_word;
1606 Register tmp = op->tmp()->as_register();
1607 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1608 __ testl(tmp, markWord::unlocked_value);
1609 __ jccb(Assembler::notZero, test_mark_word);
1610 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1611 __ bind(test_mark_word);
1612 __ testl(tmp, markWord::null_free_array_bit_in_place);
1613 }
1614
1615 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1616 Label L_oops_equal;
1617 Label L_oops_not_equal;
1618 Label L_end;
1619
1620 Register left = op->left()->as_register();
1621 Register right = op->right()->as_register();
1622
1623 __ cmpptr(left, right);
1624 __ jcc(Assembler::equal, L_oops_equal);
1625
1626 // (1) Null check -- if one of the operands is null, the other must not be null (because
1627 // the two references are not equal), so they are not substitutable,
1628 // FIXME: do null check only if the operand is nullable
1629 __ testptr(left, right);
1630 __ jcc(Assembler::zero, L_oops_not_equal);
1631
1632 ciKlass* left_klass = op->left_klass();
1633 ciKlass* right_klass = op->right_klass();
1634
1635 // (2) Inline type check -- if either of the operands is not a inline type,
1636 // they are not substitutable. We do this only if we are not sure that the
1637 // operands are inline type
1638 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1639 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1640 Register tmp1 = op->tmp1()->as_register();
1641 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1642 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1643 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1644 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1645 __ jcc(Assembler::notEqual, L_oops_not_equal);
1646 }
1647
1648 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1649 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1650 // No need to load klass -- the operands are statically known to be the same inline klass.
1651 __ jmp(*op->stub()->entry());
1652 } else {
1653 Register left_klass_op = op->left_klass_op()->as_register();
1654 Register right_klass_op = op->right_klass_op()->as_register();
1655
1656 if (UseCompressedClassPointers) {
1657 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1658 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1659 __ cmpl(left_klass_op, right_klass_op);
1660 } else {
1661 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1662 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1663 __ cmpptr(left_klass_op, right_klass_op);
1664 }
1665
1666 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1667 // fall through to L_oops_not_equal
1668 }
1669
1670 __ bind(L_oops_not_equal);
1671 move(op->not_equal_result(), op->result_opr());
1672 __ jmp(L_end);
1673
1674 __ bind(L_oops_equal);
1675 move(op->equal_result(), op->result_opr());
1676 __ jmp(L_end);
1677
1678 // We've returned from the stub. RAX contains 0x0 IFF the two
1679 // operands are not substitutable. (Don't compare against 0x1 in case the
1680 // C compiler is naughty)
1681 __ bind(*op->stub()->continuation());
1682 __ cmpl(rax, 0);
1683 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1684 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1685 // fall-through
1686 __ bind(L_end);
1687 }
1688
1689 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1690 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1691 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1692 Register newval = op->new_value()->as_register();
1693 Register cmpval = op->cmp_value()->as_register();
1694 assert(cmpval == rax, "wrong register");
1695 assert(newval != noreg, "new val must be register");
1696 assert(cmpval != newval, "cmp and new values must be in different registers");
1697 assert(cmpval != addr, "cmp and addr must be in different registers");
1698 assert(newval != addr, "new value and addr must be in different registers");
1699
1700 if (op->code() == lir_cas_obj) {
1701 if (UseCompressedOops) {
1702 __ encode_heap_oop(cmpval);
1703 __ mov(rscratch1, newval);
1704 __ encode_heap_oop(rscratch1);
1705 __ lock();
1706 // cmpval (rax) is implicitly used by this instruction
1707 __ cmpxchgl(rscratch1, Address(addr, 0));
1708 } else {
1709 __ lock();
1710 __ cmpxchgptr(newval, Address(addr, 0));
1711 }
1712 } else {
1713 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1714 __ lock();
1715 __ cmpxchgl(newval, Address(addr, 0));
1716 }
1717 } else if (op->code() == lir_cas_long) {
1718 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1719 Register newval = op->new_value()->as_register_lo();
1720 Register cmpval = op->cmp_value()->as_register_lo();
1721 assert(cmpval == rax, "wrong register");
1722 assert(newval != noreg, "new val must be register");
1723 assert(cmpval != newval, "cmp and new values must be in different registers");
1724 assert(cmpval != addr, "cmp and addr must be in different registers");
1725 assert(newval != addr, "new value and addr must be in different registers");
1726 __ lock();
1727 __ cmpxchgq(newval, Address(addr, 0));
1728 } else {
1729 Unimplemented();
1730 }
1731 }
1732
1733 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1734 assert(dst->is_cpu_register(), "must be");
1735 assert(dst->type() == src->type(), "must be");
1736
1737 if (src->is_cpu_register()) {
1738 reg2reg(src, dst);
1739 } else if (src->is_stack()) {
1740 stack2reg(src, dst, dst->type());
1741 } else if (src->is_constant()) {
1742 const2reg(src, dst, lir_patch_none, nullptr);
1743 } else {
1744 ShouldNotReachHere();
1745 }
1746 }
1747
1748 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1749 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1750 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1751
1752 Assembler::Condition acond, ncond;
1753 switch (condition) {
1754 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1755 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1756 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1757 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1758 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1759 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1760 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1761 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1762 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1763 ShouldNotReachHere();
1764 }
1765
1766 if (opr1->is_cpu_register()) {
1767 reg2reg(opr1, result);
1768 } else if (opr1->is_stack()) {
1769 stack2reg(opr1, result, result->type());
1770 } else if (opr1->is_constant()) {
1771 const2reg(opr1, result, lir_patch_none, nullptr);
1772 } else {
1773 ShouldNotReachHere();
1774 }
1775
1776 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1777 // optimized version that does not require a branch
1778 if (opr2->is_single_cpu()) {
1779 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1780 __ cmov(ncond, result->as_register(), opr2->as_register());
1781 } else if (opr2->is_double_cpu()) {
1782 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1783 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1784 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1785 } else if (opr2->is_single_stack()) {
1786 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1787 } else if (opr2->is_double_stack()) {
1788 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1789 } else {
1790 ShouldNotReachHere();
1791 }
1792
1793 } else {
1794 Label skip;
1795 __ jccb(acond, skip);
1796 if (opr2->is_cpu_register()) {
1797 reg2reg(opr2, result);
1798 } else if (opr2->is_stack()) {
1799 stack2reg(opr2, result, result->type());
1800 } else if (opr2->is_constant()) {
1801 const2reg(opr2, result, lir_patch_none, nullptr);
1802 } else {
1803 ShouldNotReachHere();
1804 }
1805 __ bind(skip);
1806 }
1807 }
1808
1809
1810 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1811 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1812
1813 if (left->is_single_cpu()) {
1814 assert(left == dest, "left and dest must be equal");
1815 Register lreg = left->as_register();
1816
1817 if (right->is_single_cpu()) {
1818 // cpu register - cpu register
1819 Register rreg = right->as_register();
1820 switch (code) {
1821 case lir_add: __ addl (lreg, rreg); break;
1822 case lir_sub: __ subl (lreg, rreg); break;
1823 case lir_mul: __ imull(lreg, rreg); break;
1824 default: ShouldNotReachHere();
1825 }
1826
1827 } else if (right->is_stack()) {
1828 // cpu register - stack
1829 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1830 switch (code) {
1831 case lir_add: __ addl(lreg, raddr); break;
1832 case lir_sub: __ subl(lreg, raddr); break;
1833 default: ShouldNotReachHere();
1834 }
1835
1836 } else if (right->is_constant()) {
1837 // cpu register - constant
1838 jint c = right->as_constant_ptr()->as_jint();
1839 switch (code) {
1840 case lir_add: {
1841 __ incrementl(lreg, c);
1842 break;
1843 }
1844 case lir_sub: {
1845 __ decrementl(lreg, c);
1846 break;
1847 }
1848 default: ShouldNotReachHere();
1849 }
1850
1851 } else {
1852 ShouldNotReachHere();
1853 }
1854
1855 } else if (left->is_double_cpu()) {
1856 assert(left == dest, "left and dest must be equal");
1857 Register lreg_lo = left->as_register_lo();
1858 Register lreg_hi = left->as_register_hi();
1859
1860 if (right->is_double_cpu()) {
1861 // cpu register - cpu register
1862 Register rreg_lo = right->as_register_lo();
1863 Register rreg_hi = right->as_register_hi();
1864 assert_different_registers(lreg_lo, rreg_lo);
1865 switch (code) {
1866 case lir_add:
1867 __ addptr(lreg_lo, rreg_lo);
1868 break;
1869 case lir_sub:
1870 __ subptr(lreg_lo, rreg_lo);
1871 break;
1872 case lir_mul:
1873 __ imulq(lreg_lo, rreg_lo);
1874 break;
1875 default:
1876 ShouldNotReachHere();
1877 }
1878
1879 } else if (right->is_constant()) {
1880 // cpu register - constant
1881 jlong c = right->as_constant_ptr()->as_jlong_bits();
1882 __ movptr(r10, (intptr_t) c);
1883 switch (code) {
1884 case lir_add:
1885 __ addptr(lreg_lo, r10);
1886 break;
1887 case lir_sub:
1888 __ subptr(lreg_lo, r10);
1889 break;
1890 default:
1891 ShouldNotReachHere();
1892 }
1893
1894 } else {
1895 ShouldNotReachHere();
1896 }
1897
1898 } else if (left->is_single_xmm()) {
1899 assert(left == dest, "left and dest must be equal");
1900 XMMRegister lreg = left->as_xmm_float_reg();
1901
1902 if (right->is_single_xmm()) {
1903 XMMRegister rreg = right->as_xmm_float_reg();
1904 switch (code) {
1905 case lir_add: __ addss(lreg, rreg); break;
1906 case lir_sub: __ subss(lreg, rreg); break;
1907 case lir_mul: __ mulss(lreg, rreg); break;
1908 case lir_div: __ divss(lreg, rreg); break;
1909 default: ShouldNotReachHere();
1910 }
1911 } else {
1912 Address raddr;
1913 if (right->is_single_stack()) {
1914 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1915 } else if (right->is_constant()) {
1916 // hack for now
1917 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1918 } else {
1919 ShouldNotReachHere();
1920 }
1921 switch (code) {
1922 case lir_add: __ addss(lreg, raddr); break;
1923 case lir_sub: __ subss(lreg, raddr); break;
1924 case lir_mul: __ mulss(lreg, raddr); break;
1925 case lir_div: __ divss(lreg, raddr); break;
1926 default: ShouldNotReachHere();
1927 }
1928 }
1929
1930 } else if (left->is_double_xmm()) {
1931 assert(left == dest, "left and dest must be equal");
1932
1933 XMMRegister lreg = left->as_xmm_double_reg();
1934 if (right->is_double_xmm()) {
1935 XMMRegister rreg = right->as_xmm_double_reg();
1936 switch (code) {
1937 case lir_add: __ addsd(lreg, rreg); break;
1938 case lir_sub: __ subsd(lreg, rreg); break;
1939 case lir_mul: __ mulsd(lreg, rreg); break;
1940 case lir_div: __ divsd(lreg, rreg); break;
1941 default: ShouldNotReachHere();
1942 }
1943 } else {
1944 Address raddr;
1945 if (right->is_double_stack()) {
1946 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1947 } else if (right->is_constant()) {
1948 // hack for now
1949 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1950 } else {
1951 ShouldNotReachHere();
1952 }
1953 switch (code) {
1954 case lir_add: __ addsd(lreg, raddr); break;
1955 case lir_sub: __ subsd(lreg, raddr); break;
1956 case lir_mul: __ mulsd(lreg, raddr); break;
1957 case lir_div: __ divsd(lreg, raddr); break;
1958 default: ShouldNotReachHere();
1959 }
1960 }
1961
1962 } else if (left->is_single_stack() || left->is_address()) {
1963 assert(left == dest, "left and dest must be equal");
1964
1965 Address laddr;
1966 if (left->is_single_stack()) {
1967 laddr = frame_map()->address_for_slot(left->single_stack_ix());
1968 } else if (left->is_address()) {
1969 laddr = as_Address(left->as_address_ptr());
1970 } else {
1971 ShouldNotReachHere();
1972 }
1973
1974 if (right->is_single_cpu()) {
1975 Register rreg = right->as_register();
1976 switch (code) {
1977 case lir_add: __ addl(laddr, rreg); break;
1978 case lir_sub: __ subl(laddr, rreg); break;
1979 default: ShouldNotReachHere();
1980 }
1981 } else if (right->is_constant()) {
1982 jint c = right->as_constant_ptr()->as_jint();
1983 switch (code) {
1984 case lir_add: {
1985 __ incrementl(laddr, c);
1986 break;
1987 }
1988 case lir_sub: {
1989 __ decrementl(laddr, c);
1990 break;
1991 }
1992 default: ShouldNotReachHere();
1993 }
1994 } else {
1995 ShouldNotReachHere();
1996 }
1997
1998 } else {
1999 ShouldNotReachHere();
2000 }
2001 }
2002
2003
2004 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2005 if (value->is_double_xmm()) {
2006 switch(code) {
2007 case lir_abs :
2008 {
2009 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2010 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2011 }
2012 assert(!tmp->is_valid(), "do not need temporary");
2013 __ andpd(dest->as_xmm_double_reg(),
2014 ExternalAddress((address)double_signmask_pool),
2015 rscratch1);
2016 }
2017 break;
2018
2019 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2020 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2021 default : ShouldNotReachHere();
2022 }
2023
2024 } else if (code == lir_f2hf) {
2025 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2026 } else if (code == lir_hf2f) {
2027 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2028 } else {
2029 Unimplemented();
2030 }
2031 }
2032
2033 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2034 // assert(left->destroys_register(), "check");
2035 if (left->is_single_cpu()) {
2036 Register reg = left->as_register();
2037 if (right->is_constant()) {
2038 int val = right->as_constant_ptr()->as_jint();
2039 switch (code) {
2040 case lir_logic_and: __ andl (reg, val); break;
2041 case lir_logic_or: __ orl (reg, val); break;
2042 case lir_logic_xor: __ xorl (reg, val); break;
2043 default: ShouldNotReachHere();
2044 }
2045 } else if (right->is_stack()) {
2046 // added support for stack operands
2047 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2048 switch (code) {
2049 case lir_logic_and: __ andl (reg, raddr); break;
2050 case lir_logic_or: __ orl (reg, raddr); break;
2051 case lir_logic_xor: __ xorl (reg, raddr); break;
2052 default: ShouldNotReachHere();
2053 }
2054 } else {
2055 Register rright = right->as_register();
2056 switch (code) {
2057 case lir_logic_and: __ andptr (reg, rright); break;
2058 case lir_logic_or : __ orptr (reg, rright); break;
2059 case lir_logic_xor: __ xorptr (reg, rright); break;
2060 default: ShouldNotReachHere();
2061 }
2062 }
2063 move_regs(reg, dst->as_register());
2064 } else {
2065 Register l_lo = left->as_register_lo();
2066 Register l_hi = left->as_register_hi();
2067 if (right->is_constant()) {
2068 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2069 switch (code) {
2070 case lir_logic_and:
2071 __ andq(l_lo, rscratch1);
2072 break;
2073 case lir_logic_or:
2074 __ orq(l_lo, rscratch1);
2075 break;
2076 case lir_logic_xor:
2077 __ xorq(l_lo, rscratch1);
2078 break;
2079 default: ShouldNotReachHere();
2080 }
2081 } else {
2082 Register r_lo;
2083 if (is_reference_type(right->type())) {
2084 r_lo = right->as_register();
2085 } else {
2086 r_lo = right->as_register_lo();
2087 }
2088 switch (code) {
2089 case lir_logic_and:
2090 __ andptr(l_lo, r_lo);
2091 break;
2092 case lir_logic_or:
2093 __ orptr(l_lo, r_lo);
2094 break;
2095 case lir_logic_xor:
2096 __ xorptr(l_lo, r_lo);
2097 break;
2098 default: ShouldNotReachHere();
2099 }
2100 }
2101
2102 Register dst_lo = dst->as_register_lo();
2103 Register dst_hi = dst->as_register_hi();
2104
2105 move_regs(l_lo, dst_lo);
2106 }
2107 }
2108
2109
2110 // we assume that rax, and rdx can be overwritten
2111 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2112
2113 assert(left->is_single_cpu(), "left must be register");
2114 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2115 assert(result->is_single_cpu(), "result must be register");
2116
2117 // assert(left->destroys_register(), "check");
2118 // assert(right->destroys_register(), "check");
2119
2120 Register lreg = left->as_register();
2121 Register dreg = result->as_register();
2122
2123 if (right->is_constant()) {
2124 jint divisor = right->as_constant_ptr()->as_jint();
2125 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2126 if (code == lir_idiv) {
2127 assert(lreg == rax, "must be rax,");
2128 assert(temp->as_register() == rdx, "tmp register must be rdx");
2129 __ cdql(); // sign extend into rdx:rax
2130 if (divisor == 2) {
2131 __ subl(lreg, rdx);
2132 } else {
2133 __ andl(rdx, divisor - 1);
2134 __ addl(lreg, rdx);
2135 }
2136 __ sarl(lreg, log2i_exact(divisor));
2137 move_regs(lreg, dreg);
2138 } else if (code == lir_irem) {
2139 Label done;
2140 __ mov(dreg, lreg);
2141 __ andl(dreg, 0x80000000 | (divisor - 1));
2142 __ jcc(Assembler::positive, done);
2143 __ decrement(dreg);
2144 __ orl(dreg, ~(divisor - 1));
2145 __ increment(dreg);
2146 __ bind(done);
2147 } else {
2148 ShouldNotReachHere();
2149 }
2150 } else {
2151 Register rreg = right->as_register();
2152 assert(lreg == rax, "left register must be rax,");
2153 assert(rreg != rdx, "right register must not be rdx");
2154 assert(temp->as_register() == rdx, "tmp register must be rdx");
2155
2156 move_regs(lreg, rax);
2157
2158 int idivl_offset = __ corrected_idivl(rreg);
2159 if (ImplicitDiv0Checks) {
2160 add_debug_info_for_div0(idivl_offset, info);
2161 }
2162 if (code == lir_irem) {
2163 move_regs(rdx, dreg); // result is in rdx
2164 } else {
2165 move_regs(rax, dreg);
2166 }
2167 }
2168 }
2169
2170
2171 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2172 if (opr1->is_single_cpu()) {
2173 Register reg1 = opr1->as_register();
2174 if (opr2->is_single_cpu()) {
2175 // cpu register - cpu register
2176 if (is_reference_type(opr1->type())) {
2177 __ cmpoop(reg1, opr2->as_register());
2178 } else {
2179 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2180 __ cmpl(reg1, opr2->as_register());
2181 }
2182 } else if (opr2->is_stack()) {
2183 // cpu register - stack
2184 if (is_reference_type(opr1->type())) {
2185 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2186 } else {
2187 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2188 }
2189 } else if (opr2->is_constant()) {
2190 // cpu register - constant
2191 LIR_Const* c = opr2->as_constant_ptr();
2192 if (c->type() == T_INT) {
2193 jint i = c->as_jint();
2194 if (i == 0) {
2195 __ testl(reg1, reg1);
2196 } else {
2197 __ cmpl(reg1, i);
2198 }
2199 } else if (c->type() == T_METADATA) {
2200 // All we need for now is a comparison with null for equality.
2201 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2202 Metadata* m = c->as_metadata();
2203 if (m == nullptr) {
2204 __ testptr(reg1, reg1);
2205 } else {
2206 ShouldNotReachHere();
2207 }
2208 } else if (is_reference_type(c->type())) {
2209 // In 64bit oops are single register
2210 jobject o = c->as_jobject();
2211 if (o == nullptr) {
2212 __ testptr(reg1, reg1);
2213 } else {
2214 __ cmpoop(reg1, o, rscratch1);
2215 }
2216 } else {
2217 fatal("unexpected type: %s", basictype_to_str(c->type()));
2218 }
2219 // cpu register - address
2220 } else if (opr2->is_address()) {
2221 if (op->info() != nullptr) {
2222 add_debug_info_for_null_check_here(op->info());
2223 }
2224 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2225 } else {
2226 ShouldNotReachHere();
2227 }
2228
2229 } else if(opr1->is_double_cpu()) {
2230 Register xlo = opr1->as_register_lo();
2231 Register xhi = opr1->as_register_hi();
2232 if (opr2->is_double_cpu()) {
2233 __ cmpptr(xlo, opr2->as_register_lo());
2234 } else if (opr2->is_constant()) {
2235 // cpu register - constant 0
2236 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2237 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2238 } else {
2239 ShouldNotReachHere();
2240 }
2241
2242 } else if (opr1->is_single_xmm()) {
2243 XMMRegister reg1 = opr1->as_xmm_float_reg();
2244 if (opr2->is_single_xmm()) {
2245 // xmm register - xmm register
2246 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2247 } else if (opr2->is_stack()) {
2248 // xmm register - stack
2249 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2250 } else if (opr2->is_constant()) {
2251 // xmm register - constant
2252 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2253 } else if (opr2->is_address()) {
2254 // xmm register - address
2255 if (op->info() != nullptr) {
2256 add_debug_info_for_null_check_here(op->info());
2257 }
2258 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2259 } else {
2260 ShouldNotReachHere();
2261 }
2262
2263 } else if (opr1->is_double_xmm()) {
2264 XMMRegister reg1 = opr1->as_xmm_double_reg();
2265 if (opr2->is_double_xmm()) {
2266 // xmm register - xmm register
2267 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2268 } else if (opr2->is_stack()) {
2269 // xmm register - stack
2270 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2271 } else if (opr2->is_constant()) {
2272 // xmm register - constant
2273 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2274 } else if (opr2->is_address()) {
2275 // xmm register - address
2276 if (op->info() != nullptr) {
2277 add_debug_info_for_null_check_here(op->info());
2278 }
2279 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2280 } else {
2281 ShouldNotReachHere();
2282 }
2283
2284 } else if (opr1->is_address() && opr2->is_constant()) {
2285 LIR_Const* c = opr2->as_constant_ptr();
2286 if (is_reference_type(c->type())) {
2287 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2288 __ movoop(rscratch1, c->as_jobject());
2289 }
2290 if (op->info() != nullptr) {
2291 add_debug_info_for_null_check_here(op->info());
2292 }
2293 // special case: address - constant
2294 LIR_Address* addr = opr1->as_address_ptr();
2295 if (c->type() == T_INT) {
2296 __ cmpl(as_Address(addr), c->as_jint());
2297 } else if (is_reference_type(c->type())) {
2298 // %%% Make this explode if addr isn't reachable until we figure out a
2299 // better strategy by giving noreg as the temp for as_Address
2300 __ cmpoop(rscratch1, as_Address(addr, noreg));
2301 } else {
2302 ShouldNotReachHere();
2303 }
2304
2305 } else {
2306 ShouldNotReachHere();
2307 }
2308 }
2309
2310 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2311 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2312 if (left->is_single_xmm()) {
2313 assert(right->is_single_xmm(), "must match");
2314 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2315 } else if (left->is_double_xmm()) {
2316 assert(right->is_double_xmm(), "must match");
2317 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2318
2319 } else {
2320 ShouldNotReachHere();
2321 }
2322 } else {
2323 assert(code == lir_cmp_l2i, "check");
2324 Label done;
2325 Register dest = dst->as_register();
2326 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2327 __ movl(dest, -1);
2328 __ jccb(Assembler::less, done);
2329 __ setb(Assembler::notZero, dest);
2330 __ movzbl(dest, dest);
2331 __ bind(done);
2332 }
2333 }
2334
2335
2336 void LIR_Assembler::align_call(LIR_Code code) {
2337 // make sure that the displacement word of the call ends up word aligned
2338 int offset = __ offset();
2339 switch (code) {
2340 case lir_static_call:
2341 case lir_optvirtual_call:
2342 case lir_dynamic_call:
2343 offset += NativeCall::displacement_offset;
2344 break;
2345 case lir_icvirtual_call:
2346 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2347 break;
2348 default: ShouldNotReachHere();
2349 }
2350 __ align(BytesPerWord, offset);
2351 }
2352
2353
2354 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2355 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2356 "must be aligned");
2357 __ call(AddressLiteral(op->addr(), rtype));
2358 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2359 __ post_call_nop();
2360 }
2361
2362
2363 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2364 __ ic_call(op->addr());
2365 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2366 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2367 "must be aligned");
2368 __ post_call_nop();
2369 }
2370
2371
2372 void LIR_Assembler::emit_static_call_stub() {
2373 address call_pc = __ pc();
2374 address stub = __ start_a_stub(call_stub_size());
2375 if (stub == nullptr) {
2376 bailout("static call stub overflow");
2377 return;
2378 }
2379
2380 int start = __ offset();
2381
2382 // make sure that the displacement word of the call ends up word aligned
2383 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2384 __ relocate(static_stub_Relocation::spec(call_pc));
2385 __ mov_metadata(rbx, (Metadata*)nullptr);
2386 // must be set to -1 at code generation time
2387 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2388 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2389 __ jump(RuntimeAddress(__ pc()));
2390
2391 assert(__ offset() - start <= call_stub_size(), "stub too big");
2392 __ end_a_stub();
2393 }
2394
2395
2396 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2397 assert(exceptionOop->as_register() == rax, "must match");
2398 assert(exceptionPC->as_register() == rdx, "must match");
2399
2400 // exception object is not added to oop map by LinearScan
2401 // (LinearScan assumes that no oops are in fixed registers)
2402 info->add_register_oop(exceptionOop);
2403 StubId unwind_id;
2404
2405 // get current pc information
2406 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2407 int pc_for_athrow_offset = __ offset();
2408 InternalAddress pc_for_athrow(__ pc());
2409 __ lea(exceptionPC->as_register(), pc_for_athrow);
2410 add_call_info(pc_for_athrow_offset, info); // for exception handler
2411
2412 __ verify_not_null_oop(rax);
2413 // search an exception handler (rax: exception oop, rdx: throwing pc)
2414 if (compilation()->has_fpu_code()) {
2415 unwind_id = StubId::c1_handle_exception_id;
2416 } else {
2417 unwind_id = StubId::c1_handle_exception_nofpu_id;
2418 }
2419 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2420
2421 // enough room for two byte trap
2422 __ nop();
2423 }
2424
2425
2426 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2427 assert(exceptionOop->as_register() == rax, "must match");
2428
2429 __ jmp(_unwind_handler_entry);
2430 }
2431
2432
2433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2434
2435 // optimized version for linear scan:
2436 // * count must be already in ECX (guaranteed by LinearScan)
2437 // * left and dest must be equal
2438 // * tmp must be unused
2439 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2440 assert(left == dest, "left and dest must be equal");
2441 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2442
2443 if (left->is_single_cpu()) {
2444 Register value = left->as_register();
2445 assert(value != SHIFT_count, "left cannot be ECX");
2446
2447 switch (code) {
2448 case lir_shl: __ shll(value); break;
2449 case lir_shr: __ sarl(value); break;
2450 case lir_ushr: __ shrl(value); break;
2451 default: ShouldNotReachHere();
2452 }
2453 } else if (left->is_double_cpu()) {
2454 Register lo = left->as_register_lo();
2455 Register hi = left->as_register_hi();
2456 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2457 switch (code) {
2458 case lir_shl: __ shlptr(lo); break;
2459 case lir_shr: __ sarptr(lo); break;
2460 case lir_ushr: __ shrptr(lo); break;
2461 default: ShouldNotReachHere();
2462 }
2463 } else {
2464 ShouldNotReachHere();
2465 }
2466 }
2467
2468
2469 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2470 if (dest->is_single_cpu()) {
2471 // first move left into dest so that left is not destroyed by the shift
2472 Register value = dest->as_register();
2473 count = count & 0x1F; // Java spec
2474
2475 move_regs(left->as_register(), value);
2476 switch (code) {
2477 case lir_shl: __ shll(value, count); break;
2478 case lir_shr: __ sarl(value, count); break;
2479 case lir_ushr: __ shrl(value, count); break;
2480 default: ShouldNotReachHere();
2481 }
2482 } else if (dest->is_double_cpu()) {
2483 // first move left into dest so that left is not destroyed by the shift
2484 Register value = dest->as_register_lo();
2485 count = count & 0x1F; // Java spec
2486
2487 move_regs(left->as_register_lo(), value);
2488 switch (code) {
2489 case lir_shl: __ shlptr(value, count); break;
2490 case lir_shr: __ sarptr(value, count); break;
2491 case lir_ushr: __ shrptr(value, count); break;
2492 default: ShouldNotReachHere();
2493 }
2494 } else {
2495 ShouldNotReachHere();
2496 }
2497 }
2498
2499
2500 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2501 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2505 }
2506
2507
2508 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2509 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2513 }
2514
2515
2516 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2517 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2521 }
2522
2523
2524 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2525 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2526 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2527 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2528 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2529 }
2530
2531
2532 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2533 if (null_check) {
2534 __ testptr(obj, obj);
2535 __ jcc(Assembler::zero, *slow_path->entry());
2536 }
2537 if (is_dest) {
2538 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2539 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2540 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2541 } else {
2542 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2543 }
2544 }
2545
2546
2547 // This code replaces a call to arraycopy; no exception may
2548 // be thrown in this code, they must be thrown in the System.arraycopy
2549 // activation frame; we could save some checks if this would not be the case
2550 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2551 ciArrayKlass* default_type = op->expected_type();
2552 Register src = op->src()->as_register();
2553 Register dst = op->dst()->as_register();
2554 Register src_pos = op->src_pos()->as_register();
2555 Register dst_pos = op->dst_pos()->as_register();
2556 Register length = op->length()->as_register();
2557 Register tmp = op->tmp()->as_register();
2558 Register tmp_load_klass = rscratch1;
2559 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2560
2561 CodeStub* stub = op->stub();
2562 int flags = op->flags();
2563 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2564 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2565
2566 if (flags & LIR_OpArrayCopy::always_slow_path) {
2567 __ jmp(*stub->entry());
2568 __ bind(*stub->continuation());
2569 return;
2570 }
2571
2572 // if we don't know anything, just go through the generic arraycopy
2573 if (default_type == nullptr) {
2574 // save outgoing arguments on stack in case call to System.arraycopy is needed
2575 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2576 // for interpreter calling conventions. Now we have to do it in new style conventions.
2577 // For the moment until C1 gets the new register allocator I just force all the
2578 // args to the right place (except the register args) and then on the back side
2579 // reload the register args properly if we go slow path. Yuck
2580
2581 // These are proper for the calling convention
2582 store_parameter(length, 2);
2583 store_parameter(dst_pos, 1);
2584 store_parameter(dst, 0);
2585
2586 // these are just temporary placements until we need to reload
2587 store_parameter(src_pos, 3);
2588 store_parameter(src, 4);
2589
2590 address copyfunc_addr = StubRoutines::generic_arraycopy();
2591 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2592
2593 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2594 // The arguments are in java calling convention so we can trivially shift them to C
2595 // convention
2596 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2597 __ mov(c_rarg0, j_rarg0);
2598 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2599 __ mov(c_rarg1, j_rarg1);
2600 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2601 __ mov(c_rarg2, j_rarg2);
2602 assert_different_registers(c_rarg3, j_rarg4);
2603 __ mov(c_rarg3, j_rarg3);
2604 #ifdef _WIN64
2605 // Allocate abi space for args but be sure to keep stack aligned
2606 __ subptr(rsp, 6*wordSize);
2607 store_parameter(j_rarg4, 4);
2608 #ifndef PRODUCT
2609 if (PrintC1Statistics) {
2610 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2611 }
2612 #endif
2613 __ call(RuntimeAddress(copyfunc_addr));
2614 __ addptr(rsp, 6*wordSize);
2615 #else
2616 __ mov(c_rarg4, j_rarg4);
2617 #ifndef PRODUCT
2618 if (PrintC1Statistics) {
2619 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2620 }
2621 #endif
2622 __ call(RuntimeAddress(copyfunc_addr));
2623 #endif // _WIN64
2624
2625 __ testl(rax, rax);
2626 __ jcc(Assembler::equal, *stub->continuation());
2627
2628 __ mov(tmp, rax);
2629 __ xorl(tmp, -1);
2630
2631 // Reload values from the stack so they are where the stub
2632 // expects them.
2633 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2634 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2635 __ movptr (length, Address(rsp, 2*BytesPerWord));
2636 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2637 __ movptr (src, Address(rsp, 4*BytesPerWord));
2638
2639 __ subl(length, tmp);
2640 __ addl(src_pos, tmp);
2641 __ addl(dst_pos, tmp);
2642 __ jmp(*stub->entry());
2643
2644 __ bind(*stub->continuation());
2645 return;
2646 }
2647
2648 // Handle inline type arrays
2649 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2650 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2651 }
2652 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2653 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2654 }
2655
2656 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2657
2658 int elem_size = type2aelembytes(basic_type);
2659 Address::ScaleFactor scale;
2660
2661 switch (elem_size) {
2662 case 1 :
2663 scale = Address::times_1;
2664 break;
2665 case 2 :
2666 scale = Address::times_2;
2667 break;
2668 case 4 :
2669 scale = Address::times_4;
2670 break;
2671 case 8 :
2672 scale = Address::times_8;
2673 break;
2674 default:
2675 scale = Address::no_scale;
2676 ShouldNotReachHere();
2677 }
2678
2679 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2680 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2681
2682 // length and pos's are all sign extended at this point on 64bit
2683
2684 // test for null
2685 if (flags & LIR_OpArrayCopy::src_null_check) {
2686 __ testptr(src, src);
2687 __ jcc(Assembler::zero, *stub->entry());
2688 }
2689 if (flags & LIR_OpArrayCopy::dst_null_check) {
2690 __ testptr(dst, dst);
2691 __ jcc(Assembler::zero, *stub->entry());
2692 }
2693
2694 // If the compiler was not able to prove that exact type of the source or the destination
2695 // of the arraycopy is an array type, check at runtime if the source or the destination is
2696 // an instance type.
2697 if (flags & LIR_OpArrayCopy::type_check) {
2698 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2699 __ load_klass(tmp, dst, tmp_load_klass);
2700 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2701 __ jcc(Assembler::greaterEqual, *stub->entry());
2702 }
2703
2704 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2705 __ load_klass(tmp, src, tmp_load_klass);
2706 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2707 __ jcc(Assembler::greaterEqual, *stub->entry());
2708 }
2709 }
2710
2711 // check if negative
2712 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2713 __ testl(src_pos, src_pos);
2714 __ jcc(Assembler::less, *stub->entry());
2715 }
2716 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2717 __ testl(dst_pos, dst_pos);
2718 __ jcc(Assembler::less, *stub->entry());
2719 }
2720
2721 if (flags & LIR_OpArrayCopy::src_range_check) {
2722 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2723 __ cmpl(tmp, src_length_addr);
2724 __ jcc(Assembler::above, *stub->entry());
2725 }
2726 if (flags & LIR_OpArrayCopy::dst_range_check) {
2727 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2728 __ cmpl(tmp, dst_length_addr);
2729 __ jcc(Assembler::above, *stub->entry());
2730 }
2731
2732 if (flags & LIR_OpArrayCopy::length_positive_check) {
2733 __ testl(length, length);
2734 __ jcc(Assembler::less, *stub->entry());
2735 }
2736
2737 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2738 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2739
2740 if (flags & LIR_OpArrayCopy::type_check) {
2741 // We don't know the array types are compatible
2742 if (basic_type != T_OBJECT) {
2743 // Simple test for basic type arrays
2744 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2745 __ jcc(Assembler::notEqual, *stub->entry());
2746 } else {
2747 // For object arrays, if src is a sub class of dst then we can
2748 // safely do the copy.
2749 Label cont, slow;
2750
2751 __ push_ppx(src);
2752 __ push_ppx(dst);
2753
2754 __ load_klass(src, src, tmp_load_klass);
2755 __ load_klass(dst, dst, tmp_load_klass);
2756
2757 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2758
2759 __ push_ppx(src);
2760 __ push_ppx(dst);
2761 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2762 __ pop_ppx(dst);
2763 __ pop_ppx(src);
2764
2765 __ testl(src, src);
2766 __ jcc(Assembler::notEqual, cont);
2767
2768 __ bind(slow);
2769 __ pop_ppx(dst);
2770 __ pop_ppx(src);
2771
2772 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2773 if (copyfunc_addr != nullptr) { // use stub if available
2774 // src is not a sub class of dst so we have to do a
2775 // per-element check.
2776
2777 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2778 if ((flags & mask) != mask) {
2779 // Check that at least both of them object arrays.
2780 assert(flags & mask, "one of the two should be known to be an object array");
2781
2782 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2783 __ load_klass(tmp, src, tmp_load_klass);
2784 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2785 __ load_klass(tmp, dst, tmp_load_klass);
2786 }
2787 int lh_offset = in_bytes(Klass::layout_helper_offset());
2788 Address klass_lh_addr(tmp, lh_offset);
2789 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2790 __ cmpl(klass_lh_addr, objArray_lh);
2791 __ jcc(Assembler::notEqual, *stub->entry());
2792 }
2793
2794 // Spill because stubs can use any register they like and it's
2795 // easier to restore just those that we care about.
2796 store_parameter(dst, 0);
2797 store_parameter(dst_pos, 1);
2798 store_parameter(length, 2);
2799 store_parameter(src_pos, 3);
2800 store_parameter(src, 4);
2801
2802 __ movl2ptr(length, length); //higher 32bits must be null
2803
2804 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2805 assert_different_registers(c_rarg0, dst, dst_pos, length);
2806 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2807 assert_different_registers(c_rarg1, dst, length);
2808
2809 __ mov(c_rarg2, length);
2810 assert_different_registers(c_rarg2, dst);
2811
2812 #ifdef _WIN64
2813 // Allocate abi space for args but be sure to keep stack aligned
2814 __ subptr(rsp, 6*wordSize);
2815 __ load_klass(c_rarg3, dst, tmp_load_klass);
2816 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2817 store_parameter(c_rarg3, 4);
2818 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2819 __ call(RuntimeAddress(copyfunc_addr));
2820 __ addptr(rsp, 6*wordSize);
2821 #else
2822 __ load_klass(c_rarg4, dst, tmp_load_klass);
2823 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2824 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2825 __ call(RuntimeAddress(copyfunc_addr));
2826 #endif
2827
2828 #ifndef PRODUCT
2829 if (PrintC1Statistics) {
2830 Label failed;
2831 __ testl(rax, rax);
2832 __ jcc(Assembler::notZero, failed);
2833 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2834 __ bind(failed);
2835 }
2836 #endif
2837
2838 __ testl(rax, rax);
2839 __ jcc(Assembler::zero, *stub->continuation());
2840
2841 #ifndef PRODUCT
2842 if (PrintC1Statistics) {
2843 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2844 }
2845 #endif
2846
2847 __ mov(tmp, rax);
2848
2849 __ xorl(tmp, -1);
2850
2851 // Restore previously spilled arguments
2852 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2853 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2854 __ movptr (length, Address(rsp, 2*BytesPerWord));
2855 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2856 __ movptr (src, Address(rsp, 4*BytesPerWord));
2857
2858
2859 __ subl(length, tmp);
2860 __ addl(src_pos, tmp);
2861 __ addl(dst_pos, tmp);
2862 }
2863
2864 __ jmp(*stub->entry());
2865
2866 __ bind(cont);
2867 __ pop(dst);
2868 __ pop(src);
2869 }
2870 }
2871
2872 #ifdef ASSERT
2873 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2874 // Sanity check the known type with the incoming class. For the
2875 // primitive case the types must match exactly with src.klass and
2876 // dst.klass each exactly matching the default type. For the
2877 // object array case, if no type check is needed then either the
2878 // dst type is exactly the expected type and the src type is a
2879 // subtype which we can't check or src is the same array as dst
2880 // but not necessarily exactly of type default_type.
2881 Label known_ok, halt;
2882
2883 __ mov_metadata(tmp, default_type->constant_encoding());
2884 if (UseCompressedClassPointers) {
2885 __ encode_klass_not_null(tmp, rscratch1);
2886 }
2887
2888 if (basic_type != T_OBJECT) {
2889 __ cmp_klass(tmp, dst, tmp2);
2890 __ jcc(Assembler::notEqual, halt);
2891 __ cmp_klass(tmp, src, tmp2);
2892 __ jcc(Assembler::equal, known_ok);
2893 } else {
2894 __ cmp_klass(tmp, dst, tmp2);
2895 __ jcc(Assembler::equal, known_ok);
2896 __ cmpptr(src, dst);
2897 __ jcc(Assembler::equal, known_ok);
2898 }
2899 __ bind(halt);
2900 __ stop("incorrect type information in arraycopy");
2901 __ bind(known_ok);
2902 }
2903 #endif
2904
2905 #ifndef PRODUCT
2906 if (PrintC1Statistics) {
2907 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2908 }
2909 #endif
2910
2911 assert_different_registers(c_rarg0, dst, dst_pos, length);
2912 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2913 assert_different_registers(c_rarg1, length);
2914 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2915 __ mov(c_rarg2, length);
2916
2917 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2918 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2919 const char *name;
2920 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2921 __ call_VM_leaf(entry, 0);
2922
2923 if (stub != nullptr) {
2924 __ bind(*stub->continuation());
2925 }
2926 }
2927
2928 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2929 assert(op->crc()->is_single_cpu(), "crc must be register");
2930 assert(op->val()->is_single_cpu(), "byte value must be register");
2931 assert(op->result_opr()->is_single_cpu(), "result must be register");
2932 Register crc = op->crc()->as_register();
2933 Register val = op->val()->as_register();
2934 Register res = op->result_opr()->as_register();
2935
2936 assert_different_registers(val, crc, res);
2937
2938 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2939 __ notl(crc); // ~crc
2940 __ update_byte_crc32(crc, val, res);
2941 __ notl(crc); // ~crc
2942 __ mov(res, crc);
2943 }
2944
2945 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2946 Register obj = op->obj_opr()->as_register(); // may not be an oop
2947 Register hdr = op->hdr_opr()->as_register();
2948 Register lock = op->lock_opr()->as_register();
2949 if (op->code() == lir_lock) {
2950 Register tmp = op->scratch_opr()->as_register();
2951 // add debug info for NullPointerException only if one is possible
2952 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2953 if (op->info() != nullptr) {
2954 add_debug_info_for_null_check(null_check_offset, op->info());
2955 }
2956 // done
2957 } else if (op->code() == lir_unlock) {
2958 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2959 } else {
2960 Unimplemented();
2961 }
2962 __ bind(*op->stub()->continuation());
2963 }
2964
2965 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2966 Register obj = op->obj()->as_pointer_register();
2967 Register result = op->result_opr()->as_pointer_register();
2968
2969 CodeEmitInfo* info = op->info();
2970 if (info != nullptr) {
2971 add_debug_info_for_null_check_here(info);
2972 }
2973
2974 __ load_klass(result, obj, rscratch1);
2975 }
2976
2977 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2978 ciMethod* method = op->profiled_method();
2979 int bci = op->profiled_bci();
2980 ciMethod* callee = op->profiled_callee();
2981 Register tmp_load_klass = rscratch1;
2982
2983 // Update counter for all call types
2984 ciMethodData* md = method->method_data_or_null();
2985 assert(md != nullptr, "Sanity");
2986 ciProfileData* data = md->bci_to_data(bci);
2987 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2988 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2989 Register mdo = op->mdo()->as_register();
2990 __ mov_metadata(mdo, md->constant_encoding());
2991 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2992 // Perform additional virtual call profiling for invokevirtual and
2993 // invokeinterface bytecodes
2994 if (op->should_profile_receiver_type()) {
2995 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2996 Register recv = op->recv()->as_register();
2997 assert_different_registers(mdo, recv);
2998 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2999 ciKlass* known_klass = op->known_holder();
3000 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
3001 // We know the type that will be seen at this call site; we can
3002 // statically update the MethodData* rather than needing to do
3003 // dynamic tests on the receiver type
3004
3005 // NOTE: we should probably put a lock around this search to
3006 // avoid collisions by concurrent compilations
3007 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3008 uint i;
3009 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3010 ciKlass* receiver = vc_data->receiver(i);
3011 if (known_klass->equals(receiver)) {
3012 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3013 __ addptr(data_addr, DataLayout::counter_increment);
3014 return;
3015 }
3016 }
3017
3018 // Receiver type not found in profile data; select an empty slot
3019
3020 // Note that this is less efficient than it should be because it
3021 // always does a write to the receiver part of the
3022 // VirtualCallData rather than just the first time
3023 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3024 ciKlass* receiver = vc_data->receiver(i);
3025 if (receiver == nullptr) {
3026 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3027 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3028 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3029 __ addptr(data_addr, DataLayout::counter_increment);
3030 return;
3031 }
3032 }
3033 } else {
3034 __ load_klass(recv, recv, tmp_load_klass);
3035 Label update_done;
3036 type_profile_helper(mdo, md, data, recv, &update_done);
3037 // Receiver did not match any saved receiver and there is no empty row for it.
3038 // Increment total counter to indicate polymorphic case.
3039 __ addptr(counter_addr, DataLayout::counter_increment);
3040
3041 __ bind(update_done);
3042 }
3043 } else {
3044 // Static call
3045 __ addptr(counter_addr, DataLayout::counter_increment);
3046 }
3047 }
3048
3049 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3050 Register obj = op->obj()->as_register();
3051 Register tmp = op->tmp()->as_pointer_register();
3052 Register tmp_load_klass = rscratch1;
3053 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3054 ciKlass* exact_klass = op->exact_klass();
3055 intptr_t current_klass = op->current_klass();
3056 bool not_null = op->not_null();
3057 bool no_conflict = op->no_conflict();
3058
3059 Label update, next, none;
3060
3061 bool do_null = !not_null;
3062 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3063 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3064
3065 assert(do_null || do_update, "why are we here?");
3066 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3067
3068 __ verify_oop(obj);
3069
3070 #ifdef ASSERT
3071 if (obj == tmp) {
3072 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3073 } else {
3074 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3075 }
3076 #endif
3077 if (do_null) {
3078 __ testptr(obj, obj);
3079 __ jccb(Assembler::notZero, update);
3080 if (!TypeEntries::was_null_seen(current_klass)) {
3081 __ testptr(mdo_addr, TypeEntries::null_seen);
3082 #ifndef ASSERT
3083 __ jccb(Assembler::notZero, next); // already set
3084 #else
3085 __ jcc(Assembler::notZero, next); // already set
3086 #endif
3087 // atomic update to prevent overwriting Klass* with 0
3088 __ lock();
3089 __ orptr(mdo_addr, TypeEntries::null_seen);
3090 }
3091 if (do_update) {
3092 #ifndef ASSERT
3093 __ jmpb(next);
3094 }
3095 #else
3096 __ jmp(next);
3097 }
3098 } else {
3099 __ testptr(obj, obj);
3100 __ jcc(Assembler::notZero, update);
3101 __ stop("unexpected null obj");
3102 #endif
3103 }
3104
3105 __ bind(update);
3106
3107 if (do_update) {
3108 #ifdef ASSERT
3109 if (exact_klass != nullptr) {
3110 Label ok;
3111 __ load_klass(tmp, obj, tmp_load_klass);
3112 __ push_ppx(tmp);
3113 __ mov_metadata(tmp, exact_klass->constant_encoding());
3114 __ cmpptr(tmp, Address(rsp, 0));
3115 __ jcc(Assembler::equal, ok);
3116 __ stop("exact klass and actual klass differ");
3117 __ bind(ok);
3118 __ pop_ppx(tmp);
3119 }
3120 #endif
3121 if (!no_conflict) {
3122 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3123 if (exact_klass != nullptr) {
3124 __ mov_metadata(tmp, exact_klass->constant_encoding());
3125 } else {
3126 __ load_klass(tmp, obj, tmp_load_klass);
3127 }
3128 __ mov(rscratch1, tmp); // save original value before XOR
3129 __ xorptr(tmp, mdo_addr);
3130 __ testptr(tmp, TypeEntries::type_klass_mask);
3131 // klass seen before, nothing to do. The unknown bit may have been
3132 // set already but no need to check.
3133 __ jccb(Assembler::zero, next);
3134
3135 __ testptr(tmp, TypeEntries::type_unknown);
3136 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3137
3138 if (TypeEntries::is_type_none(current_klass)) {
3139 __ testptr(mdo_addr, TypeEntries::type_mask);
3140 __ jccb(Assembler::zero, none);
3141 // There is a chance that the checks above (re-reading profiling
3142 // data from memory) fail if another thread has just set the
3143 // profiling to this obj's klass
3144 __ mov(tmp, rscratch1); // get back original value before XOR
3145 __ xorptr(tmp, mdo_addr);
3146 __ testptr(tmp, TypeEntries::type_klass_mask);
3147 __ jccb(Assembler::zero, next);
3148 }
3149 } else {
3150 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3151 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3152
3153 __ testptr(mdo_addr, TypeEntries::type_unknown);
3154 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3155 }
3156
3157 // different than before. Cannot keep accurate profile.
3158 __ orptr(mdo_addr, TypeEntries::type_unknown);
3159
3160 if (TypeEntries::is_type_none(current_klass)) {
3161 __ jmpb(next);
3162
3163 __ bind(none);
3164 // first time here. Set profile type.
3165 __ movptr(mdo_addr, tmp);
3166 #ifdef ASSERT
3167 __ andptr(tmp, TypeEntries::type_klass_mask);
3168 __ verify_klass_ptr(tmp);
3169 #endif
3170 }
3171 } else {
3172 // There's a single possible klass at this profile point
3173 assert(exact_klass != nullptr, "should be");
3174 if (TypeEntries::is_type_none(current_klass)) {
3175 __ mov_metadata(tmp, exact_klass->constant_encoding());
3176 __ xorptr(tmp, mdo_addr);
3177 __ testptr(tmp, TypeEntries::type_klass_mask);
3178 #ifdef ASSERT
3179 __ jcc(Assembler::zero, next);
3180
3181 {
3182 Label ok;
3183 __ push_ppx(tmp);
3184 __ testptr(mdo_addr, TypeEntries::type_mask);
3185 __ jcc(Assembler::zero, ok);
3186 // may have been set by another thread
3187 __ mov_metadata(tmp, exact_klass->constant_encoding());
3188 __ xorptr(tmp, mdo_addr);
3189 __ testptr(tmp, TypeEntries::type_mask);
3190 __ jcc(Assembler::zero, ok);
3191
3192 __ stop("unexpected profiling mismatch");
3193 __ bind(ok);
3194 __ pop_ppx(tmp);
3195 }
3196 #else
3197 __ jccb(Assembler::zero, next);
3198 #endif
3199 // first time here. Set profile type.
3200 __ movptr(mdo_addr, tmp);
3201 #ifdef ASSERT
3202 __ andptr(tmp, TypeEntries::type_klass_mask);
3203 __ verify_klass_ptr(tmp);
3204 #endif
3205 } else {
3206 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3207 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3208
3209 __ testptr(mdo_addr, TypeEntries::type_unknown);
3210 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3211
3212 __ orptr(mdo_addr, TypeEntries::type_unknown);
3213 }
3214 }
3215 }
3216 __ bind(next);
3217 }
3218
3219 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3220 Register obj = op->obj()->as_register();
3221 Register tmp = op->tmp()->as_pointer_register();
3222 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3223 bool not_null = op->not_null();
3224 int flag = op->flag();
3225
3226 Label not_inline_type;
3227 if (!not_null) {
3228 __ testptr(obj, obj);
3229 __ jccb(Assembler::zero, not_inline_type);
3230 }
3231
3232 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3233
3234 __ orb(mdo_addr, flag);
3235
3236 __ bind(not_inline_type);
3237 }
3238
3239
3240 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3241 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3242 }
3243
3244
3245 void LIR_Assembler::align_backward_branch_target() {
3246 __ align(BytesPerWord);
3247 }
3248
3249
3250 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3251 if (left->is_single_cpu()) {
3252 __ negl(left->as_register());
3253 move_regs(left->as_register(), dest->as_register());
3254
3255 } else if (left->is_double_cpu()) {
3256 Register lo = left->as_register_lo();
3257 Register dst = dest->as_register_lo();
3258 __ movptr(dst, lo);
3259 __ negptr(dst);
3260
3261 } else if (dest->is_single_xmm()) {
3262 assert(!tmp->is_valid(), "do not need temporary");
3263 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3264 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3265 }
3266 __ xorps(dest->as_xmm_float_reg(),
3267 ExternalAddress((address)float_signflip_pool),
3268 rscratch1);
3269 } else if (dest->is_double_xmm()) {
3270 assert(!tmp->is_valid(), "do not need temporary");
3271 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3272 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3273 }
3274 __ xorpd(dest->as_xmm_double_reg(),
3275 ExternalAddress((address)double_signflip_pool),
3276 rscratch1);
3277 } else {
3278 ShouldNotReachHere();
3279 }
3280 }
3281
3282
3283 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3284 assert(src->is_address(), "must be an address");
3285 assert(dest->is_register(), "must be a register");
3286
3287 PatchingStub* patch = nullptr;
3288 if (patch_code != lir_patch_none) {
3289 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3290 }
3291
3292 Register reg = dest->as_pointer_register();
3293 LIR_Address* addr = src->as_address_ptr();
3294 __ lea(reg, as_Address(addr));
3295
3296 if (patch != nullptr) {
3297 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3298 }
3299 }
3300
3301
3302
3303 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3304 assert(!tmp->is_valid(), "don't need temporary");
3305 __ call(RuntimeAddress(dest));
3306 if (info != nullptr) {
3307 add_call_info_here(info);
3308 }
3309 __ post_call_nop();
3310 }
3311
3312
3313 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3314 assert(type == T_LONG, "only for volatile long fields");
3315
3316 if (info != nullptr) {
3317 add_debug_info_for_null_check_here(info);
3318 }
3319
3320 if (src->is_double_xmm()) {
3321 if (dest->is_double_cpu()) {
3322 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3323 } else if (dest->is_double_stack()) {
3324 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3325 } else if (dest->is_address()) {
3326 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3327 } else {
3328 ShouldNotReachHere();
3329 }
3330
3331 } else if (dest->is_double_xmm()) {
3332 if (src->is_double_stack()) {
3333 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3334 } else if (src->is_address()) {
3335 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3336 } else {
3337 ShouldNotReachHere();
3338 }
3339
3340 } else {
3341 ShouldNotReachHere();
3342 }
3343 }
3344
3345 #ifdef ASSERT
3346 // emit run-time assertion
3347 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3348 assert(op->code() == lir_assert, "must be");
3349
3350 if (op->in_opr1()->is_valid()) {
3351 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3352 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3353 } else {
3354 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3355 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3356 }
3357
3358 Label ok;
3359 if (op->condition() != lir_cond_always) {
3360 Assembler::Condition acond = Assembler::zero;
3361 switch (op->condition()) {
3362 case lir_cond_equal: acond = Assembler::equal; break;
3363 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3364 case lir_cond_less: acond = Assembler::less; break;
3365 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3366 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3367 case lir_cond_greater: acond = Assembler::greater; break;
3368 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3369 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3370 default: ShouldNotReachHere();
3371 }
3372 __ jcc(acond, ok);
3373 }
3374 if (op->halt()) {
3375 const char* str = __ code_string(op->msg());
3376 __ stop(str);
3377 } else {
3378 breakpoint();
3379 }
3380 __ bind(ok);
3381 }
3382 #endif
3383
3384 void LIR_Assembler::membar() {
3385 // QQQ sparc TSO uses this,
3386 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3387 }
3388
3389 void LIR_Assembler::membar_acquire() {
3390 // No x86 machines currently require load fences
3391 }
3392
3393 void LIR_Assembler::membar_release() {
3394 // No x86 machines currently require store fences
3395 }
3396
3397 void LIR_Assembler::membar_loadload() {
3398 // no-op
3399 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3400 }
3401
3402 void LIR_Assembler::membar_storestore() {
3403 // no-op
3404 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3405 }
3406
3407 void LIR_Assembler::membar_loadstore() {
3408 // no-op
3409 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3410 }
3411
3412 void LIR_Assembler::membar_storeload() {
3413 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3414 }
3415
3416 void LIR_Assembler::on_spin_wait() {
3417 __ pause ();
3418 }
3419
3420 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3421 assert(result_reg->is_register(), "check");
3422 __ mov(result_reg->as_register(), r15_thread);
3423 }
3424
3425 void LIR_Assembler::check_orig_pc() {
3426 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3427 }
3428
3429 void LIR_Assembler::peephole(LIR_List*) {
3430 // do nothing for now
3431 }
3432
3433 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3434 assert(data == dest, "xchg/xadd uses only 2 operands");
3435
3436 if (data->type() == T_INT) {
3437 if (code == lir_xadd) {
3438 __ lock();
3439 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3440 } else {
3441 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3442 }
3443 } else if (data->is_oop()) {
3444 assert (code == lir_xchg, "xadd for oops");
3445 Register obj = data->as_register();
3446 if (UseCompressedOops) {
3447 __ encode_heap_oop(obj);
3448 __ xchgl(obj, as_Address(src->as_address_ptr()));
3449 __ decode_heap_oop(obj);
3450 } else {
3451 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3452 }
3453 } else if (data->type() == T_LONG) {
3454 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3455 if (code == lir_xadd) {
3456 __ lock();
3457 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3458 } else {
3459 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3460 }
3461 } else {
3462 ShouldNotReachHere();
3463 }
3464 }
3465
3466 #undef __