16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
60 operand[1] = hi;
61 return operand;
62 }
63
64 // Buffer for 128-bits masks used by SSE instructions.
65 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
66
67 // Static initialization during VM startup.
68 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
69 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
70 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
71 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
72
73
74 NEEDS_CLEANUP // remove this definitions ?
75 const Register SYNC_header = rax; // synchronization header
76 const Register SHIFT_count = rcx; // where count for shift operations must be
77
78 #define __ _masm->
79
80
81 static void select_different_registers(Register preserve,
82 Register extra,
83 Register &tmp1,
84 Register &tmp2) {
85 if (tmp1 == preserve) {
86 assert_different_registers(tmp1, tmp2, extra);
87 tmp1 = extra;
88 } else if (tmp2 == preserve) {
89 assert_different_registers(tmp1, tmp2, extra);
90 tmp2 = extra;
91 }
558 assert(src->is_constant(), "should not call otherwise");
559 assert(dest->is_register(), "should not call otherwise");
560 LIR_Const* c = src->as_constant_ptr();
561
562 switch (c->type()) {
563 case T_INT: {
564 assert(patch_code == lir_patch_none, "no patching handled here");
565 __ movl(dest->as_register(), c->as_jint());
566 break;
567 }
568
569 case T_ADDRESS: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 __ movptr(dest->as_register(), c->as_jint());
572 break;
573 }
574
575 case T_LONG: {
576 assert(patch_code == lir_patch_none, "no patching handled here");
577 #ifdef _LP64
578 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
579 #else
580 __ movptr(dest->as_register_lo(), c->as_jint_lo());
581 __ movptr(dest->as_register_hi(), c->as_jint_hi());
582 #endif // _LP64
583 break;
584 }
585
586 case T_OBJECT: {
587 if (patch_code != lir_patch_none) {
588 jobject2reg_with_patching(dest->as_register(), info);
589 } else {
590 __ movoop(dest->as_register(), c->as_jobject());
591 }
592 break;
593 }
594
595 case T_METADATA: {
596 if (patch_code != lir_patch_none) {
597 klass2reg_with_patching(dest->as_register(), info);
2388 #endif // _LP64
2389
2390
2391 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2392 if (value->is_double_xmm()) {
2393 switch(code) {
2394 case lir_abs :
2395 {
2396 #ifdef _LP64
2397 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2398 assert(tmp->is_valid(), "need temporary");
2399 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2400 } else
2401 #endif
2402 {
2403 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2404 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2405 }
2406 assert(!tmp->is_valid(), "do not need temporary");
2407 __ andpd(dest->as_xmm_double_reg(),
2408 ExternalAddress((address)double_signmask_pool),
2409 rscratch1);
2410 }
2411 }
2412 break;
2413
2414 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2415 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2416 default : ShouldNotReachHere();
2417 }
2418
2419 #ifndef _LP64
2420 } else if (value->is_double_fpu()) {
2421 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2422 switch(code) {
2423 case lir_abs : __ fabs() ; break;
2424 case lir_sqrt : __ fsqrt(); break;
2425 default : ShouldNotReachHere();
2426 }
2427 #endif // !_LP64
2428 } else if (code == lir_f2hf) {
3795 move_regs(lo, dest->as_register_lo());
3796 move_regs(hi, dest->as_register_hi());
3797 }
3798 #endif // _LP64
3799
3800 } else if (dest->is_single_xmm()) {
3801 #ifdef _LP64
3802 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3803 assert(tmp->is_valid(), "need temporary");
3804 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3805 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3806 }
3807 else
3808 #endif
3809 {
3810 assert(!tmp->is_valid(), "do not need temporary");
3811 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3812 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3813 }
3814 __ xorps(dest->as_xmm_float_reg(),
3815 ExternalAddress((address)float_signflip_pool),
3816 rscratch1);
3817 }
3818 } else if (dest->is_double_xmm()) {
3819 #ifdef _LP64
3820 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3821 assert(tmp->is_valid(), "need temporary");
3822 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3823 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3824 }
3825 else
3826 #endif
3827 {
3828 assert(!tmp->is_valid(), "do not need temporary");
3829 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3830 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3831 }
3832 __ xorpd(dest->as_xmm_double_reg(),
3833 ExternalAddress((address)double_signflip_pool),
3834 rscratch1);
3835 }
3836 #ifndef _LP64
3837 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3838 assert(left->fpu() == 0, "arg must be on TOS");
3839 assert(dest->fpu() == 0, "dest must be TOS");
3840 __ fchs();
3841 #endif // !_LP64
3842
3843 } else {
3844 ShouldNotReachHere();
3845 }
3846 }
3847
3848
3849 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3850 assert(src->is_address(), "must be an address");
3851 assert(dest->is_register(), "must be a register");
3852
3853 PatchingStub* patch = nullptr;
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciUtilities.hpp"
37 #include "code/SCCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
560 assert(src->is_constant(), "should not call otherwise");
561 assert(dest->is_register(), "should not call otherwise");
562 LIR_Const* c = src->as_constant_ptr();
563
564 switch (c->type()) {
565 case T_INT: {
566 assert(patch_code == lir_patch_none, "no patching handled here");
567 __ movl(dest->as_register(), c->as_jint());
568 break;
569 }
570
571 case T_ADDRESS: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movptr(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_LONG: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 #ifdef _LP64
580 if (SCCache::is_on_for_write()) {
581 // SCA needs relocation info for card table base
582 address b = c->as_pointer();
583 if (is_card_table_address(b)) {
584 __ lea(dest->as_register_lo(), ExternalAddress(b));
585 break;
586 }
587 if (AOTRuntimeConstants::contains(b)) {
588 __ load_aotrc_address(dest->as_register_lo(), b);
589 break;
590 }
591 }
592 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
593 #else
594 __ movptr(dest->as_register_lo(), c->as_jint_lo());
595 __ movptr(dest->as_register_hi(), c->as_jint_hi());
596 #endif // _LP64
597 break;
598 }
599
600 case T_OBJECT: {
601 if (patch_code != lir_patch_none) {
602 jobject2reg_with_patching(dest->as_register(), info);
603 } else {
604 __ movoop(dest->as_register(), c->as_jobject());
605 }
606 break;
607 }
608
609 case T_METADATA: {
610 if (patch_code != lir_patch_none) {
611 klass2reg_with_patching(dest->as_register(), info);
2402 #endif // _LP64
2403
2404
2405 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2406 if (value->is_double_xmm()) {
2407 switch(code) {
2408 case lir_abs :
2409 {
2410 #ifdef _LP64
2411 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2412 assert(tmp->is_valid(), "need temporary");
2413 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2414 } else
2415 #endif
2416 {
2417 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2418 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2419 }
2420 assert(!tmp->is_valid(), "do not need temporary");
2421 __ andpd(dest->as_xmm_double_reg(),
2422 ExternalAddress(LIR_Assembler::double_signmask_pool),
2423 rscratch1);
2424 }
2425 }
2426 break;
2427
2428 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2429 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2430 default : ShouldNotReachHere();
2431 }
2432
2433 #ifndef _LP64
2434 } else if (value->is_double_fpu()) {
2435 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2436 switch(code) {
2437 case lir_abs : __ fabs() ; break;
2438 case lir_sqrt : __ fsqrt(); break;
2439 default : ShouldNotReachHere();
2440 }
2441 #endif // !_LP64
2442 } else if (code == lir_f2hf) {
3809 move_regs(lo, dest->as_register_lo());
3810 move_regs(hi, dest->as_register_hi());
3811 }
3812 #endif // _LP64
3813
3814 } else if (dest->is_single_xmm()) {
3815 #ifdef _LP64
3816 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3817 assert(tmp->is_valid(), "need temporary");
3818 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3819 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3820 }
3821 else
3822 #endif
3823 {
3824 assert(!tmp->is_valid(), "do not need temporary");
3825 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3826 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3827 }
3828 __ xorps(dest->as_xmm_float_reg(),
3829 ExternalAddress(LIR_Assembler::float_signflip_pool),
3830 rscratch1);
3831 }
3832 } else if (dest->is_double_xmm()) {
3833 #ifdef _LP64
3834 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3835 assert(tmp->is_valid(), "need temporary");
3836 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3837 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3838 }
3839 else
3840 #endif
3841 {
3842 assert(!tmp->is_valid(), "do not need temporary");
3843 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3844 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3845 }
3846 __ xorpd(dest->as_xmm_double_reg(),
3847 ExternalAddress(LIR_Assembler::double_signflip_pool),
3848 rscratch1);
3849 }
3850 #ifndef _LP64
3851 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3852 assert(left->fpu() == 0, "arg must be on TOS");
3853 assert(dest->fpu() == 0, "dest must be TOS");
3854 __ fchs();
3855 #endif // !_LP64
3856
3857 } else {
3858 ShouldNotReachHere();
3859 }
3860 }
3861
3862
3863 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3864 assert(src->is_address(), "must be an address");
3865 assert(dest->is_register(), "must be a register");
3866
3867 PatchingStub* patch = nullptr;
|