16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
60 operand[1] = hi;
61 return operand;
62 }
63
64 // Buffer for 128-bits masks used by SSE instructions.
65 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
66
67 // Static initialization during VM startup.
68 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
69 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
70 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
71 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
72
73
74 NEEDS_CLEANUP // remove this definitions ?
75 const Register SYNC_header = rax; // synchronization header
76 const Register SHIFT_count = rcx; // where count for shift operations must be
77
78 #define __ _masm->
79
80
81 static void select_different_registers(Register preserve,
82 Register extra,
83 Register &tmp1,
84 Register &tmp2) {
85 if (tmp1 == preserve) {
86 assert_different_registers(tmp1, tmp2, extra);
87 tmp1 = extra;
88 } else if (tmp2 == preserve) {
89 assert_different_registers(tmp1, tmp2, extra);
90 tmp2 = extra;
91 }
558 assert(src->is_constant(), "should not call otherwise");
559 assert(dest->is_register(), "should not call otherwise");
560 LIR_Const* c = src->as_constant_ptr();
561
562 switch (c->type()) {
563 case T_INT: {
564 assert(patch_code == lir_patch_none, "no patching handled here");
565 __ movl(dest->as_register(), c->as_jint());
566 break;
567 }
568
569 case T_ADDRESS: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 __ movptr(dest->as_register(), c->as_jint());
572 break;
573 }
574
575 case T_LONG: {
576 assert(patch_code == lir_patch_none, "no patching handled here");
577 #ifdef _LP64
578 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
579 #else
580 __ movptr(dest->as_register_lo(), c->as_jint_lo());
581 __ movptr(dest->as_register_hi(), c->as_jint_hi());
582 #endif // _LP64
583 break;
584 }
585
586 case T_OBJECT: {
587 if (patch_code != lir_patch_none) {
588 jobject2reg_with_patching(dest->as_register(), info);
589 } else {
590 __ movoop(dest->as_register(), c->as_jobject());
591 }
592 break;
593 }
594
595 case T_METADATA: {
596 if (patch_code != lir_patch_none) {
597 klass2reg_with_patching(dest->as_register(), info);
2390 #endif // _LP64
2391
2392
2393 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2394 if (value->is_double_xmm()) {
2395 switch(code) {
2396 case lir_abs :
2397 {
2398 #ifdef _LP64
2399 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2400 assert(tmp->is_valid(), "need temporary");
2401 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2402 } else
2403 #endif
2404 {
2405 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2406 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2407 }
2408 assert(!tmp->is_valid(), "do not need temporary");
2409 __ andpd(dest->as_xmm_double_reg(),
2410 ExternalAddress((address)double_signmask_pool),
2411 rscratch1);
2412 }
2413 }
2414 break;
2415
2416 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2417 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2418 default : ShouldNotReachHere();
2419 }
2420
2421 #ifndef _LP64
2422 } else if (value->is_double_fpu()) {
2423 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2424 switch(code) {
2425 case lir_abs : __ fabs() ; break;
2426 case lir_sqrt : __ fsqrt(); break;
2427 default : ShouldNotReachHere();
2428 }
2429 #endif // !_LP64
2430 } else if (code == lir_f2hf) {
3813 move_regs(lo, dest->as_register_lo());
3814 move_regs(hi, dest->as_register_hi());
3815 }
3816 #endif // _LP64
3817
3818 } else if (dest->is_single_xmm()) {
3819 #ifdef _LP64
3820 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3821 assert(tmp->is_valid(), "need temporary");
3822 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3823 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3824 }
3825 else
3826 #endif
3827 {
3828 assert(!tmp->is_valid(), "do not need temporary");
3829 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3830 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3831 }
3832 __ xorps(dest->as_xmm_float_reg(),
3833 ExternalAddress((address)float_signflip_pool),
3834 rscratch1);
3835 }
3836 } else if (dest->is_double_xmm()) {
3837 #ifdef _LP64
3838 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3839 assert(tmp->is_valid(), "need temporary");
3840 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3841 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3842 }
3843 else
3844 #endif
3845 {
3846 assert(!tmp->is_valid(), "do not need temporary");
3847 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3848 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3849 }
3850 __ xorpd(dest->as_xmm_double_reg(),
3851 ExternalAddress((address)double_signflip_pool),
3852 rscratch1);
3853 }
3854 #ifndef _LP64
3855 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3856 assert(left->fpu() == 0, "arg must be on TOS");
3857 assert(dest->fpu() == 0, "dest must be TOS");
3858 __ fchs();
3859 #endif // !_LP64
3860
3861 } else {
3862 ShouldNotReachHere();
3863 }
3864 }
3865
3866
3867 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3868 assert(src->is_address(), "must be an address");
3869 assert(dest->is_register(), "must be a register");
3870
3871 PatchingStub* patch = nullptr;
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciUtilities.hpp"
37 #include "code/SCCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
560 assert(src->is_constant(), "should not call otherwise");
561 assert(dest->is_register(), "should not call otherwise");
562 LIR_Const* c = src->as_constant_ptr();
563
564 switch (c->type()) {
565 case T_INT: {
566 assert(patch_code == lir_patch_none, "no patching handled here");
567 __ movl(dest->as_register(), c->as_jint());
568 break;
569 }
570
571 case T_ADDRESS: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movptr(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_LONG: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 #ifdef _LP64
580 if (SCCache::is_on_for_write()) {
581 // SCA needs relocation info for card table base
582 address b = c->as_pointer();
583 if (is_card_table_address(b)) {
584 __ lea(dest->as_register_lo(), ExternalAddress(b));
585 break;
586 }
587 }
588 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
589 #else
590 __ movptr(dest->as_register_lo(), c->as_jint_lo());
591 __ movptr(dest->as_register_hi(), c->as_jint_hi());
592 #endif // _LP64
593 break;
594 }
595
596 case T_OBJECT: {
597 if (patch_code != lir_patch_none) {
598 jobject2reg_with_patching(dest->as_register(), info);
599 } else {
600 __ movoop(dest->as_register(), c->as_jobject());
601 }
602 break;
603 }
604
605 case T_METADATA: {
606 if (patch_code != lir_patch_none) {
607 klass2reg_with_patching(dest->as_register(), info);
2400 #endif // _LP64
2401
2402
2403 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2404 if (value->is_double_xmm()) {
2405 switch(code) {
2406 case lir_abs :
2407 {
2408 #ifdef _LP64
2409 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2410 assert(tmp->is_valid(), "need temporary");
2411 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2412 } else
2413 #endif
2414 {
2415 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2416 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2417 }
2418 assert(!tmp->is_valid(), "do not need temporary");
2419 __ andpd(dest->as_xmm_double_reg(),
2420 ExternalAddress(LIR_Assembler::double_signmask_pool),
2421 rscratch1);
2422 }
2423 }
2424 break;
2425
2426 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2427 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2428 default : ShouldNotReachHere();
2429 }
2430
2431 #ifndef _LP64
2432 } else if (value->is_double_fpu()) {
2433 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2434 switch(code) {
2435 case lir_abs : __ fabs() ; break;
2436 case lir_sqrt : __ fsqrt(); break;
2437 default : ShouldNotReachHere();
2438 }
2439 #endif // !_LP64
2440 } else if (code == lir_f2hf) {
3823 move_regs(lo, dest->as_register_lo());
3824 move_regs(hi, dest->as_register_hi());
3825 }
3826 #endif // _LP64
3827
3828 } else if (dest->is_single_xmm()) {
3829 #ifdef _LP64
3830 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3831 assert(tmp->is_valid(), "need temporary");
3832 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3833 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3834 }
3835 else
3836 #endif
3837 {
3838 assert(!tmp->is_valid(), "do not need temporary");
3839 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3840 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3841 }
3842 __ xorps(dest->as_xmm_float_reg(),
3843 ExternalAddress(LIR_Assembler::float_signflip_pool),
3844 rscratch1);
3845 }
3846 } else if (dest->is_double_xmm()) {
3847 #ifdef _LP64
3848 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3849 assert(tmp->is_valid(), "need temporary");
3850 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3851 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3852 }
3853 else
3854 #endif
3855 {
3856 assert(!tmp->is_valid(), "do not need temporary");
3857 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3858 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3859 }
3860 __ xorpd(dest->as_xmm_double_reg(),
3861 ExternalAddress(LIR_Assembler::double_signflip_pool),
3862 rscratch1);
3863 }
3864 #ifndef _LP64
3865 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3866 assert(left->fpu() == 0, "arg must be on TOS");
3867 assert(dest->fpu() == 0, "dest must be TOS");
3868 __ fchs();
3869 #endif // !_LP64
3870
3871 } else {
3872 ShouldNotReachHere();
3873 }
3874 }
3875
3876
3877 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3878 assert(src->is_address(), "must be an address");
3879 assert(dest->is_register(), "must be a register");
3880
3881 PatchingStub* patch = nullptr;
|