16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
60 operand[1] = hi;
61 return operand;
62 }
63
64 // Buffer for 128-bits masks used by SSE instructions.
65 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
66
67 // Static initialization during VM startup.
68 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
69 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
70 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
71 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
72
73
74 NEEDS_CLEANUP // remove this definitions ?
75 const Register SYNC_header = rax; // synchronization header
76 const Register SHIFT_count = rcx; // where count for shift operations must be
77
78 #define __ _masm->
79
80
81 static void select_different_registers(Register preserve,
82 Register extra,
83 Register &tmp1,
84 Register &tmp2) {
85 if (tmp1 == preserve) {
86 assert_different_registers(tmp1, tmp2, extra);
87 tmp1 = extra;
88 } else if (tmp2 == preserve) {
89 assert_different_registers(tmp1, tmp2, extra);
90 tmp2 = extra;
91 }
558 assert(src->is_constant(), "should not call otherwise");
559 assert(dest->is_register(), "should not call otherwise");
560 LIR_Const* c = src->as_constant_ptr();
561
562 switch (c->type()) {
563 case T_INT: {
564 assert(patch_code == lir_patch_none, "no patching handled here");
565 __ movl(dest->as_register(), c->as_jint());
566 break;
567 }
568
569 case T_ADDRESS: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 __ movptr(dest->as_register(), c->as_jint());
572 break;
573 }
574
575 case T_LONG: {
576 assert(patch_code == lir_patch_none, "no patching handled here");
577 #ifdef _LP64
578 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
579 #else
580 __ movptr(dest->as_register_lo(), c->as_jint_lo());
581 __ movptr(dest->as_register_hi(), c->as_jint_hi());
582 #endif // _LP64
583 break;
584 }
585
586 case T_OBJECT: {
587 if (patch_code != lir_patch_none) {
588 jobject2reg_with_patching(dest->as_register(), info);
589 } else {
590 __ movoop(dest->as_register(), c->as_jobject());
591 }
592 break;
593 }
594
595 case T_METADATA: {
596 if (patch_code != lir_patch_none) {
597 klass2reg_with_patching(dest->as_register(), info);
2391 #endif // _LP64
2392
2393
2394 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2395 if (value->is_double_xmm()) {
2396 switch(code) {
2397 case lir_abs :
2398 {
2399 #ifdef _LP64
2400 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2401 assert(tmp->is_valid(), "need temporary");
2402 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2403 } else
2404 #endif
2405 {
2406 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2407 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2408 }
2409 assert(!tmp->is_valid(), "do not need temporary");
2410 __ andpd(dest->as_xmm_double_reg(),
2411 ExternalAddress((address)double_signmask_pool),
2412 rscratch1);
2413 }
2414 }
2415 break;
2416
2417 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2418 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2419 default : ShouldNotReachHere();
2420 }
2421
2422 #ifndef _LP64
2423 } else if (value->is_double_fpu()) {
2424 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2425 switch(code) {
2426 case lir_abs : __ fabs() ; break;
2427 case lir_sqrt : __ fsqrt(); break;
2428 default : ShouldNotReachHere();
2429 }
2430 #endif // !_LP64
2431 } else if (code == lir_f2hf) {
3814 move_regs(lo, dest->as_register_lo());
3815 move_regs(hi, dest->as_register_hi());
3816 }
3817 #endif // _LP64
3818
3819 } else if (dest->is_single_xmm()) {
3820 #ifdef _LP64
3821 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3822 assert(tmp->is_valid(), "need temporary");
3823 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3824 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3825 }
3826 else
3827 #endif
3828 {
3829 assert(!tmp->is_valid(), "do not need temporary");
3830 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3831 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3832 }
3833 __ xorps(dest->as_xmm_float_reg(),
3834 ExternalAddress((address)float_signflip_pool),
3835 rscratch1);
3836 }
3837 } else if (dest->is_double_xmm()) {
3838 #ifdef _LP64
3839 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3840 assert(tmp->is_valid(), "need temporary");
3841 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3842 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3843 }
3844 else
3845 #endif
3846 {
3847 assert(!tmp->is_valid(), "do not need temporary");
3848 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3849 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3850 }
3851 __ xorpd(dest->as_xmm_double_reg(),
3852 ExternalAddress((address)double_signflip_pool),
3853 rscratch1);
3854 }
3855 #ifndef _LP64
3856 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3857 assert(left->fpu() == 0, "arg must be on TOS");
3858 assert(dest->fpu() == 0, "dest must be TOS");
3859 __ fchs();
3860 #endif // !_LP64
3861
3862 } else {
3863 ShouldNotReachHere();
3864 }
3865 }
3866
3867
3868 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3869 assert(src->is_address(), "must be an address");
3870 assert(dest->is_register(), "must be a register");
3871
3872 PatchingStub* patch = nullptr;
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciUtilities.hpp"
37 #include "code/SCCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
62 operand[1] = hi;
63 return (address)operand;
64 }
65
66 // Buffer for 128-bits masks used by SSE instructions.
67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
68
69 // Static initialization during VM startup.
70 address LIR_Assembler::float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
71 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
72 address LIR_Assembler::float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
73 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
74
75
76 NEEDS_CLEANUP // remove this definitions ?
77 const Register SYNC_header = rax; // synchronization header
78 const Register SHIFT_count = rcx; // where count for shift operations must be
79
80 #define __ _masm->
81
82
83 static void select_different_registers(Register preserve,
84 Register extra,
85 Register &tmp1,
86 Register &tmp2) {
87 if (tmp1 == preserve) {
88 assert_different_registers(tmp1, tmp2, extra);
89 tmp1 = extra;
90 } else if (tmp2 == preserve) {
91 assert_different_registers(tmp1, tmp2, extra);
92 tmp2 = extra;
93 }
560 assert(src->is_constant(), "should not call otherwise");
561 assert(dest->is_register(), "should not call otherwise");
562 LIR_Const* c = src->as_constant_ptr();
563
564 switch (c->type()) {
565 case T_INT: {
566 assert(patch_code == lir_patch_none, "no patching handled here");
567 __ movl(dest->as_register(), c->as_jint());
568 break;
569 }
570
571 case T_ADDRESS: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movptr(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_LONG: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 #ifdef _LP64
580 if (SCCache::is_on_for_write()) {
581 // SCA needs relocation info for card table base
582 address b = c->as_pointer();
583 if (is_card_table_address(b)) {
584 __ lea(dest->as_register_lo(), ExternalAddress(b));
585 break;
586 }
587 if (AOTRuntimeConstants::contains(b)) {
588 __ load_aotrc_address(dest->as_register_lo(), b);
589 break;
590 }
591 }
592 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
593 #else
594 __ movptr(dest->as_register_lo(), c->as_jint_lo());
595 __ movptr(dest->as_register_hi(), c->as_jint_hi());
596 #endif // _LP64
597 break;
598 }
599
600 case T_OBJECT: {
601 if (patch_code != lir_patch_none) {
602 jobject2reg_with_patching(dest->as_register(), info);
603 } else {
604 __ movoop(dest->as_register(), c->as_jobject());
605 }
606 break;
607 }
608
609 case T_METADATA: {
610 if (patch_code != lir_patch_none) {
611 klass2reg_with_patching(dest->as_register(), info);
2405 #endif // _LP64
2406
2407
2408 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2409 if (value->is_double_xmm()) {
2410 switch(code) {
2411 case lir_abs :
2412 {
2413 #ifdef _LP64
2414 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2415 assert(tmp->is_valid(), "need temporary");
2416 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2417 } else
2418 #endif
2419 {
2420 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2421 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2422 }
2423 assert(!tmp->is_valid(), "do not need temporary");
2424 __ andpd(dest->as_xmm_double_reg(),
2425 ExternalAddress(LIR_Assembler::double_signmask_pool),
2426 rscratch1);
2427 }
2428 }
2429 break;
2430
2431 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2432 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2433 default : ShouldNotReachHere();
2434 }
2435
2436 #ifndef _LP64
2437 } else if (value->is_double_fpu()) {
2438 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2439 switch(code) {
2440 case lir_abs : __ fabs() ; break;
2441 case lir_sqrt : __ fsqrt(); break;
2442 default : ShouldNotReachHere();
2443 }
2444 #endif // !_LP64
2445 } else if (code == lir_f2hf) {
3828 move_regs(lo, dest->as_register_lo());
3829 move_regs(hi, dest->as_register_hi());
3830 }
3831 #endif // _LP64
3832
3833 } else if (dest->is_single_xmm()) {
3834 #ifdef _LP64
3835 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3836 assert(tmp->is_valid(), "need temporary");
3837 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3838 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3839 }
3840 else
3841 #endif
3842 {
3843 assert(!tmp->is_valid(), "do not need temporary");
3844 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3845 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3846 }
3847 __ xorps(dest->as_xmm_float_reg(),
3848 ExternalAddress(LIR_Assembler::float_signflip_pool),
3849 rscratch1);
3850 }
3851 } else if (dest->is_double_xmm()) {
3852 #ifdef _LP64
3853 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3854 assert(tmp->is_valid(), "need temporary");
3855 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3856 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3857 }
3858 else
3859 #endif
3860 {
3861 assert(!tmp->is_valid(), "do not need temporary");
3862 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3863 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3864 }
3865 __ xorpd(dest->as_xmm_double_reg(),
3866 ExternalAddress(LIR_Assembler::double_signflip_pool),
3867 rscratch1);
3868 }
3869 #ifndef _LP64
3870 } else if (left->is_single_fpu() || left->is_double_fpu()) {
3871 assert(left->fpu() == 0, "arg must be on TOS");
3872 assert(dest->fpu() == 0, "dest must be TOS");
3873 __ fchs();
3874 #endif // !_LP64
3875
3876 } else {
3877 ShouldNotReachHere();
3878 }
3879 }
3880
3881
3882 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3883 assert(src->is_address(), "must be an address");
3884 assert(dest->is_register(), "must be a register");
3885
3886 PatchingStub* patch = nullptr;
|