421 source_hpp %{
422
423 #include "peephole_x86_64.hpp"
424
425 bool castLL_is_imm32(const Node* n);
426
427 %}
428
429 source %{
430
431 bool castLL_is_imm32(const Node* n) {
432 assert(n->is_CastLL(), "must be a CastLL");
433 const TypeLong* t = n->bottom_type()->is_long();
434 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
435 }
436
437 %}
438
439 // Register masks
440 source_hpp %{
441
442 extern RegMask _ANY_REG_mask;
443 extern RegMask _PTR_REG_mask;
444 extern RegMask _PTR_REG_NO_RBP_mask;
445 extern RegMask _PTR_NO_RAX_REG_mask;
446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
447 extern RegMask _LONG_REG_mask;
448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
449 extern RegMask _LONG_NO_RCX_REG_mask;
450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
451 extern RegMask _INT_REG_mask;
452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
453 extern RegMask _INT_NO_RCX_REG_mask;
454 extern RegMask _INT_NO_RBP_R13_REG_mask;
455 extern RegMask _FLOAT_REG_mask;
456
457 extern RegMask _STACK_OR_PTR_REG_mask;
458 extern RegMask _STACK_OR_LONG_REG_mask;
459 extern RegMask _STACK_OR_INT_REG_mask;
460
461 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
834 st->print("\n\t");
835 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
836 st->print("\n\t");
837 st->print("je fast_entry\t");
838 st->print("\n\t");
839 st->print("call #nmethod_entry_barrier_stub\t");
840 st->print("\n\tfast_entry:");
841 }
842 st->cr();
843 }
844 #endif
845
846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
847 Compile* C = ra_->C;
848
849 int framesize = C->output()->frame_size_in_bytes();
850 int bangsize = C->output()->bang_size_in_bytes();
851
852 if (C->clinit_barrier_on_entry()) {
853 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
854 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
855
856 Label L_skip_barrier;
857 Register klass = rscratch1;
858
859 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
860 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
861
862 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
863
864 __ bind(L_skip_barrier);
865 }
866
867 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
868
869 C->output()->set_frame_complete(__ offset());
870
871 if (C->has_mach_constant_base_node()) {
872 // NOTE: We set the table base offset here because users might be
873 // emitted before MachConstantBaseNode.
874 ConstantTable& constant_table = C->output()->constant_table();
1843 // idivq (note: must be emitted by the user of this rule)
1844 // <done>
1845 __ idivq($div$$Register);
1846 __ bind(done);
1847 %}
1848
1849 enc_class clear_avx %{
1850 DEBUG_ONLY(int off0 = __ offset());
1851 if (generate_vzeroupper(Compile::current())) {
1852 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
1853 // Clear upper bits of YMM registers when current compiled code uses
1854 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1855 __ vzeroupper();
1856 }
1857 DEBUG_ONLY(int off1 = __ offset());
1858 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1859 %}
1860
1861 enc_class Java_To_Runtime(method meth) %{
1862 // No relocation needed
1863 __ mov64(r10, (int64_t) $meth$$method);
1864 __ call(r10);
1865 __ post_call_nop();
1866 %}
1867
1868 enc_class Java_Static_Call(method meth)
1869 %{
1870 // JAVA STATIC CALL
1871 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
1872 // determine who we intended to call.
1873 if (!_method) {
1874 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
1875 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
1876 // The NOP here is purely to ensure that eliding a call to
1877 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
1878 __ addr_nop_5();
1879 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
1880 } else {
1881 int method_index = resolved_method_index(masm);
1882 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1883 : static_call_Relocation::spec(method_index);
2470 // Constant for byte-wide masking
2471 operand immL_255()
2472 %{
2473 predicate(n->get_long() == 255);
2474 match(ConL);
2475
2476 format %{ %}
2477 interface(CONST_INTER);
2478 %}
2479
2480 // Constant for short-wide masking
2481 operand immL_65535()
2482 %{
2483 predicate(n->get_long() == 65535);
2484 match(ConL);
2485
2486 format %{ %}
2487 interface(CONST_INTER);
2488 %}
2489
2490 operand kReg()
2491 %{
2492 constraint(ALLOC_IN_RC(vectmask_reg));
2493 match(RegVectMask);
2494 format %{%}
2495 interface(REG_INTER);
2496 %}
2497
2498 // Register Operands
2499 // Integer Register
2500 operand rRegI()
2501 %{
2502 constraint(ALLOC_IN_RC(int_reg));
2503 match(RegI);
2504
2505 match(rax_RegI);
2506 match(rbx_RegI);
2507 match(rcx_RegI);
2508 match(rdx_RegI);
2509 match(rdi_RegI);
4430 format %{ "movlpd $dst, $mem\t# double" %}
4431 ins_encode %{
4432 __ movdbl($dst$$XMMRegister, $mem$$Address);
4433 %}
4434 ins_pipe(pipe_slow); // XXX
4435 %}
4436
4437 instruct loadD(regD dst, memory mem)
4438 %{
4439 predicate(UseXmmLoadAndClearUpper);
4440 match(Set dst (LoadD mem));
4441
4442 ins_cost(145); // XXX
4443 format %{ "movsd $dst, $mem\t# double" %}
4444 ins_encode %{
4445 __ movdbl($dst$$XMMRegister, $mem$$Address);
4446 %}
4447 ins_pipe(pipe_slow); // XXX
4448 %}
4449
4450 // max = java.lang.Math.max(float a, float b)
4451 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
4452 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4453 match(Set dst (MaxF a b));
4454 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
4455 format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
4456 ins_encode %{
4457 __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4458 %}
4459 ins_pipe( pipe_slow );
4460 %}
4461
4462 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
4463 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4464 match(Set dst (MaxF a b));
4465 effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
4466
4467 format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
4468 ins_encode %{
4469 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
|
421 source_hpp %{
422
423 #include "peephole_x86_64.hpp"
424
425 bool castLL_is_imm32(const Node* n);
426
427 %}
428
429 source %{
430
431 bool castLL_is_imm32(const Node* n) {
432 assert(n->is_CastLL(), "must be a CastLL");
433 const TypeLong* t = n->bottom_type()->is_long();
434 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
435 }
436
437 %}
438
439 // Register masks
440 source_hpp %{
441 extern RegMask _ANY_REG_mask;
442 extern RegMask _PTR_REG_mask;
443 extern RegMask _PTR_REG_NO_RBP_mask;
444 extern RegMask _PTR_NO_RAX_REG_mask;
445 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
446 extern RegMask _LONG_REG_mask;
447 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
448 extern RegMask _LONG_NO_RCX_REG_mask;
449 extern RegMask _LONG_NO_RBP_R13_REG_mask;
450 extern RegMask _INT_REG_mask;
451 extern RegMask _INT_NO_RAX_RDX_REG_mask;
452 extern RegMask _INT_NO_RCX_REG_mask;
453 extern RegMask _INT_NO_RBP_R13_REG_mask;
454 extern RegMask _FLOAT_REG_mask;
455
456 extern RegMask _STACK_OR_PTR_REG_mask;
457 extern RegMask _STACK_OR_LONG_REG_mask;
458 extern RegMask _STACK_OR_INT_REG_mask;
459
460 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
833 st->print("\n\t");
834 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
835 st->print("\n\t");
836 st->print("je fast_entry\t");
837 st->print("\n\t");
838 st->print("call #nmethod_entry_barrier_stub\t");
839 st->print("\n\tfast_entry:");
840 }
841 st->cr();
842 }
843 #endif
844
845 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
846 Compile* C = ra_->C;
847
848 int framesize = C->output()->frame_size_in_bytes();
849 int bangsize = C->output()->bang_size_in_bytes();
850
851 if (C->clinit_barrier_on_entry()) {
852 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
853 assert(!C->method()->holder()->is_not_initialized() || C->do_clinit_barriers(), "initialization should have been started");
854
855 Label L_skip_barrier;
856 Register klass = rscratch1;
857
858 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
859 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
860
861 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
862
863 __ bind(L_skip_barrier);
864 }
865
866 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
867
868 C->output()->set_frame_complete(__ offset());
869
870 if (C->has_mach_constant_base_node()) {
871 // NOTE: We set the table base offset here because users might be
872 // emitted before MachConstantBaseNode.
873 ConstantTable& constant_table = C->output()->constant_table();
1842 // idivq (note: must be emitted by the user of this rule)
1843 // <done>
1844 __ idivq($div$$Register);
1845 __ bind(done);
1846 %}
1847
1848 enc_class clear_avx %{
1849 DEBUG_ONLY(int off0 = __ offset());
1850 if (generate_vzeroupper(Compile::current())) {
1851 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
1852 // Clear upper bits of YMM registers when current compiled code uses
1853 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1854 __ vzeroupper();
1855 }
1856 DEBUG_ONLY(int off1 = __ offset());
1857 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1858 %}
1859
1860 enc_class Java_To_Runtime(method meth) %{
1861 // No relocation needed
1862 if (AOTCodeCache::is_on_for_dump()) {
1863 // Created runtime_call_type relocation when caching code
1864 __ lea(r10, RuntimeAddress((address)$meth$$method));
1865 } else {
1866 __ mov64(r10, (int64_t) $meth$$method);
1867 }
1868 __ call(r10);
1869 __ post_call_nop();
1870 %}
1871
1872 enc_class Java_Static_Call(method meth)
1873 %{
1874 // JAVA STATIC CALL
1875 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
1876 // determine who we intended to call.
1877 if (!_method) {
1878 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
1879 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
1880 // The NOP here is purely to ensure that eliding a call to
1881 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
1882 __ addr_nop_5();
1883 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
1884 } else {
1885 int method_index = resolved_method_index(masm);
1886 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1887 : static_call_Relocation::spec(method_index);
2474 // Constant for byte-wide masking
2475 operand immL_255()
2476 %{
2477 predicate(n->get_long() == 255);
2478 match(ConL);
2479
2480 format %{ %}
2481 interface(CONST_INTER);
2482 %}
2483
2484 // Constant for short-wide masking
2485 operand immL_65535()
2486 %{
2487 predicate(n->get_long() == 65535);
2488 match(ConL);
2489
2490 format %{ %}
2491 interface(CONST_INTER);
2492 %}
2493
2494 // AOT Runtime Constants Address
2495 operand immAOTRuntimeConstantsAddress()
2496 %{
2497 // Check if the address is in the range of AOT Runtime Constants
2498 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
2499 match(ConP);
2500
2501 op_cost(0);
2502 format %{ %}
2503 interface(CONST_INTER);
2504 %}
2505
2506 operand kReg()
2507 %{
2508 constraint(ALLOC_IN_RC(vectmask_reg));
2509 match(RegVectMask);
2510 format %{%}
2511 interface(REG_INTER);
2512 %}
2513
2514 // Register Operands
2515 // Integer Register
2516 operand rRegI()
2517 %{
2518 constraint(ALLOC_IN_RC(int_reg));
2519 match(RegI);
2520
2521 match(rax_RegI);
2522 match(rbx_RegI);
2523 match(rcx_RegI);
2524 match(rdx_RegI);
2525 match(rdi_RegI);
4446 format %{ "movlpd $dst, $mem\t# double" %}
4447 ins_encode %{
4448 __ movdbl($dst$$XMMRegister, $mem$$Address);
4449 %}
4450 ins_pipe(pipe_slow); // XXX
4451 %}
4452
4453 instruct loadD(regD dst, memory mem)
4454 %{
4455 predicate(UseXmmLoadAndClearUpper);
4456 match(Set dst (LoadD mem));
4457
4458 ins_cost(145); // XXX
4459 format %{ "movsd $dst, $mem\t# double" %}
4460 ins_encode %{
4461 __ movdbl($dst$$XMMRegister, $mem$$Address);
4462 %}
4463 ins_pipe(pipe_slow); // XXX
4464 %}
4465
4466 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
4467 %{
4468 match(Set dst con);
4469
4470 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
4471
4472 ins_encode %{
4473 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
4474 %}
4475
4476 ins_pipe(ialu_reg_fat);
4477 %}
4478
4479 // max = java.lang.Math.max(float a, float b)
4480 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
4481 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4482 match(Set dst (MaxF a b));
4483 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
4484 format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
4485 ins_encode %{
4486 __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4487 %}
4488 ins_pipe( pipe_slow );
4489 %}
4490
4491 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
4492 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4493 match(Set dst (MaxF a b));
4494 effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
4495
4496 format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
4497 ins_encode %{
4498 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
|