< prev index next > src/hotspot/cpu/x86/x86_64.ad
Print this page
offset += clear_avx_size();
return offset;
}
int MachCallRuntimeNode::ret_addr_offset() {
+ if (_entry_point == nullptr) {
+ // CallLeafNoFPInDirect
+ return 3; // callq (register)
+ }
int offset = 13; // movq r10,#addr; callq (r10)
if (this->ideal_Opcode() != Op_CallLeafVector) {
offset += clear_avx_size();
}
return offset;
}
+
//
// Compute padding required for nodes which need alignment
//
// The address of the call instruction needs to be 4-byte aligned to
#ifdef ASSERT
st->print("\n\t");
st->print("# stack alignment check");
#endif
}
- if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
st->print("\n\t");
st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
st->print("\n\t");
st->print("je fast_entry\t");
st->print("\n\t");
void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
Compile* C = ra_->C;
C2_MacroAssembler _masm(&cbuf);
- int framesize = C->output()->frame_size_in_bytes();
- int bangsize = C->output()->bang_size_in_bytes();
-
- if (C->clinit_barrier_on_entry()) {
- assert(VM_Version::supports_fast_class_init_checks(), "sanity");
- assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
-
- Label L_skip_barrier;
- Register klass = rscratch1;
-
- __ mov_metadata(klass, C->method()->holder()->constant_encoding());
- __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
-
- __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
+ __ verified_entry(C);
- __ bind(L_skip_barrier);
+ if (ra_->C->stub_function() == nullptr) {
+ __ entry_barrier();
}
- __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ __ bind(*_verified_entry);
+ }
C->output()->set_frame_complete(cbuf.insts_size());
if (C->has_mach_constant_base_node()) {
// NOTE: We set the table base offset here because users might be
ConstantTable& constant_table = C->output()->constant_table();
constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
}
}
- uint MachPrologNode::size(PhaseRegAlloc* ra_) const
- {
- return MachNode::size(ra_); // too many variables; just compute it
- // the hard way
- }
-
int MachPrologNode::reloc() const
{
return 0; // a large enough number
}
// Clear upper bits of YMM registers when current compiled code uses
// wide vectors to avoid AVX <-> SSE transition penalty during call.
__ vzeroupper();
}
- int framesize = C->output()->frame_size_in_bytes();
- assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
- // Remove word for return adr already pushed
- // and RBP
- framesize -= 2*wordSize;
-
- // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
-
- if (framesize) {
- __ addq(rsp, framesize);
- }
-
- __ popq(rbp);
+ // Subtract two words to account for return address and rbp
+ int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
+ __ remove_frame(initial_framesize, C->needs_stack_repair());
if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
__ reserved_stack_check();
}
__ relocate(relocInfo::poll_return_type);
__ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
}
}
- uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
- {
- return MachNode::size(ra_); // too many variables; just compute it
- // the hard way
- }
-
int MachEpilogNode::reloc() const
{
return 2; // a large enough number
}
uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
PhaseRegAlloc* ra_,
bool do_size,
outputStream* st) const {
- assert(cbuf != NULL || st != NULL, "sanity");
+ assert(cbuf != nullptr || st != nullptr, "sanity");
// Get registers to move
OptoReg::Name src_second = ra_->get_reg_second(in(1));
OptoReg::Name src_first = ra_->get_reg_first(in(1));
OptoReg::Name dst_second = ra_->get_reg_second(this);
OptoReg::Name dst_first = ra_->get_reg_first(this);
if (src_first == dst_first && src_second == dst_second) {
// Self copy, no move
return 0;
}
- if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
+ if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
// mem -> mem
return 0;
}
#ifndef PRODUCT
void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
- implementation(NULL, ra_, false, st);
+ implementation(nullptr, ra_, false, st);
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
return MachNode::size(ra_);
}
{
int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
return (offset < 0x80) ? 5 : 8; // REX
}
+ //=============================================================================
+ #ifndef PRODUCT
+ void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
+ {
+ st->print_cr("MachVEPNode");
+ }
+ #endif
+
+ void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
+ {
+ C2_MacroAssembler _masm(&cbuf);
+ uint insts_size = cbuf.insts_size();
+ if (!_verified) {
+ if (UseCompressedClassPointers) {
+ __ load_klass(rscratch1, j_rarg0, rscratch2);
+ __ cmpptr(rax, rscratch1);
+ } else {
+ __ cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
+ }
+ __ jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ } else {
+ // TODO 8284443 Avoid creation of temporary frame
+ if (ra_->C->stub_function() == nullptr) {
+ __ verified_entry(ra_->C, 0);
+ __ entry_barrier();
+ int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
+ __ remove_frame(initial_framesize, false);
+ }
+ // Unpack inline type args passed as oop and then jump to
+ // the verified entry point (skipping the unverified entry).
+ int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
+ // Emit code for verified entry and save increment for stack repair on return
+ __ verified_entry(ra_->C, sp_inc);
+ if (Compile::current()->output()->in_scratch_emit_size()) {
+ Label dummy_verified_entry;
+ __ jmp(dummy_verified_entry);
+ } else {
+ __ jmp(*_verified_entry);
+ }
+ }
+ /* WARNING these NOPs are critical so that verified entry point is properly
+ 4 bytes aligned for patching by NativeJump::patch_verified_entry() */
+ int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
+ nops_cnt &= 0x3; // Do not add nops if code is aligned.
+ if (nops_cnt > 0) {
+ __ nop(nops_cnt);
+ }
+ }
+
//=============================================================================
#ifndef PRODUCT
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
if (UseCompressedClassPointers) {
nops_cnt &= 0x3; // Do not add nops if code is aligned.
if (nops_cnt > 0)
masm.nop(nops_cnt);
}
- uint MachUEPNode::size(PhaseRegAlloc* ra_) const
- {
- return MachNode::size(ra_); // too many variables; just compute it
- // the hard way
- }
-
-
//=============================================================================
bool Matcher::supports_vector_calling_convention(void) {
if (EnableVectorSupport && UseVectorStubs) {
return true;
Label miss;
const bool set_cond_codes = true;
MacroAssembler _masm(&cbuf);
__ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
- NULL, &miss,
+ nullptr, &miss,
/*set_cond_codes:*/ true);
if ($primary) {
__ xorptr(Rrdi, Rrdi);
}
__ bind(miss);
// a stub to the interpreter.
cbuf.shared_stub_to_interp_for(_method, call_offset);
} else {
// Emit stubs for static call.
address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
- if (stub == NULL) {
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
}
}
op_cost(10);
format %{ %}
interface(CONST_INTER);
%}
- // NULL Pointer Immediate
+ // nullptr Pointer Immediate
operand immP0()
%{
predicate(n->get_ptr() == 0);
match(ConP);
op_cost(10);
format %{ %}
interface(CONST_INTER);
%}
- // NULL Pointer Immediate
+ // nullptr Pointer Immediate
operand immN0() %{
predicate(n->get_narrowcon() == 0);
match(ConN);
op_cost(5);
scale($scale);
disp($off);
%}
%}
+ // Indirect Narrow Oop Operand
+ operand indCompressedOop(rRegN reg) %{
+ predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(DecodeN reg);
+
+ op_cost(10);
+ format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
+ interface(MEMORY_INTER) %{
+ base(0xc); // R12
+ index($reg);
+ scale(0x3);
+ disp(0x0);
+ %}
+ %}
+
// Indirect Narrow Oop Plus Offset Operand
// Note: x86 architecture doesn't support "scale * index + offset" without a base
- // we can't free r12 even with CompressedOops::base() == NULL.
+ // we can't free r12 even with CompressedOops::base() == nullptr.
operand indCompressedOopOffset(rRegN reg, immL32 off) %{
predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
constraint(ALLOC_IN_RC(ptr_reg));
match(AddP (DecodeN reg) off);
// multiple operand types with the same basic encoding and format. The classic
// case of this is memory operands.
opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
- indCompressedOopOffset,
+ indCompressedOop, indCompressedOopOffset,
indirectNarrow, indOffset8Narrow, indOffset32Narrow,
indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
//----------PIPELINE-----------------------------------------------------------
%}
instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
match(Set dst src);
effect(KILL cr);
- format %{ "xorq $dst, $src\t# compressed NULL ptr" %}
+ format %{ "xorq $dst, $src\t# compressed nullptr ptr" %}
ins_encode %{
__ xorq($dst$$Register, $dst$$Register);
%}
ins_pipe(ialu_reg);
%}
ins_cost(125);
format %{ "movl $dst, $src\t# compressed ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
__ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
}
%}
ins_cost(125);
format %{ "movl $dst, $src\t# compressed klass ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
__ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
}
%}
ins_pipe(ialu_mem_reg);
%}
instruct storeImmP0(memory mem, immP0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0);
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
match(Set mem (StoreP mem zero));
ins_cost(125); // XXX
format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
ins_encode %{
__ movq($mem$$Address, r12);
%}
ins_pipe(ialu_mem_reg);
%}
- // Store NULL Pointer, mark word, or other simple pointer constant.
+ // Store nullptr Pointer, mark word, or other simple pointer constant.
instruct storeImmP(memory mem, immP31 src)
%{
predicate(n->as_Store()->barrier_data() == 0);
match(Set mem (StoreP mem src));
ins_pipe(ialu_mem_reg);
%}
instruct storeImmN0(memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == NULL);
+ predicate(CompressedOops::base() == nullptr);
match(Set mem (StoreN mem zero));
ins_cost(125); // XXX
format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
ins_encode %{
ins_cost(150); // XXX
format %{ "movl $mem, $src\t# compressed ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
__ movl($mem$$Address, 0);
} else {
__ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
}
%}
%}
// Store Integer Immediate
instruct storeImmI0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreI mem zero));
ins_cost(125); // XXX
format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
ins_encode %{
%}
// Store Long Immediate
instruct storeImmL0(memory mem, immL0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreL mem zero));
ins_cost(125); // XXX
format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
ins_encode %{
%}
// Store Short/Char Immediate
instruct storeImmC0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreC mem zero));
ins_cost(125); // XXX
format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
ins_encode %{
%}
// Store Byte Immediate
instruct storeImmB0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreB mem zero));
ins_cost(125); // XXX
format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
ins_encode %{
%}
// Store CMS card-mark Immediate
instruct storeImmCM0_reg(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreCM mem zero));
ins_cost(125); // XXX
format %{ "movb $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
ins_encode %{
%}
// Store immediate Float value (it is faster than store from XMM register)
instruct storeF0(memory mem, immF0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreF mem zero));
ins_cost(25); // XXX
format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
ins_encode %{
%}
// Store immediate double 0.0 (it is faster than store from XMM register)
instruct storeD0_imm(memory mem, immD0 src)
%{
- predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
+ predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
match(Set mem (StoreD mem src));
ins_cost(50);
format %{ "movq $mem, $src\t# double 0." %}
ins_encode %{
ins_pipe(ialu_mem_imm);
%}
instruct storeD0(memory mem, immD0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreD mem zero));
ins_cost(25); // XXX
format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
ins_encode %{
}
%}
ins_pipe(ialu_reg_reg); // XXX
%}
+ instruct castN2X(rRegL dst, rRegN src)
+ %{
+ match(Set dst (CastP2X src));
+
+ format %{ "movq $dst, $src\t# ptr -> long" %}
+ ins_encode %{
+ if ($dst$$reg != $src$$reg) {
+ __ movptr($dst$$Register, $src$$Register);
+ }
+ %}
+ ins_pipe(ialu_reg_reg); // XXX
+ %}
+
instruct castP2X(rRegL dst, rRegP src)
%{
match(Set dst (CastP2X src));
format %{ "movq $dst, $src\t# ptr -> long" %}
__ movdq($dst$$XMMRegister, $src$$Register);
%}
ins_pipe( pipe_slow );
%}
+
// Fast clearing of an array
// Small ClearArray non-AVX512.
- instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
+ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
- predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
+ predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
+
+ format %{ $$template
+ $$emit$$"cmp InitArrayShortSize,rcx\n\t"
+ $$emit$$"jg LARGE\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"js DONE\t# Zero length\n\t"
+ $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge LOOP\n\t"
+ $$emit$$"jmp DONE\n\t"
+ $$emit$$"# LARGE:\n\t"
+ if (UseFastStosb) {
+ $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
+ $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
+ } else if (UseXMMForObjInit) {
+ $$emit$$"movdq $tmp, $val\n\t"
+ $$emit$$"punpcklqdq $tmp, $tmp\n\t"
+ $$emit$$"vinserti128_high $tmp, $tmp\n\t"
+ $$emit$$"jmpq L_zero_64_bytes\n\t"
+ $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
+ $$emit$$"add 0x40,rax\n\t"
+ $$emit$$"# L_zero_64_bytes:\n\t"
+ $$emit$$"sub 0x8,rcx\n\t"
+ $$emit$$"jge L_loop\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jl L_tail\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"add 0x20,rax\n\t"
+ $$emit$$"sub 0x4,rcx\n\t"
+ $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jle L_end\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
+ $$emit$$"vmovq xmm0,(rax)\n\t"
+ $$emit$$"add 0x8,rax\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge L_sloop\n\t"
+ $$emit$$"# L_end:\n\t"
+ } else {
+ $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
+ }
+ $$emit$$"# DONE"
+ %}
+ ins_encode %{
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, false, false);
+ %}
+ ins_pipe(pipe_slow);
+ %}
+
+ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
+ %{
+ predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
+
+ format %{ $$template
+ $$emit$$"cmp InitArrayShortSize,rcx\n\t"
+ $$emit$$"jg LARGE\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"js DONE\t# Zero length\n\t"
+ $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge LOOP\n\t"
+ $$emit$$"jmp DONE\n\t"
+ $$emit$$"# LARGE:\n\t"
+ if (UseXMMForObjInit) {
+ $$emit$$"movdq $tmp, $val\n\t"
+ $$emit$$"punpcklqdq $tmp, $tmp\n\t"
+ $$emit$$"vinserti128_high $tmp, $tmp\n\t"
+ $$emit$$"jmpq L_zero_64_bytes\n\t"
+ $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
+ $$emit$$"add 0x40,rax\n\t"
+ $$emit$$"# L_zero_64_bytes:\n\t"
+ $$emit$$"sub 0x8,rcx\n\t"
+ $$emit$$"jge L_loop\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jl L_tail\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"add 0x20,rax\n\t"
+ $$emit$$"sub 0x4,rcx\n\t"
+ $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jle L_end\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
+ $$emit$$"vmovq xmm0,(rax)\n\t"
+ $$emit$$"add 0x8,rax\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge L_sloop\n\t"
+ $$emit$$"# L_end:\n\t"
+ } else {
+ $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
+ }
+ $$emit$$"# DONE"
+ %}
+ ins_encode %{
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, false, true);
+ %}
+ ins_pipe(pipe_slow);
+ %}
+
+ // Small ClearArray AVX512 non-constant length.
+ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
+ %{
+ predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ ins_cost(125);
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"cmp InitArrayShortSize,rcx\n\t"
$$emit$$"jg LARGE\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
}
$$emit$$"# DONE"
%}
ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, false, knoreg);
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
%}
ins_pipe(pipe_slow);
%}
- // Small ClearArray AVX512 non-constant length.
- instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
- Universe dummy, rFlagsReg cr)
+ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
%{
- predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
- match(Set dummy (ClearArray cnt base));
+ predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
ins_cost(125);
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"cmp InitArrayShortSize,rcx\n\t"
$$emit$$"jg LARGE\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
}
$$emit$$"# DONE"
%}
ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, false, $ktmp$$KRegister);
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
%}
ins_pipe(pipe_slow);
%}
// Large ClearArray non-AVX512.
- instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
+ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
- predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
+ predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
+
+ format %{ $$template
+ if (UseFastStosb) {
+ $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
+ $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
+ } else if (UseXMMForObjInit) {
+ $$emit$$"movdq $tmp, $val\n\t"
+ $$emit$$"punpcklqdq $tmp, $tmp\n\t"
+ $$emit$$"vinserti128_high $tmp, $tmp\n\t"
+ $$emit$$"jmpq L_zero_64_bytes\n\t"
+ $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
+ $$emit$$"add 0x40,rax\n\t"
+ $$emit$$"# L_zero_64_bytes:\n\t"
+ $$emit$$"sub 0x8,rcx\n\t"
+ $$emit$$"jge L_loop\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jl L_tail\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"add 0x20,rax\n\t"
+ $$emit$$"sub 0x4,rcx\n\t"
+ $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jle L_end\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
+ $$emit$$"vmovq xmm0,(rax)\n\t"
+ $$emit$$"add 0x8,rax\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge L_sloop\n\t"
+ $$emit$$"# L_end:\n\t"
+ } else {
+ $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
+ }
+ %}
+ ins_encode %{
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, true, false);
+ %}
+ ins_pipe(pipe_slow);
+ %}
+
+ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
+ %{
+ predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
+
+ format %{ $$template
+ if (UseXMMForObjInit) {
+ $$emit$$"movdq $tmp, $val\n\t"
+ $$emit$$"punpcklqdq $tmp, $tmp\n\t"
+ $$emit$$"vinserti128_high $tmp, $tmp\n\t"
+ $$emit$$"jmpq L_zero_64_bytes\n\t"
+ $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
+ $$emit$$"add 0x40,rax\n\t"
+ $$emit$$"# L_zero_64_bytes:\n\t"
+ $$emit$$"sub 0x8,rcx\n\t"
+ $$emit$$"jge L_loop\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jl L_tail\n\t"
+ $$emit$$"vmovdqu $tmp,(rax)\n\t"
+ $$emit$$"add 0x20,rax\n\t"
+ $$emit$$"sub 0x4,rcx\n\t"
+ $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
+ $$emit$$"add 0x4,rcx\n\t"
+ $$emit$$"jle L_end\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
+ $$emit$$"vmovq xmm0,(rax)\n\t"
+ $$emit$$"add 0x8,rax\n\t"
+ $$emit$$"dec rcx\n\t"
+ $$emit$$"jge L_sloop\n\t"
+ $$emit$$"# L_end:\n\t"
+ } else {
+ $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
+ }
+ %}
+ ins_encode %{
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, true, true);
+ %}
+ ins_pipe(pipe_slow);
+ %}
+
+ // Large ClearArray AVX512.
+ instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
+ %{
+ predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
if (UseFastStosb) {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
}
%}
ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, true, knoreg);
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
%}
ins_pipe(pipe_slow);
%}
- // Large ClearArray AVX512.
- instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
- Universe dummy, rFlagsReg cr)
+ instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
+ Universe dummy, rFlagsReg cr)
%{
- predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
+ predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
+ match(Set dummy (ClearArray (Binary cnt base) val));
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
if (UseFastStosb) {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
}
%}
ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, true, $ktmp$$KRegister);
+ __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
+ $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
%}
ins_pipe(pipe_slow);
%}
// Small ClearArray AVX512 constant length.
- instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
+ instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
%{
- predicate(!((ClearArrayNode*)n)->is_large() &&
- ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
- match(Set dummy (ClearArray cnt base));
+ predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
+ ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
+ match(Set dummy (ClearArray (Binary cnt base) val));
ins_cost(100);
- effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
+ effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
format %{ "clear_mem_imm $base , $cnt \n\t" %}
ins_encode %{
- __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
+ __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
%}
ins_pipe(pipe_slow);
%}
instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
// This will generate a signed flags result. This should be OK since
// any compare to a zero should be eq/neq.
instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
%{
- predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
+ predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP op) zero));
ins_cost(500); // XXX
format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
ins_pipe(ialu_cr_reg_imm);
%}
instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP mem) zero));
format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
ins_encode %{
ins_pipe(ialu_cr_reg_imm);
%}
instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() != NULL);
+ predicate(CompressedOops::base() != nullptr);
match(Set cr (CmpN (LoadN mem) zero));
ins_cost(500); // XXX
format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
ins_encode %{
ins_pipe(ialu_cr_reg_mem);
%}
instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == NULL);
+ predicate(CompressedOops::base() == nullptr);
match(Set cr (CmpN (LoadN mem) zero));
format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
ins_encode %{
__ cmpl(r12, $mem$$Address);
ins_encode(Java_To_Runtime(meth));
ins_pipe(pipe_slow);
%}
// Call runtime without safepoint
+ // entry point is null, target holds the address to call
+ instruct CallLeafNoFPInDirect(rRegP target)
+ %{
+ predicate(n->as_Call()->entry_point() == nullptr);
+ match(CallLeafNoFP target);
+
+ ins_cost(300);
+ format %{ "call_leaf_nofp,runtime indirect " %}
+ ins_encode %{
+ __ call($target$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+ %}
+
instruct CallLeafNoFPDirect(method meth)
%{
+ predicate(n->as_Call()->entry_point() != nullptr);
match(CallLeafNoFP);
effect(USE meth);
ins_cost(300);
format %{ "call_leaf_nofp,runtime " %}
< prev index next >