< prev index next >

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

Print this page

   1 /*
   2  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.

   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  28 #include "gc/shenandoah/shenandoahForwarding.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  34 #include "interpreter/interpreter.hpp"
  35 #include "runtime/javaThread.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "utilities/macros.hpp"
  38 #ifdef COMPILER1
  39 #include "c1/c1_LIRAssembler.hpp"
  40 #include "c1/c1_MacroAssembler.hpp"
  41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  42 #endif
  43 
  44 #define __ masm->
  45 
  46 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  47   if (handle_gpr) {
  48     __ push_IU_state();
  49   }
  50 
  51   if (handle_fp) {
  52     // Some paths can be reached from the c2i adapter with live fp arguments in registers.
  53     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));

 103       __ movflt(xmm5, Address(rsp, xmm_size * 5));
 104       __ movflt(xmm6, Address(rsp, xmm_size * 6));
 105       __ movflt(xmm7, Address(rsp, xmm_size * 7));
 106       __ addptr(rsp, xmm_size * 8);
 107     } else {
 108       __ pop_FPU_state();
 109     }
 110   }
 111 
 112   if (handle_gpr) {
 113     __ pop_IU_state();
 114   }
 115 }
 116 
 117 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 118                                                        Register src, Register dst, Register count) {
 119 
 120   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 121 
 122   if (is_reference_type(type)) {























 123 
 124     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 125 #ifdef _LP64
 126       Register thread = r15_thread;
 127 #else
 128       Register thread = rax;
 129       if (thread == src || thread == dst || thread == count) {
 130         thread = rbx;
 131       }
 132       if (thread == src || thread == dst || thread == count) {
 133         thread = rcx;
 134       }
 135       if (thread == src || thread == dst || thread == count) {
 136         thread = rdx;
 137       }
 138       __ push(thread);
 139       __ get_thread(thread);
 140 #endif
 141       assert_different_registers(src, dst, count, thread);
 142 
 143       Label done;
 144       // Short-circuit if count == 0.
 145       __ testptr(count, count);
 146       __ jcc(Assembler::zero, done);
 147 
 148       // Avoid runtime call when not active.
 149       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 150       int flags;
 151       if (ShenandoahSATBBarrier && dest_uninitialized) {
 152         flags = ShenandoahHeap::HAS_FORWARDED;
 153       } else {
 154         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 155       }
 156       __ testb(gc_state, flags);
 157       __ jcc(Assembler::zero, done);
 158 
 159       save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 160 
 161 #ifdef _LP64
 162       assert(src == rdi, "expected");
 163       assert(dst == rsi, "expected");
 164       assert(count == rdx, "expected");
 165       if (UseCompressedOops) {
 166         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
 167                         src, dst, count);
 168       } else
 169 #endif
 170       {
 171         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 172                         src, dst, count);
 173       }
 174 
 175       restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 176 
 177       __ bind(done);
 178       NOT_LP64(__ pop(thread);)
 179     }
 180   }
 181 
 182 }
 183 






























 184 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 185                                                                  Register obj,
 186                                                                  Register pre_val,
 187                                                                  Register thread,
 188                                                                  Register tmp,
 189                                                                  bool tosca_live,
 190                                                                  bool expand_call) {
 191 
 192   if (ShenandoahSATBBarrier) {
 193     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 194   }
 195 }
 196 
 197 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 198                                                            Register obj,
 199                                                            Register pre_val,
 200                                                            Register thread,
 201                                                            Register tmp,
 202                                                            bool tosca_live,
 203                                                            bool expand_call) {

 573     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 574     assert_different_registers(dst, tmp1, tmp_thread);
 575     if (!thread->is_valid()) {
 576       thread = rdx;
 577     }
 578     NOT_LP64(__ get_thread(thread));
 579     // Generate the SATB pre-barrier code to log the value of
 580     // the referent field in an SATB buffer.
 581     shenandoah_write_barrier_pre(masm /* masm */,
 582                                  noreg /* obj */,
 583                                  dst /* pre_val */,
 584                                  thread /* thread */,
 585                                  tmp1 /* tmp */,
 586                                  true /* tosca_live */,
 587                                  true /* expand_call */);
 588 
 589     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 590   }
 591 }
 592 











































 593 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 594               Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 595 
 596   bool on_oop = is_reference_type(type);
 597   bool in_heap = (decorators & IN_HEAP) != 0;
 598   bool as_normal = (decorators & AS_NORMAL) != 0;
 599   if (on_oop && in_heap) {
 600     bool needs_pre_barrier = as_normal;
 601 
 602     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 603     // flatten object address if needed
 604     // We do it regardless of precise because we need the registers
 605     if (dst.index() == noreg && dst.disp() == 0) {
 606       if (dst.base() != tmp1) {
 607         __ movptr(tmp1, dst.base());
 608       }
 609     } else {
 610       __ lea(tmp1, dst);
 611     }
 612 

 615 #ifndef _LP64
 616     __ get_thread(rthread);
 617     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 618     imasm->save_bcp();
 619 #endif
 620 
 621     if (needs_pre_barrier) {
 622       shenandoah_write_barrier_pre(masm /*masm*/,
 623                                    tmp1 /* obj */,
 624                                    tmp2 /* pre_val */,
 625                                    rthread /* thread */,
 626                                    tmp3  /* tmp */,
 627                                    val != noreg /* tosca_live */,
 628                                    false /* expand_call */);
 629     }
 630     if (val == noreg) {
 631       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
 632     } else {
 633       iu_barrier(masm, val, tmp3);
 634       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);



 635     }
 636     NOT_LP64(imasm->restore_bcp());
 637   } else {
 638     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
 639   }
 640 }
 641 
 642 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 643                                                                   Register obj, Register tmp, Label& slowpath) {
 644   Label done;
 645   // Resolve jobject
 646   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 647 
 648   // Check for null.
 649   __ testptr(obj, obj);
 650   __ jcc(Assembler::zero, done);
 651 
 652   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 653   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 654   __ jccb(Assembler::notZero, slowpath);

 810   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 811   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 812 
 813   if (exchange) {
 814     __ bind(L_failure);
 815     __ bind(L_success);
 816   } else {
 817     assert(res != noreg, "need result register");
 818 
 819     Label exit;
 820     __ bind(L_failure);
 821     __ xorptr(res, res);
 822     __ jmpb(exit);
 823 
 824     __ bind(L_success);
 825     __ movptr(res, 1);
 826     __ bind(exit);
 827   }
 828 }
 829 

























































 830 #undef __
 831 
 832 #ifdef COMPILER1
 833 
 834 #define __ ce->masm()->
 835 
 836 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 837   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 838   // At this point we know that marking is in progress.
 839   // If do_load() is true then we have to emit the
 840   // load of the previous value; otherwise it has already
 841   // been loaded into _pre_val.
 842 
 843   __ bind(*stub->entry());
 844   assert(stub->pre_val()->is_register(), "Precondition.");
 845 
 846   Register pre_val_reg = stub->pre_val()->as_register();
 847 
 848   if (stub->do_load()) {
 849     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);

   1 /*
   2  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
   3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahRuntime.hpp"
  33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  35 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "runtime/javaThread.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "utilities/macros.hpp"
  40 #ifdef COMPILER1
  41 #include "c1/c1_LIRAssembler.hpp"
  42 #include "c1/c1_MacroAssembler.hpp"
  43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  44 #endif
  45 
  46 #define __ masm->
  47 
  48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  49   if (handle_gpr) {
  50     __ push_IU_state();
  51   }
  52 
  53   if (handle_fp) {
  54     // Some paths can be reached from the c2i adapter with live fp arguments in registers.
  55     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));

 105       __ movflt(xmm5, Address(rsp, xmm_size * 5));
 106       __ movflt(xmm6, Address(rsp, xmm_size * 6));
 107       __ movflt(xmm7, Address(rsp, xmm_size * 7));
 108       __ addptr(rsp, xmm_size * 8);
 109     } else {
 110       __ pop_FPU_state();
 111     }
 112   }
 113 
 114   if (handle_gpr) {
 115     __ pop_IU_state();
 116   }
 117 }
 118 
 119 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 120                                                        Register src, Register dst, Register count) {
 121 
 122   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 123 
 124   if (is_reference_type(type)) {
 125     if (ShenandoahCardBarrier) {
 126       bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 127       bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 128       bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 129 
 130       // We need to save the original element count because the array copy stub
 131       // will destroy the value and we need it for the card marking barrier.
 132 #ifdef _LP64
 133       if (!checkcast) {
 134         if (!obj_int) {
 135           // Save count for barrier
 136           __ movptr(r11, count);
 137         } else if (disjoint) {
 138           // Save dst in r11 in the disjoint case
 139           __ movq(r11, dst);
 140         }
 141       }
 142 #else
 143       if (disjoint) {
 144         __ mov(rdx, dst);          // save 'to'
 145       }
 146 #endif
 147     }
 148 
 149     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 150 #ifdef _LP64
 151       Register thread = r15_thread;
 152 #else
 153       Register thread = rax;
 154       if (thread == src || thread == dst || thread == count) {
 155         thread = rbx;
 156       }
 157       if (thread == src || thread == dst || thread == count) {
 158         thread = rcx;
 159       }
 160       if (thread == src || thread == dst || thread == count) {
 161         thread = rdx;
 162       }
 163       __ push(thread);
 164       __ get_thread(thread);
 165 #endif
 166       assert_different_registers(src, dst, count, thread);
 167 
 168       Label L_done;
 169       // Short-circuit if count == 0.
 170       __ testptr(count, count);
 171       __ jcc(Assembler::zero, L_done);
 172 
 173       // Avoid runtime call when not active.
 174       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 175       int flags;
 176       if (ShenandoahSATBBarrier && dest_uninitialized) {
 177         flags = ShenandoahHeap::HAS_FORWARDED;
 178       } else {
 179         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 180       }
 181       __ testb(gc_state, flags);
 182       __ jcc(Assembler::zero, L_done);
 183 
 184       save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 185 
 186 #ifdef _LP64
 187       assert(src == rdi, "expected");
 188       assert(dst == rsi, "expected");
 189       assert(count == rdx, "expected");
 190       if (UseCompressedOops) {
 191         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
 192                         src, dst, count);
 193       } else
 194 #endif
 195       {
 196         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 197                         src, dst, count);
 198       }
 199 
 200       restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 201 
 202       __ bind(L_done);
 203       NOT_LP64(__ pop(thread);)
 204     }
 205   }
 206 
 207 }
 208 
 209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 210                                                        Register src, Register dst, Register count) {
 211 
 212   if (ShenandoahCardBarrier && is_reference_type(type)) {
 213     bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 214     bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 215     bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 216     Register tmp = rax;
 217 
 218 #ifdef _LP64
 219     if (!checkcast) {
 220       if (!obj_int) {
 221         // Save count for barrier
 222         count = r11;
 223       } else if (disjoint) {
 224         // Use the saved dst in the disjoint case
 225         dst = r11;
 226       }
 227     } else {
 228       tmp = rscratch1;
 229     }
 230 #else
 231     if (disjoint) {
 232       __ mov(dst, rdx); // restore 'to'
 233     }
 234 #endif
 235     gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
 236   }
 237 }
 238 
 239 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 240                                                                  Register obj,
 241                                                                  Register pre_val,
 242                                                                  Register thread,
 243                                                                  Register tmp,
 244                                                                  bool tosca_live,
 245                                                                  bool expand_call) {
 246 
 247   if (ShenandoahSATBBarrier) {
 248     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 249   }
 250 }
 251 
 252 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 253                                                            Register obj,
 254                                                            Register pre_val,
 255                                                            Register thread,
 256                                                            Register tmp,
 257                                                            bool tosca_live,
 258                                                            bool expand_call) {

 628     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 629     assert_different_registers(dst, tmp1, tmp_thread);
 630     if (!thread->is_valid()) {
 631       thread = rdx;
 632     }
 633     NOT_LP64(__ get_thread(thread));
 634     // Generate the SATB pre-barrier code to log the value of
 635     // the referent field in an SATB buffer.
 636     shenandoah_write_barrier_pre(masm /* masm */,
 637                                  noreg /* obj */,
 638                                  dst /* pre_val */,
 639                                  thread /* thread */,
 640                                  tmp1 /* tmp */,
 641                                  true /* tosca_live */,
 642                                  true /* expand_call */);
 643 
 644     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 645   }
 646 }
 647 
 648 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
 649   assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
 650 
 651   // Does a store check for the oop in register obj. The content of
 652   // register obj is destroyed afterwards.
 653 
 654   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 655   CardTable* ct = ctbs->card_table();
 656 
 657   __ shrptr(obj, CardTable::card_shift());
 658 
 659   Address card_addr;
 660 
 661   // The calculation for byte_map_base is as follows:
 662   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
 663   // So this essentially converts an address to a displacement and it will
 664   // never need to be relocated. On 64-bit however the value may be too
 665   // large for a 32-bit displacement.
 666   intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
 667   if (__ is_simm32(byte_map_base)) {
 668     card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
 669   } else {
 670     // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
 671     // displacement and done in a single instruction given favorable mapping and a
 672     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
 673     // entry and that entry is not properly handled by the relocation code.
 674     AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
 675     Address index(noreg, obj, Address::times_1);
 676     card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
 677   }
 678 
 679   int dirty = CardTable::dirty_card_val();
 680   if (UseCondCardMark) {
 681     Label L_already_dirty;
 682     __ cmpb(card_addr, dirty);
 683     __ jccb(Assembler::equal, L_already_dirty);
 684     __ movb(card_addr, dirty);
 685     __ bind(L_already_dirty);
 686   } else {
 687     __ movb(card_addr, dirty);
 688   }
 689 }
 690 
 691 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 692               Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 693 
 694   bool on_oop = is_reference_type(type);
 695   bool in_heap = (decorators & IN_HEAP) != 0;
 696   bool as_normal = (decorators & AS_NORMAL) != 0;
 697   if (on_oop && in_heap) {
 698     bool needs_pre_barrier = as_normal;
 699 
 700     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 701     // flatten object address if needed
 702     // We do it regardless of precise because we need the registers
 703     if (dst.index() == noreg && dst.disp() == 0) {
 704       if (dst.base() != tmp1) {
 705         __ movptr(tmp1, dst.base());
 706       }
 707     } else {
 708       __ lea(tmp1, dst);
 709     }
 710 

 713 #ifndef _LP64
 714     __ get_thread(rthread);
 715     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 716     imasm->save_bcp();
 717 #endif
 718 
 719     if (needs_pre_barrier) {
 720       shenandoah_write_barrier_pre(masm /*masm*/,
 721                                    tmp1 /* obj */,
 722                                    tmp2 /* pre_val */,
 723                                    rthread /* thread */,
 724                                    tmp3  /* tmp */,
 725                                    val != noreg /* tosca_live */,
 726                                    false /* expand_call */);
 727     }
 728     if (val == noreg) {
 729       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
 730     } else {
 731       iu_barrier(masm, val, tmp3);
 732       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
 733       if (ShenandoahCardBarrier) {
 734         store_check(masm, tmp1);
 735       }
 736     }
 737     NOT_LP64(imasm->restore_bcp());
 738   } else {
 739     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
 740   }
 741 }
 742 
 743 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 744                                                                   Register obj, Register tmp, Label& slowpath) {
 745   Label done;
 746   // Resolve jobject
 747   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 748 
 749   // Check for null.
 750   __ testptr(obj, obj);
 751   __ jcc(Assembler::zero, done);
 752 
 753   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 754   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 755   __ jccb(Assembler::notZero, slowpath);

 911   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 912   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 913 
 914   if (exchange) {
 915     __ bind(L_failure);
 916     __ bind(L_success);
 917   } else {
 918     assert(res != noreg, "need result register");
 919 
 920     Label exit;
 921     __ bind(L_failure);
 922     __ xorptr(res, res);
 923     __ jmpb(exit);
 924 
 925     __ bind(L_success);
 926     __ movptr(res, 1);
 927     __ bind(exit);
 928   }
 929 }
 930 
 931 #ifdef PRODUCT
 932 #define BLOCK_COMMENT(str) /* nothing */
 933 #else
 934 #define BLOCK_COMMENT(str) __ block_comment(str)
 935 #endif
 936 
 937 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 938 
 939 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
 940 
 941 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 942                                                                      Register addr, Register count,
 943                                                                      Register tmp) {
 944   assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
 945 
 946   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 947   CardTable* ct = bs->card_table();
 948   intptr_t disp = (intptr_t) ct->byte_map_base();
 949 
 950   Label L_loop, L_done;
 951   const Register end = count;
 952   assert_different_registers(addr, end);
 953 
 954   // Zero count? Nothing to do.
 955   __ testl(count, count);
 956   __ jccb(Assembler::zero, L_done);
 957 
 958 #ifdef _LP64
 959   __ leaq(end, Address(addr, count, TIMES_OOP, 0));  // end == addr+count*oop_size
 960   __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
 961   __ shrptr(addr, CardTable::card_shift());
 962   __ shrptr(end, CardTable::card_shift());
 963   __ subptr(end, addr); // end --> cards count
 964 
 965   __ mov64(tmp, disp);
 966   __ addptr(addr, tmp);
 967 
 968   __ BIND(L_loop);
 969   __ movb(Address(addr, count, Address::times_1), 0);
 970   __ decrement(count);
 971   __ jccb(Assembler::greaterEqual, L_loop);
 972 #else
 973   __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
 974   __ shrptr(addr, CardTable::card_shift());
 975   __ shrptr(end,  CardTable::card_shift());
 976   __ subptr(end, addr); // end --> count
 977 
 978   __ BIND(L_loop);
 979   Address cardtable(addr, count, Address::times_1, disp);
 980   __ movb(cardtable, 0);
 981   __ decrement(count);
 982   __ jccb(Assembler::greaterEqual, L_loop);
 983 #endif
 984 
 985   __ BIND(L_done);
 986 }
 987 
 988 #undef __
 989 
 990 #ifdef COMPILER1
 991 
 992 #define __ ce->masm()->
 993 
 994 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 995   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 996   // At this point we know that marking is in progress.
 997   // If do_load() is true then we have to emit the
 998   // load of the previous value; otherwise it has already
 999   // been loaded into _pre_val.
1000 
1001   __ bind(*stub->entry());
1002   assert(stub->pre_val()->is_register(), "Precondition.");
1003 
1004   Register pre_val_reg = stub->pre_val()->as_register();
1005 
1006   if (stub->do_load()) {
1007     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
< prev index next >