< prev index next >

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

Print this page

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  28 #include "gc/shenandoah/shenandoahForwarding.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  34 #include "interpreter/interpreter.hpp"
  35 #include "runtime/sharedRuntime.hpp"
  36 #include "runtime/thread.hpp"
  37 #include "utilities/macros.hpp"
  38 #ifdef COMPILER1
  39 #include "c1/c1_LIRAssembler.hpp"
  40 #include "c1/c1_MacroAssembler.hpp"
  41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  42 #endif
  43 
  44 #define __ masm->
  45 
  46 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  47   if (handle_gpr) {
  48     __ push_IU_state();
  49   }
  50 
  51   if (handle_fp) {
  52     // Some paths can be reached from the c2i adapter with live fp arguments in registers.
  53     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));

 103       __ movflt(xmm5, Address(rsp, xmm_size * 5));
 104       __ movflt(xmm6, Address(rsp, xmm_size * 6));
 105       __ movflt(xmm7, Address(rsp, xmm_size * 7));
 106       __ addptr(rsp, xmm_size * 8);
 107     } else {
 108       __ pop_FPU_state();
 109     }
 110   }
 111 
 112   if (handle_gpr) {
 113     __ pop_IU_state();
 114   }
 115 }
 116 
 117 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 118                                                        Register src, Register dst, Register count) {
 119 
 120   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 121 
 122   if (is_reference_type(type)) {
























 123 
 124     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 125 #ifdef _LP64
 126       Register thread = r15_thread;
 127 #else
 128       Register thread = rax;
 129       if (thread == src || thread == dst || thread == count) {
 130         thread = rbx;
 131       }
 132       if (thread == src || thread == dst || thread == count) {
 133         thread = rcx;
 134       }
 135       if (thread == src || thread == dst || thread == count) {
 136         thread = rdx;
 137       }
 138       __ push(thread);
 139       __ get_thread(thread);
 140 #endif
 141       assert_different_registers(src, dst, count, thread);
 142 
 143       Label done;
 144       // Short-circuit if count == 0.
 145       __ testptr(count, count);
 146       __ jcc(Assembler::zero, done);
 147 
 148       // Avoid runtime call when not active.
 149       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 150       int flags;
 151       if (ShenandoahSATBBarrier && dest_uninitialized) {
 152         flags = ShenandoahHeap::HAS_FORWARDED;
 153       } else {
 154         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 155       }
 156       __ testb(gc_state, flags);
 157       __ jcc(Assembler::zero, done);
 158 
 159       save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 160 
 161 #ifdef _LP64
 162       assert(src == rdi, "expected");
 163       assert(dst == rsi, "expected");
 164       assert(count == rdx, "expected");
 165       if (UseCompressedOops) {
 166         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
 167                         src, dst, count);
 168       } else
 169 #endif
 170       {
 171         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 172                         src, dst, count);
 173       }
 174 
 175       restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 176 
 177       __ bind(done);
 178       NOT_LP64(__ pop(thread);)
 179     }
 180   }
 181 
 182 }
 183 





























 184 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 185                                                                  Register obj,
 186                                                                  Register pre_val,
 187                                                                  Register thread,
 188                                                                  Register tmp,
 189                                                                  bool tosca_live,
 190                                                                  bool expand_call) {
 191 
 192   if (ShenandoahSATBBarrier) {
 193     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 194   }
 195 }
 196 
 197 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 198                                                            Register obj,
 199                                                            Register pre_val,
 200                                                            Register thread,
 201                                                            Register tmp,
 202                                                            bool tosca_live,
 203                                                            bool expand_call) {

 207 
 208 #ifdef _LP64
 209   assert(thread == r15_thread, "must be");
 210 #endif // _LP64
 211 
 212   Label done;
 213   Label runtime;
 214 
 215   assert(pre_val != noreg, "check this code");
 216 
 217   if (obj != noreg) {
 218     assert_different_registers(obj, pre_val, tmp);
 219     assert(pre_val != rax, "check this code");
 220   }
 221 
 222   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 223   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 224   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 225 
 226   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 227   __ testb(gc_state, ShenandoahHeap::MARKING);
 228   __ jcc(Assembler::zero, done);
 229 
 230   // Do we need to load the previous value?
 231   if (obj != noreg) {
 232     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 233   }
 234 
 235   // Is the previous value null?
 236   __ cmpptr(pre_val, (int32_t) NULL_WORD);
 237   __ jcc(Assembler::equal, done);
 238 
 239   // Can we store original value in the thread's buffer?
 240   // Is index == 0?
 241   // (The index field is typed as size_t.)
 242 
 243   __ movptr(tmp, index);                   // tmp := *index_adr
 244   __ cmpptr(tmp, 0);                       // tmp == 0?
 245   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 246 
 247   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize

 573     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 574     assert_different_registers(dst, tmp1, tmp_thread);
 575     if (!thread->is_valid()) {
 576       thread = rdx;
 577     }
 578     NOT_LP64(__ get_thread(thread));
 579     // Generate the SATB pre-barrier code to log the value of
 580     // the referent field in an SATB buffer.
 581     shenandoah_write_barrier_pre(masm /* masm */,
 582                                  noreg /* obj */,
 583                                  dst /* pre_val */,
 584                                  thread /* thread */,
 585                                  tmp1 /* tmp */,
 586                                  true /* tosca_live */,
 587                                  true /* expand_call */);
 588 
 589     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 590   }
 591 }
 592 













































 593 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 594               Address dst, Register val, Register tmp1, Register tmp2) {
 595 
 596   bool on_oop = is_reference_type(type);
 597   bool in_heap = (decorators & IN_HEAP) != 0;
 598   bool as_normal = (decorators & AS_NORMAL) != 0;
 599   if (on_oop && in_heap) {
 600     bool needs_pre_barrier = as_normal;
 601 
 602     Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
 603     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 604     // flatten object address if needed
 605     // We do it regardless of precise because we need the registers
 606     if (dst.index() == noreg && dst.disp() == 0) {
 607       if (dst.base() != tmp1) {
 608         __ movptr(tmp1, dst.base());
 609       }
 610     } else {
 611       __ lea(tmp1, dst);
 612     }
 613 
 614     assert_different_registers(val, tmp1, tmp2, tmp3, rthread);

 616 #ifndef _LP64
 617     __ get_thread(rthread);
 618     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 619     imasm->save_bcp();
 620 #endif
 621 
 622     if (needs_pre_barrier) {
 623       shenandoah_write_barrier_pre(masm /*masm*/,
 624                                    tmp1 /* obj */,
 625                                    tmp2 /* pre_val */,
 626                                    rthread /* thread */,
 627                                    tmp3  /* tmp */,
 628                                    val != noreg /* tosca_live */,
 629                                    false /* expand_call */);
 630     }
 631     if (val == noreg) {
 632       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 633     } else {
 634       iu_barrier(masm, val, tmp3);
 635       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);

 636     }
 637     NOT_LP64(imasm->restore_bcp());
 638   } else {
 639     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
 640   }
 641 }
 642 
 643 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 644                                                                   Register obj, Register tmp, Label& slowpath) {
 645   Label done;
 646   // Resolve jobject
 647   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 648 
 649   // Check for null.
 650   __ testptr(obj, obj);
 651   __ jcc(Assembler::zero, done);
 652 
 653   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 654   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 655   __ jccb(Assembler::notZero, slowpath);

 811   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 812   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 813 
 814   if (exchange) {
 815     __ bind(L_failure);
 816     __ bind(L_success);
 817   } else {
 818     assert(res != NULL, "need result register");
 819 
 820     Label exit;
 821     __ bind(L_failure);
 822     __ xorptr(res, res);
 823     __ jmpb(exit);
 824 
 825     __ bind(L_success);
 826     __ movptr(res, 1);
 827     __ bind(exit);
 828   }
 829 }
 830 























































 831 #undef __
 832 
 833 #ifdef COMPILER1
 834 
 835 #define __ ce->masm()->
 836 
 837 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 838   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 839   // At this point we know that marking is in progress.
 840   // If do_load() is true then we have to emit the
 841   // load of the previous value; otherwise it has already
 842   // been loaded into _pre_val.
 843 
 844   __ bind(*stub->entry());
 845   assert(stub->pre_val()->is_register(), "Precondition.");
 846 
 847   Register pre_val_reg = stub->pre_val()->as_register();
 848 
 849   if (stub->do_load()) {
 850     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);

 926   __ prologue("shenandoah_pre_barrier", false);
 927   // arg0 : previous value of memory
 928 
 929   __ push(rax);
 930   __ push(rdx);
 931 
 932   const Register pre_val = rax;
 933   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 934   const Register tmp = rdx;
 935 
 936   NOT_LP64(__ get_thread(thread);)
 937 
 938   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 939   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 940 
 941   Label done;
 942   Label runtime;
 943 
 944   // Is SATB still active?
 945   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 946   __ testb(gc_state, ShenandoahHeap::MARKING);
 947   __ jcc(Assembler::zero, done);
 948 
 949   // Can we store original value in the thread's buffer?
 950 
 951   __ movptr(tmp, queue_index);
 952   __ testptr(tmp, tmp);
 953   __ jcc(Assembler::zero, runtime);
 954   __ subptr(tmp, wordSize);
 955   __ movptr(queue_index, tmp);
 956   __ addptr(tmp, buffer);
 957 
 958   // prev_val (rax)
 959   __ load_parameter(0, pre_val);
 960   __ movptr(Address(tmp, 0), pre_val);
 961   __ jmp(done);
 962 
 963   __ bind(runtime);
 964 
 965   __ save_live_registers_no_oop_map(true);
 966 

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  28 #include "gc/shenandoah/shenandoahForwarding.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  34 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/thread.hpp"
  38 #include "utilities/macros.hpp"
  39 #ifdef COMPILER1
  40 #include "c1/c1_LIRAssembler.hpp"
  41 #include "c1/c1_MacroAssembler.hpp"
  42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  43 #endif
  44 
  45 #define __ masm->
  46 
  47 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  48   if (handle_gpr) {
  49     __ push_IU_state();
  50   }
  51 
  52   if (handle_fp) {
  53     // Some paths can be reached from the c2i adapter with live fp arguments in registers.
  54     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));

 104       __ movflt(xmm5, Address(rsp, xmm_size * 5));
 105       __ movflt(xmm6, Address(rsp, xmm_size * 6));
 106       __ movflt(xmm7, Address(rsp, xmm_size * 7));
 107       __ addptr(rsp, xmm_size * 8);
 108     } else {
 109       __ pop_FPU_state();
 110     }
 111   }
 112 
 113   if (handle_gpr) {
 114     __ pop_IU_state();
 115   }
 116 }
 117 
 118 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 119                                                        Register src, Register dst, Register count) {
 120 
 121   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 122 
 123   if (is_reference_type(type)) {
 124     if (ShenandoahHeap::heap()->mode()->is_generational()) {
 125       bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 126       bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 127       bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 128 
 129       // We need to squirrel away the original element count because the
 130       // array copy assembly will destroy the value and we need it for the
 131       // card marking barrier.
 132 #ifdef _LP64
 133       if (!checkcast) {
 134         if (!obj_int) {
 135           // Save count for barrier
 136           __ movptr(r11, count);
 137         } else if (disjoint) {
 138           // Save dst in r11 in the disjoint case
 139           __ movq(r11, dst);
 140         }
 141       }
 142 #else
 143 if (disjoint) {
 144         __ mov(rdx, dst);          // save 'to'
 145       }
 146 #endif
 147     }
 148 
 149     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 150 #ifdef _LP64
 151       Register thread = r15_thread;
 152 #else
 153       Register thread = rax;
 154       if (thread == src || thread == dst || thread == count) {
 155         thread = rbx;
 156       }
 157       if (thread == src || thread == dst || thread == count) {
 158         thread = rcx;
 159       }
 160       if (thread == src || thread == dst || thread == count) {
 161         thread = rdx;
 162       }
 163       __ push(thread);
 164       __ get_thread(thread);
 165 #endif
 166       assert_different_registers(src, dst, count, thread);
 167 
 168       Label done;
 169       // Short-circuit if count == 0.
 170       __ testptr(count, count);
 171       __ jcc(Assembler::zero, done);
 172 
 173       // Avoid runtime call when not active.
 174       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 175       int flags;
 176       if (ShenandoahSATBBarrier && dest_uninitialized) {
 177         flags = ShenandoahHeap::HAS_FORWARDED;
 178       } else {
 179         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
 180       }
 181       __ testb(gc_state, flags);
 182       __ jcc(Assembler::zero, done);
 183 
 184       save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 185 
 186 #ifdef _LP64
 187       assert(src == rdi, "expected");
 188       assert(dst == rsi, "expected");
 189       assert(count == rdx, "expected");
 190       if (UseCompressedOops) {
 191         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
 192                         src, dst, count);
 193       } else
 194 #endif
 195       {
 196         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 197                         src, dst, count);
 198       }
 199 
 200       restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 201 
 202       __ bind(done);
 203       NOT_LP64(__ pop(thread);)
 204     }
 205   }
 206 
 207 }
 208 
 209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 210                                                        Register src, Register dst, Register count) {
 211   bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 212   bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 213   bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 214   Register tmp = rax;
 215 
 216 if (is_reference_type(type)) {
 217 #ifdef _LP64
 218     if (!checkcast) {
 219       if (!obj_int) {
 220         // Save count for barrier
 221         count = r11;
 222       } else if (disjoint) {
 223         // Use the saved dst in the disjoint case
 224         dst = r11;
 225       }
 226     } else {
 227       tmp = rscratch1;
 228     }
 229 #else
 230     if (disjoint) {
 231       __ mov(dst, rdx); // restore 'to'
 232     }
 233 #endif
 234     gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
 235   }
 236 }
 237 
 238 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 239                                                                  Register obj,
 240                                                                  Register pre_val,
 241                                                                  Register thread,
 242                                                                  Register tmp,
 243                                                                  bool tosca_live,
 244                                                                  bool expand_call) {
 245 
 246   if (ShenandoahSATBBarrier) {
 247     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 248   }
 249 }
 250 
 251 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 252                                                            Register obj,
 253                                                            Register pre_val,
 254                                                            Register thread,
 255                                                            Register tmp,
 256                                                            bool tosca_live,
 257                                                            bool expand_call) {

 261 
 262 #ifdef _LP64
 263   assert(thread == r15_thread, "must be");
 264 #endif // _LP64
 265 
 266   Label done;
 267   Label runtime;
 268 
 269   assert(pre_val != noreg, "check this code");
 270 
 271   if (obj != noreg) {
 272     assert_different_registers(obj, pre_val, tmp);
 273     assert(pre_val != rax, "check this code");
 274   }
 275 
 276   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 277   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 278   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 279 
 280   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 281   __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 282   __ jcc(Assembler::zero, done);
 283 
 284   // Do we need to load the previous value?
 285   if (obj != noreg) {
 286     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 287   }
 288 
 289   // Is the previous value null?
 290   __ cmpptr(pre_val, (int32_t) NULL_WORD);
 291   __ jcc(Assembler::equal, done);
 292 
 293   // Can we store original value in the thread's buffer?
 294   // Is index == 0?
 295   // (The index field is typed as size_t.)
 296 
 297   __ movptr(tmp, index);                   // tmp := *index_adr
 298   __ cmpptr(tmp, 0);                       // tmp == 0?
 299   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 300 
 301   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize

 627     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 628     assert_different_registers(dst, tmp1, tmp_thread);
 629     if (!thread->is_valid()) {
 630       thread = rdx;
 631     }
 632     NOT_LP64(__ get_thread(thread));
 633     // Generate the SATB pre-barrier code to log the value of
 634     // the referent field in an SATB buffer.
 635     shenandoah_write_barrier_pre(masm /* masm */,
 636                                  noreg /* obj */,
 637                                  dst /* pre_val */,
 638                                  thread /* thread */,
 639                                  tmp1 /* tmp */,
 640                                  true /* tosca_live */,
 641                                  true /* expand_call */);
 642 
 643     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 644   }
 645 }
 646 
 647 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
 648   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 649     return;
 650   }
 651 
 652   // Does a store check for the oop in register obj. The content of
 653   // register obj is destroyed afterwards.
 654 
 655   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 656   CardTable* ct = ctbs->card_table();
 657 
 658   __ shrptr(obj, CardTable::card_shift);
 659 
 660   Address card_addr;
 661 
 662   // The calculation for byte_map_base is as follows:
 663   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
 664   // So this essentially converts an address to a displacement and it will
 665   // never need to be relocated. On 64bit however the value may be too
 666   // large for a 32bit displacement.
 667   intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
 668   if (__ is_simm32(byte_map_base)) {
 669     card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
 670   } else {
 671     // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
 672     // displacement and done in a single instruction given favorable mapping and a
 673     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
 674     // entry and that entry is not properly handled by the relocation code.
 675     AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
 676     Address index(noreg, obj, Address::times_1);
 677     card_addr = __ as_Address(ArrayAddress(cardtable, index));
 678   }
 679 
 680   int dirty = CardTable::dirty_card_val();
 681   if (UseCondCardMark) {
 682     Label L_already_dirty;
 683     __ cmpb(card_addr, dirty);
 684     __ jcc(Assembler::equal, L_already_dirty);
 685     __ movb(card_addr, dirty);
 686     __ bind(L_already_dirty);
 687   } else {
 688     __ movb(card_addr, dirty);
 689   }
 690 }
 691 
 692 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 693                                              Address dst, Register val, Register tmp1, Register tmp2) {
 694 
 695   bool on_oop = is_reference_type(type);
 696   bool in_heap = (decorators & IN_HEAP) != 0;
 697   bool as_normal = (decorators & AS_NORMAL) != 0;
 698   if (on_oop && in_heap) {
 699     bool needs_pre_barrier = as_normal;
 700 
 701     Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
 702     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 703     // flatten object address if needed
 704     // We do it regardless of precise because we need the registers
 705     if (dst.index() == noreg && dst.disp() == 0) {
 706       if (dst.base() != tmp1) {
 707         __ movptr(tmp1, dst.base());
 708       }
 709     } else {
 710       __ lea(tmp1, dst);
 711     }
 712 
 713     assert_different_registers(val, tmp1, tmp2, tmp3, rthread);

 715 #ifndef _LP64
 716     __ get_thread(rthread);
 717     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 718     imasm->save_bcp();
 719 #endif
 720 
 721     if (needs_pre_barrier) {
 722       shenandoah_write_barrier_pre(masm /*masm*/,
 723                                    tmp1 /* obj */,
 724                                    tmp2 /* pre_val */,
 725                                    rthread /* thread */,
 726                                    tmp3  /* tmp */,
 727                                    val != noreg /* tosca_live */,
 728                                    false /* expand_call */);
 729     }
 730     if (val == noreg) {
 731       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 732     } else {
 733       iu_barrier(masm, val, tmp3);
 734       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 735       store_check(masm, tmp1);
 736     }
 737     NOT_LP64(imasm->restore_bcp());
 738   } else {
 739     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
 740   }
 741 }
 742 
 743 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 744                                                                   Register obj, Register tmp, Label& slowpath) {
 745   Label done;
 746   // Resolve jobject
 747   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 748 
 749   // Check for null.
 750   __ testptr(obj, obj);
 751   __ jcc(Assembler::zero, done);
 752 
 753   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 754   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 755   __ jccb(Assembler::notZero, slowpath);

 911   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 912   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 913 
 914   if (exchange) {
 915     __ bind(L_failure);
 916     __ bind(L_success);
 917   } else {
 918     assert(res != NULL, "need result register");
 919 
 920     Label exit;
 921     __ bind(L_failure);
 922     __ xorptr(res, res);
 923     __ jmpb(exit);
 924 
 925     __ bind(L_success);
 926     __ movptr(res, 1);
 927     __ bind(exit);
 928   }
 929 }
 930 
 931 #ifdef PRODUCT
 932 #define BLOCK_COMMENT(str) /* nothing */
 933 #else
 934 #define BLOCK_COMMENT(str) __ block_comment(str)
 935 #endif
 936 
 937 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 938 
 939 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
 940 
 941 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp) {
 942   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 943     return;
 944   }
 945 
 946   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 947   CardTable* ct = bs->card_table();
 948   intptr_t disp = (intptr_t) ct->byte_map_base();
 949 
 950   Label L_loop, L_done;
 951   const Register end = count;
 952   assert_different_registers(addr, end);
 953 
 954   __ testl(count, count);
 955   __ jcc(Assembler::zero, L_done); // zero count - nothing to do
 956 
 957 
 958 #ifdef _LP64
 959   __ leaq(end, Address(addr, count, TIMES_OOP, 0));  // end == addr+count*oop_size
 960   __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
 961   __ shrptr(addr, CardTable::card_shift);
 962   __ shrptr(end, CardTable::card_shift);
 963   __ subptr(end, addr); // end --> cards count
 964 
 965   __ mov64(tmp, disp);
 966   __ addptr(addr, tmp);
 967 __ BIND(L_loop);
 968   __ movb(Address(addr, count, Address::times_1), 0);
 969   __ decrement(count);
 970   __ jcc(Assembler::greaterEqual, L_loop);
 971 #else
 972   __ lea(end,  Address(addr, count, Address::times_ptr, -wordSize));
 973   __ shrptr(addr, CardTable::card_shift);
 974   __ shrptr(end,   CardTable::card_shift);
 975   __ subptr(end, addr); // end --> count
 976 __ BIND(L_loop);
 977   Address cardtable(addr, count, Address::times_1, disp);
 978   __ movb(cardtable, 0);
 979   __ decrement(count);
 980   __ jcc(Assembler::greaterEqual, L_loop);
 981 #endif
 982 
 983 __ BIND(L_done);
 984 }
 985 
 986 #undef __
 987 
 988 #ifdef COMPILER1
 989 
 990 #define __ ce->masm()->
 991 
 992 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 993   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 994   // At this point we know that marking is in progress.
 995   // If do_load() is true then we have to emit the
 996   // load of the previous value; otherwise it has already
 997   // been loaded into _pre_val.
 998 
 999   __ bind(*stub->entry());
1000   assert(stub->pre_val()->is_register(), "Precondition.");
1001 
1002   Register pre_val_reg = stub->pre_val()->as_register();
1003 
1004   if (stub->do_load()) {
1005     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);

1081   __ prologue("shenandoah_pre_barrier", false);
1082   // arg0 : previous value of memory
1083 
1084   __ push(rax);
1085   __ push(rdx);
1086 
1087   const Register pre_val = rax;
1088   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1089   const Register tmp = rdx;
1090 
1091   NOT_LP64(__ get_thread(thread);)
1092 
1093   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1094   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1095 
1096   Label done;
1097   Label runtime;
1098 
1099   // Is SATB still active?
1100   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1101   __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
1102   __ jcc(Assembler::zero, done);
1103 
1104   // Can we store original value in the thread's buffer?
1105 
1106   __ movptr(tmp, queue_index);
1107   __ testptr(tmp, tmp);
1108   __ jcc(Assembler::zero, runtime);
1109   __ subptr(tmp, wordSize);
1110   __ movptr(queue_index, tmp);
1111   __ addptr(tmp, buffer);
1112 
1113   // prev_val (rax)
1114   __ load_parameter(0, pre_val);
1115   __ movptr(Address(tmp, 0), pre_val);
1116   __ jmp(done);
1117 
1118   __ bind(runtime);
1119 
1120   __ save_live_registers_no_oop_map(true);
1121 
< prev index next >