1 /*
   2  * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  27 #include "gc/shenandoah/shenandoahForwarding.hpp"
  28 #include "gc/shenandoah/shenandoahHeap.hpp"
  29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  30 #include "gc/shenandoah/shenandoahRuntime.hpp"
  31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  32 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interp_masm.hpp"
  35 #include "runtime/sharedRuntime.hpp"
  36 #include "runtime/thread.hpp"
  37 #include "utilities/macros.hpp"
  38 #include "vmreg_x86.inline.hpp"
  39 #ifdef COMPILER1
  40 #include "c1/c1_LIRAssembler.hpp"
  41 #include "c1/c1_MacroAssembler.hpp"
  42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  43 #endif
  44 
  45 #define __ masm->
  46 
  47 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
  48 
  49 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
  50                                                        Register src, Register dst, Register count) {
  51 
  52   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  53 
  54   if (type == T_OBJECT || type == T_ARRAY) {
  55 
  56     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
  57 #ifdef _LP64
  58       Register thread = r15_thread;
  59 #else
  60       Register thread = rax;
  61       if (thread == src || thread == dst || thread == count) {
  62         thread = rbx;
  63       }
  64       if (thread == src || thread == dst || thread == count) {
  65         thread = rcx;
  66       }
  67       if (thread == src || thread == dst || thread == count) {
  68         thread = rdx;
  69       }
  70       __ push(thread);
  71       __ get_thread(thread);
  72 #endif
  73       assert_different_registers(src, dst, count, thread);
  74 
  75       Label done;
  76       // Short-circuit if count == 0.
  77       __ testptr(count, count);
  78       __ jcc(Assembler::zero, done);
  79 
  80       // Avoid runtime call when not active.
  81       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
  82       int flags;
  83       if (ShenandoahSATBBarrier && dest_uninitialized) {
  84         flags = ShenandoahHeap::HAS_FORWARDED;
  85       } else {
  86         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
  87       }
  88       __ testb(gc_state, flags);
  89       __ jcc(Assembler::zero, done);
  90 
  91       __ pusha();                      // push registers
  92 
  93 #ifdef _LP64
  94       assert(src == rdi, "expected");
  95       assert(dst == rsi, "expected");
  96       assert(count == rdx, "expected");
  97       if (UseCompressedOops) {
  98         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
  99                         src, dst, count);
 100       } else
 101 #endif
 102       {
 103         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 104                         src, dst, count);
 105       }
 106 
 107       __ popa();
 108       __ bind(done);
 109       NOT_LP64(__ pop(thread);)
 110     }
 111   }
 112 
 113 }
 114 
 115 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 116                                                                  Register obj,
 117                                                                  Register pre_val,
 118                                                                  Register thread,
 119                                                                  Register tmp,
 120                                                                  bool tosca_live,
 121                                                                  bool expand_call) {
 122 
 123   if (ShenandoahSATBBarrier) {
 124     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 125   }
 126 }
 127 
 128 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 129                                                            Register obj,
 130                                                            Register pre_val,
 131                                                            Register thread,
 132                                                            Register tmp,
 133                                                            bool tosca_live,
 134                                                            bool expand_call) {
 135   // If expand_call is true then we expand the call_VM_leaf macro
 136   // directly to skip generating the check by
 137   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
 138 
 139 #ifdef _LP64
 140   assert(thread == r15_thread, "must be");
 141 #endif // _LP64
 142 
 143   Label done;
 144   Label runtime;
 145 
 146   assert(pre_val != noreg, "check this code");
 147 
 148   if (obj != noreg) {
 149     assert_different_registers(obj, pre_val, tmp);
 150     assert(pre_val != rax, "check this code");
 151   }
 152 
 153   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 154   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 155   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 156 
 157   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 158   __ testb(gc_state, ShenandoahHeap::MARKING);
 159   __ jcc(Assembler::zero, done);
 160 
 161   // Do we need to load the previous value?
 162   if (obj != noreg) {
 163     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 164   }
 165 
 166   // Is the previous value null?
 167   __ cmpptr(pre_val, (int32_t) NULL_WORD);
 168   __ jcc(Assembler::equal, done);
 169 
 170   // Can we store original value in the thread's buffer?
 171   // Is index == 0?
 172   // (The index field is typed as size_t.)
 173 
 174   __ movptr(tmp, index);                   // tmp := *index_adr
 175   __ cmpptr(tmp, 0);                       // tmp == 0?
 176   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 177 
 178   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
 179   __ movptr(index, tmp);                   // *index_adr := tmp
 180   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
 181 
 182   // Record the previous value
 183   __ movptr(Address(tmp, 0), pre_val);
 184   __ jmp(done);
 185 
 186   __ bind(runtime);
 187   // save the live input values
 188   if(tosca_live) __ push(rax);
 189 
 190   if (obj != noreg && obj != rax)
 191     __ push(obj);
 192 
 193   if (pre_val != rax)
 194     __ push(pre_val);
 195 
 196   // Calling the runtime using the regular call_VM_leaf mechanism generates
 197   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
 198   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
 199   //
 200   // If we care generating the pre-barrier without a frame (e.g. in the
 201   // intrinsified Reference.get() routine) then ebp might be pointing to
 202   // the caller frame and so this check will most likely fail at runtime.
 203   //
 204   // Expanding the call directly bypasses the generation of the check.
 205   // So when we do not have have a full interpreter frame on the stack
 206   // expand_call should be passed true.
 207 
 208   NOT_LP64( __ push(thread); )
 209 
 210 #ifdef _LP64
 211   // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
 212   // pre_val be c_rarg1 (where the call prologue would copy thread argument).
 213   // Note: this should not accidentally smash thread, because thread is always r15.
 214   assert(thread != c_rarg0, "smashed arg");
 215   if (c_rarg0 != pre_val) {
 216     __ mov(c_rarg0, pre_val);
 217   }
 218 #endif
 219 
 220   if (expand_call) {
 221     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
 222 #ifdef _LP64
 223     if (c_rarg1 != thread) {
 224       __ mov(c_rarg1, thread);
 225     }
 226     // Already moved pre_val into c_rarg0 above
 227 #else
 228     __ push(thread);
 229     __ push(pre_val);
 230 #endif
 231     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
 232   } else {
 233     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
 234   }
 235 
 236   NOT_LP64( __ pop(thread); )
 237 
 238   // save the live input values
 239   if (pre_val != rax)
 240     __ pop(pre_val);
 241 
 242   if (obj != noreg && obj != rax)
 243     __ pop(obj);
 244 
 245   if(tosca_live) __ pop(rax);
 246 
 247   __ bind(done);
 248 }
 249 
 250 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src) {
 251   assert(ShenandoahLoadRefBarrier, "Should be enabled");
 252 
 253   Label done;
 254 
 255 #ifdef _LP64
 256   Register thread = r15_thread;
 257 #else
 258   Register thread = rcx;
 259   if (thread == dst) {
 260     thread = rbx;
 261   }
 262   __ push(thread);
 263   __ get_thread(thread);
 264 #endif
 265 
 266   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 267   __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
 268   __ jccb(Assembler::zero, done);
 269 
 270   // Use rsi for src address
 271   const Register src_addr = rsi;
 272   // Setup address parameter first, if it does not clobber oop in dst
 273   bool need_addr_setup = (src_addr != dst);
 274 
 275   if (need_addr_setup) {
 276     __ push(src_addr);
 277     __ lea(src_addr, src);
 278 
 279     if (dst != rax) {
 280       // Move obj into rax and save rax
 281       __ push(rax);
 282       __ movptr(rax, dst);
 283     }
 284   } else {
 285     // dst == rsi
 286     __ push(rax);
 287     __ movptr(rax, dst);
 288 
 289     // we can clobber it, since it is outgoing register
 290     __ lea(src_addr, src);
 291   }
 292 
 293   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
 294 
 295   if (need_addr_setup) {
 296     if (dst != rax) {
 297       __ movptr(dst, rax);
 298       __ pop(rax);
 299     }
 300     __ pop(src_addr);
 301   } else {
 302     __ movptr(dst, rax);
 303     __ pop(rax);
 304   }
 305 
 306   __ bind(done);
 307 
 308 #ifndef _LP64
 309     __ pop(thread);
 310 #endif
 311 }
 312 
 313 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
 314   if (ShenandoahStoreValEnqueueBarrier) {
 315     storeval_barrier_impl(masm, dst, tmp);
 316   }
 317 }
 318 
 319 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
 320   assert(ShenandoahStoreValEnqueueBarrier, "should be enabled");
 321 
 322   if (dst == noreg) return;
 323 
 324   if (ShenandoahStoreValEnqueueBarrier) {
 325     // The set of registers to be saved+restored is the same as in the write-barrier above.
 326     // Those are the commonly used registers in the interpreter.
 327     __ pusha();
 328     // __ push_callee_saved_registers();
 329     __ subptr(rsp, 2 * Interpreter::stackElementSize);
 330     __ movdbl(Address(rsp, 0), xmm0);
 331 
 332 #ifdef _LP64
 333     Register thread = r15_thread;
 334 #else
 335     Register thread = rcx;
 336     if (thread == dst || thread == tmp) {
 337       thread = rdi;
 338     }
 339     if (thread == dst || thread == tmp) {
 340       thread = rbx;
 341     }
 342     __ get_thread(thread);
 343 #endif
 344     assert_different_registers(dst, tmp, thread);
 345 
 346     satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false);
 347     __ movdbl(xmm0, Address(rsp, 0));
 348     __ addptr(rsp, 2 * Interpreter::stackElementSize);
 349     //__ pop_callee_saved_registers();
 350     __ popa();
 351   }
 352 }
 353 
 354 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src) {
 355   if (ShenandoahLoadRefBarrier) {
 356     Label done;
 357     __ testptr(dst, dst);
 358     __ jcc(Assembler::zero, done);
 359     load_reference_barrier_not_null(masm, dst, src);
 360     __ bind(done);
 361   }
 362 }
 363 
 364 //
 365 // Arguments:
 366 //
 367 // Inputs:
 368 //   src:        oop location, might be clobbered
 369 //   tmp1:       scratch register, might not be valid.
 370 //
 371 // Output:
 372 //   dst:        oop loaded from src location
 373 //
 374 // Kill:
 375 //   tmp1 (if it is valid)
 376 //
 377 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 378              Register dst, Address src, Register tmp1, Register tmp_thread) {
 379   // 1: non-reference load, no additional barrier is needed
 380   if (!is_reference_type(type)) {
 381     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 382     return;
 383   }
 384 
 385   assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected");
 386 
 387   // 2: load a reference from src location and apply LRB if needed
 388   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 389     Register result_dst = dst;
 390     bool use_tmp1_for_dst = false;
 391 
 392     // Preserve src location for LRB
 393     if (dst == src.base() || dst == src.index()) {
 394       // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
 395       if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
 396         dst = tmp1;
 397         use_tmp1_for_dst = true;
 398       } else {
 399         dst = rdi;
 400         __ push(dst);
 401       }
 402       assert_different_registers(dst, src.base(), src.index());
 403     }
 404 
 405     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 406 
 407     load_reference_barrier(masm, dst, src);
 408 
 409     // Move loaded oop to final destination
 410     if (dst != result_dst) {
 411       __ movptr(result_dst, dst);
 412 
 413       if (!use_tmp1_for_dst) {
 414         __ pop(dst);
 415       }
 416 
 417       dst = result_dst;
 418     }
 419   } else {
 420     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 421   }
 422 
 423   // 3: apply keep-alive barrier if needed
 424   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 425     __ push_IU_state();
 426     // That path can be reached from the c2i adapter with live fp
 427     // arguments in registers.
 428     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
 429     __ subptr(rsp, 64);
 430     __ movdbl(Address(rsp, 0), xmm0);
 431     __ movdbl(Address(rsp, 8), xmm1);
 432     __ movdbl(Address(rsp, 16), xmm2);
 433     __ movdbl(Address(rsp, 24), xmm3);
 434     __ movdbl(Address(rsp, 32), xmm4);
 435     __ movdbl(Address(rsp, 40), xmm5);
 436     __ movdbl(Address(rsp, 48), xmm6);
 437     __ movdbl(Address(rsp, 56), xmm7);
 438 
 439     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 440     assert_different_registers(dst, tmp1, tmp_thread);
 441     if (!thread->is_valid()) {
 442       thread = rdx;
 443     }
 444     NOT_LP64(__ get_thread(thread));
 445     // Generate the SATB pre-barrier code to log the value of
 446     // the referent field in an SATB buffer.
 447     shenandoah_write_barrier_pre(masm /* masm */,
 448                                  noreg /* obj */,
 449                                  dst /* pre_val */,
 450                                  thread /* thread */,
 451                                  tmp1 /* tmp */,
 452                                  true /* tosca_live */,
 453                                  true /* expand_call */);
 454     __ movdbl(xmm0, Address(rsp, 0));
 455     __ movdbl(xmm1, Address(rsp, 8));
 456     __ movdbl(xmm2, Address(rsp, 16));
 457     __ movdbl(xmm3, Address(rsp, 24));
 458     __ movdbl(xmm4, Address(rsp, 32));
 459     __ movdbl(xmm5, Address(rsp, 40));
 460     __ movdbl(xmm6, Address(rsp, 48));
 461     __ movdbl(xmm7, Address(rsp, 56));
 462     __ addptr(rsp, 64);
 463     __ pop_IU_state();
 464   }
 465 }
 466 
 467 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 468               Address dst, Register val, Register tmp1, Register tmp2) {
 469 
 470   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 471   bool in_heap = (decorators & IN_HEAP) != 0;
 472   bool as_normal = (decorators & AS_NORMAL) != 0;
 473   if (on_oop && in_heap) {
 474     bool needs_pre_barrier = as_normal;
 475 
 476     Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
 477     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 478     // flatten object address if needed
 479     // We do it regardless of precise because we need the registers
 480     if (dst.index() == noreg && dst.disp() == 0) {
 481       if (dst.base() != tmp1) {
 482         __ movptr(tmp1, dst.base());
 483       }
 484     } else {
 485       __ lea(tmp1, dst);
 486     }
 487 
 488     assert_different_registers(val, tmp1, tmp2, tmp3, rthread);
 489 
 490 #ifndef _LP64
 491     __ get_thread(rthread);
 492     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 493     imasm->save_bcp();
 494 #endif
 495 
 496     if (needs_pre_barrier) {
 497       shenandoah_write_barrier_pre(masm /*masm*/,
 498                                    tmp1 /* obj */,
 499                                    tmp2 /* pre_val */,
 500                                    rthread /* thread */,
 501                                    tmp3  /* tmp */,
 502                                    val != noreg /* tosca_live */,
 503                                    false /* expand_call */);
 504     }
 505     if (val == noreg) {
 506       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 507     } else {
 508       storeval_barrier(masm, val, tmp3);
 509       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 510     }
 511     NOT_LP64(imasm->restore_bcp());
 512   } else {
 513     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
 514   }
 515 }
 516 
 517 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 518                                                                   Register obj, Register tmp, Label& slowpath) {
 519   Label done;
 520   // Resolve jobject
 521   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 522 
 523   // Check for null.
 524   __ testptr(obj, obj);
 525   __ jcc(Assembler::zero, done);
 526 
 527   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 528   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 529   __ jccb(Assembler::notZero, slowpath);
 530   __ bind(done);
 531 }
 532 
 533 // Special Shenandoah CAS implementation that handles false negatives
 534 // due to concurrent evacuation.
 535 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
 536                                                 Register res, Address addr, Register oldval, Register newval,
 537                                                 bool exchange, Register tmp1, Register tmp2) {
 538   assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
 539   assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
 540   assert_different_registers(oldval, newval, tmp1, tmp2);
 541 
 542   Label L_success, L_failure;
 543 
 544   // Remember oldval for retry logic below
 545 #ifdef _LP64
 546   if (UseCompressedOops) {
 547     __ movl(tmp1, oldval);
 548   } else
 549 #endif
 550   {
 551     __ movptr(tmp1, oldval);
 552   }
 553 
 554   // Step 1. Fast-path.
 555   //
 556   // Try to CAS with given arguments. If successful, then we are done.
 557 
 558   if (os::is_MP()) __ lock();
 559 #ifdef _LP64
 560   if (UseCompressedOops) {
 561     __ cmpxchgl(newval, addr);
 562   } else
 563 #endif
 564   {
 565     __ cmpxchgptr(newval, addr);
 566   }
 567   __ jcc(Assembler::equal, L_success);
 568 
 569   // Step 2. CAS had failed. This may be a false negative.
 570   //
 571   // The trouble comes when we compare the to-space pointer with the from-space
 572   // pointer to the same object. To resolve this, it will suffice to resolve
 573   // the value from memory -- this will give both to-space pointers.
 574   // If they mismatch, then it was a legitimate failure.
 575   //
 576   // Before reaching to resolve sequence, see if we can avoid the whole shebang
 577   // with filters.
 578 
 579   // Filter: when offending in-memory value is NULL, the failure is definitely legitimate
 580   __ testptr(oldval, oldval);
 581   __ jcc(Assembler::zero, L_failure);
 582 
 583   // Filter: when heap is stable, the failure is definitely legitimate
 584 #ifdef _LP64
 585   const Register thread = r15_thread;
 586 #else
 587   const Register thread = tmp2;
 588   __ get_thread(thread);
 589 #endif
 590   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 591   __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
 592   __ jcc(Assembler::zero, L_failure);
 593 
 594 #ifdef _LP64
 595   if (UseCompressedOops) {
 596     __ movl(tmp2, oldval);
 597     __ decode_heap_oop(tmp2);
 598   } else
 599 #endif
 600   {
 601     __ movptr(tmp2, oldval);
 602   }
 603 
 604   // Decode offending in-memory value.
 605   // Test if-forwarded
 606   __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markOopDesc::marked_value);
 607   __ jcc(Assembler::noParity, L_failure);  // When odd number of bits, then not forwarded
 608   __ jcc(Assembler::zero, L_failure);      // When it is 00, then also not forwarded
 609 
 610   // Load and mask forwarding pointer
 611   __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
 612   __ shrptr(tmp2, 2);
 613   __ shlptr(tmp2, 2);
 614 
 615 #ifdef _LP64
 616   if (UseCompressedOops) {
 617     __ decode_heap_oop(tmp1); // decode for comparison
 618   }
 619 #endif
 620 
 621   // Now we have the forwarded offender in tmp2.
 622   // Compare and if they don't match, we have legitimate failure
 623   __ cmpptr(tmp1, tmp2);
 624   __ jcc(Assembler::notEqual, L_failure);
 625 
 626   // Step 3. Need to fix the memory ptr before continuing.
 627   //
 628   // At this point, we have from-space oldval in the register, and its to-space
 629   // address is in tmp2. Let's try to update it into memory. We don't care if it
 630   // succeeds or not. If it does, then the retrying CAS would see it and succeed.
 631   // If this fixup fails, this means somebody else beat us to it, and necessarily
 632   // with to-space ptr store. We still have to do the retry, because the GC might
 633   // have updated the reference for us.
 634 
 635 #ifdef _LP64
 636   if (UseCompressedOops) {
 637     __ encode_heap_oop(tmp2); // previously decoded at step 2.
 638   }
 639 #endif
 640 
 641   if (os::is_MP()) __ lock();
 642 #ifdef _LP64
 643   if (UseCompressedOops) {
 644     __ cmpxchgl(tmp2, addr);
 645   } else
 646 #endif
 647   {
 648     __ cmpxchgptr(tmp2, addr);
 649   }
 650 
 651   // Step 4. Try to CAS again.
 652   //
 653   // This is guaranteed not to have false negatives, because oldval is definitely
 654   // to-space, and memory pointer is to-space as well. Nothing is able to store
 655   // from-space ptr into memory anymore. Make sure oldval is restored, after being
 656   // garbled during retries.
 657   //
 658 #ifdef _LP64
 659   if (UseCompressedOops) {
 660     __ movl(oldval, tmp2);
 661   } else
 662 #endif
 663   {
 664     __ movptr(oldval, tmp2);
 665   }
 666 
 667   if (os::is_MP()) __ lock();
 668 #ifdef _LP64
 669   if (UseCompressedOops) {
 670     __ cmpxchgl(newval, addr);
 671   } else
 672 #endif
 673   {
 674     __ cmpxchgptr(newval, addr);
 675   }
 676   if (!exchange) {
 677     __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
 678   }
 679 
 680   // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
 681   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 682   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 683 
 684   if (exchange) {
 685     __ bind(L_failure);
 686     __ bind(L_success);
 687   } else {
 688     assert(res != NULL, "need result register");
 689 
 690     Label exit;
 691     __ bind(L_failure);
 692     __ xorptr(res, res);
 693     __ jmpb(exit);
 694 
 695     __ bind(L_success);
 696     __ movptr(res, 1);
 697     __ bind(exit);
 698   }
 699 }
 700 
 701 #ifdef _LP64
 702 // Copied from sharedRuntime_x86_64.cpp
 703 static int reg2offset_in(VMReg r) {
 704   // Account for saved rbp and return address
 705   // This should really be in_preserve_stack_slots
 706   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
 707 }
 708 
 709 // Copied from sharedRuntime_x86_64.cpp
 710 static int reg2offset_out(VMReg r) {
 711   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 712 }
 713 
 714 // Copied from sharedRuntime_x86_64.cpp
 715 static void move_ptr(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 716   if (src.first()->is_stack()) {
 717     if (dst.first()->is_stack()) {
 718       // stack to stack
 719       __ movq(rax, Address(rbp, reg2offset_in(src.first())));
 720       __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
 721     } else {
 722       // stack to reg
 723       __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 724     }
 725   } else if (dst.first()->is_stack()) {
 726     // reg to stack
 727     __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 728   } else {
 729     if (dst.first() != src.first()) {
 730       __ movq(dst.first()->as_Register(), src.first()->as_Register());
 731     }
 732   }
 733 }
 734 
 735 // Pin incoming array argument of java critical method
 736 void ShenandoahBarrierSetAssembler::pin_critical_native_array(MacroAssembler* masm,
 737                                                               VMRegPair reg,
 738                                                               int& pinned_slot) {
 739   __ block_comment("pin_critical_native_array {");
 740   Register tmp_reg = rax;
 741 
 742   Label is_null;
 743   VMRegPair tmp;
 744   VMRegPair in_reg = reg;
 745   bool on_stack = false;
 746 
 747   tmp.set_ptr(tmp_reg->as_VMReg());
 748   if (reg.first()->is_stack()) {
 749     // Load the arg up from the stack
 750     move_ptr(masm, reg, tmp);
 751     reg = tmp;
 752     on_stack = true;
 753   } else {
 754     __ movptr(rax, reg.first()->as_Register());
 755   }
 756   __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
 757   __ jccb(Assembler::equal, is_null);
 758 
 759   __ push(c_rarg0);
 760   __ push(c_rarg1);
 761   __ push(c_rarg2);
 762   __ push(c_rarg3);
 763 #ifdef _WIN64
 764   // caller-saved registers on Windows
 765   __ push(r10);
 766   __ push(r11);
 767 #else
 768   __ push(c_rarg4);
 769   __ push(c_rarg5);
 770 #endif
 771 
 772   if (reg.first()->as_Register() != c_rarg1) {
 773     __ movptr(c_rarg1, reg.first()->as_Register());
 774   }
 775   __ movptr(c_rarg0, r15_thread);
 776   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::pin_object)));
 777 
 778 #ifdef _WIN64
 779   __ pop(r11);
 780   __ pop(r10);
 781 #else
 782   __ pop(c_rarg5);
 783   __ pop(c_rarg4);
 784 #endif
 785   __ pop(c_rarg3);
 786   __ pop(c_rarg2);
 787   __ pop(c_rarg1);
 788   __ pop(c_rarg0);
 789 
 790   if (on_stack) {
 791     __ movptr(Address(rbp, reg2offset_in(in_reg.first())), rax);
 792     __ bind(is_null);
 793   } else {
 794     __ movptr(reg.first()->as_Register(), rax);
 795 
 796     // save on stack for unpinning later
 797     __ bind(is_null);
 798     assert(reg.first()->is_Register(), "Must be a register");
 799     int offset = pinned_slot * VMRegImpl::stack_slot_size;
 800     pinned_slot += VMRegImpl::slots_per_word;
 801     __ movq(Address(rsp, offset), rax);
 802   }
 803   __ block_comment("} pin_critical_native_array");
 804 }
 805 
 806 // Unpin array argument of java critical method
 807 void ShenandoahBarrierSetAssembler::unpin_critical_native_array(MacroAssembler* masm,
 808                                                                 VMRegPair reg,
 809                                                                 int& pinned_slot) {
 810   __ block_comment("unpin_critical_native_array {");
 811   Label is_null;
 812 
 813   if (reg.first()->is_stack()) {
 814     __ movptr(c_rarg1, Address(rbp, reg2offset_in(reg.first())));
 815   } else {
 816     int offset = pinned_slot * VMRegImpl::stack_slot_size;
 817     pinned_slot += VMRegImpl::slots_per_word;
 818     __ movq(c_rarg1, Address(rsp, offset));
 819   }
 820   __ testptr(c_rarg1, c_rarg1);
 821   __ jccb(Assembler::equal, is_null);
 822 
 823   __ movptr(c_rarg0, r15_thread);
 824   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object)));
 825 
 826   __ bind(is_null);
 827   __ block_comment("} unpin_critical_native_array");
 828 }
 829 #else
 830 // Copied from sharedRuntime_x86_32.cpp
 831 static int reg2offset_in(VMReg r) {
 832   // Account for saved rbp, and return address
 833   // This should really be in_preserve_stack_slots
 834   return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;
 835 }
 836 
 837 // Copied from sharedRuntime_x86_32.cpp
 838 static int reg2offset_out(VMReg r) {
 839   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 840 }
 841 
 842 // Copied from sharedRuntime_x86_32.cpp
 843 static void simple_move32(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 844   if (src.first()->is_stack()) {
 845     if (dst.first()->is_stack()) {
 846       // stack to stack
 847       // __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
 848       // __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
 849       __ movl2ptr(rax, Address(rbp, reg2offset_in(src.first())));
 850       __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
 851     } else {
 852       // stack to reg
 853       __ movl2ptr(dst.first()->as_Register(),  Address(rbp, reg2offset_in(src.first())));
 854     }
 855   } else if (dst.first()->is_stack()) {
 856     // reg to stack
 857     // no need to sign extend on 64bit
 858     __ movptr(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 859   } else {
 860     if (dst.first() != src.first()) {
 861       __ mov(dst.first()->as_Register(), src.first()->as_Register());
 862     }
 863   }
 864 }
 865 
 866 // Registers need to be saved for runtime call
 867 static Register caller_saved_registers[] = {
 868   rcx, rdx, rsi, rdi
 869 };
 870 
 871 // Save caller saved registers except r1 and r2
 872 static void save_registers_except(MacroAssembler* masm, Register r1, Register r2) {
 873   int reg_len = (int)(sizeof(caller_saved_registers) / sizeof(Register));
 874   for (int index = 0; index < reg_len; index ++) {
 875     Register this_reg = caller_saved_registers[index];
 876     if (this_reg != r1 && this_reg != r2) {
 877       __ push(this_reg);
 878     }
 879   }
 880 }
 881 
 882 // Restore caller saved registers except r1 and r2
 883 static void restore_registers_except(MacroAssembler* masm, Register r1, Register r2) {
 884   int reg_len = (int)(sizeof(caller_saved_registers) / sizeof(Register));
 885   for (int index = reg_len - 1; index >= 0; index --) {
 886     Register this_reg = caller_saved_registers[index];
 887     if (this_reg != r1 && this_reg != r2) {
 888       __ pop(this_reg);
 889     }
 890   }
 891 }
 892 
 893 // Pin object, return pinned object or null in rax
 894 void ShenandoahBarrierSetAssembler::gen_pin_object(MacroAssembler* masm,
 895                                                    Register thread, VMRegPair reg) {
 896   __ block_comment("gen_pin_object {");
 897 
 898   Label is_null;
 899   Register tmp_reg = rax;
 900   VMRegPair tmp(tmp_reg->as_VMReg());
 901   if (reg.first()->is_stack()) {
 902     // Load the arg up from the stack
 903     simple_move32(masm, reg, tmp);
 904     reg = tmp;
 905   } else {
 906     __ movl(tmp_reg, reg.first()->as_Register());
 907   }
 908   __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
 909   __ jccb(Assembler::equal, is_null);
 910 
 911   // Save registers that may be used by runtime call
 912   Register arg = reg.first()->is_Register() ? reg.first()->as_Register() : noreg;
 913   save_registers_except(masm, arg, thread);
 914 
 915   __ call_VM_leaf(
 916     CAST_FROM_FN_PTR(address, SharedRuntime::pin_object),
 917     thread, reg.first()->as_Register());
 918 
 919   // Restore saved registers
 920   restore_registers_except(masm, arg, thread);
 921 
 922   __ bind(is_null);
 923   __ block_comment("} gen_pin_object");
 924 }
 925 
 926 // Unpin object
 927 void ShenandoahBarrierSetAssembler::gen_unpin_object(MacroAssembler* masm,
 928                                                      Register thread, VMRegPair reg) {
 929   __ block_comment("gen_unpin_object {");
 930   Label is_null;
 931 
 932   // temp register
 933   __ push(rax);
 934   Register tmp_reg = rax;
 935   VMRegPair tmp(tmp_reg->as_VMReg());
 936 
 937   simple_move32(masm, reg, tmp);
 938 
 939   __ testptr(rax, rax);
 940   __ jccb(Assembler::equal, is_null);
 941 
 942   // Save registers that may be used by runtime call
 943   Register arg = reg.first()->is_Register() ? reg.first()->as_Register() : noreg;
 944   save_registers_except(masm, arg, thread);
 945 
 946   __ call_VM_leaf(
 947     CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object),
 948     thread, rax);
 949 
 950   // Restore saved registers
 951   restore_registers_except(masm, arg, thread);
 952   __ bind(is_null);
 953   __ pop(rax);
 954   __ block_comment("} gen_unpin_object");
 955 }
 956 #endif
 957 
 958 #undef __
 959 
 960 #ifdef COMPILER1
 961 
 962 #define __ ce->masm()->
 963 
 964 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 965   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 966   // At this point we know that marking is in progress.
 967   // If do_load() is true then we have to emit the
 968   // load of the previous value; otherwise it has already
 969   // been loaded into _pre_val.
 970 
 971   __ bind(*stub->entry());
 972   assert(stub->pre_val()->is_register(), "Precondition.");
 973 
 974   Register pre_val_reg = stub->pre_val()->as_register();
 975 
 976   if (stub->do_load()) {
 977     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
 978   }
 979 
 980   __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
 981   __ jcc(Assembler::equal, *stub->continuation());
 982   ce->store_parameter(stub->pre_val()->as_register(), 0);
 983   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
 984   __ jmp(*stub->continuation());
 985 
 986 }
 987 
 988 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
 989   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 990   __ bind(*stub->entry());
 991 
 992   Register obj = stub->obj()->as_register();
 993   Register res = stub->result()->as_register();
 994   Register addr = stub->addr()->as_pointer_register();
 995   Register tmp1 = stub->tmp1()->as_register();
 996   Register tmp2 = stub->tmp2()->as_register();
 997   assert_different_registers(obj, res, addr, tmp1, tmp2);
 998 
 999   Label slow_path;
1000 
1001   assert(res == rax, "result must arrive in rax");
1002 
1003   if (res != obj) {
1004     __ mov(res, obj);
1005   }
1006 
1007   // Check for null.
1008   __ testptr(res, res);
1009   __ jcc(Assembler::zero, *stub->continuation());
1010 
1011   // Check for object being in the collection set.
1012   __ mov(tmp1, res);
1013   __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1014   __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1015 #ifdef _LP64
1016   __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
1017   __ testbool(tmp2);
1018 #else
1019   // On x86_32, C1 register allocator can give us the register without 8-bit support.
1020   // Do the full-register access and test to avoid compilation failures.
1021   __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
1022   __ testptr(tmp2, 0xFF);
1023 #endif
1024   __ jcc(Assembler::zero, *stub->continuation());
1025 
1026   __ bind(slow_path);
1027   ce->store_parameter(res, 0);
1028   ce->store_parameter(addr, 1);
1029   __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
1030 
1031   __ jmp(*stub->continuation());
1032 }
1033 
1034 #undef __
1035 
1036 #define __ sasm->
1037 
1038 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
1039   __ prologue("shenandoah_pre_barrier", false);
1040   // arg0 : previous value of memory
1041 
1042   __ push(rax);
1043   __ push(rdx);
1044 
1045   const Register pre_val = rax;
1046   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1047   const Register tmp = rdx;
1048 
1049   NOT_LP64(__ get_thread(thread);)
1050 
1051   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1052   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1053 
1054   Label done;
1055   Label runtime;
1056 
1057   // Is SATB still active?
1058   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1059   __ testb(gc_state, ShenandoahHeap::MARKING);
1060   __ jcc(Assembler::zero, done);
1061 
1062   // Can we store original value in the thread's buffer?
1063 
1064   __ movptr(tmp, queue_index);
1065   __ testptr(tmp, tmp);
1066   __ jcc(Assembler::zero, runtime);
1067   __ subptr(tmp, wordSize);
1068   __ movptr(queue_index, tmp);
1069   __ addptr(tmp, buffer);
1070 
1071   // prev_val (rax)
1072   __ load_parameter(0, pre_val);
1073   __ movptr(Address(tmp, 0), pre_val);
1074   __ jmp(done);
1075 
1076   __ bind(runtime);
1077 
1078   __ save_live_registers_no_oop_map(true);
1079 
1080   // load the pre-value
1081   __ load_parameter(0, rcx);
1082   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
1083 
1084   __ restore_live_registers(true);
1085 
1086   __ bind(done);
1087 
1088   __ pop(rdx);
1089   __ pop(rax);
1090 
1091   __ epilogue();
1092 }
1093 
1094 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
1095   __ prologue("shenandoah_load_reference_barrier", false);
1096   // arg0 : object to be resolved
1097 
1098   __ save_live_registers_no_oop_map(true);
1099 
1100 #ifdef _LP64
1101   __ load_parameter(0, c_rarg0);
1102   __ load_parameter(1, c_rarg1);
1103   if (UseCompressedOops) {
1104     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
1105   } else {
1106     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
1107   }
1108 #else
1109   __ load_parameter(0, rax);
1110   __ load_parameter(1, rbx);
1111   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
1112 #endif
1113 
1114   __ restore_live_registers_except_rax(true);
1115 
1116   __ epilogue();
1117 }
1118 
1119 #undef __
1120 
1121 #endif // COMPILER1
1122 
1123 address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
1124   assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
1125   return _shenandoah_lrb;
1126 }
1127 
1128 #define __ cgen->assembler()->
1129 
1130 /*
1131  *  Incoming parameters:
1132  *  rax: oop
1133  *  rsi: load address
1134  */
1135 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
1136   __ align(CodeEntryAlignment);
1137   StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
1138   address start = __ pc();
1139 
1140   Label resolve_oop, slow_path;
1141 
1142   // We use RDI, which also serves as argument register for slow call.
1143   // RAX always holds the src object ptr, except after the slow call,
1144   // then it holds the result. R8/RBX is used as temporary register.
1145 
1146   Register tmp1 = rdi;
1147   Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
1148 
1149   __ push(tmp1);
1150   __ push(tmp2);
1151 
1152   // Check for object being in the collection set.
1153   __ mov(tmp1, rax);
1154   __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1155   __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1156   __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
1157   __ testbool(tmp2);
1158   __ jccb(Assembler::notZero, resolve_oop);
1159   __ pop(tmp2);
1160   __ pop(tmp1);
1161   __ ret(0);
1162 
1163   // Test if object is already resolved.
1164   __ bind(resolve_oop);
1165   __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
1166   // Test if both lowest bits are set. We trick it by negating the bits
1167   // then test for both bits clear.
1168   __ notptr(tmp2);
1169   __ testb(tmp2, markOopDesc::marked_value);
1170   __ jccb(Assembler::notZero, slow_path);
1171   // Clear both lower bits. It's still inverted, so set them, and then invert back.
1172   __ orptr(tmp2, markOopDesc::marked_value);
1173   __ notptr(tmp2);
1174   // At this point, tmp2 contains the decoded forwarding pointer.
1175   __ mov(rax, tmp2);
1176 
1177   __ pop(tmp2);
1178   __ pop(tmp1);
1179   __ ret(0);
1180 
1181   __ bind(slow_path);
1182 
1183   __ push(rcx);
1184   __ push(rdx);
1185   __ push(rdi);
1186 #ifdef _LP64
1187   __ push(r8);
1188   __ push(r9);
1189   __ push(r10);
1190   __ push(r11);
1191   __ push(r12);
1192   __ push(r13);
1193   __ push(r14);
1194   __ push(r15);
1195 #endif
1196   __ push(rbp);
1197   __ movptr(rbp, rsp);
1198   __ andptr(rsp, -StackAlignmentInBytes);
1199   __ push_FPU_state();
1200   if (UseCompressedOops) {
1201     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), rax, rsi);
1202   } else {
1203     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rsi);
1204   }
1205   __ pop_FPU_state();
1206   __ movptr(rsp, rbp);
1207   __ pop(rbp);
1208 #ifdef _LP64
1209   __ pop(r15);
1210   __ pop(r14);
1211   __ pop(r13);
1212   __ pop(r12);
1213   __ pop(r11);
1214   __ pop(r10);
1215   __ pop(r9);
1216   __ pop(r8);
1217 #endif
1218   __ pop(rdi);
1219   __ pop(rdx);
1220   __ pop(rcx);
1221 
1222   __ pop(tmp2);
1223   __ pop(tmp1);
1224   __ ret(0);
1225 
1226   return start;
1227 }
1228 
1229 #undef __
1230 
1231 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1232   if (ShenandoahLoadRefBarrier) {
1233     int stub_code_size = 4096;
1234     ResourceMark rm;
1235     BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1236     CodeBuffer buf(bb);
1237     StubCodeGenerator cgen(&buf);
1238     _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
1239   }
1240 }