1 /*
   2  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
   3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahRuntime.hpp"
  33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  35 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "runtime/javaThread.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "utilities/macros.hpp"
  40 #ifdef COMPILER1
  41 #include "c1/c1_LIRAssembler.hpp"
  42 #include "c1/c1_MacroAssembler.hpp"
  43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  44 #endif
  45 
  46 #define __ masm->
  47 
  48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  49   if (handle_gpr) {
  50     __ push_IU_state();
  51   }
  52 
  53   if (handle_fp) {
  54     // Some paths can be reached from the c2i adapter with live fp arguments in registers.
  55     LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
  56 
  57     if (UseSSE >= 2) {
  58       const int xmm_size = wordSize * LP64_ONLY(2) NOT_LP64(4);
  59       __ subptr(rsp, xmm_size * 8);
  60       __ movdbl(Address(rsp, xmm_size * 0), xmm0);
  61       __ movdbl(Address(rsp, xmm_size * 1), xmm1);
  62       __ movdbl(Address(rsp, xmm_size * 2), xmm2);
  63       __ movdbl(Address(rsp, xmm_size * 3), xmm3);
  64       __ movdbl(Address(rsp, xmm_size * 4), xmm4);
  65       __ movdbl(Address(rsp, xmm_size * 5), xmm5);
  66       __ movdbl(Address(rsp, xmm_size * 6), xmm6);
  67       __ movdbl(Address(rsp, xmm_size * 7), xmm7);
  68     } else if (UseSSE >= 1) {
  69       const int xmm_size = wordSize * LP64_ONLY(1) NOT_LP64(2);
  70       __ subptr(rsp, xmm_size * 8);
  71       __ movflt(Address(rsp, xmm_size * 0), xmm0);
  72       __ movflt(Address(rsp, xmm_size * 1), xmm1);
  73       __ movflt(Address(rsp, xmm_size * 2), xmm2);
  74       __ movflt(Address(rsp, xmm_size * 3), xmm3);
  75       __ movflt(Address(rsp, xmm_size * 4), xmm4);
  76       __ movflt(Address(rsp, xmm_size * 5), xmm5);
  77       __ movflt(Address(rsp, xmm_size * 6), xmm6);
  78       __ movflt(Address(rsp, xmm_size * 7), xmm7);
  79     } else {
  80       __ push_FPU_state();
  81     }
  82   }
  83 }
  84 
  85 static void restore_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
  86   if (handle_fp) {
  87     if (UseSSE >= 2) {
  88       const int xmm_size = wordSize * LP64_ONLY(2) NOT_LP64(4);
  89       __ movdbl(xmm0, Address(rsp, xmm_size * 0));
  90       __ movdbl(xmm1, Address(rsp, xmm_size * 1));
  91       __ movdbl(xmm2, Address(rsp, xmm_size * 2));
  92       __ movdbl(xmm3, Address(rsp, xmm_size * 3));
  93       __ movdbl(xmm4, Address(rsp, xmm_size * 4));
  94       __ movdbl(xmm5, Address(rsp, xmm_size * 5));
  95       __ movdbl(xmm6, Address(rsp, xmm_size * 6));
  96       __ movdbl(xmm7, Address(rsp, xmm_size * 7));
  97       __ addptr(rsp, xmm_size * 8);
  98     } else if (UseSSE >= 1) {
  99       const int xmm_size = wordSize * LP64_ONLY(1) NOT_LP64(2);
 100       __ movflt(xmm0, Address(rsp, xmm_size * 0));
 101       __ movflt(xmm1, Address(rsp, xmm_size * 1));
 102       __ movflt(xmm2, Address(rsp, xmm_size * 2));
 103       __ movflt(xmm3, Address(rsp, xmm_size * 3));
 104       __ movflt(xmm4, Address(rsp, xmm_size * 4));
 105       __ movflt(xmm5, Address(rsp, xmm_size * 5));
 106       __ movflt(xmm6, Address(rsp, xmm_size * 6));
 107       __ movflt(xmm7, Address(rsp, xmm_size * 7));
 108       __ addptr(rsp, xmm_size * 8);
 109     } else {
 110       __ pop_FPU_state();
 111     }
 112   }
 113 
 114   if (handle_gpr) {
 115     __ pop_IU_state();
 116   }
 117 }
 118 
 119 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 120                                                        Register src, Register dst, Register count) {
 121 
 122   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 123 
 124   if (is_reference_type(type)) {
 125     if (ShenandoahHeap::heap()->mode()->is_generational()) {
 126       bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 127       bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 128       bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 129 
 130       // We need to save the original element count because the array copy stub
 131       // will destroy the value and we need it for the card marking barrier.
 132 #ifdef _LP64
 133       if (!checkcast) {
 134         if (!obj_int) {
 135           // Save count for barrier
 136           __ movptr(r11, count);
 137         } else if (disjoint) {
 138           // Save dst in r11 in the disjoint case
 139           __ movq(r11, dst);
 140         }
 141       }
 142 #else
 143       if (disjoint) {
 144         __ mov(rdx, dst);          // save 'to'
 145       }
 146 #endif
 147     }
 148 
 149     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 150 #ifdef _LP64
 151       Register thread = r15_thread;
 152 #else
 153       Register thread = rax;
 154       if (thread == src || thread == dst || thread == count) {
 155         thread = rbx;
 156       }
 157       if (thread == src || thread == dst || thread == count) {
 158         thread = rcx;
 159       }
 160       if (thread == src || thread == dst || thread == count) {
 161         thread = rdx;
 162       }
 163       __ push(thread);
 164       __ get_thread(thread);
 165 #endif
 166       assert_different_registers(src, dst, count, thread);
 167 
 168       Label L_done;
 169       // Short-circuit if count == 0.
 170       __ testptr(count, count);
 171       __ jcc(Assembler::zero, L_done);
 172 
 173       // Avoid runtime call when not active.
 174       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 175       int flags;
 176       if (ShenandoahSATBBarrier && dest_uninitialized) {
 177         flags = ShenandoahHeap::HAS_FORWARDED;
 178       } else {
 179         flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
 180       }
 181       __ testb(gc_state, flags);
 182       __ jcc(Assembler::zero, L_done);
 183 
 184       save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 185 
 186 #ifdef _LP64
 187       assert(src == rdi, "expected");
 188       assert(dst == rsi, "expected");
 189       assert(count == rdx, "expected");
 190       if (UseCompressedOops) {
 191         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
 192                         src, dst, count);
 193       } else
 194 #endif
 195       {
 196         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
 197                         src, dst, count);
 198       }
 199 
 200       restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
 201 
 202       __ bind(L_done);
 203       NOT_LP64(__ pop(thread);)
 204     }
 205   }
 206 
 207 }
 208 
 209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 210                                                        Register src, Register dst, Register count) {
 211   bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 212   bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 213   bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 214   Register tmp = rax;
 215 
 216   if (is_reference_type(type)) {
 217 #ifdef _LP64
 218     if (!checkcast) {
 219       if (!obj_int) {
 220         // Save count for barrier
 221         count = r11;
 222       } else if (disjoint) {
 223         // Use the saved dst in the disjoint case
 224         dst = r11;
 225       }
 226     } else {
 227       tmp = rscratch1;
 228     }
 229 #else
 230     if (disjoint) {
 231       __ mov(dst, rdx); // restore 'to'
 232     }
 233 #endif
 234     gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
 235   }
 236 }
 237 
 238 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 239                                                                  Register obj,
 240                                                                  Register pre_val,
 241                                                                  Register thread,
 242                                                                  Register tmp,
 243                                                                  bool tosca_live,
 244                                                                  bool expand_call) {
 245 
 246   if (ShenandoahSATBBarrier) {
 247     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 248   }
 249 }
 250 
 251 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 252                                                            Register obj,
 253                                                            Register pre_val,
 254                                                            Register thread,
 255                                                            Register tmp,
 256                                                            bool tosca_live,
 257                                                            bool expand_call) {
 258   // If expand_call is true then we expand the call_VM_leaf macro
 259   // directly to skip generating the check by
 260   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
 261 
 262 #ifdef _LP64
 263   assert(thread == r15_thread, "must be");
 264 #endif // _LP64
 265 
 266   Label done;
 267   Label runtime;
 268 
 269   assert(pre_val != noreg, "check this code");
 270 
 271   if (obj != noreg) {
 272     assert_different_registers(obj, pre_val, tmp);
 273     assert(pre_val != rax, "check this code");
 274   }
 275 
 276   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 277   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 278   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 279 
 280   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 281   __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 282   __ jcc(Assembler::zero, done);
 283 
 284   // Do we need to load the previous value?
 285   if (obj != noreg) {
 286     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 287   }
 288 
 289   // Is the previous value null?
 290   __ cmpptr(pre_val, NULL_WORD);
 291   __ jcc(Assembler::equal, done);
 292 
 293   // Can we store original value in the thread's buffer?
 294   // Is index == 0?
 295   // (The index field is typed as size_t.)
 296 
 297   __ movptr(tmp, index);                   // tmp := *index_adr
 298   __ cmpptr(tmp, 0);                       // tmp == 0?
 299   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 300 
 301   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
 302   __ movptr(index, tmp);                   // *index_adr := tmp
 303   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
 304 
 305   // Record the previous value
 306   __ movptr(Address(tmp, 0), pre_val);
 307   __ jmp(done);
 308 
 309   __ bind(runtime);
 310   // save the live input values
 311   if(tosca_live) __ push(rax);
 312 
 313   if (obj != noreg && obj != rax)
 314     __ push(obj);
 315 
 316   if (pre_val != rax)
 317     __ push(pre_val);
 318 
 319   // Calling the runtime using the regular call_VM_leaf mechanism generates
 320   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
 321   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
 322   //
 323   // If we care generating the pre-barrier without a frame (e.g. in the
 324   // intrinsified Reference.get() routine) then ebp might be pointing to
 325   // the caller frame and so this check will most likely fail at runtime.
 326   //
 327   // Expanding the call directly bypasses the generation of the check.
 328   // So when we do not have have a full interpreter frame on the stack
 329   // expand_call should be passed true.
 330 
 331   NOT_LP64( __ push(thread); )
 332 
 333 #ifdef _LP64
 334   // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
 335   // pre_val be c_rarg1 (where the call prologue would copy thread argument).
 336   // Note: this should not accidentally smash thread, because thread is always r15.
 337   assert(thread != c_rarg0, "smashed arg");
 338   if (c_rarg0 != pre_val) {
 339     __ mov(c_rarg0, pre_val);
 340   }
 341 #endif
 342 
 343   if (expand_call) {
 344     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
 345 #ifdef _LP64
 346     if (c_rarg1 != thread) {
 347       __ mov(c_rarg1, thread);
 348     }
 349     // Already moved pre_val into c_rarg0 above
 350 #else
 351     __ push(thread);
 352     __ push(pre_val);
 353 #endif
 354     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
 355   } else {
 356     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
 357   }
 358 
 359   NOT_LP64( __ pop(thread); )
 360 
 361   // save the live input values
 362   if (pre_val != rax)
 363     __ pop(pre_val);
 364 
 365   if (obj != noreg && obj != rax)
 366     __ pop(obj);
 367 
 368   if(tosca_live) __ pop(rax);
 369 
 370   __ bind(done);
 371 }
 372 
 373 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) {
 374   assert(ShenandoahLoadRefBarrier, "Should be enabled");
 375 
 376   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 377   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 378   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 379   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 380   bool is_narrow  = UseCompressedOops && !is_native;
 381 
 382   Label heap_stable, not_cset;
 383 
 384   __ block_comment("load_reference_barrier { ");
 385 
 386   // Check if GC is active
 387 #ifdef _LP64
 388   Register thread = r15_thread;
 389 #else
 390   Register thread = rcx;
 391   if (thread == dst) {
 392     thread = rbx;
 393   }
 394   __ push(thread);
 395   __ get_thread(thread);
 396 #endif
 397 
 398   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 399   int flags = ShenandoahHeap::HAS_FORWARDED;
 400   if (!is_strong) {
 401     flags |= ShenandoahHeap::WEAK_ROOTS;
 402   }
 403   __ testb(gc_state, flags);
 404   __ jcc(Assembler::zero, heap_stable);
 405 
 406   Register tmp1 = noreg, tmp2 = noreg;
 407   if (is_strong) {
 408     // Test for object in cset
 409     // Allocate temporary registers
 410     for (int i = 0; i < 8; i++) {
 411       Register r = as_Register(i);
 412       if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) {
 413         if (tmp1 == noreg) {
 414           tmp1 = r;
 415         } else {
 416           tmp2 = r;
 417           break;
 418         }
 419       }
 420     }
 421     assert(tmp1 != noreg, "tmp1 allocated");
 422     assert(tmp2 != noreg, "tmp2 allocated");
 423     assert_different_registers(tmp1, tmp2, src.base(), src.index());
 424     assert_different_registers(tmp1, tmp2, dst);
 425 
 426     __ push(tmp1);
 427     __ push(tmp2);
 428 
 429     // Optimized cset-test
 430     __ movptr(tmp1, dst);
 431     __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 432     __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
 433     __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1));
 434     __ testbool(tmp1);
 435     __ jcc(Assembler::zero, not_cset);
 436   }
 437 
 438   save_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true);
 439 
 440   // The rest is saved with the optimized path
 441 
 442   uint num_saved_regs = 4 + (dst != rax ? 1 : 0) LP64_ONLY(+4);
 443   __ subptr(rsp, num_saved_regs * wordSize);
 444   uint slot = num_saved_regs;
 445   if (dst != rax) {
 446     __ movptr(Address(rsp, (--slot) * wordSize), rax);
 447   }
 448   __ movptr(Address(rsp, (--slot) * wordSize), rcx);
 449   __ movptr(Address(rsp, (--slot) * wordSize), rdx);
 450   __ movptr(Address(rsp, (--slot) * wordSize), rdi);
 451   __ movptr(Address(rsp, (--slot) * wordSize), rsi);
 452 #ifdef _LP64
 453   __ movptr(Address(rsp, (--slot) * wordSize), r8);
 454   __ movptr(Address(rsp, (--slot) * wordSize), r9);
 455   __ movptr(Address(rsp, (--slot) * wordSize), r10);
 456   __ movptr(Address(rsp, (--slot) * wordSize), r11);
 457   // r12-r15 are callee saved in all calling conventions
 458 #endif
 459   assert(slot == 0, "must use all slots");
 460 
 461   // Shuffle registers such that dst is in c_rarg0 and addr in c_rarg1.
 462 #ifdef _LP64
 463   Register arg0 = c_rarg0, arg1 = c_rarg1;
 464 #else
 465   Register arg0 = rdi, arg1 = rsi;
 466 #endif
 467   if (dst == arg1) {
 468     __ lea(arg0, src);
 469     __ xchgptr(arg1, arg0);
 470   } else {
 471     __ lea(arg1, src);
 472     __ movptr(arg0, dst);
 473   }
 474 
 475   if (is_strong) {
 476     if (is_narrow) {
 477       __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1);
 478     } else {
 479       __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1);
 480     }
 481   } else if (is_weak) {
 482     if (is_narrow) {
 483       __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
 484     } else {
 485       __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
 486     }
 487   } else {
 488     assert(is_phantom, "only remaining strength");
 489     assert(!is_narrow, "phantom access cannot be narrow");
 490     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1);
 491   }
 492 
 493 #ifdef _LP64
 494   __ movptr(r11, Address(rsp, (slot++) * wordSize));
 495   __ movptr(r10, Address(rsp, (slot++) * wordSize));
 496   __ movptr(r9,  Address(rsp, (slot++) * wordSize));
 497   __ movptr(r8,  Address(rsp, (slot++) * wordSize));
 498 #endif
 499   __ movptr(rsi, Address(rsp, (slot++) * wordSize));
 500   __ movptr(rdi, Address(rsp, (slot++) * wordSize));
 501   __ movptr(rdx, Address(rsp, (slot++) * wordSize));
 502   __ movptr(rcx, Address(rsp, (slot++) * wordSize));
 503 
 504   if (dst != rax) {
 505     __ movptr(dst, rax);
 506     __ movptr(rax, Address(rsp, (slot++) * wordSize));
 507   }
 508 
 509   assert(slot == num_saved_regs, "must use all slots");
 510   __ addptr(rsp, num_saved_regs * wordSize);
 511 
 512   restore_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true);
 513 
 514   __ bind(not_cset);
 515 
 516   if  (is_strong) {
 517     __ pop(tmp2);
 518     __ pop(tmp1);
 519   }
 520 
 521   __ bind(heap_stable);
 522 
 523   __ block_comment("} load_reference_barrier");
 524 
 525 #ifndef _LP64
 526     __ pop(thread);
 527 #endif
 528 }
 529 
 530 void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) {
 531   if (ShenandoahIUBarrier) {
 532     iu_barrier_impl(masm, dst, tmp);
 533   }
 534 }
 535 
 536 void ShenandoahBarrierSetAssembler::iu_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
 537   assert(ShenandoahIUBarrier, "should be enabled");
 538 
 539   if (dst == noreg) return;
 540 
 541   if (ShenandoahIUBarrier) {
 542     save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 543 
 544 #ifdef _LP64
 545     Register thread = r15_thread;
 546 #else
 547     Register thread = rcx;
 548     if (thread == dst || thread == tmp) {
 549       thread = rdi;
 550     }
 551     if (thread == dst || thread == tmp) {
 552       thread = rbx;
 553     }
 554     __ get_thread(thread);
 555 #endif
 556     assert_different_registers(dst, tmp, thread);
 557 
 558     satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false);
 559 
 560     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 561   }
 562 }
 563 
 564 //
 565 // Arguments:
 566 //
 567 // Inputs:
 568 //   src:        oop location, might be clobbered
 569 //   tmp1:       scratch register, might not be valid.
 570 //
 571 // Output:
 572 //   dst:        oop loaded from src location
 573 //
 574 // Kill:
 575 //   tmp1 (if it is valid)
 576 //
 577 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 578              Register dst, Address src, Register tmp1, Register tmp_thread) {
 579   // 1: non-reference load, no additional barrier is needed
 580   if (!is_reference_type(type)) {
 581     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 582     return;
 583   }
 584 
 585   assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected");
 586 
 587   // 2: load a reference from src location and apply LRB if needed
 588   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 589     Register result_dst = dst;
 590     bool use_tmp1_for_dst = false;
 591 
 592     // Preserve src location for LRB
 593     if (dst == src.base() || dst == src.index()) {
 594     // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
 595       if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
 596         dst = tmp1;
 597         use_tmp1_for_dst = true;
 598       } else {
 599         dst = rdi;
 600         __ push(dst);
 601       }
 602       assert_different_registers(dst, src.base(), src.index());
 603     }
 604 
 605     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 606 
 607     load_reference_barrier(masm, dst, src, decorators);
 608 
 609     // Move loaded oop to final destination
 610     if (dst != result_dst) {
 611       __ movptr(result_dst, dst);
 612 
 613       if (!use_tmp1_for_dst) {
 614         __ pop(dst);
 615       }
 616 
 617       dst = result_dst;
 618     }
 619   } else {
 620     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 621   }
 622 
 623   // 3: apply keep-alive barrier if needed
 624   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 625     save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 626 
 627     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 628     assert_different_registers(dst, tmp1, tmp_thread);
 629     if (!thread->is_valid()) {
 630       thread = rdx;
 631     }
 632     NOT_LP64(__ get_thread(thread));
 633     // Generate the SATB pre-barrier code to log the value of
 634     // the referent field in an SATB buffer.
 635     shenandoah_write_barrier_pre(masm /* masm */,
 636                                  noreg /* obj */,
 637                                  dst /* pre_val */,
 638                                  thread /* thread */,
 639                                  tmp1 /* tmp */,
 640                                  true /* tosca_live */,
 641                                  true /* expand_call */);
 642 
 643     restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
 644   }
 645 }
 646 
 647 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
 648   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 649     return;
 650   }
 651 
 652   // Does a store check for the oop in register obj. The content of
 653   // register obj is destroyed afterwards.
 654 
 655   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 656   CardTable* ct = ctbs->card_table();
 657 
 658   __ shrptr(obj, CardTable::card_shift());
 659 
 660   Address card_addr;
 661 
 662   // The calculation for byte_map_base is as follows:
 663   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
 664   // So this essentially converts an address to a displacement and it will
 665   // never need to be relocated. On 64-bit however the value may be too
 666   // large for a 32-bit displacement.
 667   intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
 668   if (__ is_simm32(byte_map_base)) {
 669     card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
 670   } else {
 671     // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
 672     // displacement and done in a single instruction given favorable mapping and a
 673     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
 674     // entry and that entry is not properly handled by the relocation code.
 675     AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
 676     Address index(noreg, obj, Address::times_1);
 677     card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
 678   }
 679 
 680   int dirty = CardTable::dirty_card_val();
 681   if (UseCondCardMark) {
 682     Label L_already_dirty;
 683     __ cmpb(card_addr, dirty);
 684     __ jccb(Assembler::equal, L_already_dirty);
 685     __ movb(card_addr, dirty);
 686     __ bind(L_already_dirty);
 687   } else {
 688     __ movb(card_addr, dirty);
 689   }
 690 }
 691 
 692 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 693               Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 694 
 695   bool on_oop = is_reference_type(type);
 696   bool in_heap = (decorators & IN_HEAP) != 0;
 697   bool as_normal = (decorators & AS_NORMAL) != 0;
 698   if (on_oop && in_heap) {
 699     bool needs_pre_barrier = as_normal;
 700 
 701     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 702     // flatten object address if needed
 703     // We do it regardless of precise because we need the registers
 704     if (dst.index() == noreg && dst.disp() == 0) {
 705       if (dst.base() != tmp1) {
 706         __ movptr(tmp1, dst.base());
 707       }
 708     } else {
 709       __ lea(tmp1, dst);
 710     }
 711 
 712     assert_different_registers(val, tmp1, tmp2, tmp3, rthread);
 713 
 714 #ifndef _LP64
 715     __ get_thread(rthread);
 716     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 717     imasm->save_bcp();
 718 #endif
 719 
 720     if (needs_pre_barrier) {
 721       shenandoah_write_barrier_pre(masm /*masm*/,
 722                                    tmp1 /* obj */,
 723                                    tmp2 /* pre_val */,
 724                                    rthread /* thread */,
 725                                    tmp3  /* tmp */,
 726                                    val != noreg /* tosca_live */,
 727                                    false /* expand_call */);
 728     }
 729     if (val == noreg) {
 730       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
 731     } else {
 732       iu_barrier(masm, val, tmp3);
 733       // TODO: store_check missing in upstream
 734       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
 735       store_check(masm, tmp1);
 736     }
 737     NOT_LP64(imasm->restore_bcp());
 738   } else {
 739     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
 740   }
 741 }
 742 
 743 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 744                                                                   Register obj, Register tmp, Label& slowpath) {
 745   Label done;
 746   // Resolve jobject
 747   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 748 
 749   // Check for null.
 750   __ testptr(obj, obj);
 751   __ jcc(Assembler::zero, done);
 752 
 753   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
 754   __ testb(gc_state, ShenandoahHeap::EVACUATION);
 755   __ jccb(Assembler::notZero, slowpath);
 756   __ bind(done);
 757 }
 758 
 759 // Special Shenandoah CAS implementation that handles false negatives
 760 // due to concurrent evacuation.
 761 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
 762                                                 Register res, Address addr, Register oldval, Register newval,
 763                                                 bool exchange, Register tmp1, Register tmp2) {
 764   assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
 765   assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
 766   assert_different_registers(oldval, tmp1, tmp2);
 767   assert_different_registers(newval, tmp1, tmp2);
 768 
 769   Label L_success, L_failure;
 770 
 771   // Remember oldval for retry logic below
 772 #ifdef _LP64
 773   if (UseCompressedOops) {
 774     __ movl(tmp1, oldval);
 775   } else
 776 #endif
 777   {
 778     __ movptr(tmp1, oldval);
 779   }
 780 
 781   // Step 1. Fast-path.
 782   //
 783   // Try to CAS with given arguments. If successful, then we are done.
 784 
 785 #ifdef _LP64
 786   if (UseCompressedOops) {
 787     __ lock();
 788     __ cmpxchgl(newval, addr);
 789   } else
 790 #endif
 791   {
 792     __ lock();
 793     __ cmpxchgptr(newval, addr);
 794   }
 795   __ jcc(Assembler::equal, L_success);
 796 
 797   // Step 2. CAS had failed. This may be a false negative.
 798   //
 799   // The trouble comes when we compare the to-space pointer with the from-space
 800   // pointer to the same object. To resolve this, it will suffice to resolve
 801   // the value from memory -- this will give both to-space pointers.
 802   // If they mismatch, then it was a legitimate failure.
 803   //
 804   // Before reaching to resolve sequence, see if we can avoid the whole shebang
 805   // with filters.
 806 
 807   // Filter: when offending in-memory value is null, the failure is definitely legitimate
 808   __ testptr(oldval, oldval);
 809   __ jcc(Assembler::zero, L_failure);
 810 
 811   // Filter: when heap is stable, the failure is definitely legitimate
 812 #ifdef _LP64
 813   const Register thread = r15_thread;
 814 #else
 815   const Register thread = tmp2;
 816   __ get_thread(thread);
 817 #endif
 818   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 819   __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
 820   __ jcc(Assembler::zero, L_failure);
 821 
 822 #ifdef _LP64
 823   if (UseCompressedOops) {
 824     __ movl(tmp2, oldval);
 825     __ decode_heap_oop(tmp2);
 826   } else
 827 #endif
 828   {
 829     __ movptr(tmp2, oldval);
 830   }
 831 
 832   // Decode offending in-memory value.
 833   // Test if-forwarded
 834   __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
 835   __ jcc(Assembler::noParity, L_failure);  // When odd number of bits, then not forwarded
 836   __ jcc(Assembler::zero, L_failure);      // When it is 00, then also not forwarded
 837 
 838   // Load and mask forwarding pointer
 839   __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
 840   __ shrptr(tmp2, 2);
 841   __ shlptr(tmp2, 2);
 842 
 843 #ifdef _LP64
 844   if (UseCompressedOops) {
 845     __ decode_heap_oop(tmp1); // decode for comparison
 846   }
 847 #endif
 848 
 849   // Now we have the forwarded offender in tmp2.
 850   // Compare and if they don't match, we have legitimate failure
 851   __ cmpptr(tmp1, tmp2);
 852   __ jcc(Assembler::notEqual, L_failure);
 853 
 854   // Step 3. Need to fix the memory ptr before continuing.
 855   //
 856   // At this point, we have from-space oldval in the register, and its to-space
 857   // address is in tmp2. Let's try to update it into memory. We don't care if it
 858   // succeeds or not. If it does, then the retrying CAS would see it and succeed.
 859   // If this fixup fails, this means somebody else beat us to it, and necessarily
 860   // with to-space ptr store. We still have to do the retry, because the GC might
 861   // have updated the reference for us.
 862 
 863 #ifdef _LP64
 864   if (UseCompressedOops) {
 865     __ encode_heap_oop(tmp2); // previously decoded at step 2.
 866   }
 867 #endif
 868 
 869 #ifdef _LP64
 870   if (UseCompressedOops) {
 871     __ lock();
 872     __ cmpxchgl(tmp2, addr);
 873   } else
 874 #endif
 875   {
 876     __ lock();
 877     __ cmpxchgptr(tmp2, addr);
 878   }
 879 
 880   // Step 4. Try to CAS again.
 881   //
 882   // This is guaranteed not to have false negatives, because oldval is definitely
 883   // to-space, and memory pointer is to-space as well. Nothing is able to store
 884   // from-space ptr into memory anymore. Make sure oldval is restored, after being
 885   // garbled during retries.
 886   //
 887 #ifdef _LP64
 888   if (UseCompressedOops) {
 889     __ movl(oldval, tmp2);
 890   } else
 891 #endif
 892   {
 893     __ movptr(oldval, tmp2);
 894   }
 895 
 896 #ifdef _LP64
 897   if (UseCompressedOops) {
 898     __ lock();
 899     __ cmpxchgl(newval, addr);
 900   } else
 901 #endif
 902   {
 903     __ lock();
 904     __ cmpxchgptr(newval, addr);
 905   }
 906   if (!exchange) {
 907     __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
 908   }
 909 
 910   // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
 911   // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
 912   // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
 913 
 914   if (exchange) {
 915     __ bind(L_failure);
 916     __ bind(L_success);
 917   } else {
 918     assert(res != noreg, "need result register");
 919 
 920     Label exit;
 921     __ bind(L_failure);
 922     __ xorptr(res, res);
 923     __ jmpb(exit);
 924 
 925     __ bind(L_success);
 926     __ movptr(res, 1);
 927     __ bind(exit);
 928   }
 929 }
 930 
 931 #ifdef PRODUCT
 932 #define BLOCK_COMMENT(str) /* nothing */
 933 #else
 934 #define BLOCK_COMMENT(str) __ block_comment(str)
 935 #endif
 936 
 937 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 938 
 939 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
 940 
 941 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 942                                                                      Register addr, Register count,
 943                                                                      Register tmp) {
 944   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 945     return;
 946   }
 947 
 948   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 949   CardTable* ct = bs->card_table();
 950   intptr_t disp = (intptr_t) ct->byte_map_base();
 951 
 952   Label L_loop, L_done;
 953   const Register end = count;
 954   assert_different_registers(addr, end);
 955 
 956   // Zero count? Nothing to do.
 957   __ testl(count, count);
 958   __ jccb(Assembler::zero, L_done);
 959 
 960 #ifdef _LP64
 961   __ leaq(end, Address(addr, count, TIMES_OOP, 0));  // end == addr+count*oop_size
 962   __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
 963   __ shrptr(addr, CardTable::card_shift());
 964   __ shrptr(end, CardTable::card_shift());
 965   __ subptr(end, addr); // end --> cards count
 966 
 967   __ mov64(tmp, disp);
 968   __ addptr(addr, tmp);
 969 
 970   __ BIND(L_loop);
 971   __ movb(Address(addr, count, Address::times_1), 0);
 972   __ decrement(count);
 973   __ jccb(Assembler::greaterEqual, L_loop);
 974 #else
 975   __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
 976   __ shrptr(addr, CardTable::card_shift());
 977   __ shrptr(end,  CardTable::card_shift());
 978   __ subptr(end, addr); // end --> count
 979 
 980   __ BIND(L_loop);
 981   Address cardtable(addr, count, Address::times_1, disp);
 982   __ movb(cardtable, 0);
 983   __ decrement(count);
 984   __ jccb(Assembler::greaterEqual, L_loop);
 985 #endif
 986 
 987   __ BIND(L_done);
 988 }
 989 
 990 #undef __
 991 
 992 #ifdef COMPILER1
 993 
 994 #define __ ce->masm()->
 995 
 996 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 997   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 998   // At this point we know that marking is in progress.
 999   // If do_load() is true then we have to emit the
1000   // load of the previous value; otherwise it has already
1001   // been loaded into _pre_val.
1002 
1003   __ bind(*stub->entry());
1004   assert(stub->pre_val()->is_register(), "Precondition.");
1005 
1006   Register pre_val_reg = stub->pre_val()->as_register();
1007 
1008   if (stub->do_load()) {
1009     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
1010   }
1011 
1012   __ cmpptr(pre_val_reg, NULL_WORD);
1013   __ jcc(Assembler::equal, *stub->continuation());
1014   ce->store_parameter(stub->pre_val()->as_register(), 0);
1015   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
1016   __ jmp(*stub->continuation());
1017 
1018 }
1019 
1020 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
1021   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
1022   __ bind(*stub->entry());
1023 
1024   DecoratorSet decorators = stub->decorators();
1025   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
1026   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
1027   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1028   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
1029 
1030   Register obj = stub->obj()->as_register();
1031   Register res = stub->result()->as_register();
1032   Register addr = stub->addr()->as_pointer_register();
1033   Register tmp1 = stub->tmp1()->as_register();
1034   Register tmp2 = stub->tmp2()->as_register();
1035   assert_different_registers(obj, res, addr, tmp1, tmp2);
1036 
1037   Label slow_path;
1038 
1039   assert(res == rax, "result must arrive in rax");
1040 
1041   if (res != obj) {
1042     __ mov(res, obj);
1043   }
1044 
1045   if (is_strong) {
1046     // Check for object being in the collection set.
1047     __ mov(tmp1, res);
1048     __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1049     __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1050 #ifdef _LP64
1051     __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
1052     __ testbool(tmp2);
1053 #else
1054     // On x86_32, C1 register allocator can give us the register without 8-bit support.
1055     // Do the full-register access and test to avoid compilation failures.
1056     __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
1057     __ testptr(tmp2, 0xFF);
1058 #endif
1059     __ jcc(Assembler::zero, *stub->continuation());
1060   }
1061 
1062   __ bind(slow_path);
1063   ce->store_parameter(res, 0);
1064   ce->store_parameter(addr, 1);
1065   if (is_strong) {
1066     if (is_native) {
1067       __ call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
1068     } else {
1069       __ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
1070     }
1071   } else if (is_weak) {
1072     __ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
1073   } else {
1074     assert(is_phantom, "only remaining strength");
1075     __ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
1076   }
1077   __ jmp(*stub->continuation());
1078 }
1079 
1080 #undef __
1081 
1082 #define __ sasm->
1083 
1084 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
1085   __ prologue("shenandoah_pre_barrier", false);
1086   // arg0 : previous value of memory
1087 
1088   __ push(rax);
1089   __ push(rdx);
1090 
1091   const Register pre_val = rax;
1092   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1093   const Register tmp = rdx;
1094 
1095   NOT_LP64(__ get_thread(thread);)
1096 
1097   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1098   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1099 
1100   Label done;
1101   Label runtime;
1102 
1103   // Is SATB still active?
1104   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1105   __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
1106   __ jcc(Assembler::zero, done);
1107 
1108   // Can we store original value in the thread's buffer?
1109 
1110   __ movptr(tmp, queue_index);
1111   __ testptr(tmp, tmp);
1112   __ jcc(Assembler::zero, runtime);
1113   __ subptr(tmp, wordSize);
1114   __ movptr(queue_index, tmp);
1115   __ addptr(tmp, buffer);
1116 
1117   // prev_val (rax)
1118   __ load_parameter(0, pre_val);
1119   __ movptr(Address(tmp, 0), pre_val);
1120   __ jmp(done);
1121 
1122   __ bind(runtime);
1123 
1124   __ save_live_registers_no_oop_map(true);
1125 
1126   // load the pre-value
1127   __ load_parameter(0, rcx);
1128   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
1129 
1130   __ restore_live_registers(true);
1131 
1132   __ bind(done);
1133 
1134   __ pop(rdx);
1135   __ pop(rax);
1136 
1137   __ epilogue();
1138 }
1139 
1140 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
1141   __ prologue("shenandoah_load_reference_barrier", false);
1142   // arg0 : object to be resolved
1143 
1144   __ save_live_registers_no_oop_map(true);
1145 
1146   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
1147   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
1148   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1149   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
1150 
1151 #ifdef _LP64
1152   __ load_parameter(0, c_rarg0);
1153   __ load_parameter(1, c_rarg1);
1154   if (is_strong) {
1155     if (is_native) {
1156       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
1157     } else {
1158       if (UseCompressedOops) {
1159         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0, c_rarg1);
1160       } else {
1161         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
1162       }
1163     }
1164   } else if (is_weak) {
1165     assert(!is_native, "weak must not be called off-heap");
1166     if (UseCompressedOops) {
1167       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
1168     } else {
1169       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1170     }
1171   } else {
1172     assert(is_phantom, "only remaining strength");
1173     assert(is_native, "phantom must only be called off-heap");
1174     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1175   }
1176 #else
1177   __ load_parameter(0, rax);
1178   __ load_parameter(1, rbx);
1179   if (is_strong) {
1180     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), rax, rbx);
1181   } else if (is_weak) {
1182     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
1183   } else {
1184     assert(is_phantom, "only remaining strength");
1185     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), rax, rbx);
1186   }
1187 #endif
1188 
1189   __ restore_live_registers_except_rax(true);
1190 
1191   __ epilogue();
1192 }
1193 
1194 #undef __
1195 
1196 #endif // COMPILER1