1 /* 2 * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 29 #include "gc/shenandoah/shenandoahForwarding.hpp" 30 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 31 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 32 #include "gc/shenandoah/shenandoahRuntime.hpp" 33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" 35 #include "gc/shenandoah/mode/shenandoahMode.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "runtime/javaThread.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "utilities/macros.hpp" 40 #ifdef COMPILER1 41 #include "c1/c1_LIRAssembler.hpp" 42 #include "c1/c1_MacroAssembler.hpp" 43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" 44 #endif 45 46 #define __ masm-> 47 48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) { 49 if (handle_gpr) { 50 __ push_IU_state(); 51 } 52 53 if (handle_fp) { 54 // Some paths can be reached from the c2i adapter with live fp arguments in registers. 55 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call")); 56 57 if (UseSSE >= 2) { 58 const int xmm_size = wordSize * LP64_ONLY(2) NOT_LP64(4); 59 __ subptr(rsp, xmm_size * 8); 60 __ movdbl(Address(rsp, xmm_size * 0), xmm0); 61 __ movdbl(Address(rsp, xmm_size * 1), xmm1); 62 __ movdbl(Address(rsp, xmm_size * 2), xmm2); 63 __ movdbl(Address(rsp, xmm_size * 3), xmm3); 64 __ movdbl(Address(rsp, xmm_size * 4), xmm4); 65 __ movdbl(Address(rsp, xmm_size * 5), xmm5); 66 __ movdbl(Address(rsp, xmm_size * 6), xmm6); 67 __ movdbl(Address(rsp, xmm_size * 7), xmm7); 68 } else if (UseSSE >= 1) { 69 const int xmm_size = wordSize * LP64_ONLY(1) NOT_LP64(2); 70 __ subptr(rsp, xmm_size * 8); 71 __ movflt(Address(rsp, xmm_size * 0), xmm0); 72 __ movflt(Address(rsp, xmm_size * 1), xmm1); 73 __ movflt(Address(rsp, xmm_size * 2), xmm2); 74 __ movflt(Address(rsp, xmm_size * 3), xmm3); 75 __ movflt(Address(rsp, xmm_size * 4), xmm4); 76 __ movflt(Address(rsp, xmm_size * 5), xmm5); 77 __ movflt(Address(rsp, xmm_size * 6), xmm6); 78 __ movflt(Address(rsp, xmm_size * 7), xmm7); 79 } else { 80 __ push_FPU_state(); 81 } 82 } 83 } 84 85 static void restore_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) { 86 if (handle_fp) { 87 if (UseSSE >= 2) { 88 const int xmm_size = wordSize * LP64_ONLY(2) NOT_LP64(4); 89 __ movdbl(xmm0, Address(rsp, xmm_size * 0)); 90 __ movdbl(xmm1, Address(rsp, xmm_size * 1)); 91 __ movdbl(xmm2, Address(rsp, xmm_size * 2)); 92 __ movdbl(xmm3, Address(rsp, xmm_size * 3)); 93 __ movdbl(xmm4, Address(rsp, xmm_size * 4)); 94 __ movdbl(xmm5, Address(rsp, xmm_size * 5)); 95 __ movdbl(xmm6, Address(rsp, xmm_size * 6)); 96 __ movdbl(xmm7, Address(rsp, xmm_size * 7)); 97 __ addptr(rsp, xmm_size * 8); 98 } else if (UseSSE >= 1) { 99 const int xmm_size = wordSize * LP64_ONLY(1) NOT_LP64(2); 100 __ movflt(xmm0, Address(rsp, xmm_size * 0)); 101 __ movflt(xmm1, Address(rsp, xmm_size * 1)); 102 __ movflt(xmm2, Address(rsp, xmm_size * 2)); 103 __ movflt(xmm3, Address(rsp, xmm_size * 3)); 104 __ movflt(xmm4, Address(rsp, xmm_size * 4)); 105 __ movflt(xmm5, Address(rsp, xmm_size * 5)); 106 __ movflt(xmm6, Address(rsp, xmm_size * 6)); 107 __ movflt(xmm7, Address(rsp, xmm_size * 7)); 108 __ addptr(rsp, xmm_size * 8); 109 } else { 110 __ pop_FPU_state(); 111 } 112 } 113 114 if (handle_gpr) { 115 __ pop_IU_state(); 116 } 117 } 118 119 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 120 Register src, Register dst, Register count) { 121 122 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 123 124 if (is_reference_type(type)) { 125 if (ShenandoahHeap::heap()->mode()->is_generational()) { 126 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; 127 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; 128 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); 129 130 // We need to save the original element count because the array copy stub 131 // will destroy the value and we need it for the card marking barrier. 132 #ifdef _LP64 133 if (!checkcast) { 134 if (!obj_int) { 135 // Save count for barrier 136 __ movptr(r11, count); 137 } else if (disjoint) { 138 // Save dst in r11 in the disjoint case 139 __ movq(r11, dst); 140 } 141 } 142 #else 143 if (disjoint) { 144 __ mov(rdx, dst); // save 'to' 145 } 146 #endif 147 } 148 149 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) { 150 #ifdef _LP64 151 Register thread = r15_thread; 152 #else 153 Register thread = rax; 154 if (thread == src || thread == dst || thread == count) { 155 thread = rbx; 156 } 157 if (thread == src || thread == dst || thread == count) { 158 thread = rcx; 159 } 160 if (thread == src || thread == dst || thread == count) { 161 thread = rdx; 162 } 163 __ push(thread); 164 __ get_thread(thread); 165 #endif 166 assert_different_registers(src, dst, count, thread); 167 168 Label L_done; 169 // Short-circuit if count == 0. 170 __ testptr(count, count); 171 __ jcc(Assembler::zero, L_done); 172 173 // Avoid runtime call when not active. 174 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 175 int flags; 176 if (ShenandoahSATBBarrier && dest_uninitialized) { 177 flags = ShenandoahHeap::HAS_FORWARDED; 178 } else { 179 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING; 180 } 181 __ testb(gc_state, flags); 182 __ jcc(Assembler::zero, L_done); 183 184 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false); 185 186 #ifdef _LP64 187 assert(src == rdi, "expected"); 188 assert(dst == rsi, "expected"); 189 assert(count == rdx, "expected"); 190 if (UseCompressedOops) { 191 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), 192 src, dst, count); 193 } else 194 #endif 195 { 196 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), 197 src, dst, count); 198 } 199 200 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false); 201 202 __ bind(L_done); 203 NOT_LP64(__ pop(thread);) 204 } 205 } 206 207 } 208 209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 210 Register src, Register dst, Register count) { 211 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; 212 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; 213 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); 214 Register tmp = rax; 215 216 if (is_reference_type(type)) { 217 #ifdef _LP64 218 if (!checkcast) { 219 if (!obj_int) { 220 // Save count for barrier 221 count = r11; 222 } else if (disjoint) { 223 // Use the saved dst in the disjoint case 224 dst = r11; 225 } 226 } else { 227 tmp = rscratch1; 228 } 229 #else 230 if (disjoint) { 231 __ mov(dst, rdx); // restore 'to' 232 } 233 #endif 234 gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp); 235 } 236 } 237 238 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, 239 Register obj, 240 Register pre_val, 241 Register thread, 242 Register tmp, 243 bool tosca_live, 244 bool expand_call) { 245 246 if (ShenandoahSATBBarrier) { 247 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); 248 } 249 } 250 251 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, 252 Register obj, 253 Register pre_val, 254 Register thread, 255 Register tmp, 256 bool tosca_live, 257 bool expand_call) { 258 // If expand_call is true then we expand the call_VM_leaf macro 259 // directly to skip generating the check by 260 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 261 262 #ifdef _LP64 263 assert(thread == r15_thread, "must be"); 264 #endif // _LP64 265 266 Label done; 267 Label runtime; 268 269 assert(pre_val != noreg, "check this code"); 270 271 if (obj != noreg) { 272 assert_different_registers(obj, pre_val, tmp); 273 assert(pre_val != rax, "check this code"); 274 } 275 276 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); 277 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 278 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 279 280 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 281 __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); 282 __ jcc(Assembler::zero, done); 283 284 // Do we need to load the previous value? 285 if (obj != noreg) { 286 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 287 } 288 289 // Is the previous value null? 290 __ cmpptr(pre_val, NULL_WORD); 291 __ jcc(Assembler::equal, done); 292 293 // Can we store original value in the thread's buffer? 294 // Is index == 0? 295 // (The index field is typed as size_t.) 296 297 __ movptr(tmp, index); // tmp := *index_adr 298 __ cmpptr(tmp, 0); // tmp == 0? 299 __ jcc(Assembler::equal, runtime); // If yes, goto runtime 300 301 __ subptr(tmp, wordSize); // tmp := tmp - wordSize 302 __ movptr(index, tmp); // *index_adr := tmp 303 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr 304 305 // Record the previous value 306 __ movptr(Address(tmp, 0), pre_val); 307 __ jmp(done); 308 309 __ bind(runtime); 310 // save the live input values 311 if(tosca_live) __ push(rax); 312 313 if (obj != noreg && obj != rax) 314 __ push(obj); 315 316 if (pre_val != rax) 317 __ push(pre_val); 318 319 // Calling the runtime using the regular call_VM_leaf mechanism generates 320 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 321 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr. 322 // 323 // If we care generating the pre-barrier without a frame (e.g. in the 324 // intrinsified Reference.get() routine) then ebp might be pointing to 325 // the caller frame and so this check will most likely fail at runtime. 326 // 327 // Expanding the call directly bypasses the generation of the check. 328 // So when we do not have have a full interpreter frame on the stack 329 // expand_call should be passed true. 330 331 NOT_LP64( __ push(thread); ) 332 333 #ifdef _LP64 334 // We move pre_val into c_rarg0 early, in order to avoid smashing it, should 335 // pre_val be c_rarg1 (where the call prologue would copy thread argument). 336 // Note: this should not accidentally smash thread, because thread is always r15. 337 assert(thread != c_rarg0, "smashed arg"); 338 if (c_rarg0 != pre_val) { 339 __ mov(c_rarg0, pre_val); 340 } 341 #endif 342 343 if (expand_call) { 344 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) 345 #ifdef _LP64 346 if (c_rarg1 != thread) { 347 __ mov(c_rarg1, thread); 348 } 349 // Already moved pre_val into c_rarg0 above 350 #else 351 __ push(thread); 352 __ push(pre_val); 353 #endif 354 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2); 355 } else { 356 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread); 357 } 358 359 NOT_LP64( __ pop(thread); ) 360 361 // save the live input values 362 if (pre_val != rax) 363 __ pop(pre_val); 364 365 if (obj != noreg && obj != rax) 366 __ pop(obj); 367 368 if(tosca_live) __ pop(rax); 369 370 __ bind(done); 371 } 372 373 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) { 374 assert(ShenandoahLoadRefBarrier, "Should be enabled"); 375 376 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 377 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 378 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 379 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 380 bool is_narrow = UseCompressedOops && !is_native; 381 382 Label heap_stable, not_cset; 383 384 __ block_comment("load_reference_barrier { "); 385 386 // Check if GC is active 387 #ifdef _LP64 388 Register thread = r15_thread; 389 #else 390 Register thread = rcx; 391 if (thread == dst) { 392 thread = rbx; 393 } 394 __ push(thread); 395 __ get_thread(thread); 396 #endif 397 398 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 399 int flags = ShenandoahHeap::HAS_FORWARDED; 400 if (!is_strong) { 401 flags |= ShenandoahHeap::WEAK_ROOTS; 402 } 403 __ testb(gc_state, flags); 404 __ jcc(Assembler::zero, heap_stable); 405 406 Register tmp1 = noreg, tmp2 = noreg; 407 if (is_strong) { 408 // Test for object in cset 409 // Allocate temporary registers 410 for (int i = 0; i < 8; i++) { 411 Register r = as_Register(i); 412 if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) { 413 if (tmp1 == noreg) { 414 tmp1 = r; 415 } else { 416 tmp2 = r; 417 break; 418 } 419 } 420 } 421 assert(tmp1 != noreg, "tmp1 allocated"); 422 assert(tmp2 != noreg, "tmp2 allocated"); 423 assert_different_registers(tmp1, tmp2, src.base(), src.index()); 424 assert_different_registers(tmp1, tmp2, dst); 425 426 __ push(tmp1); 427 __ push(tmp2); 428 429 // Optimized cset-test 430 __ movptr(tmp1, dst); 431 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 432 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); 433 __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1)); 434 __ testbool(tmp1); 435 __ jcc(Assembler::zero, not_cset); 436 } 437 438 save_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true); 439 440 // The rest is saved with the optimized path 441 442 uint num_saved_regs = 4 + (dst != rax ? 1 : 0) LP64_ONLY(+4); 443 __ subptr(rsp, num_saved_regs * wordSize); 444 uint slot = num_saved_regs; 445 if (dst != rax) { 446 __ movptr(Address(rsp, (--slot) * wordSize), rax); 447 } 448 __ movptr(Address(rsp, (--slot) * wordSize), rcx); 449 __ movptr(Address(rsp, (--slot) * wordSize), rdx); 450 __ movptr(Address(rsp, (--slot) * wordSize), rdi); 451 __ movptr(Address(rsp, (--slot) * wordSize), rsi); 452 #ifdef _LP64 453 __ movptr(Address(rsp, (--slot) * wordSize), r8); 454 __ movptr(Address(rsp, (--slot) * wordSize), r9); 455 __ movptr(Address(rsp, (--slot) * wordSize), r10); 456 __ movptr(Address(rsp, (--slot) * wordSize), r11); 457 // r12-r15 are callee saved in all calling conventions 458 #endif 459 assert(slot == 0, "must use all slots"); 460 461 // Shuffle registers such that dst is in c_rarg0 and addr in c_rarg1. 462 #ifdef _LP64 463 Register arg0 = c_rarg0, arg1 = c_rarg1; 464 #else 465 Register arg0 = rdi, arg1 = rsi; 466 #endif 467 if (dst == arg1) { 468 __ lea(arg0, src); 469 __ xchgptr(arg1, arg0); 470 } else { 471 __ lea(arg1, src); 472 __ movptr(arg0, dst); 473 } 474 475 if (is_strong) { 476 if (is_narrow) { 477 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1); 478 } else { 479 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1); 480 } 481 } else if (is_weak) { 482 if (is_narrow) { 483 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1); 484 } else { 485 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1); 486 } 487 } else { 488 assert(is_phantom, "only remaining strength"); 489 assert(!is_narrow, "phantom access cannot be narrow"); 490 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1); 491 } 492 493 #ifdef _LP64 494 __ movptr(r11, Address(rsp, (slot++) * wordSize)); 495 __ movptr(r10, Address(rsp, (slot++) * wordSize)); 496 __ movptr(r9, Address(rsp, (slot++) * wordSize)); 497 __ movptr(r8, Address(rsp, (slot++) * wordSize)); 498 #endif 499 __ movptr(rsi, Address(rsp, (slot++) * wordSize)); 500 __ movptr(rdi, Address(rsp, (slot++) * wordSize)); 501 __ movptr(rdx, Address(rsp, (slot++) * wordSize)); 502 __ movptr(rcx, Address(rsp, (slot++) * wordSize)); 503 504 if (dst != rax) { 505 __ movptr(dst, rax); 506 __ movptr(rax, Address(rsp, (slot++) * wordSize)); 507 } 508 509 assert(slot == num_saved_regs, "must use all slots"); 510 __ addptr(rsp, num_saved_regs * wordSize); 511 512 restore_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true); 513 514 __ bind(not_cset); 515 516 if (is_strong) { 517 __ pop(tmp2); 518 __ pop(tmp1); 519 } 520 521 __ bind(heap_stable); 522 523 __ block_comment("} load_reference_barrier"); 524 525 #ifndef _LP64 526 __ pop(thread); 527 #endif 528 } 529 530 void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) { 531 if (ShenandoahIUBarrier) { 532 iu_barrier_impl(masm, dst, tmp); 533 } 534 } 535 536 void ShenandoahBarrierSetAssembler::iu_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { 537 assert(ShenandoahIUBarrier, "should be enabled"); 538 539 if (dst == noreg) return; 540 541 if (ShenandoahIUBarrier) { 542 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true); 543 544 #ifdef _LP64 545 Register thread = r15_thread; 546 #else 547 Register thread = rcx; 548 if (thread == dst || thread == tmp) { 549 thread = rdi; 550 } 551 if (thread == dst || thread == tmp) { 552 thread = rbx; 553 } 554 __ get_thread(thread); 555 #endif 556 assert_different_registers(dst, tmp, thread); 557 558 satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false); 559 560 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true); 561 } 562 } 563 564 // 565 // Arguments: 566 // 567 // Inputs: 568 // src: oop location, might be clobbered 569 // tmp1: scratch register, might not be valid. 570 // 571 // Output: 572 // dst: oop loaded from src location 573 // 574 // Kill: 575 // tmp1 (if it is valid) 576 // 577 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 578 Register dst, Address src, Register tmp1, Register tmp_thread) { 579 // 1: non-reference load, no additional barrier is needed 580 if (!is_reference_type(type)) { 581 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 582 return; 583 } 584 585 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected"); 586 587 // 2: load a reference from src location and apply LRB if needed 588 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { 589 Register result_dst = dst; 590 bool use_tmp1_for_dst = false; 591 592 // Preserve src location for LRB 593 if (dst == src.base() || dst == src.index()) { 594 // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at() 595 if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) { 596 dst = tmp1; 597 use_tmp1_for_dst = true; 598 } else { 599 dst = rdi; 600 __ push(dst); 601 } 602 assert_different_registers(dst, src.base(), src.index()); 603 } 604 605 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 606 607 load_reference_barrier(masm, dst, src, decorators); 608 609 // Move loaded oop to final destination 610 if (dst != result_dst) { 611 __ movptr(result_dst, dst); 612 613 if (!use_tmp1_for_dst) { 614 __ pop(dst); 615 } 616 617 dst = result_dst; 618 } 619 } else { 620 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 621 } 622 623 // 3: apply keep-alive barrier if needed 624 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { 625 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true); 626 627 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); 628 assert_different_registers(dst, tmp1, tmp_thread); 629 if (!thread->is_valid()) { 630 thread = rdx; 631 } 632 NOT_LP64(__ get_thread(thread)); 633 // Generate the SATB pre-barrier code to log the value of 634 // the referent field in an SATB buffer. 635 shenandoah_write_barrier_pre(masm /* masm */, 636 noreg /* obj */, 637 dst /* pre_val */, 638 thread /* thread */, 639 tmp1 /* tmp */, 640 true /* tosca_live */, 641 true /* expand_call */); 642 643 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true); 644 } 645 } 646 647 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) { 648 if (!ShenandoahHeap::heap()->mode()->is_generational()) { 649 return; 650 } 651 652 // Does a store check for the oop in register obj. The content of 653 // register obj is destroyed afterwards. 654 655 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); 656 CardTable* ct = ctbs->card_table(); 657 658 __ shrptr(obj, CardTable::card_shift()); 659 660 Address card_addr; 661 662 // The calculation for byte_map_base is as follows: 663 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); 664 // So this essentially converts an address to a displacement and it will 665 // never need to be relocated. On 64-bit however the value may be too 666 // large for a 32-bit displacement. 667 intptr_t byte_map_base = (intptr_t)ct->byte_map_base(); 668 if (__ is_simm32(byte_map_base)) { 669 card_addr = Address(noreg, obj, Address::times_1, byte_map_base); 670 } else { 671 // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative 672 // displacement and done in a single instruction given favorable mapping and a 673 // smarter version of as_Address. However, 'ExternalAddress' generates a relocation 674 // entry and that entry is not properly handled by the relocation code. 675 AddressLiteral cardtable((address)byte_map_base, relocInfo::none); 676 Address index(noreg, obj, Address::times_1); 677 card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1); 678 } 679 680 int dirty = CardTable::dirty_card_val(); 681 if (UseCondCardMark) { 682 Label L_already_dirty; 683 __ cmpb(card_addr, dirty); 684 __ jccb(Assembler::equal, L_already_dirty); 685 __ movb(card_addr, dirty); 686 __ bind(L_already_dirty); 687 } else { 688 __ movb(card_addr, dirty); 689 } 690 } 691 692 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 693 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 694 695 bool on_oop = is_reference_type(type); 696 bool in_heap = (decorators & IN_HEAP) != 0; 697 bool as_normal = (decorators & AS_NORMAL) != 0; 698 if (on_oop && in_heap) { 699 bool needs_pre_barrier = as_normal; 700 701 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); 702 // flatten object address if needed 703 // We do it regardless of precise because we need the registers 704 if (dst.index() == noreg && dst.disp() == 0) { 705 if (dst.base() != tmp1) { 706 __ movptr(tmp1, dst.base()); 707 } 708 } else { 709 __ lea(tmp1, dst); 710 } 711 712 assert_different_registers(val, tmp1, tmp2, tmp3, rthread); 713 714 #ifndef _LP64 715 __ get_thread(rthread); 716 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm); 717 imasm->save_bcp(); 718 #endif 719 720 if (needs_pre_barrier) { 721 shenandoah_write_barrier_pre(masm /*masm*/, 722 tmp1 /* obj */, 723 tmp2 /* pre_val */, 724 rthread /* thread */, 725 tmp3 /* tmp */, 726 val != noreg /* tosca_live */, 727 false /* expand_call */); 728 } 729 if (val == noreg) { 730 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); 731 } else { 732 iu_barrier(masm, val, tmp3); 733 // TODO: store_check missing in upstream 734 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); 735 store_check(masm, tmp1); 736 } 737 NOT_LP64(imasm->restore_bcp()); 738 } else { 739 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); 740 } 741 } 742 743 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 744 Register obj, Register tmp, Label& slowpath) { 745 Label done; 746 // Resolve jobject 747 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); 748 749 // Check for null. 750 __ testptr(obj, obj); 751 __ jcc(Assembler::zero, done); 752 753 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); 754 __ testb(gc_state, ShenandoahHeap::EVACUATION); 755 __ jccb(Assembler::notZero, slowpath); 756 __ bind(done); 757 } 758 759 // Special Shenandoah CAS implementation that handles false negatives 760 // due to concurrent evacuation. 761 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, 762 Register res, Address addr, Register oldval, Register newval, 763 bool exchange, Register tmp1, Register tmp2) { 764 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); 765 assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); 766 assert_different_registers(oldval, tmp1, tmp2); 767 assert_different_registers(newval, tmp1, tmp2); 768 769 Label L_success, L_failure; 770 771 // Remember oldval for retry logic below 772 #ifdef _LP64 773 if (UseCompressedOops) { 774 __ movl(tmp1, oldval); 775 } else 776 #endif 777 { 778 __ movptr(tmp1, oldval); 779 } 780 781 // Step 1. Fast-path. 782 // 783 // Try to CAS with given arguments. If successful, then we are done. 784 785 #ifdef _LP64 786 if (UseCompressedOops) { 787 __ lock(); 788 __ cmpxchgl(newval, addr); 789 } else 790 #endif 791 { 792 __ lock(); 793 __ cmpxchgptr(newval, addr); 794 } 795 __ jcc(Assembler::equal, L_success); 796 797 // Step 2. CAS had failed. This may be a false negative. 798 // 799 // The trouble comes when we compare the to-space pointer with the from-space 800 // pointer to the same object. To resolve this, it will suffice to resolve 801 // the value from memory -- this will give both to-space pointers. 802 // If they mismatch, then it was a legitimate failure. 803 // 804 // Before reaching to resolve sequence, see if we can avoid the whole shebang 805 // with filters. 806 807 // Filter: when offending in-memory value is null, the failure is definitely legitimate 808 __ testptr(oldval, oldval); 809 __ jcc(Assembler::zero, L_failure); 810 811 // Filter: when heap is stable, the failure is definitely legitimate 812 #ifdef _LP64 813 const Register thread = r15_thread; 814 #else 815 const Register thread = tmp2; 816 __ get_thread(thread); 817 #endif 818 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 819 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED); 820 __ jcc(Assembler::zero, L_failure); 821 822 #ifdef _LP64 823 if (UseCompressedOops) { 824 __ movl(tmp2, oldval); 825 __ decode_heap_oop(tmp2); 826 } else 827 #endif 828 { 829 __ movptr(tmp2, oldval); 830 } 831 832 // Decode offending in-memory value. 833 // Test if-forwarded 834 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value); 835 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded 836 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded 837 838 // Load and mask forwarding pointer 839 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes())); 840 __ shrptr(tmp2, 2); 841 __ shlptr(tmp2, 2); 842 843 #ifdef _LP64 844 if (UseCompressedOops) { 845 __ decode_heap_oop(tmp1); // decode for comparison 846 } 847 #endif 848 849 // Now we have the forwarded offender in tmp2. 850 // Compare and if they don't match, we have legitimate failure 851 __ cmpptr(tmp1, tmp2); 852 __ jcc(Assembler::notEqual, L_failure); 853 854 // Step 3. Need to fix the memory ptr before continuing. 855 // 856 // At this point, we have from-space oldval in the register, and its to-space 857 // address is in tmp2. Let's try to update it into memory. We don't care if it 858 // succeeds or not. If it does, then the retrying CAS would see it and succeed. 859 // If this fixup fails, this means somebody else beat us to it, and necessarily 860 // with to-space ptr store. We still have to do the retry, because the GC might 861 // have updated the reference for us. 862 863 #ifdef _LP64 864 if (UseCompressedOops) { 865 __ encode_heap_oop(tmp2); // previously decoded at step 2. 866 } 867 #endif 868 869 #ifdef _LP64 870 if (UseCompressedOops) { 871 __ lock(); 872 __ cmpxchgl(tmp2, addr); 873 } else 874 #endif 875 { 876 __ lock(); 877 __ cmpxchgptr(tmp2, addr); 878 } 879 880 // Step 4. Try to CAS again. 881 // 882 // This is guaranteed not to have false negatives, because oldval is definitely 883 // to-space, and memory pointer is to-space as well. Nothing is able to store 884 // from-space ptr into memory anymore. Make sure oldval is restored, after being 885 // garbled during retries. 886 // 887 #ifdef _LP64 888 if (UseCompressedOops) { 889 __ movl(oldval, tmp2); 890 } else 891 #endif 892 { 893 __ movptr(oldval, tmp2); 894 } 895 896 #ifdef _LP64 897 if (UseCompressedOops) { 898 __ lock(); 899 __ cmpxchgl(newval, addr); 900 } else 901 #endif 902 { 903 __ lock(); 904 __ cmpxchgptr(newval, addr); 905 } 906 if (!exchange) { 907 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump 908 } 909 910 // Step 5. If we need a boolean result out of CAS, set the flag appropriately. 911 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS. 912 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return. 913 914 if (exchange) { 915 __ bind(L_failure); 916 __ bind(L_success); 917 } else { 918 assert(res != noreg, "need result register"); 919 920 Label exit; 921 __ bind(L_failure); 922 __ xorptr(res, res); 923 __ jmpb(exit); 924 925 __ bind(L_success); 926 __ movptr(res, 1); 927 __ bind(exit); 928 } 929 } 930 931 #ifdef PRODUCT 932 #define BLOCK_COMMENT(str) /* nothing */ 933 #else 934 #define BLOCK_COMMENT(str) __ block_comment(str) 935 #endif 936 937 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 938 939 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) 940 941 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 942 Register addr, Register count, 943 Register tmp) { 944 if (!ShenandoahHeap::heap()->mode()->is_generational()) { 945 return; 946 } 947 948 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 949 CardTable* ct = bs->card_table(); 950 intptr_t disp = (intptr_t) ct->byte_map_base(); 951 952 Label L_loop, L_done; 953 const Register end = count; 954 assert_different_registers(addr, end); 955 956 // Zero count? Nothing to do. 957 __ testl(count, count); 958 __ jccb(Assembler::zero, L_done); 959 960 #ifdef _LP64 961 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size 962 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive 963 __ shrptr(addr, CardTable::card_shift()); 964 __ shrptr(end, CardTable::card_shift()); 965 __ subptr(end, addr); // end --> cards count 966 967 __ mov64(tmp, disp); 968 __ addptr(addr, tmp); 969 970 __ BIND(L_loop); 971 __ movb(Address(addr, count, Address::times_1), 0); 972 __ decrement(count); 973 __ jccb(Assembler::greaterEqual, L_loop); 974 #else 975 __ lea(end, Address(addr, count, Address::times_ptr, -wordSize)); 976 __ shrptr(addr, CardTable::card_shift()); 977 __ shrptr(end, CardTable::card_shift()); 978 __ subptr(end, addr); // end --> count 979 980 __ BIND(L_loop); 981 Address cardtable(addr, count, Address::times_1, disp); 982 __ movb(cardtable, 0); 983 __ decrement(count); 984 __ jccb(Assembler::greaterEqual, L_loop); 985 #endif 986 987 __ BIND(L_done); 988 } 989 990 #undef __ 991 992 #ifdef COMPILER1 993 994 #define __ ce->masm()-> 995 996 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { 997 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 998 // At this point we know that marking is in progress. 999 // If do_load() is true then we have to emit the 1000 // load of the previous value; otherwise it has already 1001 // been loaded into _pre_val. 1002 1003 __ bind(*stub->entry()); 1004 assert(stub->pre_val()->is_register(), "Precondition."); 1005 1006 Register pre_val_reg = stub->pre_val()->as_register(); 1007 1008 if (stub->do_load()) { 1009 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); 1010 } 1011 1012 __ cmpptr(pre_val_reg, NULL_WORD); 1013 __ jcc(Assembler::equal, *stub->continuation()); 1014 ce->store_parameter(stub->pre_val()->as_register(), 0); 1015 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 1016 __ jmp(*stub->continuation()); 1017 1018 } 1019 1020 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { 1021 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 1022 __ bind(*stub->entry()); 1023 1024 DecoratorSet decorators = stub->decorators(); 1025 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 1026 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 1027 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 1028 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 1029 1030 Register obj = stub->obj()->as_register(); 1031 Register res = stub->result()->as_register(); 1032 Register addr = stub->addr()->as_pointer_register(); 1033 Register tmp1 = stub->tmp1()->as_register(); 1034 Register tmp2 = stub->tmp2()->as_register(); 1035 assert_different_registers(obj, res, addr, tmp1, tmp2); 1036 1037 Label slow_path; 1038 1039 assert(res == rax, "result must arrive in rax"); 1040 1041 if (res != obj) { 1042 __ mov(res, obj); 1043 } 1044 1045 if (is_strong) { 1046 // Check for object being in the collection set. 1047 __ mov(tmp1, res); 1048 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 1049 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); 1050 #ifdef _LP64 1051 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); 1052 __ testbool(tmp2); 1053 #else 1054 // On x86_32, C1 register allocator can give us the register without 8-bit support. 1055 // Do the full-register access and test to avoid compilation failures. 1056 __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1)); 1057 __ testptr(tmp2, 0xFF); 1058 #endif 1059 __ jcc(Assembler::zero, *stub->continuation()); 1060 } 1061 1062 __ bind(slow_path); 1063 ce->store_parameter(res, 0); 1064 ce->store_parameter(addr, 1); 1065 if (is_strong) { 1066 if (is_native) { 1067 __ call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); 1068 } else { 1069 __ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); 1070 } 1071 } else if (is_weak) { 1072 __ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); 1073 } else { 1074 assert(is_phantom, "only remaining strength"); 1075 __ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); 1076 } 1077 __ jmp(*stub->continuation()); 1078 } 1079 1080 #undef __ 1081 1082 #define __ sasm-> 1083 1084 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 1085 __ prologue("shenandoah_pre_barrier", false); 1086 // arg0 : previous value of memory 1087 1088 __ push(rax); 1089 __ push(rdx); 1090 1091 const Register pre_val = rax; 1092 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); 1093 const Register tmp = rdx; 1094 1095 NOT_LP64(__ get_thread(thread);) 1096 1097 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 1098 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 1099 1100 Label done; 1101 Label runtime; 1102 1103 // Is SATB still active? 1104 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 1105 __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); 1106 __ jcc(Assembler::zero, done); 1107 1108 // Can we store original value in the thread's buffer? 1109 1110 __ movptr(tmp, queue_index); 1111 __ testptr(tmp, tmp); 1112 __ jcc(Assembler::zero, runtime); 1113 __ subptr(tmp, wordSize); 1114 __ movptr(queue_index, tmp); 1115 __ addptr(tmp, buffer); 1116 1117 // prev_val (rax) 1118 __ load_parameter(0, pre_val); 1119 __ movptr(Address(tmp, 0), pre_val); 1120 __ jmp(done); 1121 1122 __ bind(runtime); 1123 1124 __ save_live_registers_no_oop_map(true); 1125 1126 // load the pre-value 1127 __ load_parameter(0, rcx); 1128 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread); 1129 1130 __ restore_live_registers(true); 1131 1132 __ bind(done); 1133 1134 __ pop(rdx); 1135 __ pop(rax); 1136 1137 __ epilogue(); 1138 } 1139 1140 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) { 1141 __ prologue("shenandoah_load_reference_barrier", false); 1142 // arg0 : object to be resolved 1143 1144 __ save_live_registers_no_oop_map(true); 1145 1146 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 1147 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 1148 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 1149 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 1150 1151 #ifdef _LP64 1152 __ load_parameter(0, c_rarg0); 1153 __ load_parameter(1, c_rarg1); 1154 if (is_strong) { 1155 if (is_native) { 1156 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1); 1157 } else { 1158 if (UseCompressedOops) { 1159 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0, c_rarg1); 1160 } else { 1161 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1); 1162 } 1163 } 1164 } else if (is_weak) { 1165 assert(!is_native, "weak must not be called off-heap"); 1166 if (UseCompressedOops) { 1167 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1); 1168 } else { 1169 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1); 1170 } 1171 } else { 1172 assert(is_phantom, "only remaining strength"); 1173 assert(is_native, "phantom must only be called off-heap"); 1174 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1); 1175 } 1176 #else 1177 __ load_parameter(0, rax); 1178 __ load_parameter(1, rbx); 1179 if (is_strong) { 1180 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), rax, rbx); 1181 } else if (is_weak) { 1182 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx); 1183 } else { 1184 assert(is_phantom, "only remaining strength"); 1185 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), rax, rbx); 1186 } 1187 #endif 1188 1189 __ restore_live_registers_except_rax(true); 1190 1191 __ epilogue(); 1192 } 1193 1194 #undef __ 1195 1196 #endif // COMPILER1