1 /* 2 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. 3 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 30 #include "gc/shenandoah/shenandoahForwarding.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 33 #include "gc/shenandoah/shenandoahRuntime.hpp" 34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "interpreter/interp_masm.hpp" 38 #include "runtime/javaThread.hpp" 39 #include "runtime/sharedRuntime.hpp" 40 #ifdef COMPILER1 41 #include "c1/c1_LIRAssembler.hpp" 42 #include "c1/c1_MacroAssembler.hpp" 43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" 44 #endif 45 46 #define __ masm-> 47 48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, 49 Register src, Register dst, Register count, RegSet saved_regs) { 50 if (is_oop) { 51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) { 53 54 Label done; 55 56 // Avoid calling runtime if count == 0 57 __ beqz(count, done); 58 59 // Is GC active? 60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 61 assert_different_registers(src, dst, count, t0); 62 63 __ lbu(t0, gc_state); 64 if (ShenandoahSATBBarrier && dest_uninitialized) { 65 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS); 66 __ beqz(t0, done); 67 } else { 68 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); 69 __ beqz(t0, done); 70 } 71 72 __ push_reg(saved_regs, sp); 73 if (UseCompressedOops) { 74 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), 75 src, dst, count); 76 } else { 77 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count); 78 } 79 __ pop_reg(saved_regs, sp); 80 __ bind(done); 81 } 82 } 83 } 84 85 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, 86 Register obj, 87 Register pre_val, 88 Register thread, 89 Register tmp, 90 bool tosca_live, 91 bool expand_call) { 92 if (ShenandoahSATBBarrier) { 93 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, t0, tosca_live, expand_call); 94 } 95 } 96 97 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, 98 Register obj, 99 Register pre_val, 100 Register thread, 101 Register tmp1, 102 Register tmp2, 103 bool tosca_live, 104 bool expand_call) { 105 // If expand_call is true then we expand the call_VM_leaf macro 106 // directly to skip generating the check by 107 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 108 assert(thread == xthread, "must be"); 109 110 Label done; 111 Label runtime; 112 113 assert_different_registers(obj, pre_val, tmp1, tmp2); 114 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); 115 116 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); 117 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 118 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 119 120 // Is marking active? 121 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 122 __ lwu(tmp1, in_progress); 123 } else { 124 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 125 __ lbu(tmp1, in_progress); 126 } 127 __ beqz(tmp1, done); 128 129 // Do we need to load the previous value? 130 if (obj != noreg) { 131 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 132 } 133 134 // Is the previous value null? 135 __ beqz(pre_val, done); 136 137 // Can we store original value in the thread's buffer? 138 // Is index == 0? 139 // (The index field is typed as size_t.) 140 __ ld(tmp1, index); // tmp := *index_adr 141 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime 142 143 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize 144 __ sd(tmp1, index); // *index_adr := tmp 145 __ ld(tmp2, buffer); 146 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr 147 148 // Record the previous value 149 __ sd(pre_val, Address(tmp1, 0)); 150 __ j(done); 151 152 __ bind(runtime); 153 // save the live input values 154 RegSet saved = RegSet::of(pre_val); 155 if (tosca_live) saved += RegSet::of(x10); 156 if (obj != noreg) saved += RegSet::of(obj); 157 158 __ push_reg(saved, sp); 159 160 // Calling the runtime using the regular call_VM_leaf mechanism generates 161 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 162 // that checks that the *(rfp+frame::interpreter_frame_last_sp) is null. 163 // 164 // If we care generating the pre-barrier without a frame (e.g. in the 165 // intrinsified Reference.get() routine) then ebp might be pointing to 166 // the caller frame and so this check will most likely fail at runtime. 167 // 168 // Expanding the call directly bypasses the generation of the check. 169 // So when we do not have have a full interpreter frame on the stack 170 // expand_call should be passed true. 171 if (expand_call) { 172 assert(pre_val != c_rarg1, "smashed arg"); 173 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); 174 } else { 175 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); 176 } 177 178 __ pop_reg(saved, sp); 179 180 __ bind(done); 181 } 182 183 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { 184 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); 185 186 Label is_null; 187 __ beqz(dst, is_null); 188 resolve_forward_pointer_not_null(masm, dst, tmp); 189 __ bind(is_null); 190 } 191 192 // IMPORTANT: This must preserve all registers, even t0 and t1, except those explicitly 193 // passed in. 194 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) { 195 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); 196 // The below loads the mark word, checks if the lowest two bits are 197 // set, and if so, clear the lowest two bits and copy the result 198 // to dst. Otherwise it leaves dst alone. 199 // Implementing this is surprisingly awkward. I do it here by: 200 // - Inverting the mark word 201 // - Test lowest two bits == 0 202 // - If so, set the lowest two bits 203 // - Invert the result back, and copy to dst 204 RegSet saved_regs = RegSet::of(t2); 205 bool borrow_reg = (tmp == noreg); 206 if (borrow_reg) { 207 // No free registers available. Make one useful. 208 tmp = t0; 209 if (tmp == dst) { 210 tmp = t1; 211 } 212 saved_regs += RegSet::of(tmp); 213 } 214 215 assert_different_registers(tmp, dst, t2); 216 __ push_reg(saved_regs, sp); 217 218 Label done; 219 __ ld(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); 220 __ xori(tmp, tmp, -1); // eon with 0 is equivalent to XOR with -1 221 __ andi(t2, tmp, markWord::lock_mask_in_place); 222 __ bnez(t2, done); 223 __ ori(tmp, tmp, markWord::marked_value); 224 __ xori(dst, tmp, -1); // eon with 0 is equivalent to XOR with -1 225 __ bind(done); 226 227 __ pop_reg(saved_regs, sp); 228 } 229 230 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, 231 Register dst, 232 Address load_addr, 233 DecoratorSet decorators) { 234 assert(ShenandoahLoadRefBarrier, "Should be enabled"); 235 assert(dst != t1 && load_addr.base() != t1, "need t1"); 236 assert_different_registers(load_addr.base(), t0, t1); 237 238 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 239 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 240 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 241 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 242 bool is_narrow = UseCompressedOops && !is_native; 243 244 Label heap_stable, not_cset; 245 __ enter(); 246 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 247 __ lbu(t1, gc_state); 248 249 // Check for heap stability 250 if (is_strong) { 251 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS); 252 __ beqz(t1, heap_stable); 253 } else { 254 Label lrb; 255 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS); 256 __ bnez(t0, lrb); 257 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS); 258 __ beqz(t0, heap_stable); 259 __ bind(lrb); 260 } 261 262 // use x11 for load address 263 Register result_dst = dst; 264 if (dst == x11) { 265 __ mv(t1, dst); 266 dst = t1; 267 } 268 269 // Save x10 and x11, unless it is an output register 270 RegSet saved_regs = RegSet::of(x10, x11) - result_dst; 271 __ push_reg(saved_regs, sp); 272 __ la(x11, load_addr); 273 __ mv(x10, dst); 274 275 // Test for in-cset 276 if (is_strong) { 277 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr()); 278 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 279 __ add(t1, t1, t0); 280 __ lbu(t1, Address(t1)); 281 __ test_bit(t0, t1, 0); 282 __ beqz(t0, not_cset); 283 } 284 285 __ push_call_clobbered_registers(); 286 address target = nullptr; 287 if (is_strong) { 288 if (is_narrow) { 289 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); 290 } else { 291 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); 292 } 293 } else if (is_weak) { 294 if (is_narrow) { 295 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); 296 } else { 297 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); 298 } 299 } else { 300 assert(is_phantom, "only remaining strength"); 301 assert(!is_narrow, "phantom access cannot be narrow"); 302 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); 303 } 304 __ call(target); 305 __ mv(t0, x10); 306 __ pop_call_clobbered_registers(); 307 __ mv(x10, t0); 308 __ bind(not_cset); 309 __ mv(result_dst, x10); 310 __ pop_reg(saved_regs, sp); 311 312 __ bind(heap_stable); 313 __ leave(); 314 } 315 316 void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) { 317 if (ShenandoahIUBarrier) { 318 __ push_call_clobbered_registers(); 319 320 satb_write_barrier_pre(masm, noreg, dst, xthread, tmp, t0, true, false); 321 322 __ pop_call_clobbered_registers(); 323 } 324 } 325 326 // 327 // Arguments: 328 // 329 // Inputs: 330 // src: oop location to load from, might be clobbered 331 // 332 // Output: 333 // dst: oop loaded from src location 334 // 335 // Kill: 336 // x30 (tmp reg) 337 // 338 // Alias: 339 // dst: x30 (might use x30 as temporary output register to avoid clobbering src) 340 // 341 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, 342 DecoratorSet decorators, 343 BasicType type, 344 Register dst, 345 Address src, 346 Register tmp1, 347 Register tmp2) { 348 // 1: non-reference load, no additional barrier is needed 349 if (!is_reference_type(type)) { 350 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 351 return; 352 } 353 354 // 2: load a reference from src location and apply LRB if needed 355 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { 356 Register result_dst = dst; 357 358 // Preserve src location for LRB 359 RegSet saved_regs; 360 if (dst == src.base()) { 361 dst = (src.base() == x28) ? x29 : x28; 362 saved_regs = RegSet::of(dst); 363 __ push_reg(saved_regs, sp); 364 } 365 assert_different_registers(dst, src.base()); 366 367 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 368 369 load_reference_barrier(masm, dst, src, decorators); 370 371 if (dst != result_dst) { 372 __ mv(result_dst, dst); 373 dst = result_dst; 374 } 375 376 if (saved_regs.bits() != 0) { 377 __ pop_reg(saved_regs, sp); 378 } 379 } else { 380 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 381 } 382 383 // 3: apply keep-alive barrier if needed 384 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { 385 __ enter(); 386 __ push_call_clobbered_registers(); 387 satb_write_barrier_pre(masm /* masm */, 388 noreg /* obj */, 389 dst /* pre_val */, 390 xthread /* thread */, 391 tmp1 /* tmp1 */, 392 tmp2 /* tmp2 */, 393 true /* tosca_live */, 394 true /* expand_call */); 395 __ pop_call_clobbered_registers(); 396 __ leave(); 397 } 398 } 399 400 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 401 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 402 bool on_oop = is_reference_type(type); 403 if (!on_oop) { 404 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); 405 return; 406 } 407 408 // flatten object address if needed 409 if (dst.offset() == 0) { 410 if (dst.base() != tmp3) { 411 __ mv(tmp3, dst.base()); 412 } 413 } else { 414 __ la(tmp3, dst); 415 } 416 417 shenandoah_write_barrier_pre(masm, 418 tmp3 /* obj */, 419 tmp2 /* pre_val */, 420 xthread /* thread */, 421 tmp1 /* tmp */, 422 val != noreg /* tosca_live */, 423 false /* expand_call */); 424 425 if (val == noreg) { 426 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg); 427 } else { 428 iu_barrier(masm, val, tmp1); 429 // G1 barrier needs uncompressed oop for region cross check. 430 Register new_val = val; 431 if (UseCompressedOops) { 432 new_val = t1; 433 __ mv(new_val, val); 434 } 435 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg); 436 } 437 } 438 439 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 440 Register obj, Register tmp, Label& slowpath) { 441 Label done; 442 // Resolve jobject 443 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); 444 445 // Check for null. 446 __ beqz(obj, done); 447 448 assert(obj != t1, "need t1"); 449 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); 450 __ lbu(t1, gc_state); 451 452 // Check for heap in evacuation phase 453 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS); 454 __ bnez(t0, slowpath); 455 456 __ bind(done); 457 } 458 459 // Special Shenandoah CAS implementation that handles false negatives due 460 // to concurrent evacuation. The service is more complex than a 461 // traditional CAS operation because the CAS operation is intended to 462 // succeed if the reference at addr exactly matches expected or if the 463 // reference at addr holds a pointer to a from-space object that has 464 // been relocated to the location named by expected. There are two 465 // races that must be addressed: 466 // a) A parallel thread may mutate the contents of addr so that it points 467 // to a different object. In this case, the CAS operation should fail. 468 // b) A parallel thread may heal the contents of addr, replacing a 469 // from-space pointer held in addr with the to-space pointer 470 // representing the new location of the object. 471 // Upon entry to cmpxchg_oop, it is assured that new_val equals null 472 // or it refers to an object that is not being evacuated out of 473 // from-space, or it refers to the to-space version of an object that 474 // is being evacuated out of from-space. 475 // 476 // By default the value held in the result register following execution 477 // of the generated code sequence is 0 to indicate failure of CAS, 478 // non-zero to indicate success. If is_cae, the result is the value most 479 // recently fetched from addr rather than a boolean success indicator. 480 // 481 // Clobbers t0, t1 482 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, 483 Register addr, 484 Register expected, 485 Register new_val, 486 Assembler::Aqrl acquire, 487 Assembler::Aqrl release, 488 bool is_cae, 489 Register result) { 490 bool is_narrow = UseCompressedOops; 491 Assembler::operand_size size = is_narrow ? Assembler::uint32 : Assembler::int64; 492 493 assert_different_registers(addr, expected, t0, t1); 494 assert_different_registers(addr, new_val, t0, t1); 495 496 Label retry, success, fail, done; 497 498 __ bind(retry); 499 500 // Step1: Try to CAS. 501 __ cmpxchg(addr, expected, new_val, size, acquire, release, /* result */ t1); 502 503 // If success, then we are done. 504 __ beq(expected, t1, success); 505 506 // Step2: CAS failed, check the forwarded pointer. 507 __ mv(t0, t1); 508 509 if (is_narrow) { 510 __ decode_heap_oop(t0, t0); 511 } 512 resolve_forward_pointer(masm, t0); 513 514 __ encode_heap_oop(t0, t0); 515 516 // Report failure when the forwarded oop was not expected. 517 __ bne(t0, expected, fail); 518 519 // Step 3: CAS again using the forwarded oop. 520 __ cmpxchg(addr, t1, new_val, size, acquire, release, /* result */ t0); 521 522 // Retry when failed. 523 __ bne(t0, t1, retry); 524 525 __ bind(success); 526 if (is_cae) { 527 __ mv(result, expected); 528 } else { 529 __ mv(result, 1); 530 } 531 __ j(done); 532 533 __ bind(fail); 534 if (is_cae) { 535 __ mv(result, t0); 536 } else { 537 __ mv(result, zr); 538 } 539 540 __ bind(done); 541 } 542 543 #undef __ 544 545 #ifdef COMPILER1 546 547 #define __ ce->masm()-> 548 549 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { 550 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 551 // At this point we know that marking is in progress. 552 // If do_load() is true then we have to emit the 553 // load of the previous value; otherwise it has already 554 // been loaded into _pre_val. 555 __ bind(*stub->entry()); 556 557 assert(stub->pre_val()->is_register(), "Precondition."); 558 559 Register pre_val_reg = stub->pre_val()->as_register(); 560 561 if (stub->do_load()) { 562 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */); 563 } 564 __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true); 565 ce->store_parameter(stub->pre_val()->as_register(), 0); 566 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 567 __ j(*stub->continuation()); 568 } 569 570 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, 571 ShenandoahLoadReferenceBarrierStub* stub) { 572 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 573 __ bind(*stub->entry()); 574 575 DecoratorSet decorators = stub->decorators(); 576 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 577 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 578 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 579 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 580 581 Register obj = stub->obj()->as_register(); 582 Register res = stub->result()->as_register(); 583 Register addr = stub->addr()->as_pointer_register(); 584 Register tmp1 = stub->tmp1()->as_register(); 585 Register tmp2 = stub->tmp2()->as_register(); 586 587 assert(res == x10, "result must arrive in x10"); 588 assert_different_registers(tmp1, tmp2, t0); 589 590 if (res != obj) { 591 __ mv(res, obj); 592 } 593 594 if (is_strong) { 595 // Check for object in cset. 596 __ mv(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); 597 __ srli(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 598 __ add(tmp2, tmp2, tmp1); 599 __ lbu(tmp2, Address(tmp2)); 600 __ beqz(tmp2, *stub->continuation(), true /* is_far */); 601 } 602 603 ce->store_parameter(res, 0); 604 ce->store_parameter(addr, 1); 605 606 if (is_strong) { 607 if (is_native) { 608 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); 609 } else { 610 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); 611 } 612 } else if (is_weak) { 613 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); 614 } else { 615 assert(is_phantom, "only remaining strength"); 616 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); 617 } 618 619 __ j(*stub->continuation()); 620 } 621 622 #undef __ 623 624 #define __ sasm-> 625 626 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 627 __ prologue("shenandoah_pre_barrier", false); 628 629 // arg0 : previous value of memory 630 631 BarrierSet* bs = BarrierSet::barrier_set(); 632 633 const Register pre_val = x10; 634 const Register thread = xthread; 635 const Register tmp = t0; 636 637 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 638 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 639 640 Label done; 641 Label runtime; 642 643 // Is marking still active? 644 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 645 __ lb(tmp, gc_state); 646 __ andi(tmp, tmp, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); 647 __ beqz(tmp, done); 648 649 // Can we store original value in the thread's buffer? 650 __ ld(tmp, queue_index); 651 __ beqz(tmp, runtime); 652 653 __ sub(tmp, tmp, wordSize); 654 __ sd(tmp, queue_index); 655 __ ld(t1, buffer); 656 __ add(tmp, tmp, t1); 657 __ load_parameter(0, t1); 658 __ sd(t1, Address(tmp, 0)); 659 __ j(done); 660 661 __ bind(runtime); 662 __ push_call_clobbered_registers(); 663 __ load_parameter(0, pre_val); 664 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); 665 __ pop_call_clobbered_registers(); 666 __ bind(done); 667 668 __ epilogue(); 669 } 670 671 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, 672 DecoratorSet decorators) { 673 __ prologue("shenandoah_load_reference_barrier", false); 674 // arg0 : object to be resolved 675 676 __ push_call_clobbered_registers(); 677 __ load_parameter(0, x10); 678 __ load_parameter(1, x11); 679 680 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 681 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 682 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 683 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 684 address target = nullptr; 685 if (is_strong) { 686 if (is_native) { 687 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); 688 } else { 689 if (UseCompressedOops) { 690 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); 691 } else { 692 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); 693 } 694 } 695 } else if (is_weak) { 696 assert(!is_native, "weak must not be called off-heap"); 697 if (UseCompressedOops) { 698 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); 699 } else { 700 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); 701 } 702 } else { 703 assert(is_phantom, "only remaining strength"); 704 assert(is_native, "phantom must only be called off-heap"); 705 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom); 706 } 707 __ call(target); 708 __ mv(t0, x10); 709 __ pop_call_clobbered_registers(); 710 __ mv(x10, t0); 711 712 __ epilogue(); 713 } 714 715 #undef __ 716 717 #endif // COMPILER1