1 /* 2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" 27 #include "gc/shenandoah/mode/shenandoahMode.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 30 #include "gc/shenandoah/shenandoahForwarding.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 33 #include "gc/shenandoah/shenandoahRuntime.hpp" 34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 35 #include "interpreter/interp_masm.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "runtime/javaThread.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #ifdef COMPILER1 40 #include "c1/c1_LIRAssembler.hpp" 41 #include "c1/c1_MacroAssembler.hpp" 42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" 43 #endif 44 #ifdef COMPILER2 45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 46 #endif 47 48 #define __ masm-> 49 50 #ifdef PRODUCT 51 #define BLOCK_COMMENT(str) /* nothing */ 52 #else 53 #define BLOCK_COMMENT(str) __ block_comment(str) 54 #endif 55 56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, 57 Register src, Register dst, Register count, RegSet saved_regs) { 58 if (is_oop) { 59 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 60 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { 61 62 Label done; 63 64 // Avoid calling runtime if count == 0 65 __ cbz(count, done); 66 67 // Is GC active? 68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 69 __ ldrb(rscratch1, gc_state); 70 if (ShenandoahSATBBarrier && dest_uninitialized) { 71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); 72 } else { 73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); 74 __ tst(rscratch1, rscratch2); 75 __ br(Assembler::EQ, done); 76 } 77 78 __ push(saved_regs, sp); 79 if (UseCompressedOops) { 80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count); 81 } else { 82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count); 83 } 84 __ pop(saved_regs, sp); 85 __ bind(done); 86 } 87 } 88 } 89 90 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, 91 Register start, Register count, Register tmp) { 92 if (ShenandoahCardBarrier && is_oop) { 93 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp); 94 } 95 } 96 97 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm, 98 Register obj, 99 Register pre_val, 100 Register thread, 101 Register tmp1, 102 Register tmp2, 103 bool tosca_live, 104 bool expand_call) { 105 assert(ShenandoahSATBBarrier, "Should be checked by caller"); 106 107 // If expand_call is true then we expand the call_VM_leaf macro 108 // directly to skip generating the check by 109 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 110 111 assert(thread == rthread, "must be"); 112 113 Label done; 114 Label runtime; 115 116 assert_different_registers(obj, pre_val, tmp1, tmp2); 117 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); 118 119 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 120 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 121 122 // Is marking active? 123 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 124 __ ldrb(tmp1, gc_state); 125 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done); 126 127 // Do we need to load the previous value? 128 if (obj != noreg) { 129 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 130 } 131 132 // Is the previous value null? 133 __ cbz(pre_val, done); 134 135 // Can we store original value in the thread's buffer? 136 // Is index == 0? 137 // (The index field is typed as size_t.) 138 139 __ ldr(tmp1, index); // tmp := *index_adr 140 __ cbz(tmp1, runtime); // tmp == 0? 141 // If yes, goto runtime 142 143 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize 144 __ str(tmp1, index); // *index_adr := tmp 145 __ ldr(tmp2, buffer); 146 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr 147 148 // Record the previous value 149 __ str(pre_val, Address(tmp1, 0)); 150 __ b(done); 151 152 __ bind(runtime); 153 // save the live input values 154 RegSet saved = RegSet::of(pre_val); 155 if (tosca_live) saved += RegSet::of(r0); 156 if (obj != noreg) saved += RegSet::of(obj); 157 158 __ push(saved, sp); 159 160 // Calling the runtime using the regular call_VM_leaf mechanism generates 161 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 162 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr. 163 // 164 // If we care generating the pre-barrier without a frame (e.g. in the 165 // intrinsified Reference.get() routine) then rfp might be pointing to 166 // the caller frame and so this check will most likely fail at runtime. 167 // 168 // Expanding the call directly bypasses the generation of the check. 169 // So when we do not have have a full interpreter frame on the stack 170 // expand_call should be passed true. 171 172 if (expand_call) { 173 assert(pre_val != c_rarg1, "smashed arg"); 174 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val); 175 } else { 176 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val); 177 } 178 179 __ pop(saved, sp); 180 181 __ bind(done); 182 } 183 184 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { 185 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); 186 Label is_null; 187 __ cbz(dst, is_null); 188 resolve_forward_pointer_not_null(masm, dst, tmp); 189 __ bind(is_null); 190 } 191 192 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitly 193 // passed in. 194 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) { 195 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); 196 // The below loads the mark word, checks if the lowest two bits are 197 // set, and if so, clear the lowest two bits and copy the result 198 // to dst. Otherwise it leaves dst alone. 199 // Implementing this is surprisingly awkward. I do it here by: 200 // - Inverting the mark word 201 // - Test lowest two bits == 0 202 // - If so, set the lowest two bits 203 // - Invert the result back, and copy to dst 204 205 bool borrow_reg = (tmp == noreg); 206 if (borrow_reg) { 207 // No free registers available. Make one useful. 208 tmp = rscratch1; 209 if (tmp == dst) { 210 tmp = rscratch2; 211 } 212 __ push(RegSet::of(tmp), sp); 213 } 214 215 assert_different_registers(tmp, dst); 216 217 Label done; 218 __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); 219 __ eon(tmp, tmp, zr); 220 __ ands(zr, tmp, markWord::lock_mask_in_place); 221 __ br(Assembler::NE, done); 222 __ orr(tmp, tmp, markWord::marked_value); 223 __ eon(dst, tmp, zr); 224 __ bind(done); 225 226 if (borrow_reg) { 227 __ pop(RegSet::of(tmp), sp); 228 } 229 } 230 231 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) { 232 assert(ShenandoahLoadRefBarrier, "Should be enabled"); 233 assert(dst != rscratch2, "need rscratch2"); 234 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2); 235 236 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 237 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 238 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 239 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 240 bool is_narrow = UseCompressedOops && !is_native; 241 242 Label heap_stable, not_cset; 243 __ enter(/*strip_ret_addr*/true); 244 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 245 __ ldrb(rscratch2, gc_state); 246 247 // Check for heap stability 248 if (is_strong) { 249 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable); 250 } else { 251 Label lrb; 252 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb); 253 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable); 254 __ bind(lrb); 255 } 256 257 // use r1 for load address 258 Register result_dst = dst; 259 if (dst == r1) { 260 __ mov(rscratch1, dst); 261 dst = rscratch1; 262 } 263 264 // Save r0 and r1, unless it is an output register 265 RegSet to_save = RegSet::of(r0, r1) - result_dst; 266 __ push(to_save, sp); 267 __ lea(r1, load_addr); 268 __ mov(r0, dst); 269 270 // Test for in-cset 271 if (is_strong) { 272 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); 273 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 274 __ ldrb(rscratch2, Address(rscratch2, rscratch1)); 275 __ tbz(rscratch2, 0, not_cset); 276 } 277 278 __ push_call_clobbered_registers(); 279 if (is_strong) { 280 if (is_narrow) { 281 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); 282 } else { 283 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); 284 } 285 } else if (is_weak) { 286 if (is_narrow) { 287 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); 288 } else { 289 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); 290 } 291 } else { 292 assert(is_phantom, "only remaining strength"); 293 assert(!is_narrow, "phantom access cannot be narrow"); 294 // AOT saved adapters need relocation for this call. 295 __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom))); 296 } 297 __ blr(lr); 298 __ mov(rscratch1, r0); 299 __ pop_call_clobbered_registers(); 300 __ mov(r0, rscratch1); 301 302 __ bind(not_cset); 303 304 __ mov(result_dst, r0); 305 __ pop(to_save, sp); 306 307 __ bind(heap_stable); 308 __ leave(); 309 } 310 311 // 312 // Arguments: 313 // 314 // Inputs: 315 // src: oop location to load from, might be clobbered 316 // 317 // Output: 318 // dst: oop loaded from src location 319 // 320 // Kill: 321 // rscratch1 (scratch reg) 322 // 323 // Alias: 324 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src) 325 // 326 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 327 Register dst, Address src, Register tmp1, Register tmp2) { 328 // 1: non-reference load, no additional barrier is needed 329 if (!is_reference_type(type)) { 330 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 331 return; 332 } 333 334 // 2: load a reference from src location and apply LRB if needed 335 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { 336 Register result_dst = dst; 337 338 // Preserve src location for LRB 339 if (dst == src.base() || dst == src.index()) { 340 dst = rscratch1; 341 } 342 assert_different_registers(dst, src.base(), src.index()); 343 344 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 345 346 load_reference_barrier(masm, dst, src, decorators); 347 348 if (dst != result_dst) { 349 __ mov(result_dst, dst); 350 dst = result_dst; 351 } 352 } else { 353 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 354 } 355 356 // 3: apply keep-alive barrier if needed 357 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { 358 __ enter(/*strip_ret_addr*/true); 359 __ push_call_clobbered_registers(); 360 satb_barrier(masm /* masm */, 361 noreg /* obj */, 362 dst /* pre_val */, 363 rthread /* thread */, 364 tmp1 /* tmp1 */, 365 tmp2 /* tmp2 */, 366 true /* tosca_live */, 367 true /* expand_call */); 368 __ pop_call_clobbered_registers(); 369 __ leave(); 370 } 371 } 372 373 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) { 374 assert(ShenandoahCardBarrier, "Should have been checked by caller"); 375 376 __ lsr(obj, obj, CardTable::card_shift()); 377 378 assert(CardTable::dirty_card_val() == 0, "must be"); 379 380 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset())); 381 __ ldr(rscratch1, curr_ct_holder_addr); 382 383 if (UseCondCardMark) { 384 Label L_already_dirty; 385 __ ldrb(rscratch2, Address(obj, rscratch1)); 386 __ cbz(rscratch2, L_already_dirty); 387 __ strb(zr, Address(obj, rscratch1)); 388 __ bind(L_already_dirty); 389 } else { 390 __ strb(zr, Address(obj, rscratch1)); 391 } 392 } 393 394 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 395 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 396 // 1: non-reference types require no barriers 397 if (!is_reference_type(type)) { 398 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); 399 return; 400 } 401 402 // Flatten object address right away for simplicity: likely needed by barriers 403 if (dst.index() == noreg && dst.offset() == 0) { 404 if (dst.base() != tmp3) { 405 __ mov(tmp3, dst.base()); 406 } 407 } else { 408 __ lea(tmp3, dst); 409 } 410 411 bool storing_non_null = (val != noreg); 412 413 // 2: pre-barrier: SATB needs the previous value 414 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) { 415 satb_barrier(masm, 416 tmp3 /* obj */, 417 tmp2 /* pre_val */, 418 rthread /* thread */, 419 tmp1 /* tmp */, 420 rscratch1 /* tmp2 */, 421 storing_non_null /* tosca_live */, 422 false /* expand_call */); 423 } 424 425 // Store! 426 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg); 427 428 // 3: post-barrier: card barrier needs store address 429 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) { 430 card_barrier(masm, tmp3); 431 } 432 } 433 434 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 435 Register obj, Register tmp, Label& slowpath) { 436 Label done; 437 // Resolve jobject 438 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); 439 440 // Check for null. 441 __ cbz(obj, done); 442 443 assert(obj != rscratch2, "need rscratch2"); 444 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); 445 __ lea(rscratch2, gc_state); 446 __ ldrb(rscratch2, Address(rscratch2)); 447 448 // Check for heap in evacuation phase 449 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath); 450 451 __ bind(done); 452 } 453 454 // Special Shenandoah CAS implementation that handles false negatives due 455 // to concurrent evacuation. The service is more complex than a 456 // traditional CAS operation because the CAS operation is intended to 457 // succeed if the reference at addr exactly matches expected or if the 458 // reference at addr holds a pointer to a from-space object that has 459 // been relocated to the location named by expected. There are two 460 // races that must be addressed: 461 // a) A parallel thread may mutate the contents of addr so that it points 462 // to a different object. In this case, the CAS operation should fail. 463 // b) A parallel thread may heal the contents of addr, replacing a 464 // from-space pointer held in addr with the to-space pointer 465 // representing the new location of the object. 466 // Upon entry to cmpxchg_oop, it is assured that new_val equals null 467 // or it refers to an object that is not being evacuated out of 468 // from-space, or it refers to the to-space version of an object that 469 // is being evacuated out of from-space. 470 // 471 // By default the value held in the result register following execution 472 // of the generated code sequence is 0 to indicate failure of CAS, 473 // non-zero to indicate success. If is_cae, the result is the value most 474 // recently fetched from addr rather than a boolean success indicator. 475 // 476 // Clobbers rscratch1, rscratch2 477 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, 478 Register addr, 479 Register expected, 480 Register new_val, 481 bool acquire, bool release, 482 bool is_cae, 483 Register result) { 484 Register tmp1 = rscratch1; 485 Register tmp2 = rscratch2; 486 bool is_narrow = UseCompressedOops; 487 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword; 488 489 assert_different_registers(addr, expected, tmp1, tmp2); 490 assert_different_registers(addr, new_val, tmp1, tmp2); 491 492 Label step4, done; 493 494 // There are two ways to reach this label. Initial entry into the 495 // cmpxchg_oop code expansion starts at step1 (which is equivalent 496 // to label step4). Additionally, in the rare case that four steps 497 // are required to perform the requested operation, the fourth step 498 // is the same as the first. On a second pass through step 1, 499 // control may flow through step 2 on its way to failure. It will 500 // not flow from step 2 to step 3 since we are assured that the 501 // memory at addr no longer holds a from-space pointer. 502 // 503 // The comments that immediately follow the step4 label apply only 504 // to the case in which control reaches this label by branch from 505 // step 3. 506 507 __ bind (step4); 508 509 // Step 4. CAS has failed because the value most recently fetched 510 // from addr is no longer the from-space pointer held in tmp2. If a 511 // different thread replaced the in-memory value with its equivalent 512 // to-space pointer, then CAS may still be able to succeed. The 513 // value held in the expected register has not changed. 514 // 515 // It is extremely rare we reach this point. For this reason, the 516 // implementation opts for smaller rather than potentially faster 517 // code. Ultimately, smaller code for this rare case most likely 518 // delivers higher overall throughput by enabling improved icache 519 // performance. 520 521 // Step 1. Fast-path. 522 // 523 // Try to CAS with given arguments. If successful, then we are done. 524 // 525 // No label required for step 1. 526 527 __ cmpxchg(addr, expected, new_val, size, acquire, release, false, tmp2); 528 // EQ flag set iff success. tmp2 holds value fetched. 529 530 // If expected equals null but tmp2 does not equal null, the 531 // following branches to done to report failure of CAS. If both 532 // expected and tmp2 equal null, the following branches to done to 533 // report success of CAS. There's no need for a special test of 534 // expected equal to null. 535 536 __ br(Assembler::EQ, done); 537 // if CAS failed, fall through to step 2 538 539 // Step 2. CAS has failed because the value held at addr does not 540 // match expected. This may be a false negative because the value fetched 541 // from addr (now held in tmp2) may be a from-space pointer to the 542 // original copy of same object referenced by to-space pointer expected. 543 // 544 // To resolve this, it suffices to find the forward pointer associated 545 // with fetched value. If this matches expected, retry CAS with new 546 // parameters. If this mismatches, then we have a legitimate 547 // failure, and we're done. 548 // 549 // No need for step2 label. 550 551 // overwrite tmp1 with from-space pointer fetched from memory 552 __ mov(tmp1, tmp2); 553 554 if (is_narrow) { 555 // Decode tmp1 in order to resolve its forward pointer 556 __ decode_heap_oop(tmp1, tmp1); 557 } 558 resolve_forward_pointer(masm, tmp1); 559 // Encode tmp1 to compare against expected. 560 __ encode_heap_oop(tmp1, tmp1); 561 562 // Does forwarded value of fetched from-space pointer match original 563 // value of expected? If tmp1 holds null, this comparison will fail 564 // because we know from step1 that expected is not null. There is 565 // no need for a separate test for tmp1 (the value originally held 566 // in memory) equal to null. 567 __ cmp(tmp1, expected); 568 569 // If not, then the failure was legitimate and we're done. 570 // Branching to done with NE condition denotes failure. 571 __ br(Assembler::NE, done); 572 573 // Fall through to step 3. No need for step3 label. 574 575 // Step 3. We've confirmed that the value originally held in memory 576 // (now held in tmp2) pointed to from-space version of original 577 // expected value. Try the CAS again with the from-space expected 578 // value. If it now succeeds, we're good. 579 // 580 // Note: tmp2 holds encoded from-space pointer that matches to-space 581 // object residing at expected. tmp2 is the new "expected". 582 583 // Note that macro implementation of __cmpxchg cannot use same register 584 // tmp2 for result and expected since it overwrites result before it 585 // compares result with expected. 586 __ cmpxchg(addr, tmp2, new_val, size, acquire, release, false, noreg); 587 // EQ flag set iff success. tmp2 holds value fetched, tmp1 (rscratch1) clobbered. 588 589 // If fetched value did not equal the new expected, this could 590 // still be a false negative because some other thread may have 591 // newly overwritten the memory value with its to-space equivalent. 592 __ br(Assembler::NE, step4); 593 594 if (is_cae) { 595 // We're falling through to done to indicate success. Success 596 // with is_cae is denoted by returning the value of expected as 597 // result. 598 __ mov(tmp2, expected); 599 } 600 601 __ bind(done); 602 // At entry to done, the Z (EQ) flag is on iff if the CAS 603 // operation was successful. Additionally, if is_cae, tmp2 holds 604 // the value most recently fetched from addr. In this case, success 605 // is denoted by tmp2 matching expected. 606 607 if (is_cae) { 608 __ mov(result, tmp2); 609 } else { 610 __ cset(result, Assembler::EQ); 611 } 612 } 613 614 #ifdef COMPILER2 615 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) { 616 assert_different_registers(obj, addr); 617 BLOCK_COMMENT("load_ref_barrier_c2 {"); 618 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) { 619 return; 620 } 621 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 622 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow); 623 624 // Don't preserve the obj across the runtime call, we override it from the 625 // return value anyway. 626 stub->dont_preserve(obj); 627 stub->dont_preserve(gc_state); 628 629 // Check if GC marking is in progress or we are handling a weak reference, 630 // otherwise we don't have to do anything. The code below was optimized to 631 // use less registers and instructions as possible at the expense of always 632 // having a branch instruction. The reason why we use this particular branch 633 // scheme is because the stub entry may be too far for the tbnz to jump to. 634 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0; 635 if (is_strong) { 636 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation()); 637 __ b(*stub->entry()); 638 } else { 639 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below."); 640 __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS); 641 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation()); 642 __ b(*stub->entry()); 643 } 644 645 __ bind(*stub->continuation()); 646 BLOCK_COMMENT("} load_ref_barrier_c2"); 647 } 648 649 /** 650 * The logic implemented here relies on certain flags being on specific 651 * positions of the GCState. Also note that all pointer values in register are 652 * guaranteed to be 'to-space' addresses. The algorithm is as follows. If the 653 * CAS succeed: 654 * - 'res' will be set to 1. 655 * - We need to check SATB flag (index 1 of GCState). If the flag is active 656 * need to store 'oldval' in the buffer. 657 * - We wrote 'newval' to 'addr', therefore we need to mark the corresponding 658 * card in the card table for 'addr' as dirty. 659 * If the CAS failed: 660 * - 'res' will be set to 0. 661 * - If the GCState FORWARDING bit (index 0 of GCState) is set we'll need to 662 * retry the CAS, because the failure may be because the value in 'addr' is 663 * the (outdated) 'from-space' version of 'expected'. The retry is done in a 664 * stub. If the retry succeed then we need to do the steps described above 665 * too for CAS succeed too. 666 * - If FORWARDING bit is clear there is nothing else to do. 667 */ 668 void ShenandoahBarrierSetAssembler::cae_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool maybe_null, bool narrow, bool acquire, bool release, bool weak) { 669 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 670 Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword; 671 672 // We need to do a combination of short and long jump because the stub entry 673 // may be too far to jump directly to. 674 Label short_branch; 675 676 // Assuming just for now that we need both barriers 677 ShenandoahCASBarrierSlowStubC2* const cmpx = ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, narrow, exchange, acquire, release, weak); 678 679 // Issue cmpxchg first, res will have the failure witness if CAS fails 680 __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, exchange ? res : tmp2); 681 682 // Load GC state in tmp1 683 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 684 __ ldrb(tmp1, gcs_addr); 685 686 // First CAS attempt. If successful, then we are done. 687 // EQ flag set iff success. 688 __ cset(exchange? tmp2 : res, Assembler::EQ); 689 690 // Shift GCState right according to result of previous CAS. Luckily, the 691 // boolean result of the CAS also matches the index of the bit that we need 692 // to test later on. 693 __ lsrv(tmp1, tmp1, exchange ? tmp2 : res); 694 695 // Test bit '0' of tmp1, which at this point will be FORWARDING bit if CAS 696 // failed, or SATB bit if CAS succeded. 697 __ tbz(tmp1, 0x0, short_branch); 698 __ b(*cmpx->entry()); 699 700 // Would be nice to fold this in the comparison above, but how? 701 // Skip Card Table dirtying if CAS failed. 702 __ bind(short_branch); 703 __ cbz(exchange ? tmp2 : res, *cmpx->continuation()); 704 card_barrier_c2(node, masm, addr, tmp1); 705 706 __ bind(*cmpx->continuation()); 707 } 708 709 710 711 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, 712 Register dst, bool dst_narrow, 713 Register src, bool src_narrow, 714 Register tmp, Register pre_val, 715 bool is_volatile) { 716 717 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 718 __ ldrb(tmp, gcs_addr); 719 720 satb_barrier_c2(node, masm, dst, pre_val, tmp, src_narrow); 721 722 card_barrier_c2(node, masm, dst, tmp); 723 724 // Need to encode into tmp, because we cannot clobber src. 725 // TODO: Maybe there is a matcher way to test that src is unused after this? 726 if (dst_narrow && !src_narrow) { 727 __ mov(tmp, src); 728 if (ShenandoahStoreBarrierStubC2::src_not_null(node)) { 729 __ encode_heap_oop_not_null(tmp); 730 } else { 731 __ encode_heap_oop(tmp); 732 } 733 src = tmp; 734 } 735 736 // Do the actual store 737 if (dst_narrow) { 738 if (is_volatile) { 739 __ stlrw(src, dst); 740 } else { 741 __ strw(src, dst); 742 } 743 } else { 744 if (is_volatile) { 745 __ stlr(src, dst); 746 } else { 747 __ str(src, dst); 748 } 749 } 750 } 751 752 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val, 753 Register gc_state, bool encoded_preval) { 754 BLOCK_COMMENT("satb_barrier_c2 {"); 755 assert_different_registers(addr, pre_val, rscratch1, rscratch2); 756 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) { 757 return; 758 } 759 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 760 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval); 761 762 // Check if GC marking is in progress, otherwise we don't have to do 763 // anything. 764 __ tstw(gc_state, ShenandoahHeap::MARKING); 765 __ br(Assembler::NE, *stub->entry()); 766 __ bind(*stub->continuation()); 767 BLOCK_COMMENT("} satb_barrier_c2"); 768 } 769 770 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) { 771 if (!ShenandoahCardBarrier || 772 (node->barrier_data() & ShenandoahBarrierCardMark) == 0) { 773 return; 774 } 775 776 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 777 __ lsr(tmp, addr, CardTable::card_shift()); 778 779 assert(CardTable::dirty_card_val() == 0, "must be"); 780 781 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset())); 782 __ ldr(rscratch1, curr_ct_holder_addr); 783 784 if (UseCondCardMark) { 785 Label L_already_dirty; 786 __ ldrb(rscratch2, Address(tmp, rscratch1)); 787 __ cbz(rscratch2, L_already_dirty); 788 __ strb(zr, Address(tmp, rscratch1)); 789 __ bind(L_already_dirty); 790 } else { 791 __ strb(zr, Address(tmp, rscratch1)); 792 } 793 } 794 #undef __ 795 #define __ masm. 796 797 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) { 798 BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {"); 799 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm); 800 __ bind(*entry()); 801 Register obj = _obj; 802 if (_narrow) { 803 __ decode_heap_oop(_tmp1, _obj); 804 obj = _tmp1; 805 } 806 // Weak/phantom loads always need to go to runtime. 807 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) { 808 // Check for object in cset. 809 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); 810 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 811 __ ldrb(rscratch2, Address(rscratch2, rscratch1)); 812 __ cbz(rscratch2, *continuation()); 813 } 814 { 815 SaveLiveRegisters save_registers(&masm, this); 816 if (c_rarg0 != obj) { 817 if (c_rarg0 == _addr) { 818 __ mov(rscratch1, _addr); 819 _addr = rscratch1; 820 } 821 __ mov(c_rarg0, obj); 822 } 823 __ mov(c_rarg1, _addr); 824 825 if (_narrow) { 826 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) { 827 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); 828 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) { 829 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); 830 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) { 831 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow)); 832 } 833 } else { 834 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) { 835 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); 836 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) { 837 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); 838 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) { 839 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)); 840 } 841 } 842 __ blr(rscratch1); 843 __ mov(_obj, r0); 844 } 845 if (_narrow) { 846 __ encode_heap_oop(_obj); 847 } 848 __ b(*continuation()); 849 BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code"); 850 } 851 852 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) { 853 BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {"); 854 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm); 855 __ bind(*entry()); 856 857 // The tmp register that we receive is usually a register holding the 858 // "gc_state" which may be required by subsequent memory operations in their 859 // fastpath. 860 RegSet saved = RegSet::of(_tmp); 861 __ push(saved, sp); 862 863 // Do we need to load the previous value? 864 if (_addr != noreg) { 865 __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW); 866 } else { 867 if (_encoded_preval) { 868 __ decode_heap_oop(_tmp, _preval); 869 } else { 870 _tmp = _preval; 871 } 872 } 873 874 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 875 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 876 Label runtime; 877 __ ldr(rscratch1, index); 878 // If buffer is full, call into runtime. 879 __ cbz(rscratch1, runtime); 880 881 // The buffer is not full, store value into it. 882 __ sub(rscratch1, rscratch1, wordSize); 883 __ str(rscratch1, index); 884 __ ldr(rscratch2, buffer); 885 __ str(_tmp, Address(rscratch2, rscratch1)); 886 __ pop(saved, sp); 887 __ b(*continuation()); 888 889 // Runtime call 890 __ bind(runtime); 891 { 892 SaveLiveRegisters save_registers(&masm, this); 893 __ mov(c_rarg0, _tmp); 894 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2)); 895 __ blr(rscratch1); 896 } 897 __ pop(saved, sp); 898 __ b(*continuation()); 899 BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code"); 900 } 901 902 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) { 903 Unimplemented(); 904 } 905 906 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) { 907 Unimplemented(); 908 } 909 910 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) { 911 __ bind(*entry()); 912 913 // Non-strong references should always go to runtime. We do not expect 914 // CASes over non-strong locations. 915 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes"); 916 917 Label L_final; 918 Label L_succeded; 919 920 // check if first CAS succeded, if it did we just need to write to SATB 921 __ cbnz(_cae ? _tmp2 : _result, L_succeded); 922 923 // First CAS attempt did not succed. Execute LRB on 'addr' and retry CAS. 924 if (!_cae) { 925 __ mov(_result, _tmp2); 926 } 927 928 // (Compressed) failure witness is in _result. 929 // Unpack it and check if it is in collection set. 930 // We need to backup the compressed version to use in the LRB. 931 if (UseCompressedOops) { 932 __ decode_heap_oop(_result); 933 } 934 935 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr()); 936 __ lsr(_tmp2, _result, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 937 __ ldrb(_tmp1, Address(_tmp1, _tmp2)); 938 __ cbz(_tmp1, L_final); 939 940 { 941 SaveLiveRegisters save_registers(&masm, this); 942 // Load up failure witness again. 943 __ mov(c_rarg0, _result); 944 __ mov(c_rarg1, _addr_reg); 945 946 if (UseCompressedOops) { 947 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2); 948 } else { 949 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2); 950 } 951 // We have called LRB to fix up the heap location. We do not care about its 952 // result, as we will just try to CAS the location again. 953 } 954 955 __ bind(L_final); 956 957 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword; 958 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result); 959 960 if (!_cae) { 961 __ cset(_result, Assembler::EQ); 962 } 963 // If the retry did not succeed skip SATB 964 __ br(Assembler::NE, *continuation()); 965 966 967 968 969 __ bind(L_succeded); 970 if (_narrow) { 971 __ decode_heap_oop(_tmp1, _expected); 972 } else { 973 _tmp1 = _expected; 974 } 975 976 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 977 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 978 Label runtime; 979 __ ldr(rscratch1, index); 980 // If buffer is full, call into runtime. 981 __ cbz(rscratch1, runtime); 982 983 // The buffer is not full, store value into it. 984 __ sub(rscratch1, rscratch1, wordSize); 985 __ str(rscratch1, index); 986 __ ldr(rscratch2, buffer); 987 __ str(_tmp1, Address(rscratch2, rscratch1)); 988 __ b(*continuation()); 989 990 // Runtime call 991 __ bind(runtime); 992 { 993 SaveLiveRegisters save_registers(&masm, this); 994 __ mov(c_rarg0, _tmp1); 995 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2)); 996 __ blr(rscratch1); 997 } 998 999 1000 __ b(*continuation()); 1001 } 1002 #undef __ 1003 #define __ masm-> 1004 #endif // COMPILER2 1005 1006 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 1007 Register start, Register count, Register scratch) { 1008 assert(ShenandoahCardBarrier, "Should have been checked by caller"); 1009 1010 Label L_loop, L_done; 1011 const Register end = count; 1012 1013 // Zero count? Nothing to do. 1014 __ cbz(count, L_done); 1015 1016 // end = start + count << LogBytesPerHeapOop 1017 // last element address to make inclusive 1018 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); 1019 __ sub(end, end, BytesPerHeapOop); 1020 __ lsr(start, start, CardTable::card_shift()); 1021 __ lsr(end, end, CardTable::card_shift()); 1022 1023 // number of bytes to copy 1024 __ sub(count, end, start); 1025 1026 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset())); 1027 __ ldr(scratch, curr_ct_holder_addr); 1028 __ add(start, start, scratch); 1029 __ bind(L_loop); 1030 __ strb(zr, Address(start, count)); 1031 __ subs(count, count, 1); 1032 __ br(Assembler::GE, L_loop); 1033 __ bind(L_done); 1034 } 1035 1036 #undef __ 1037 1038 #ifdef COMPILER1 1039 1040 #define __ ce->masm()-> 1041 1042 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { 1043 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 1044 // At this point we know that marking is in progress. 1045 // If do_load() is true then we have to emit the 1046 // load of the previous value; otherwise it has already 1047 // been loaded into _pre_val. 1048 1049 __ bind(*stub->entry()); 1050 1051 assert(stub->pre_val()->is_register(), "Precondition."); 1052 1053 Register pre_val_reg = stub->pre_val()->as_register(); 1054 1055 if (stub->do_load()) { 1056 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); 1057 } 1058 __ cbz(pre_val_reg, *stub->continuation()); 1059 ce->store_parameter(stub->pre_val()->as_register(), 0); 1060 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 1061 __ b(*stub->continuation()); 1062 } 1063 1064 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { 1065 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 1066 __ bind(*stub->entry()); 1067 1068 DecoratorSet decorators = stub->decorators(); 1069 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 1070 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 1071 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 1072 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 1073 1074 Register obj = stub->obj()->as_register(); 1075 Register res = stub->result()->as_register(); 1076 Register addr = stub->addr()->as_pointer_register(); 1077 Register tmp1 = stub->tmp1()->as_register(); 1078 Register tmp2 = stub->tmp2()->as_register(); 1079 1080 assert(res == r0, "result must arrive in r0"); 1081 1082 if (res != obj) { 1083 __ mov(res, obj); 1084 } 1085 1086 if (is_strong) { 1087 // Check for object in cset. 1088 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); 1089 __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 1090 __ ldrb(tmp2, Address(tmp2, tmp1)); 1091 __ cbz(tmp2, *stub->continuation()); 1092 } 1093 1094 ce->store_parameter(res, 0); 1095 ce->store_parameter(addr, 1); 1096 if (is_strong) { 1097 if (is_native) { 1098 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); 1099 } else { 1100 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); 1101 } 1102 } else if (is_weak) { 1103 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); 1104 } else { 1105 assert(is_phantom, "only remaining strength"); 1106 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); 1107 } 1108 1109 __ b(*stub->continuation()); 1110 } 1111 1112 #undef __ 1113 1114 #define __ sasm-> 1115 1116 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 1117 __ prologue("shenandoah_pre_barrier", false); 1118 1119 // arg0 : previous value of memory 1120 1121 BarrierSet* bs = BarrierSet::barrier_set(); 1122 1123 const Register pre_val = r0; 1124 const Register thread = rthread; 1125 const Register tmp = rscratch1; 1126 1127 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 1128 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 1129 1130 Label done; 1131 Label runtime; 1132 1133 // Is marking still active? 1134 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 1135 __ ldrb(tmp, gc_state); 1136 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done); 1137 1138 // Can we store original value in the thread's buffer? 1139 __ ldr(tmp, queue_index); 1140 __ cbz(tmp, runtime); 1141 1142 __ sub(tmp, tmp, wordSize); 1143 __ str(tmp, queue_index); 1144 __ ldr(rscratch2, buffer); 1145 __ add(tmp, tmp, rscratch2); 1146 __ load_parameter(0, rscratch2); 1147 __ str(rscratch2, Address(tmp, 0)); 1148 __ b(done); 1149 1150 __ bind(runtime); 1151 __ push_call_clobbered_registers(); 1152 __ load_parameter(0, pre_val); 1153 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val); 1154 __ pop_call_clobbered_registers(); 1155 __ bind(done); 1156 1157 __ epilogue(); 1158 } 1159 1160 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) { 1161 __ prologue("shenandoah_load_reference_barrier", false); 1162 // arg0 : object to be resolved 1163 1164 __ push_call_clobbered_registers(); 1165 __ load_parameter(0, r0); 1166 __ load_parameter(1, r1); 1167 1168 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 1169 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 1170 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 1171 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 1172 if (is_strong) { 1173 if (is_native) { 1174 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); 1175 } else { 1176 if (UseCompressedOops) { 1177 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); 1178 } else { 1179 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); 1180 } 1181 } 1182 } else if (is_weak) { 1183 assert(!is_native, "weak must not be called off-heap"); 1184 if (UseCompressedOops) { 1185 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); 1186 } else { 1187 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); 1188 } 1189 } else { 1190 assert(is_phantom, "only remaining strength"); 1191 assert(is_native, "phantom must only be called off-heap"); 1192 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)); 1193 } 1194 __ blr(lr); 1195 __ mov(rscratch1, r0); 1196 __ pop_call_clobbered_registers(); 1197 __ mov(r0, rscratch1); 1198 1199 __ epilogue(); 1200 } 1201 1202 #undef __ 1203 1204 #endif // COMPILER1 --- EOF ---