1 /* 2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #if INCLUDE_CDS 28 #include "code/SCCache.hpp" 29 #endif 30 #include "gc/g1/g1BarrierSet.hpp" 31 #include "gc/g1/g1BarrierSetAssembler.hpp" 32 #include "gc/g1/g1BarrierSetRuntime.hpp" 33 #include "gc/g1/g1CardTable.hpp" 34 #include "gc/g1/g1HeapRegion.hpp" 35 #include "gc/g1/g1ThreadLocalData.hpp" 36 #include "gc/shared/collectedHeap.hpp" 37 #include "interpreter/interp_masm.hpp" 38 #include "runtime/javaThread.hpp" 39 #include "runtime/sharedRuntime.hpp" 40 #ifdef COMPILER1 41 #include "c1/c1_LIRAssembler.hpp" 42 #include "c1/c1_MacroAssembler.hpp" 43 #include "gc/g1/c1/g1BarrierSetC1.hpp" 44 #endif 45 46 #define __ masm-> 47 48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 49 Register addr, Register count, RegSet saved_regs) { 50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 51 if (!dest_uninitialized) { 52 Label done; 53 Address in_progress(rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 54 55 // Is marking active? 56 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 57 __ ldrw(rscratch1, in_progress); 58 } else { 59 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 60 __ ldrb(rscratch1, in_progress); 61 } 62 __ cbzw(rscratch1, done); 63 64 __ push(saved_regs, sp); 65 if (count == c_rarg0) { 66 if (addr == c_rarg1) { 67 // exactly backwards!! 68 __ mov(rscratch1, c_rarg0); 69 __ mov(c_rarg0, c_rarg1); 70 __ mov(c_rarg1, rscratch1); 71 } else { 72 __ mov(c_rarg1, count); 73 __ mov(c_rarg0, addr); 74 } 75 } else { 76 __ mov(c_rarg0, addr); 77 __ mov(c_rarg1, count); 78 } 79 if (UseCompressedOops) { 80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2); 81 } else { 82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2); 83 } 84 __ pop(saved_regs, sp); 85 86 __ bind(done); 87 } 88 } 89 90 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 91 Register start, Register count, Register scratch, RegSet saved_regs) { 92 __ push(saved_regs, sp); 93 assert_different_registers(start, count, scratch); 94 assert_different_registers(c_rarg0, count); 95 __ mov(c_rarg0, start); 96 __ mov(c_rarg1, count); 97 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2); 98 __ pop(saved_regs, sp); 99 } 100 101 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, 102 Register obj, 103 Register pre_val, 104 Register thread, 105 Register tmp1, 106 Register tmp2, 107 bool tosca_live, 108 bool expand_call) { 109 // If expand_call is true then we expand the call_VM_leaf macro 110 // directly to skip generating the check by 111 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 112 113 assert(thread == rthread, "must be"); 114 115 Label done; 116 Label runtime; 117 118 assert_different_registers(obj, pre_val, tmp1, tmp2); 119 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); 120 121 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 122 Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 123 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 124 125 // Is marking active? 126 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 127 __ ldrw(tmp1, in_progress); 128 } else { 129 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 130 __ ldrb(tmp1, in_progress); 131 } 132 __ cbzw(tmp1, done); 133 134 // Do we need to load the previous value? 135 if (obj != noreg) { 136 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 137 } 138 139 // Is the previous value null? 140 __ cbz(pre_val, done); 141 142 // Can we store original value in the thread's buffer? 143 // Is index == 0? 144 // (The index field is typed as size_t.) 145 146 __ ldr(tmp1, index); // tmp := *index_adr 147 __ cbz(tmp1, runtime); // tmp == 0? 148 // If yes, goto runtime 149 150 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize 151 __ str(tmp1, index); // *index_adr := tmp 152 __ ldr(tmp2, buffer); 153 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr 154 155 // Record the previous value 156 __ str(pre_val, Address(tmp1, 0)); 157 __ b(done); 158 159 __ bind(runtime); 160 161 __ push_call_clobbered_registers(); 162 163 // Calling the runtime using the regular call_VM_leaf mechanism generates 164 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 165 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr. 166 // 167 // If we care generating the pre-barrier without a frame (e.g. in the 168 // intrinsified Reference.get() routine) then rfp might be pointing to 169 // the caller frame and so this check will most likely fail at runtime. 170 // 171 // Expanding the call directly bypasses the generation of the check. 172 // So when we do not have have a full interpreter frame on the stack 173 // expand_call should be passed true. 174 175 if (expand_call) { 176 assert(pre_val != c_rarg1, "smashed arg"); 177 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 178 } else { 179 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 180 } 181 182 __ pop_call_clobbered_registers(); 183 184 __ bind(done); 185 186 } 187 188 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, 189 Register store_addr, 190 Register new_val, 191 Register thread, 192 Register tmp1, 193 Register tmp2) { 194 assert(thread == rthread, "must be"); 195 assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, 196 rscratch1); 197 assert(store_addr != noreg && new_val != noreg && tmp1 != noreg 198 && tmp2 != noreg, "expecting a register"); 199 200 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 201 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 202 203 BarrierSet* bs = BarrierSet::barrier_set(); 204 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 205 CardTable* ct = ctbs->card_table(); 206 207 Label done; 208 Label runtime; 209 210 // Does store cross heap regions? 211 212 #if INCLUDE_CDS 213 // AOT code needs to load the barrier grain shift from the aot 214 // runtime constants area in the code cache otherwise we can compile 215 // it as an immediate operand 216 if (SCCache::is_on_for_write()) { 217 address grain_shift_address = (address)AOTRuntimeConstants::grain_shift_address(); 218 __ eor(tmp1, store_addr, new_val); 219 __ lea(tmp2, ExternalAddress(grain_shift_address)); 220 __ ldrb(tmp2, tmp2); 221 __ lsrv(tmp1, tmp1, tmp2); 222 __ cbz(tmp1, done); 223 } else 224 #endif 225 { 226 __ eor(tmp1, store_addr, new_val); 227 __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); 228 __ cbz(tmp1, done); 229 } 230 // crosses regions, storing null? 231 232 __ cbz(new_val, done); 233 234 // storing region crossing non-null, is card already dirty? 235 236 const Register card_addr = tmp1; 237 238 #if INCLUDE_CDS 239 // AOT code needs to load the barrier card shift from the aot 240 // runtime constants area in the code cache otherwise we can compile 241 // it as an immediate operand 242 if (SCCache::is_on_for_write()) { 243 address card_shift_address = (address)AOTRuntimeConstants::card_shift_address(); 244 __ lea(tmp2, ExternalAddress(card_shift_address)); 245 __ ldrb(tmp2, tmp2); 246 __ lsrv(card_addr, store_addr, tmp2); 247 } else 248 #endif 249 { 250 __ lsr(card_addr, store_addr, CardTable::card_shift()); 251 } 252 253 // get the address of the card 254 __ load_byte_map_base(tmp2); 255 __ add(card_addr, card_addr, tmp2); 256 __ ldrb(tmp2, Address(card_addr)); 257 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); 258 __ br(Assembler::EQ, done); 259 260 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 261 262 __ membar(Assembler::StoreLoad); 263 264 __ ldrb(tmp2, Address(card_addr)); 265 __ cbzw(tmp2, done); 266 267 // storing a region crossing, non-null oop, card is clean. 268 // dirty card and log. 269 270 __ strb(zr, Address(card_addr)); 271 272 __ ldr(rscratch1, queue_index); 273 __ cbz(rscratch1, runtime); 274 __ sub(rscratch1, rscratch1, wordSize); 275 __ str(rscratch1, queue_index); 276 277 __ ldr(tmp2, buffer); 278 __ str(card_addr, Address(tmp2, rscratch1)); 279 __ b(done); 280 281 __ bind(runtime); 282 // save the live input values 283 RegSet saved = RegSet::of(store_addr); 284 __ push(saved, sp); 285 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); 286 __ pop(saved, sp); 287 288 __ bind(done); 289 } 290 291 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 292 Register dst, Address src, Register tmp1, Register tmp2) { 293 bool on_oop = is_reference_type(type); 294 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; 295 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; 296 bool on_reference = on_weak || on_phantom; 297 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); 298 if (on_oop && on_reference) { 299 // LR is live. It must be saved around calls. 300 __ enter(/*strip_ret_addr*/true); // barrier may call runtime 301 // Generate the G1 pre-barrier code to log the value of 302 // the referent field in an SATB buffer. 303 g1_write_barrier_pre(masm /* masm */, 304 noreg /* obj */, 305 dst /* pre_val */, 306 rthread /* thread */, 307 tmp1 /* tmp1 */, 308 tmp2 /* tmp2 */, 309 true /* tosca_live */, 310 true /* expand_call */); 311 __ leave(); 312 } 313 } 314 315 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 316 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 317 // flatten object address if needed 318 if (dst.index() == noreg && dst.offset() == 0) { 319 if (dst.base() != tmp3) { 320 __ mov(tmp3, dst.base()); 321 } 322 } else { 323 __ lea(tmp3, dst); 324 } 325 326 g1_write_barrier_pre(masm, 327 tmp3 /* obj */, 328 tmp2 /* pre_val */, 329 rthread /* thread */, 330 tmp1 /* tmp1 */, 331 rscratch2 /* tmp2 */, 332 val != noreg /* tosca_live */, 333 false /* expand_call */); 334 335 if (val == noreg) { 336 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg); 337 } else { 338 // G1 barrier needs uncompressed oop for region cross check. 339 Register new_val = val; 340 if (UseCompressedOops) { 341 new_val = rscratch2; 342 __ mov(new_val, val); 343 } 344 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg); 345 g1_write_barrier_post(masm, 346 tmp3 /* store_adr */, 347 new_val /* new_val */, 348 rthread /* thread */, 349 tmp1 /* tmp1 */, 350 tmp2 /* tmp2 */); 351 } 352 353 } 354 355 #ifdef COMPILER1 356 357 #undef __ 358 #define __ ce->masm()-> 359 360 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 361 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 362 // At this point we know that marking is in progress. 363 // If do_load() is true then we have to emit the 364 // load of the previous value; otherwise it has already 365 // been loaded into _pre_val. 366 367 __ bind(*stub->entry()); 368 369 assert(stub->pre_val()->is_register(), "Precondition."); 370 371 Register pre_val_reg = stub->pre_val()->as_register(); 372 373 if (stub->do_load()) { 374 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); 375 } 376 __ cbz(pre_val_reg, *stub->continuation()); 377 ce->store_parameter(stub->pre_val()->as_register(), 0); 378 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 379 __ b(*stub->continuation()); 380 } 381 382 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 383 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 384 __ bind(*stub->entry()); 385 assert(stub->addr()->is_register(), "Precondition."); 386 assert(stub->new_val()->is_register(), "Precondition."); 387 Register new_val_reg = stub->new_val()->as_register(); 388 __ cbz(new_val_reg, *stub->continuation()); 389 ce->store_parameter(stub->addr()->as_pointer_register(), 0); 390 __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); 391 __ b(*stub->continuation()); 392 } 393 394 #undef __ 395 396 #define __ sasm-> 397 398 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 399 __ prologue("g1_pre_barrier", false); 400 401 // arg0 : previous value of memory 402 403 BarrierSet* bs = BarrierSet::barrier_set(); 404 405 const Register pre_val = r0; 406 const Register thread = rthread; 407 const Register tmp = rscratch1; 408 409 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 410 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 411 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 412 413 Label done; 414 Label runtime; 415 416 // Is marking still active? 417 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 418 __ ldrw(tmp, in_progress); 419 } else { 420 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 421 __ ldrb(tmp, in_progress); 422 } 423 __ cbzw(tmp, done); 424 425 // Can we store original value in the thread's buffer? 426 __ ldr(tmp, queue_index); 427 __ cbz(tmp, runtime); 428 429 __ sub(tmp, tmp, wordSize); 430 __ str(tmp, queue_index); 431 __ ldr(rscratch2, buffer); 432 __ add(tmp, tmp, rscratch2); 433 __ load_parameter(0, rscratch2); 434 __ str(rscratch2, Address(tmp, 0)); 435 __ b(done); 436 437 __ bind(runtime); 438 __ push_call_clobbered_registers(); 439 __ load_parameter(0, pre_val); 440 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 441 __ pop_call_clobbered_registers(); 442 __ bind(done); 443 444 __ epilogue(); 445 } 446 447 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 448 __ prologue("g1_post_barrier", false); 449 450 // arg0: store_address 451 Address store_addr(rfp, 2*BytesPerWord); 452 453 BarrierSet* bs = BarrierSet::barrier_set(); 454 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 455 CardTable* ct = ctbs->card_table(); 456 457 Label done; 458 Label runtime; 459 460 // At this point we know new_value is non-null and the new_value crosses regions. 461 // Must check to see if card is already dirty 462 463 const Register thread = rthread; 464 465 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 466 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 467 468 const Register card_offset = rscratch2; 469 // LR is free here, so we can use it to hold the byte_map_base. 470 const Register byte_map_base = lr; 471 472 assert_different_registers(card_offset, byte_map_base, rscratch1); 473 474 __ load_parameter(0, card_offset); 475 __ lsr(card_offset, card_offset, CardTable::card_shift()); 476 __ load_byte_map_base(byte_map_base); 477 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 478 __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val()); 479 __ br(Assembler::EQ, done); 480 481 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 482 483 __ membar(Assembler::StoreLoad); 484 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 485 __ cbzw(rscratch1, done); 486 487 // storing region crossing non-null, card is clean. 488 // dirty card and log. 489 __ strb(zr, Address(byte_map_base, card_offset)); 490 491 // Convert card offset into an address in card_addr 492 Register card_addr = card_offset; 493 __ add(card_addr, byte_map_base, card_addr); 494 495 __ ldr(rscratch1, queue_index); 496 __ cbz(rscratch1, runtime); 497 __ sub(rscratch1, rscratch1, wordSize); 498 __ str(rscratch1, queue_index); 499 500 // Reuse LR to hold buffer_addr 501 const Register buffer_addr = lr; 502 503 __ ldr(buffer_addr, buffer); 504 __ str(card_addr, Address(buffer_addr, rscratch1)); 505 __ b(done); 506 507 __ bind(runtime); 508 __ push_call_clobbered_registers(); 509 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); 510 __ pop_call_clobbered_registers(); 511 __ bind(done); 512 __ epilogue(); 513 } 514 515 #undef __ 516 517 #endif // COMPILER1