1 /* 2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "gc/g1/g1BarrierSet.hpp" 28 #include "gc/g1/g1BarrierSetAssembler.hpp" 29 #include "gc/g1/g1BarrierSetRuntime.hpp" 30 #include "gc/g1/g1CardTable.hpp" 31 #include "gc/g1/g1HeapRegion.hpp" 32 #include "gc/g1/g1ThreadLocalData.hpp" 33 #include "interpreter/interp_masm.hpp" 34 #include "runtime/sharedRuntime.hpp" 35 #include "utilities/debug.hpp" 36 #include "utilities/macros.hpp" 37 #ifdef COMPILER1 38 #include "c1/c1_LIRAssembler.hpp" 39 #include "c1/c1_MacroAssembler.hpp" 40 #include "gc/g1/c1/g1BarrierSetC1.hpp" 41 #endif // COMPILER1 42 #ifdef COMPILER2 43 #include "gc/g1/c2/g1BarrierSetC2.hpp" 44 #endif // COMPILER2 45 46 #define __ masm-> 47 48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 49 Register addr, Register count) { 50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 51 52 if (!dest_uninitialized) { 53 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); 54 #ifndef _LP64 55 __ push(thread); 56 __ get_thread(thread); 57 #endif 58 59 Label filtered; 60 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 61 // Is marking active? 62 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 63 __ cmpl(in_progress, 0); 64 } else { 65 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 66 __ cmpb(in_progress, 0); 67 } 68 69 NOT_LP64(__ pop(thread);) 70 71 __ jcc(Assembler::equal, filtered); 72 73 __ push_call_clobbered_registers(false /* save_fpu */); 74 #ifdef _LP64 75 if (count == c_rarg0) { 76 if (addr == c_rarg1) { 77 // exactly backwards!! 78 __ xchgptr(c_rarg1, c_rarg0); 79 } else { 80 __ movptr(c_rarg1, count); 81 __ movptr(c_rarg0, addr); 82 } 83 } else { 84 __ movptr(c_rarg0, addr); 85 __ movptr(c_rarg1, count); 86 } 87 if (UseCompressedOops) { 88 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2); 89 } else { 90 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2); 91 } 92 #else 93 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 94 addr, count); 95 #endif 96 __ pop_call_clobbered_registers(false /* save_fpu */); 97 98 __ bind(filtered); 99 } 100 } 101 102 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 103 Register addr, Register count, Register tmp) { 104 __ push_call_clobbered_registers(false /* save_fpu */); 105 #ifdef _LP64 106 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx 107 assert_different_registers(c_rarg1, addr); 108 __ mov(c_rarg1, count); 109 __ mov(c_rarg0, addr); 110 } else { 111 assert_different_registers(c_rarg0, count); 112 __ mov(c_rarg0, addr); 113 __ mov(c_rarg1, count); 114 } 115 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2); 116 #else 117 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 118 addr, count); 119 #endif 120 __ pop_call_clobbered_registers(false /* save_fpu */); 121 } 122 123 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 124 Register dst, Address src, Register tmp1, Register tmp_thread) { 125 bool on_oop = is_reference_type(type); 126 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; 127 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; 128 bool on_reference = on_weak || on_phantom; 129 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 130 if (on_oop && on_reference) { 131 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); 132 133 #ifndef _LP64 134 // Work around the x86_32 bug that only manifests with Loom for some reason. 135 // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg. 136 if (thread == noreg) { 137 if (dst != rcx && tmp1 != rcx) { 138 thread = rcx; 139 } else if (dst != rdx && tmp1 != rdx) { 140 thread = rdx; 141 } else if (dst != rdi && tmp1 != rdi) { 142 thread = rdi; 143 } 144 } 145 assert_different_registers(dst, tmp1, thread); 146 __ push(thread); 147 __ get_thread(thread); 148 #endif 149 150 // Generate the G1 pre-barrier code to log the value of 151 // the referent field in an SATB buffer. 152 g1_write_barrier_pre(masm /* masm */, 153 noreg /* obj */, 154 dst /* pre_val */, 155 thread /* thread */, 156 tmp1 /* tmp */, 157 true /* tosca_live */, 158 true /* expand_call */); 159 160 #ifndef _LP64 161 __ pop(thread); 162 #endif 163 } 164 } 165 166 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, 167 const Register thread, const Register value, const Register temp) { 168 // This code assumes that buffer index is pointer sized. 169 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t)); 170 // Can we store a value in the given thread's buffer? 171 // (The index field is typed as size_t.) 172 __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address) 173 __ testptr(temp, temp); // index == 0? 174 __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer) 175 // The buffer is not full, store value into it. 176 __ subptr(temp, wordSize); // temp := next index 177 __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index 178 __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index 179 __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value 180 } 181 182 static void generate_pre_barrier_fast_path(MacroAssembler* masm, 183 const Register thread) { 184 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 185 // Is marking active? 186 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 187 __ cmpl(in_progress, 0); 188 } else { 189 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 190 __ cmpb(in_progress, 0); 191 } 192 } 193 194 static void generate_pre_barrier_slow_path(MacroAssembler* masm, 195 const Register obj, 196 const Register pre_val, 197 const Register thread, 198 const Register tmp, 199 Label& done, 200 Label& runtime) { 201 // Do we need to load the previous value? 202 if (obj != noreg) { 203 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 204 } 205 // Is the previous value null? 206 __ cmpptr(pre_val, NULL_WORD); 207 __ jcc(Assembler::equal, done); 208 generate_queue_insertion(masm, 209 G1ThreadLocalData::satb_mark_queue_index_offset(), 210 G1ThreadLocalData::satb_mark_queue_buffer_offset(), 211 runtime, 212 thread, pre_val, tmp); 213 __ jmp(done); 214 } 215 216 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, 217 Register obj, 218 Register pre_val, 219 Register thread, 220 Register tmp, 221 bool tosca_live, 222 bool expand_call) { 223 // If expand_call is true then we expand the call_VM_leaf macro 224 // directly to skip generating the check by 225 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 226 227 #ifdef _LP64 228 assert(thread == r15_thread, "must be"); 229 #endif // _LP64 230 231 Label done; 232 Label runtime; 233 234 assert(pre_val != noreg, "check this code"); 235 236 if (obj != noreg) { 237 assert_different_registers(obj, pre_val, tmp); 238 assert(pre_val != rax, "check this code"); 239 } 240 241 generate_pre_barrier_fast_path(masm, thread); 242 // If marking is not active (*(mark queue active address) == 0), jump to done 243 __ jcc(Assembler::equal, done); 244 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime); 245 246 __ bind(runtime); 247 248 if (EnableValhalla && InlineTypePassFieldsAsArgs) { 249 // Barriers might be emitted when converting between (scalarized) calling conventions for inline 250 // types. Save all argument registers before calling into the runtime. 251 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 ) 252 __ pusha(); 253 __ subptr(rsp, 64); 254 __ movdbl(Address(rsp, 0), j_farg0); 255 __ movdbl(Address(rsp, 8), j_farg1); 256 __ movdbl(Address(rsp, 16), j_farg2); 257 __ movdbl(Address(rsp, 24), j_farg3); 258 __ movdbl(Address(rsp, 32), j_farg4); 259 __ movdbl(Address(rsp, 40), j_farg5); 260 __ movdbl(Address(rsp, 48), j_farg6); 261 __ movdbl(Address(rsp, 56), j_farg7); 262 } else { 263 // Determine and save the live input values 264 __ push_call_clobbered_registers(); 265 } 266 267 // Calling the runtime using the regular call_VM_leaf mechanism generates 268 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 269 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr. 270 // 271 // If we care generating the pre-barrier without a frame (e.g. in the 272 // intrinsified Reference.get() routine) then ebp might be pointing to 273 // the caller frame and so this check will most likely fail at runtime. 274 // 275 // Expanding the call directly bypasses the generation of the check. 276 // So when we do not have have a full interpreter frame on the stack 277 // expand_call should be passed true. 278 279 if (expand_call) { 280 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) 281 #ifdef _LP64 282 if (c_rarg1 != thread) { 283 __ mov(c_rarg1, thread); 284 } 285 if (c_rarg0 != pre_val) { 286 __ mov(c_rarg0, pre_val); 287 } 288 #else 289 __ push(thread); 290 __ push(pre_val); 291 #endif 292 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2); 293 } else { 294 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 295 } 296 297 if (EnableValhalla && InlineTypePassFieldsAsArgs) { 298 // Restore registers 299 __ movdbl(j_farg0, Address(rsp, 0)); 300 __ movdbl(j_farg1, Address(rsp, 8)); 301 __ movdbl(j_farg2, Address(rsp, 16)); 302 __ movdbl(j_farg3, Address(rsp, 24)); 303 __ movdbl(j_farg4, Address(rsp, 32)); 304 __ movdbl(j_farg5, Address(rsp, 40)); 305 __ movdbl(j_farg6, Address(rsp, 48)); 306 __ movdbl(j_farg7, Address(rsp, 56)); 307 __ addptr(rsp, 64); 308 __ popa(); 309 } else { 310 __ pop_call_clobbered_registers(); 311 } 312 313 __ bind(done); 314 } 315 316 static void generate_post_barrier_fast_path(MacroAssembler* masm, 317 const Register store_addr, 318 const Register new_val, 319 const Register tmp, 320 const Register tmp2, 321 Label& done, 322 bool new_val_may_be_null) { 323 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); 324 // Does store cross heap regions? 325 __ movptr(tmp, store_addr); // tmp := store address 326 __ xorptr(tmp, new_val); // tmp := store address ^ new value 327 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0? 328 __ jcc(Assembler::equal, done); 329 // Crosses regions, storing null? 330 if (new_val_may_be_null) { 331 __ cmpptr(new_val, NULL_WORD); // new value == null? 332 __ jcc(Assembler::equal, done); 333 } 334 // Storing region crossing non-null, is card young? 335 __ movptr(tmp, store_addr); // tmp := store address 336 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base 337 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT 338 // a valid address and therefore is not properly handled by the relocation code. 339 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address 340 __ addptr(tmp, tmp2); // tmp := card address 341 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val? 342 } 343 344 static void generate_post_barrier_slow_path(MacroAssembler* masm, 345 const Register thread, 346 const Register tmp, 347 const Register tmp2, 348 Label& done, 349 Label& runtime) { 350 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar 351 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val? 352 __ jcc(Assembler::equal, done); 353 // Storing a region crossing, non-null oop, card is clean. 354 // Dirty card and log. 355 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val 356 generate_queue_insertion(masm, 357 G1ThreadLocalData::dirty_card_queue_index_offset(), 358 G1ThreadLocalData::dirty_card_queue_buffer_offset(), 359 runtime, 360 thread, tmp, tmp2); 361 __ jmp(done); 362 } 363 364 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, 365 Register store_addr, 366 Register new_val, 367 Register thread, 368 Register tmp, 369 Register tmp2) { 370 #ifdef _LP64 371 assert(thread == r15_thread, "must be"); 372 #endif // _LP64 373 374 Label done; 375 Label runtime; 376 377 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */); 378 // If card is young, jump to done 379 __ jcc(Assembler::equal, done); 380 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime); 381 382 __ bind(runtime); 383 // Barriers might be emitted when converting between (scalarized) calling conventions for inline 384 // types. Save all argument registers before calling into the runtime. 385 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64) 386 __ pusha(); 387 __ subptr(rsp, 64); 388 __ movdbl(Address(rsp, 0), j_farg0); 389 __ movdbl(Address(rsp, 8), j_farg1); 390 __ movdbl(Address(rsp, 16), j_farg2); 391 __ movdbl(Address(rsp, 24), j_farg3); 392 __ movdbl(Address(rsp, 32), j_farg4); 393 __ movdbl(Address(rsp, 40), j_farg5); 394 __ movdbl(Address(rsp, 48), j_farg6); 395 __ movdbl(Address(rsp, 56), j_farg7); 396 397 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread); 398 399 // Restore registers 400 __ movdbl(j_farg0, Address(rsp, 0)); 401 __ movdbl(j_farg1, Address(rsp, 8)); 402 __ movdbl(j_farg2, Address(rsp, 16)); 403 __ movdbl(j_farg3, Address(rsp, 24)); 404 __ movdbl(j_farg4, Address(rsp, 32)); 405 __ movdbl(j_farg5, Address(rsp, 40)); 406 __ movdbl(j_farg6, Address(rsp, 48)); 407 __ movdbl(j_farg7, Address(rsp, 56)); 408 __ addptr(rsp, 64); 409 __ popa(); 410 411 __ bind(done); 412 } 413 414 #if defined(COMPILER2) 415 416 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { 417 #ifdef _LP64 418 SaveLiveRegisters save_registers(masm, stub); 419 if (c_rarg0 != arg) { 420 __ mov(c_rarg0, arg); 421 } 422 __ mov(c_rarg1, r15_thread); 423 // rax is a caller-saved, non-argument-passing register, so it does not 424 // interfere with c_rarg0 or c_rarg1. If it contained any live value before 425 // entering this stub, it is saved at this point, and restored after the 426 // call. If it did not contain any live value, it is free to be used. In 427 // either case, it is safe to use it here as a call scratch register. 428 __ call(RuntimeAddress(runtime_path), rax); 429 #else 430 Unimplemented(); 431 #endif // _LP64 432 } 433 434 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, 435 Register obj, 436 Register pre_val, 437 Register thread, 438 Register tmp, 439 G1PreBarrierStubC2* stub) { 440 #ifdef _LP64 441 assert(thread == r15_thread, "must be"); 442 #endif // _LP64 443 assert(pre_val != noreg, "check this code"); 444 if (obj != noreg) { 445 assert_different_registers(obj, pre_val, tmp); 446 } 447 448 stub->initialize_registers(obj, pre_val, thread, tmp); 449 450 generate_pre_barrier_fast_path(masm, thread); 451 // If marking is active (*(mark queue active address) != 0), jump to stub (slow path) 452 __ jcc(Assembler::notEqual, *stub->entry()); 453 454 __ bind(*stub->continuation()); 455 } 456 457 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, 458 G1PreBarrierStubC2* stub) const { 459 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 460 Label runtime; 461 Register obj = stub->obj(); 462 Register pre_val = stub->pre_val(); 463 Register thread = stub->thread(); 464 Register tmp = stub->tmp1(); 465 assert(stub->tmp2() == noreg, "not needed in this platform"); 466 467 __ bind(*stub->entry()); 468 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime); 469 470 __ bind(runtime); 471 generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); 472 __ jmp(*stub->continuation()); 473 } 474 475 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, 476 Register store_addr, 477 Register new_val, 478 Register thread, 479 Register tmp, 480 Register tmp2, 481 G1PostBarrierStubC2* stub) { 482 #ifdef _LP64 483 assert(thread == r15_thread, "must be"); 484 #endif // _LP64 485 486 stub->initialize_registers(thread, tmp, tmp2); 487 488 bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0; 489 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null); 490 // If card is not young, jump to stub (slow path) 491 __ jcc(Assembler::notEqual, *stub->entry()); 492 493 __ bind(*stub->continuation()); 494 } 495 496 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, 497 G1PostBarrierStubC2* stub) const { 498 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 499 Label runtime; 500 Register thread = stub->thread(); 501 Register tmp = stub->tmp1(); // tmp holds the card address. 502 Register tmp2 = stub->tmp2(); 503 assert(stub->tmp3() == noreg, "not needed in this platform"); 504 505 __ bind(*stub->entry()); 506 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime); 507 508 __ bind(runtime); 509 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); 510 __ jmp(*stub->continuation()); 511 } 512 513 #endif // COMPILER2 514 515 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 516 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 517 bool in_heap = (decorators & IN_HEAP) != 0; 518 bool as_normal = (decorators & AS_NORMAL) != 0; 519 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 520 521 bool needs_pre_barrier = as_normal && !dest_uninitialized; 522 bool needs_post_barrier = val != noreg && in_heap; 523 524 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); 525 // flatten object address if needed 526 // We do it regardless of precise because we need the registers 527 if (dst.index() == noreg && dst.disp() == 0) { 528 if (dst.base() != tmp1) { 529 __ movptr(tmp1, dst.base()); 530 } 531 } else { 532 __ lea(tmp1, dst); 533 } 534 535 #ifndef _LP64 536 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm); 537 #endif 538 539 NOT_LP64(__ get_thread(rcx)); 540 NOT_LP64(imasm->save_bcp()); 541 542 if (needs_pre_barrier) { 543 g1_write_barrier_pre(masm /*masm*/, 544 tmp1 /* obj */, 545 tmp2 /* pre_val */, 546 rthread /* thread */, 547 tmp3 /* tmp */, 548 val != noreg /* tosca_live */, 549 false /* expand_call */); 550 } 551 if (val == noreg) { 552 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); 553 } else { 554 Register new_val = val; 555 if (needs_post_barrier) { 556 // G1 barrier needs uncompressed oop for region cross check. 557 if (UseCompressedOops) { 558 new_val = tmp2; 559 __ movptr(new_val, val); 560 } 561 } 562 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); 563 if (needs_post_barrier) { 564 g1_write_barrier_post(masm /*masm*/, 565 tmp1 /* store_adr */, 566 new_val /* new_val */, 567 rthread /* thread */, 568 tmp3 /* tmp */, 569 tmp2 /* tmp2 */); 570 } 571 } 572 NOT_LP64(imasm->restore_bcp()); 573 } 574 575 #ifdef COMPILER1 576 577 #undef __ 578 #define __ ce->masm()-> 579 580 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 581 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 582 // At this point we know that marking is in progress. 583 // If do_load() is true then we have to emit the 584 // load of the previous value; otherwise it has already 585 // been loaded into _pre_val. 586 587 __ bind(*stub->entry()); 588 assert(stub->pre_val()->is_register(), "Precondition."); 589 590 Register pre_val_reg = stub->pre_val()->as_register(); 591 592 if (stub->do_load()) { 593 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); 594 } 595 596 __ cmpptr(pre_val_reg, NULL_WORD); 597 __ jcc(Assembler::equal, *stub->continuation()); 598 ce->store_parameter(stub->pre_val()->as_register(), 0); 599 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 600 __ jmp(*stub->continuation()); 601 602 } 603 604 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 605 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 606 __ bind(*stub->entry()); 607 assert(stub->addr()->is_register(), "Precondition."); 608 assert(stub->new_val()->is_register(), "Precondition."); 609 Register new_val_reg = stub->new_val()->as_register(); 610 __ cmpptr(new_val_reg, NULL_WORD); 611 __ jcc(Assembler::equal, *stub->continuation()); 612 ce->store_parameter(stub->addr()->as_pointer_register(), 0); 613 __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); 614 __ jmp(*stub->continuation()); 615 } 616 617 #undef __ 618 619 #define __ sasm-> 620 621 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 622 // Generated code assumes that buffer index is pointer sized. 623 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t)); 624 625 __ prologue("g1_pre_barrier", false); 626 // arg0 : previous value of memory 627 628 __ push(rax); 629 __ push(rdx); 630 631 const Register pre_val = rax; 632 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); 633 const Register tmp = rdx; 634 635 NOT_LP64(__ get_thread(thread);) 636 637 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 638 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 639 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 640 641 Label done; 642 Label runtime; 643 644 // Is marking still active? 645 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 646 __ cmpl(queue_active, 0); 647 } else { 648 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 649 __ cmpb(queue_active, 0); 650 } 651 __ jcc(Assembler::equal, done); 652 653 // Can we store original value in the thread's buffer? 654 655 __ movptr(tmp, queue_index); 656 __ testptr(tmp, tmp); 657 __ jcc(Assembler::zero, runtime); 658 __ subptr(tmp, wordSize); 659 __ movptr(queue_index, tmp); 660 __ addptr(tmp, buffer); 661 662 // prev_val (rax) 663 __ load_parameter(0, pre_val); 664 __ movptr(Address(tmp, 0), pre_val); 665 __ jmp(done); 666 667 __ bind(runtime); 668 669 __ push_call_clobbered_registers(); 670 671 // load the pre-value 672 __ load_parameter(0, rcx); 673 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread); 674 675 __ pop_call_clobbered_registers(); 676 677 __ bind(done); 678 679 __ pop(rdx); 680 __ pop(rax); 681 682 __ epilogue(); 683 } 684 685 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 686 __ prologue("g1_post_barrier", false); 687 688 CardTableBarrierSet* ct = 689 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); 690 691 Label done; 692 Label enqueued; 693 Label runtime; 694 695 // At this point we know new_value is non-null and the new_value crosses regions. 696 // Must check to see if card is already dirty 697 698 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); 699 700 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 701 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 702 703 __ push(rax); 704 __ push(rcx); 705 706 const Register cardtable = rax; 707 const Register card_addr = rcx; 708 709 __ load_parameter(0, card_addr); 710 __ shrptr(card_addr, CardTable::card_shift()); 711 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT 712 // a valid address and therefore is not properly handled by the relocation code. 713 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base()); 714 __ addptr(card_addr, cardtable); 715 716 NOT_LP64(__ get_thread(thread);) 717 718 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val()); 719 __ jcc(Assembler::equal, done); 720 721 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); 722 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val()); 723 __ jcc(Assembler::equal, done); 724 725 // storing region crossing non-null, card is clean. 726 // dirty card and log. 727 728 __ movb(Address(card_addr, 0), CardTable::dirty_card_val()); 729 730 const Register tmp = rdx; 731 __ push(rdx); 732 733 __ movptr(tmp, queue_index); 734 __ testptr(tmp, tmp); 735 __ jcc(Assembler::zero, runtime); 736 __ subptr(tmp, wordSize); 737 __ movptr(queue_index, tmp); 738 __ addptr(tmp, buffer); 739 __ movptr(Address(tmp, 0), card_addr); 740 __ jmp(enqueued); 741 742 __ bind(runtime); 743 __ push_call_clobbered_registers(); 744 745 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); 746 747 __ pop_call_clobbered_registers(); 748 749 __ bind(enqueued); 750 __ pop(rdx); 751 752 __ bind(done); 753 __ pop(rcx); 754 __ pop(rax); 755 756 __ epilogue(); 757 } 758 759 #undef __ 760 761 #endif // COMPILER1 --- EOF ---