1 // 2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. 3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 // 5 // This code is free software; you can redistribute it and/or modify it 6 // under the terms of the GNU General Public License version 2 only, as 7 // published by the Free Software Foundation. 8 // 9 // This code is distributed in the hope that it will be useful, but WITHOUT 10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 // version 2 for more details (a copy is included in the LICENSE file that 13 // accompanied this code). 14 // 15 // You should have received a copy of the GNU General Public License version 16 // 2 along with this work; if not, write to the Free Software Foundation, 17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 // 19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 // or visit www.oracle.com if you need additional information or have any 21 // questions. 22 // 23 24 source_hpp %{ 25 26 #include "gc/g1/c2/g1BarrierSetC2.hpp" 27 #include "gc/shared/gc_globals.hpp" 28 29 %} 30 31 source %{ 32 33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp" 34 #include "gc/g1/g1BarrierSetRuntime.hpp" 35 36 static void write_barrier_pre(MacroAssembler* masm, 37 const MachNode* node, 38 Register obj, 39 Register pre_val, 40 Register tmp, 41 RegSet preserve = RegSet(), 42 RegSet no_preserve = RegSet()) { 43 if (!G1PreBarrierStubC2::needs_barrier(node)) { 44 return; 45 } 46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler()); 48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node); 49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) { 50 stub->preserve(*reg); 51 } 52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) { 53 stub->dont_preserve(*reg); 54 } 55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, r15_thread, tmp, stub); 56 } 57 58 static void write_barrier_post(MacroAssembler* masm, 59 const MachNode* node, 60 Register store_addr, 61 Register new_val, 62 Register tmp1, 63 Register tmp2) { 64 if (!G1PostBarrierStubC2::needs_barrier(node)) { 65 return; 66 } 67 Assembler::InlineSkippedInstructionsCounter skip_counter(masm); 68 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler()); 69 G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node); 70 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, stub); 71 } 72 73 %} 74 75 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr) 76 %{ 77 predicate(UseG1GC && n->as_Store()->barrier_data() != 0); 78 match(Set mem (StoreP mem src)); 79 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 80 ins_cost(125); // XXX 81 format %{ "movq $mem, $src\t# ptr" %} 82 ins_encode %{ 83 // Materialize the store address internally (as opposed to defining 'mem' as 84 // an indirect memory operand) to reduce the overhead of LCM when processing 85 // large basic blocks with many stores. Such basic blocks arise, for 86 // instance, from static initializations of large String arrays. 87 // The same holds for g1StoreN and g1EncodePAndStoreN. 88 __ lea($tmp1$$Register, $mem$$Address); 89 write_barrier_pre(masm, this, 90 $tmp1$$Register /* obj */, 91 $tmp2$$Register /* pre_val */, 92 $tmp3$$Register /* tmp */, 93 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */); 94 __ movq(Address($tmp1$$Register, 0), $src$$Register); 95 write_barrier_post(masm, this, 96 $tmp1$$Register /* store_addr */, 97 $src$$Register /* new_val */, 98 $tmp3$$Register /* tmp1 */, 99 $tmp2$$Register /* tmp2 */); 100 %} 101 ins_pipe(ialu_mem_reg); 102 %} 103 104 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr) 105 %{ 106 predicate(UseG1GC && n->as_Store()->barrier_data() != 0); 107 match(Set mem (StoreN mem src)); 108 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 109 ins_cost(125); // XXX 110 format %{ "movl $mem, $src\t# ptr" %} 111 ins_encode %{ 112 __ lea($tmp1$$Register, $mem$$Address); 113 write_barrier_pre(masm, this, 114 $tmp1$$Register /* obj */, 115 $tmp2$$Register /* pre_val */, 116 $tmp3$$Register /* tmp */, 117 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */); 118 __ movl(Address($tmp1$$Register, 0), $src$$Register); 119 if ((barrier_data() & G1C2BarrierPost) != 0) { 120 __ movl($tmp2$$Register, $src$$Register); 121 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { 122 __ decode_heap_oop($tmp2$$Register); 123 } else { 124 __ decode_heap_oop_not_null($tmp2$$Register); 125 } 126 } 127 write_barrier_post(masm, this, 128 $tmp1$$Register /* store_addr */, 129 $tmp2$$Register /* new_val */, 130 $tmp3$$Register /* tmp1 */, 131 $tmp2$$Register /* tmp2 */); 132 %} 133 ins_pipe(ialu_mem_reg); 134 %} 135 136 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr) 137 %{ 138 predicate(UseG1GC && n->as_Store()->barrier_data() != 0); 139 match(Set mem (StoreN mem (EncodeP src))); 140 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 141 ins_cost(125); // XXX 142 format %{ "encode_heap_oop $src\n\t" 143 "movl $mem, $src\t# ptr" %} 144 ins_encode %{ 145 __ lea($tmp1$$Register, $mem$$Address); 146 write_barrier_pre(masm, this, 147 $tmp1$$Register /* obj */, 148 $tmp2$$Register /* pre_val */, 149 $tmp3$$Register /* tmp */, 150 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */); 151 __ movq($tmp2$$Register, $src$$Register); 152 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { 153 __ encode_heap_oop($tmp2$$Register); 154 } else { 155 __ encode_heap_oop_not_null($tmp2$$Register); 156 } 157 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register); 158 write_barrier_post(masm, this, 159 $tmp1$$Register /* store_addr */, 160 $src$$Register /* new_val */, 161 $tmp3$$Register /* tmp1 */, 162 $tmp2$$Register /* tmp2 */); 163 %} 164 ins_pipe(ialu_mem_reg); 165 %} 166 167 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr) 168 %{ 169 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 170 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); 171 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 172 format %{ "lock\n\t" 173 "cmpxchgq $newval, $mem" %} 174 ins_encode %{ 175 assert_different_registers($oldval$$Register, $mem$$Register); 176 // Pass $oldval to the pre-barrier (instead of loading from $mem), because 177 // $oldval is the only value that can be overwritten. 178 // The same holds for g1CompareAndSwapP. 179 write_barrier_pre(masm, this, 180 noreg /* obj */, 181 $oldval$$Register /* pre_val */, 182 $tmp3$$Register /* tmp */, 183 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */); 184 __ movq($tmp1$$Register, $newval$$Register); 185 __ lock(); 186 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0)); 187 write_barrier_post(masm, this, 188 $mem$$Register /* store_addr */, 189 $tmp1$$Register /* new_val */, 190 $tmp2$$Register /* tmp1 */, 191 $tmp3$$Register /* tmp2 */); 192 %} 193 ins_pipe(pipe_cmpxchg); 194 %} 195 196 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr) 197 %{ 198 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 199 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval))); 200 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 201 format %{ "lock\n\t" 202 "cmpxchgq $newval, $mem" %} 203 ins_encode %{ 204 assert_different_registers($oldval$$Register, $mem$$Register); 205 write_barrier_pre(masm, this, 206 $mem$$Register /* obj */, 207 $tmp2$$Register /* pre_val */, 208 $tmp3$$Register /* tmp */, 209 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */); 210 __ movl($tmp1$$Register, $newval$$Register); 211 __ lock(); 212 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0)); 213 __ decode_heap_oop($tmp1$$Register); 214 write_barrier_post(masm, this, 215 $mem$$Register /* store_addr */, 216 $tmp1$$Register /* new_val */, 217 $tmp2$$Register /* tmp1 */, 218 $tmp3$$Register /* tmp2 */); 219 %} 220 ins_pipe(pipe_cmpxchg); 221 %} 222 223 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr) 224 %{ 225 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 226 match(Set res (CompareAndSwapP mem (Binary oldval newval))); 227 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); 228 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr); 229 format %{ "lock\n\t" 230 "cmpxchgq $newval, $mem\n\t" 231 "sete $res\n\t" 232 "movzbl $res, $res" %} 233 ins_encode %{ 234 assert_different_registers($oldval$$Register, $mem$$Register); 235 write_barrier_pre(masm, this, 236 noreg /* obj */, 237 $oldval$$Register /* pre_val */, 238 $tmp3$$Register /* tmp */, 239 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */, 240 RegSet::of($res$$Register) /* no_preserve */); 241 __ movq($tmp1$$Register, $newval$$Register); 242 __ lock(); 243 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0)); 244 __ setb(Assembler::equal, $res$$Register); 245 __ movzbl($res$$Register, $res$$Register); 246 write_barrier_post(masm, this, 247 $mem$$Register /* store_addr */, 248 $tmp1$$Register /* new_val */, 249 $tmp2$$Register /* tmp1 */, 250 $tmp3$$Register /* tmp2 */); 251 %} 252 ins_pipe(pipe_cmpxchg); 253 %} 254 255 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr) 256 %{ 257 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 258 match(Set res (CompareAndSwapN mem (Binary oldval newval))); 259 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); 260 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr); 261 format %{ "lock\n\t" 262 "cmpxchgq $newval, $mem\n\t" 263 "sete $res\n\t" 264 "movzbl $res, $res" %} 265 ins_encode %{ 266 assert_different_registers($oldval$$Register, $mem$$Register); 267 write_barrier_pre(masm, this, 268 $mem$$Register /* obj */, 269 $tmp2$$Register /* pre_val */, 270 $tmp3$$Register /* tmp */, 271 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */, 272 RegSet::of($res$$Register) /* no_preserve */); 273 __ movl($tmp1$$Register, $newval$$Register); 274 __ lock(); 275 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0)); 276 __ setb(Assembler::equal, $res$$Register); 277 __ movzbl($res$$Register, $res$$Register); 278 __ decode_heap_oop($tmp1$$Register); 279 write_barrier_post(masm, this, 280 $mem$$Register /* store_addr */, 281 $tmp1$$Register /* new_val */, 282 $tmp2$$Register /* tmp1 */, 283 $tmp3$$Register /* tmp2 */); 284 %} 285 ins_pipe(pipe_cmpxchg); 286 %} 287 288 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr) 289 %{ 290 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 291 match(Set newval (GetAndSetP mem newval)); 292 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 293 format %{ "xchgq $newval, $mem" %} 294 ins_encode %{ 295 assert_different_registers($mem$$Register, $newval$$Register); 296 write_barrier_pre(masm, this, 297 $mem$$Register /* obj */, 298 $tmp2$$Register /* pre_val */, 299 $tmp3$$Register /* tmp */, 300 RegSet::of($mem$$Register, $newval$$Register) /* preserve */); 301 __ movq($tmp1$$Register, $newval$$Register); 302 __ xchgq($newval$$Register, Address($mem$$Register, 0)); 303 write_barrier_post(masm, this, 304 $mem$$Register /* store_addr */, 305 $tmp1$$Register /* new_val */, 306 $tmp2$$Register /* tmp1 */, 307 $tmp3$$Register /* tmp2 */); 308 %} 309 ins_pipe(pipe_cmpxchg); 310 %} 311 312 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr) 313 %{ 314 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); 315 match(Set newval (GetAndSetN mem newval)); 316 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); 317 format %{ "xchgq $newval, $mem" %} 318 ins_encode %{ 319 assert_different_registers($mem$$Register, $newval$$Register); 320 write_barrier_pre(masm, this, 321 $mem$$Register /* obj */, 322 $tmp2$$Register /* pre_val */, 323 $tmp3$$Register /* tmp */, 324 RegSet::of($mem$$Register, $newval$$Register) /* preserve */); 325 __ movl($tmp1$$Register, $newval$$Register); 326 __ decode_heap_oop($tmp1$$Register); 327 __ xchgl($newval$$Register, Address($mem$$Register, 0)); 328 write_barrier_post(masm, this, 329 $mem$$Register /* store_addr */, 330 $tmp1$$Register /* new_val */, 331 $tmp2$$Register /* tmp1 */, 332 $tmp3$$Register /* tmp2 */); 333 %} 334 ins_pipe(pipe_cmpxchg); 335 %} 336 337 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr) 338 %{ 339 predicate(UseG1GC && n->as_Load()->barrier_data() != 0); 340 match(Set dst (LoadP mem)); 341 effect(TEMP dst, TEMP tmp, KILL cr); 342 ins_cost(125); // XXX 343 format %{ "movq $dst, $mem\t# ptr" %} 344 ins_encode %{ 345 __ movq($dst$$Register, $mem$$Address); 346 write_barrier_pre(masm, this, 347 noreg /* obj */, 348 $dst$$Register /* pre_val */, 349 $tmp$$Register /* tmp */); 350 %} 351 ins_pipe(ialu_reg_mem); // XXX 352 %} 353 354 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr) 355 %{ 356 predicate(UseG1GC && n->as_Load()->barrier_data() != 0); 357 match(Set dst (LoadN mem)); 358 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr); 359 ins_cost(125); // XXX 360 format %{ "movl $dst, $mem\t# compressed ptr" %} 361 ins_encode %{ 362 __ movl($dst$$Register, $mem$$Address); 363 __ movl($tmp1$$Register, $dst$$Register); 364 __ decode_heap_oop($tmp1$$Register); 365 write_barrier_pre(masm, this, 366 noreg /* obj */, 367 $tmp1$$Register /* pre_val */, 368 $tmp2$$Register /* tmp */); 369 %} 370 ins_pipe(ialu_reg_mem); // XXX 371 %}