185
186 const Register thread = r15_thread;
187
188 Label done;
189 Label runtime;
190
191 assert(pre_val != noreg, "check this code");
192
193 if (obj != noreg) {
194 assert_different_registers(obj, pre_val, tmp);
195 assert(pre_val != rax, "check this code");
196 }
197
198 generate_pre_barrier_fast_path(masm, thread);
199 // If marking is not active (*(mark queue active address) == 0), jump to done
200 __ jcc(Assembler::equal, done);
201 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
202
203 __ bind(runtime);
204
205 // Determine and save the live input values
206 __ push_call_clobbered_registers();
207
208 // Calling the runtime using the regular call_VM_leaf mechanism generates
209 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
210 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
211 //
212 // If we care generating the pre-barrier without a frame (e.g. in the
213 // intrinsified Reference.get() routine) then ebp might be pointing to
214 // the caller frame and so this check will most likely fail at runtime.
215 //
216 // Expanding the call directly bypasses the generation of the check.
217 // So when we do not have have a full interpreter frame on the stack
218 // expand_call should be passed true.
219
220 if (expand_call) {
221 assert(pre_val != c_rarg1, "smashed arg");
222 if (c_rarg1 != thread) {
223 __ mov(c_rarg1, thread);
224 }
225 if (c_rarg0 != pre_val) {
226 __ mov(c_rarg0, pre_val);
227 }
228 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
229 } else {
230 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
231 }
232
233 __ pop_call_clobbered_registers();
234
235 __ bind(done);
236 }
237
238 static void generate_post_barrier_fast_path(MacroAssembler* masm,
239 const Register store_addr,
240 const Register new_val,
241 const Register tmp,
242 const Register tmp2,
243 Label& done,
244 bool new_val_may_be_null) {
245 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
246 // Does store cross heap regions?
247 __ movptr(tmp, store_addr); // tmp := store address
248 __ xorptr(tmp, new_val); // tmp := store address ^ new value
249 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
250 __ jcc(Assembler::equal, done);
251 // Crosses regions, storing null?
252 if (new_val_may_be_null) {
253 __ cmpptr(new_val, NULL_WORD); // new value == null?
282 thread, tmp, tmp2);
283 __ jmp(done);
284 }
285
286 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
287 Register store_addr,
288 Register new_val,
289 Register tmp,
290 Register tmp2) {
291 const Register thread = r15_thread;
292
293 Label done;
294 Label runtime;
295
296 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
297 // If card is young, jump to done
298 __ jcc(Assembler::equal, done);
299 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
300
301 __ bind(runtime);
302 // save the live input values
303 RegSet saved = RegSet::of(store_addr);
304 __ push_set(saved);
305 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
306 __ pop_set(saved);
307
308 __ bind(done);
309 }
310
311 #if defined(COMPILER2)
312
313 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
314 SaveLiveRegisters save_registers(masm, stub);
315 if (c_rarg0 != arg) {
316 __ mov(c_rarg0, arg);
317 }
318 __ mov(c_rarg1, r15_thread);
319 // rax is a caller-saved, non-argument-passing register, so it does not
320 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
321 // entering this stub, it is saved at this point, and restored after the
322 // call. If it did not contain any live value, it is free to be used. In
323 // either case, it is safe to use it here as a call scratch register.
324 __ call(RuntimeAddress(runtime_path), rax);
325 }
326
386 Label runtime;
387 Register thread = stub->thread();
388 Register tmp = stub->tmp1(); // tmp holds the card address.
389 Register tmp2 = stub->tmp2();
390 assert(stub->tmp3() == noreg, "not needed in this platform");
391
392 __ bind(*stub->entry());
393 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
394
395 __ bind(runtime);
396 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
397 __ jmp(*stub->continuation());
398 }
399
400 #endif // COMPILER2
401
402 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
403 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
404 bool in_heap = (decorators & IN_HEAP) != 0;
405 bool as_normal = (decorators & AS_NORMAL) != 0;
406
407 bool needs_pre_barrier = as_normal;
408 bool needs_post_barrier = val != noreg && in_heap;
409
410 // flatten object address if needed
411 // We do it regardless of precise because we need the registers
412 if (dst.index() == noreg && dst.disp() == 0) {
413 if (dst.base() != tmp1) {
414 __ movptr(tmp1, dst.base());
415 }
416 } else {
417 __ lea(tmp1, dst);
418 }
419
420 if (needs_pre_barrier) {
421 g1_write_barrier_pre(masm /*masm*/,
422 tmp1 /* obj */,
423 tmp2 /* pre_val */,
424 tmp3 /* tmp */,
425 val != noreg /* tosca_live */,
426 false /* expand_call */);
427 }
|
185
186 const Register thread = r15_thread;
187
188 Label done;
189 Label runtime;
190
191 assert(pre_val != noreg, "check this code");
192
193 if (obj != noreg) {
194 assert_different_registers(obj, pre_val, tmp);
195 assert(pre_val != rax, "check this code");
196 }
197
198 generate_pre_barrier_fast_path(masm, thread);
199 // If marking is not active (*(mark queue active address) == 0), jump to done
200 __ jcc(Assembler::equal, done);
201 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
202
203 __ bind(runtime);
204
205 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
206 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
207 // types. Save all argument registers before calling into the runtime.
208 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
209 __ pusha();
210 __ subptr(rsp, 64);
211 __ movdbl(Address(rsp, 0), j_farg0);
212 __ movdbl(Address(rsp, 8), j_farg1);
213 __ movdbl(Address(rsp, 16), j_farg2);
214 __ movdbl(Address(rsp, 24), j_farg3);
215 __ movdbl(Address(rsp, 32), j_farg4);
216 __ movdbl(Address(rsp, 40), j_farg5);
217 __ movdbl(Address(rsp, 48), j_farg6);
218 __ movdbl(Address(rsp, 56), j_farg7);
219 } else {
220 // Determine and save the live input values
221 __ push_call_clobbered_registers();
222 }
223
224 // Calling the runtime using the regular call_VM_leaf mechanism generates
225 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
226 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
227 //
228 // If we care generating the pre-barrier without a frame (e.g. in the
229 // intrinsified Reference.get() routine) then ebp might be pointing to
230 // the caller frame and so this check will most likely fail at runtime.
231 //
232 // Expanding the call directly bypasses the generation of the check.
233 // So when we do not have have a full interpreter frame on the stack
234 // expand_call should be passed true.
235
236 if (expand_call) {
237 assert(pre_val != c_rarg1, "smashed arg");
238 if (c_rarg1 != thread) {
239 __ mov(c_rarg1, thread);
240 }
241 if (c_rarg0 != pre_val) {
242 __ mov(c_rarg0, pre_val);
243 }
244 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
245 } else {
246 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
247 }
248
249 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
250 // Restore registers
251 __ movdbl(j_farg0, Address(rsp, 0));
252 __ movdbl(j_farg1, Address(rsp, 8));
253 __ movdbl(j_farg2, Address(rsp, 16));
254 __ movdbl(j_farg3, Address(rsp, 24));
255 __ movdbl(j_farg4, Address(rsp, 32));
256 __ movdbl(j_farg5, Address(rsp, 40));
257 __ movdbl(j_farg6, Address(rsp, 48));
258 __ movdbl(j_farg7, Address(rsp, 56));
259 __ addptr(rsp, 64);
260 __ popa();
261 } else {
262 __ pop_call_clobbered_registers();
263 }
264
265 __ bind(done);
266 }
267
268 static void generate_post_barrier_fast_path(MacroAssembler* masm,
269 const Register store_addr,
270 const Register new_val,
271 const Register tmp,
272 const Register tmp2,
273 Label& done,
274 bool new_val_may_be_null) {
275 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
276 // Does store cross heap regions?
277 __ movptr(tmp, store_addr); // tmp := store address
278 __ xorptr(tmp, new_val); // tmp := store address ^ new value
279 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
280 __ jcc(Assembler::equal, done);
281 // Crosses regions, storing null?
282 if (new_val_may_be_null) {
283 __ cmpptr(new_val, NULL_WORD); // new value == null?
312 thread, tmp, tmp2);
313 __ jmp(done);
314 }
315
316 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
317 Register store_addr,
318 Register new_val,
319 Register tmp,
320 Register tmp2) {
321 const Register thread = r15_thread;
322
323 Label done;
324 Label runtime;
325
326 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
327 // If card is young, jump to done
328 __ jcc(Assembler::equal, done);
329 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
330
331 __ bind(runtime);
332 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
333 // types. Save all argument registers before calling into the runtime.
334 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
335 __ pusha();
336 __ subptr(rsp, 64);
337 __ movdbl(Address(rsp, 0), j_farg0);
338 __ movdbl(Address(rsp, 8), j_farg1);
339 __ movdbl(Address(rsp, 16), j_farg2);
340 __ movdbl(Address(rsp, 24), j_farg3);
341 __ movdbl(Address(rsp, 32), j_farg4);
342 __ movdbl(Address(rsp, 40), j_farg5);
343 __ movdbl(Address(rsp, 48), j_farg6);
344 __ movdbl(Address(rsp, 56), j_farg7);
345
346 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
347
348 // Restore registers
349 __ movdbl(j_farg0, Address(rsp, 0));
350 __ movdbl(j_farg1, Address(rsp, 8));
351 __ movdbl(j_farg2, Address(rsp, 16));
352 __ movdbl(j_farg3, Address(rsp, 24));
353 __ movdbl(j_farg4, Address(rsp, 32));
354 __ movdbl(j_farg5, Address(rsp, 40));
355 __ movdbl(j_farg6, Address(rsp, 48));
356 __ movdbl(j_farg7, Address(rsp, 56));
357 __ addptr(rsp, 64);
358 __ popa();
359
360 __ bind(done);
361 }
362
363 #if defined(COMPILER2)
364
365 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
366 SaveLiveRegisters save_registers(masm, stub);
367 if (c_rarg0 != arg) {
368 __ mov(c_rarg0, arg);
369 }
370 __ mov(c_rarg1, r15_thread);
371 // rax is a caller-saved, non-argument-passing register, so it does not
372 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
373 // entering this stub, it is saved at this point, and restored after the
374 // call. If it did not contain any live value, it is free to be used. In
375 // either case, it is safe to use it here as a call scratch register.
376 __ call(RuntimeAddress(runtime_path), rax);
377 }
378
438 Label runtime;
439 Register thread = stub->thread();
440 Register tmp = stub->tmp1(); // tmp holds the card address.
441 Register tmp2 = stub->tmp2();
442 assert(stub->tmp3() == noreg, "not needed in this platform");
443
444 __ bind(*stub->entry());
445 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
446
447 __ bind(runtime);
448 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
449 __ jmp(*stub->continuation());
450 }
451
452 #endif // COMPILER2
453
454 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
455 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
456 bool in_heap = (decorators & IN_HEAP) != 0;
457 bool as_normal = (decorators & AS_NORMAL) != 0;
458 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
459
460 bool needs_pre_barrier = as_normal && !dest_uninitialized;
461 bool needs_post_barrier = val != noreg && in_heap;
462
463 // flatten object address if needed
464 // We do it regardless of precise because we need the registers
465 if (dst.index() == noreg && dst.disp() == 0) {
466 if (dst.base() != tmp1) {
467 __ movptr(tmp1, dst.base());
468 }
469 } else {
470 __ lea(tmp1, dst);
471 }
472
473 if (needs_pre_barrier) {
474 g1_write_barrier_pre(masm /*masm*/,
475 tmp1 /* obj */,
476 tmp2 /* pre_val */,
477 tmp3 /* tmp */,
478 val != noreg /* tosca_live */,
479 false /* expand_call */);
480 }
|