228 assert(thread == r15_thread, "must be");
229 #endif // _LP64
230
231 Label done;
232 Label runtime;
233
234 assert(pre_val != noreg, "check this code");
235
236 if (obj != noreg) {
237 assert_different_registers(obj, pre_val, tmp);
238 assert(pre_val != rax, "check this code");
239 }
240
241 generate_pre_barrier_fast_path(masm, thread);
242 // If marking is not active (*(mark queue active address) == 0), jump to done
243 __ jcc(Assembler::equal, done);
244 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
245
246 __ bind(runtime);
247
248 // Determine and save the live input values
249 __ push_call_clobbered_registers();
250
251 // Calling the runtime using the regular call_VM_leaf mechanism generates
252 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
253 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
254 //
255 // If we care generating the pre-barrier without a frame (e.g. in the
256 // intrinsified Reference.get() routine) then ebp might be pointing to
257 // the caller frame and so this check will most likely fail at runtime.
258 //
259 // Expanding the call directly bypasses the generation of the check.
260 // So when we do not have have a full interpreter frame on the stack
261 // expand_call should be passed true.
262
263 if (expand_call) {
264 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
265 #ifdef _LP64
266 if (c_rarg1 != thread) {
267 __ mov(c_rarg1, thread);
268 }
269 if (c_rarg0 != pre_val) {
270 __ mov(c_rarg0, pre_val);
271 }
272 #else
273 __ push(thread);
274 __ push(pre_val);
275 #endif
276 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
277 } else {
278 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
279 }
280
281 __ pop_call_clobbered_registers();
282
283 __ bind(done);
284 }
285
286 static void generate_post_barrier_fast_path(MacroAssembler* masm,
287 const Register store_addr,
288 const Register new_val,
289 const Register tmp,
290 const Register tmp2,
291 Label& done,
292 bool new_val_may_be_null) {
293 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
294 // Does store cross heap regions?
295 __ movptr(tmp, store_addr); // tmp := store address
296 __ xorptr(tmp, new_val); // tmp := store address ^ new value
297 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
298 __ jcc(Assembler::equal, done);
299 // Crosses regions, storing null?
300 if (new_val_may_be_null) {
301 __ cmpptr(new_val, NULL_WORD); // new value == null?
333
334 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
335 Register store_addr,
336 Register new_val,
337 Register thread,
338 Register tmp,
339 Register tmp2) {
340 #ifdef _LP64
341 assert(thread == r15_thread, "must be");
342 #endif // _LP64
343
344 Label done;
345 Label runtime;
346
347 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
348 // If card is young, jump to done
349 __ jcc(Assembler::equal, done);
350 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
351
352 __ bind(runtime);
353 // save the live input values
354 RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
355 __ push_set(saved);
356 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
357 __ pop_set(saved);
358
359 __ bind(done);
360 }
361
362 #if defined(COMPILER2)
363
364 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
365 #ifdef _LP64
366 SaveLiveRegisters save_registers(masm, stub);
367 if (c_rarg0 != arg) {
368 __ mov(c_rarg0, arg);
369 }
370 __ mov(c_rarg1, r15_thread);
371 // rax is a caller-saved, non-argument-passing register, so it does not
372 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
373 // entering this stub, it is saved at this point, and restored after the
374 // call. If it did not contain any live value, it is free to be used. In
375 // either case, it is safe to use it here as a call scratch register.
376 __ call(RuntimeAddress(runtime_path), rax);
377 #else
447 Label runtime;
448 Register thread = stub->thread();
449 Register tmp = stub->tmp1(); // tmp holds the card address.
450 Register tmp2 = stub->tmp2();
451 assert(stub->tmp3() == noreg, "not needed in this platform");
452
453 __ bind(*stub->entry());
454 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
455
456 __ bind(runtime);
457 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
458 __ jmp(*stub->continuation());
459 }
460
461 #endif // COMPILER2
462
463 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
464 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
465 bool in_heap = (decorators & IN_HEAP) != 0;
466 bool as_normal = (decorators & AS_NORMAL) != 0;
467
468 bool needs_pre_barrier = as_normal;
469 bool needs_post_barrier = val != noreg && in_heap;
470
471 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
472 // flatten object address if needed
473 // We do it regardless of precise because we need the registers
474 if (dst.index() == noreg && dst.disp() == 0) {
475 if (dst.base() != tmp1) {
476 __ movptr(tmp1, dst.base());
477 }
478 } else {
479 __ lea(tmp1, dst);
480 }
481
482 #ifndef _LP64
483 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
484 #endif
485
486 NOT_LP64(__ get_thread(rcx));
487 NOT_LP64(imasm->save_bcp());
488
|
228 assert(thread == r15_thread, "must be");
229 #endif // _LP64
230
231 Label done;
232 Label runtime;
233
234 assert(pre_val != noreg, "check this code");
235
236 if (obj != noreg) {
237 assert_different_registers(obj, pre_val, tmp);
238 assert(pre_val != rax, "check this code");
239 }
240
241 generate_pre_barrier_fast_path(masm, thread);
242 // If marking is not active (*(mark queue active address) == 0), jump to done
243 __ jcc(Assembler::equal, done);
244 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
245
246 __ bind(runtime);
247
248 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
249 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
250 // types. Save all argument registers before calling into the runtime.
251 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
252 __ pusha();
253 __ subptr(rsp, 64);
254 __ movdbl(Address(rsp, 0), j_farg0);
255 __ movdbl(Address(rsp, 8), j_farg1);
256 __ movdbl(Address(rsp, 16), j_farg2);
257 __ movdbl(Address(rsp, 24), j_farg3);
258 __ movdbl(Address(rsp, 32), j_farg4);
259 __ movdbl(Address(rsp, 40), j_farg5);
260 __ movdbl(Address(rsp, 48), j_farg6);
261 __ movdbl(Address(rsp, 56), j_farg7);
262 } else {
263 // Determine and save the live input values
264 __ push_call_clobbered_registers();
265 }
266
267 // Calling the runtime using the regular call_VM_leaf mechanism generates
268 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
269 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
270 //
271 // If we care generating the pre-barrier without a frame (e.g. in the
272 // intrinsified Reference.get() routine) then ebp might be pointing to
273 // the caller frame and so this check will most likely fail at runtime.
274 //
275 // Expanding the call directly bypasses the generation of the check.
276 // So when we do not have have a full interpreter frame on the stack
277 // expand_call should be passed true.
278
279 if (expand_call) {
280 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
281 #ifdef _LP64
282 if (c_rarg1 != thread) {
283 __ mov(c_rarg1, thread);
284 }
285 if (c_rarg0 != pre_val) {
286 __ mov(c_rarg0, pre_val);
287 }
288 #else
289 __ push(thread);
290 __ push(pre_val);
291 #endif
292 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
293 } else {
294 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
295 }
296
297 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
298 // Restore registers
299 __ movdbl(j_farg0, Address(rsp, 0));
300 __ movdbl(j_farg1, Address(rsp, 8));
301 __ movdbl(j_farg2, Address(rsp, 16));
302 __ movdbl(j_farg3, Address(rsp, 24));
303 __ movdbl(j_farg4, Address(rsp, 32));
304 __ movdbl(j_farg5, Address(rsp, 40));
305 __ movdbl(j_farg6, Address(rsp, 48));
306 __ movdbl(j_farg7, Address(rsp, 56));
307 __ addptr(rsp, 64);
308 __ popa();
309 } else {
310 __ pop_call_clobbered_registers();
311 }
312
313 __ bind(done);
314 }
315
316 static void generate_post_barrier_fast_path(MacroAssembler* masm,
317 const Register store_addr,
318 const Register new_val,
319 const Register tmp,
320 const Register tmp2,
321 Label& done,
322 bool new_val_may_be_null) {
323 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
324 // Does store cross heap regions?
325 __ movptr(tmp, store_addr); // tmp := store address
326 __ xorptr(tmp, new_val); // tmp := store address ^ new value
327 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
328 __ jcc(Assembler::equal, done);
329 // Crosses regions, storing null?
330 if (new_val_may_be_null) {
331 __ cmpptr(new_val, NULL_WORD); // new value == null?
363
364 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
365 Register store_addr,
366 Register new_val,
367 Register thread,
368 Register tmp,
369 Register tmp2) {
370 #ifdef _LP64
371 assert(thread == r15_thread, "must be");
372 #endif // _LP64
373
374 Label done;
375 Label runtime;
376
377 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
378 // If card is young, jump to done
379 __ jcc(Assembler::equal, done);
380 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
381
382 __ bind(runtime);
383 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
384 // types. Save all argument registers before calling into the runtime.
385 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
386 __ pusha();
387 __ subptr(rsp, 64);
388 __ movdbl(Address(rsp, 0), j_farg0);
389 __ movdbl(Address(rsp, 8), j_farg1);
390 __ movdbl(Address(rsp, 16), j_farg2);
391 __ movdbl(Address(rsp, 24), j_farg3);
392 __ movdbl(Address(rsp, 32), j_farg4);
393 __ movdbl(Address(rsp, 40), j_farg5);
394 __ movdbl(Address(rsp, 48), j_farg6);
395 __ movdbl(Address(rsp, 56), j_farg7);
396
397 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
398
399 // Restore registers
400 __ movdbl(j_farg0, Address(rsp, 0));
401 __ movdbl(j_farg1, Address(rsp, 8));
402 __ movdbl(j_farg2, Address(rsp, 16));
403 __ movdbl(j_farg3, Address(rsp, 24));
404 __ movdbl(j_farg4, Address(rsp, 32));
405 __ movdbl(j_farg5, Address(rsp, 40));
406 __ movdbl(j_farg6, Address(rsp, 48));
407 __ movdbl(j_farg7, Address(rsp, 56));
408 __ addptr(rsp, 64);
409 __ popa();
410
411 __ bind(done);
412 }
413
414 #if defined(COMPILER2)
415
416 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
417 #ifdef _LP64
418 SaveLiveRegisters save_registers(masm, stub);
419 if (c_rarg0 != arg) {
420 __ mov(c_rarg0, arg);
421 }
422 __ mov(c_rarg1, r15_thread);
423 // rax is a caller-saved, non-argument-passing register, so it does not
424 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
425 // entering this stub, it is saved at this point, and restored after the
426 // call. If it did not contain any live value, it is free to be used. In
427 // either case, it is safe to use it here as a call scratch register.
428 __ call(RuntimeAddress(runtime_path), rax);
429 #else
499 Label runtime;
500 Register thread = stub->thread();
501 Register tmp = stub->tmp1(); // tmp holds the card address.
502 Register tmp2 = stub->tmp2();
503 assert(stub->tmp3() == noreg, "not needed in this platform");
504
505 __ bind(*stub->entry());
506 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
507
508 __ bind(runtime);
509 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
510 __ jmp(*stub->continuation());
511 }
512
513 #endif // COMPILER2
514
515 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
516 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
517 bool in_heap = (decorators & IN_HEAP) != 0;
518 bool as_normal = (decorators & AS_NORMAL) != 0;
519 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
520
521 bool needs_pre_barrier = as_normal && !dest_uninitialized;
522 bool needs_post_barrier = val != noreg && in_heap;
523
524 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
525 // flatten object address if needed
526 // We do it regardless of precise because we need the registers
527 if (dst.index() == noreg && dst.disp() == 0) {
528 if (dst.base() != tmp1) {
529 __ movptr(tmp1, dst.base());
530 }
531 } else {
532 __ lea(tmp1, dst);
533 }
534
535 #ifndef _LP64
536 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
537 #endif
538
539 NOT_LP64(__ get_thread(rcx));
540 NOT_LP64(imasm->save_bcp());
541
|