227 assert(thread == r15_thread, "must be");
228 #endif // _LP64
229
230 Label done;
231 Label runtime;
232
233 assert(pre_val != noreg, "check this code");
234
235 if (obj != noreg) {
236 assert_different_registers(obj, pre_val, tmp);
237 assert(pre_val != rax, "check this code");
238 }
239
240 generate_pre_barrier_fast_path(masm, thread);
241 // If marking is not active (*(mark queue active address) == 0), jump to done
242 __ jcc(Assembler::equal, done);
243 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
244
245 __ bind(runtime);
246
247 // Determine and save the live input values
248 __ push_call_clobbered_registers();
249
250 // Calling the runtime using the regular call_VM_leaf mechanism generates
251 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
252 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
253 //
254 // If we care generating the pre-barrier without a frame (e.g. in the
255 // intrinsified Reference.get() routine) then ebp might be pointing to
256 // the caller frame and so this check will most likely fail at runtime.
257 //
258 // Expanding the call directly bypasses the generation of the check.
259 // So when we do not have have a full interpreter frame on the stack
260 // expand_call should be passed true.
261
262 if (expand_call) {
263 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
264 #ifdef _LP64
265 if (c_rarg1 != thread) {
266 __ mov(c_rarg1, thread);
267 }
268 if (c_rarg0 != pre_val) {
269 __ mov(c_rarg0, pre_val);
270 }
271 #else
272 __ push(thread);
273 __ push(pre_val);
274 #endif
275 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
276 } else {
277 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
278 }
279
280 __ pop_call_clobbered_registers();
281
282 __ bind(done);
283 }
284
285 static void generate_post_barrier_fast_path(MacroAssembler* masm,
286 const Register store_addr,
287 const Register new_val,
288 const Register tmp,
289 const Register tmp2,
290 Label& done,
291 bool new_val_may_be_null) {
292 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
293 // Does store cross heap regions?
294 __ movptr(tmp, store_addr); // tmp := store address
295 __ xorptr(tmp, new_val); // tmp := store address ^ new value
296 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
297 __ jcc(Assembler::equal, done);
298 // Crosses regions, storing null?
299 if (new_val_may_be_null) {
300 __ cmpptr(new_val, NULL_WORD); // new value == null?
332
333 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
334 Register store_addr,
335 Register new_val,
336 Register thread,
337 Register tmp,
338 Register tmp2) {
339 #ifdef _LP64
340 assert(thread == r15_thread, "must be");
341 #endif // _LP64
342
343 Label done;
344 Label runtime;
345
346 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
347 // If card is young, jump to done
348 __ jcc(Assembler::equal, done);
349 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
350
351 __ bind(runtime);
352 // save the live input values
353 RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
354 __ push_set(saved);
355 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
356 __ pop_set(saved);
357
358 __ bind(done);
359 }
360
361 #if defined(COMPILER2)
362
363 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
364 #ifdef _LP64
365 SaveLiveRegisters save_registers(masm, stub);
366 if (c_rarg0 != arg) {
367 __ mov(c_rarg0, arg);
368 }
369 __ mov(c_rarg1, r15_thread);
370 // rax is a caller-saved, non-argument-passing register, so it does not
371 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
372 // entering this stub, it is saved at this point, and restored after the
373 // call. If it did not contain any live value, it is free to be used. In
374 // either case, it is safe to use it here as a call scratch register.
375 __ call(RuntimeAddress(runtime_path), rax);
376 #else
446 Label runtime;
447 Register thread = stub->thread();
448 Register tmp = stub->tmp1(); // tmp holds the card address.
449 Register tmp2 = stub->tmp2();
450 assert(stub->tmp3() == noreg, "not needed in this platform");
451
452 __ bind(*stub->entry());
453 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
454
455 __ bind(runtime);
456 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
457 __ jmp(*stub->continuation());
458 }
459
460 #endif // COMPILER2
461
462 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
463 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
464 bool in_heap = (decorators & IN_HEAP) != 0;
465 bool as_normal = (decorators & AS_NORMAL) != 0;
466
467 bool needs_pre_barrier = as_normal;
468 bool needs_post_barrier = val != noreg && in_heap;
469
470 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
471 // flatten object address if needed
472 // We do it regardless of precise because we need the registers
473 if (dst.index() == noreg && dst.disp() == 0) {
474 if (dst.base() != tmp1) {
475 __ movptr(tmp1, dst.base());
476 }
477 } else {
478 __ lea(tmp1, dst);
479 }
480
481 #ifndef _LP64
482 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
483 #endif
484
485 NOT_LP64(__ get_thread(rcx));
486 NOT_LP64(imasm->save_bcp());
487
|
227 assert(thread == r15_thread, "must be");
228 #endif // _LP64
229
230 Label done;
231 Label runtime;
232
233 assert(pre_val != noreg, "check this code");
234
235 if (obj != noreg) {
236 assert_different_registers(obj, pre_val, tmp);
237 assert(pre_val != rax, "check this code");
238 }
239
240 generate_pre_barrier_fast_path(masm, thread);
241 // If marking is not active (*(mark queue active address) == 0), jump to done
242 __ jcc(Assembler::equal, done);
243 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
244
245 __ bind(runtime);
246
247 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
248 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
249 // types. Save all argument registers before calling into the runtime.
250 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
251 __ pusha();
252 __ subptr(rsp, 64);
253 __ movdbl(Address(rsp, 0), j_farg0);
254 __ movdbl(Address(rsp, 8), j_farg1);
255 __ movdbl(Address(rsp, 16), j_farg2);
256 __ movdbl(Address(rsp, 24), j_farg3);
257 __ movdbl(Address(rsp, 32), j_farg4);
258 __ movdbl(Address(rsp, 40), j_farg5);
259 __ movdbl(Address(rsp, 48), j_farg6);
260 __ movdbl(Address(rsp, 56), j_farg7);
261 } else {
262 // Determine and save the live input values
263 __ push_call_clobbered_registers();
264 }
265
266 // Calling the runtime using the regular call_VM_leaf mechanism generates
267 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
268 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
269 //
270 // If we care generating the pre-barrier without a frame (e.g. in the
271 // intrinsified Reference.get() routine) then ebp might be pointing to
272 // the caller frame and so this check will most likely fail at runtime.
273 //
274 // Expanding the call directly bypasses the generation of the check.
275 // So when we do not have have a full interpreter frame on the stack
276 // expand_call should be passed true.
277
278 if (expand_call) {
279 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
280 #ifdef _LP64
281 if (c_rarg1 != thread) {
282 __ mov(c_rarg1, thread);
283 }
284 if (c_rarg0 != pre_val) {
285 __ mov(c_rarg0, pre_val);
286 }
287 #else
288 __ push(thread);
289 __ push(pre_val);
290 #endif
291 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
292 } else {
293 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
294 }
295
296 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
297 // Restore registers
298 __ movdbl(j_farg0, Address(rsp, 0));
299 __ movdbl(j_farg1, Address(rsp, 8));
300 __ movdbl(j_farg2, Address(rsp, 16));
301 __ movdbl(j_farg3, Address(rsp, 24));
302 __ movdbl(j_farg4, Address(rsp, 32));
303 __ movdbl(j_farg5, Address(rsp, 40));
304 __ movdbl(j_farg6, Address(rsp, 48));
305 __ movdbl(j_farg7, Address(rsp, 56));
306 __ addptr(rsp, 64);
307 __ popa();
308 } else {
309 __ pop_call_clobbered_registers();
310 }
311
312 __ bind(done);
313 }
314
315 static void generate_post_barrier_fast_path(MacroAssembler* masm,
316 const Register store_addr,
317 const Register new_val,
318 const Register tmp,
319 const Register tmp2,
320 Label& done,
321 bool new_val_may_be_null) {
322 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
323 // Does store cross heap regions?
324 __ movptr(tmp, store_addr); // tmp := store address
325 __ xorptr(tmp, new_val); // tmp := store address ^ new value
326 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
327 __ jcc(Assembler::equal, done);
328 // Crosses regions, storing null?
329 if (new_val_may_be_null) {
330 __ cmpptr(new_val, NULL_WORD); // new value == null?
362
363 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
364 Register store_addr,
365 Register new_val,
366 Register thread,
367 Register tmp,
368 Register tmp2) {
369 #ifdef _LP64
370 assert(thread == r15_thread, "must be");
371 #endif // _LP64
372
373 Label done;
374 Label runtime;
375
376 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
377 // If card is young, jump to done
378 __ jcc(Assembler::equal, done);
379 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
380
381 __ bind(runtime);
382 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
383 // types. Save all argument registers before calling into the runtime.
384 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
385 __ pusha();
386 __ subptr(rsp, 64);
387 __ movdbl(Address(rsp, 0), j_farg0);
388 __ movdbl(Address(rsp, 8), j_farg1);
389 __ movdbl(Address(rsp, 16), j_farg2);
390 __ movdbl(Address(rsp, 24), j_farg3);
391 __ movdbl(Address(rsp, 32), j_farg4);
392 __ movdbl(Address(rsp, 40), j_farg5);
393 __ movdbl(Address(rsp, 48), j_farg6);
394 __ movdbl(Address(rsp, 56), j_farg7);
395
396 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
397
398 // Restore registers
399 __ movdbl(j_farg0, Address(rsp, 0));
400 __ movdbl(j_farg1, Address(rsp, 8));
401 __ movdbl(j_farg2, Address(rsp, 16));
402 __ movdbl(j_farg3, Address(rsp, 24));
403 __ movdbl(j_farg4, Address(rsp, 32));
404 __ movdbl(j_farg5, Address(rsp, 40));
405 __ movdbl(j_farg6, Address(rsp, 48));
406 __ movdbl(j_farg7, Address(rsp, 56));
407 __ addptr(rsp, 64);
408 __ popa();
409
410 __ bind(done);
411 }
412
413 #if defined(COMPILER2)
414
415 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
416 #ifdef _LP64
417 SaveLiveRegisters save_registers(masm, stub);
418 if (c_rarg0 != arg) {
419 __ mov(c_rarg0, arg);
420 }
421 __ mov(c_rarg1, r15_thread);
422 // rax is a caller-saved, non-argument-passing register, so it does not
423 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
424 // entering this stub, it is saved at this point, and restored after the
425 // call. If it did not contain any live value, it is free to be used. In
426 // either case, it is safe to use it here as a call scratch register.
427 __ call(RuntimeAddress(runtime_path), rax);
428 #else
498 Label runtime;
499 Register thread = stub->thread();
500 Register tmp = stub->tmp1(); // tmp holds the card address.
501 Register tmp2 = stub->tmp2();
502 assert(stub->tmp3() == noreg, "not needed in this platform");
503
504 __ bind(*stub->entry());
505 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
506
507 __ bind(runtime);
508 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
509 __ jmp(*stub->continuation());
510 }
511
512 #endif // COMPILER2
513
514 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
515 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
516 bool in_heap = (decorators & IN_HEAP) != 0;
517 bool as_normal = (decorators & AS_NORMAL) != 0;
518 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
519
520 bool needs_pre_barrier = as_normal && !dest_uninitialized;
521 bool needs_post_barrier = val != noreg && in_heap;
522
523 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
524 // flatten object address if needed
525 // We do it regardless of precise because we need the registers
526 if (dst.index() == noreg && dst.disp() == 0) {
527 if (dst.base() != tmp1) {
528 __ movptr(tmp1, dst.base());
529 }
530 } else {
531 __ lea(tmp1, dst);
532 }
533
534 #ifndef _LP64
535 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
536 #endif
537
538 NOT_LP64(__ get_thread(rcx));
539 NOT_LP64(imasm->save_bcp());
540
|