< prev index next >

src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp

Print this page

173   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
174     __ cmpl(in_progress, 0);
175   } else {
176     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
177     __ cmpb(in_progress, 0);
178   }
179   __ jcc(Assembler::equal, done);
180 
181   // Do we need to load the previous value?
182   if (obj != noreg) {
183     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
184   }
185 
186   // Is the previous value null?
187   __ cmpptr(pre_val, (int32_t) NULL_WORD);
188   __ jcc(Assembler::equal, done);
189 
190   // Can we store original value in the thread's buffer?
191   // Is index == 0?
192   // (The index field is typed as size_t.)
193 
194   __ movptr(tmp, index);                   // tmp := *index_adr
195   __ cmpptr(tmp, 0);                       // tmp == 0?
196   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
197 
198   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
199   __ movptr(index, tmp);                   // *index_adr := tmp
200   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
201 
202   // Record the previous value
203   __ movptr(Address(tmp, 0), pre_val);
204   __ jmp(done);
205 
206   __ bind(runtime);
207   // save the live input values
208   if(tosca_live) __ push(rax);
209 
210   if (obj != noreg && obj != rax)
211     __ push(obj);
212 
213   if (pre_val != rax)
214     __ push(pre_val);




215 
216   // Calling the runtime using the regular call_VM_leaf mechanism generates
217   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
218   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
219   //
220   // If we care generating the pre-barrier without a frame (e.g. in the
221   // intrinsified Reference.get() routine) then ebp might be pointing to
222   // the caller frame and so this check will most likely fail at runtime.
223   //
224   // Expanding the call directly bypasses the generation of the check.
225   // So when we do not have have a full interpreter frame on the stack
226   // expand_call should be passed true.
227 
228   NOT_LP64( __ push(thread); )
229 
230   if (expand_call) {
231     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
232 #ifdef _LP64
233     if (c_rarg1 != thread) {
234       __ mov(c_rarg1, thread);
235     }
236     if (c_rarg0 != pre_val) {
237       __ mov(c_rarg0, pre_val);
238     }
239 #else
240     __ push(thread);
241     __ push(pre_val);
242 #endif
243     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
244   } else {
245     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
246   }
247 
248   NOT_LP64( __ pop(thread); )
249 
250   // save the live input values
251   if (pre_val != rax)
252     __ pop(pre_val);
253 
254   if (obj != noreg && obj != rax)
255     __ pop(obj);
256 
257   if(tosca_live) __ pop(rax);

258 
259   __ bind(done);
260 }
261 
262 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
263                                                   Register store_addr,
264                                                   Register new_val,
265                                                   Register thread,
266                                                   Register tmp,
267                                                   Register tmp2) {
268   // Generated code assumes that buffer index is pointer sized.
269   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
270 #ifdef _LP64
271   assert(thread == r15_thread, "must be");
272 #endif // _LP64
273 
274   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
275   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
276 
277   CardTableBarrierSet* ct =

310   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
311   __ cmpb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
312   __ jcc(Assembler::equal, done);
313 
314 
315   // storing a region crossing, non-NULL oop, card is clean.
316   // dirty card and log.
317 
318   __ movb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
319 
320   __ movptr(tmp2, queue_index);
321   __ testptr(tmp2, tmp2);
322   __ jcc(Assembler::zero, runtime);
323   __ subptr(tmp2, wordSize);
324   __ movptr(queue_index, tmp2);
325   __ addptr(tmp2, buffer);
326   __ movptr(Address(tmp2, 0), card_addr);
327   __ jmp(done);
328 
329   __ bind(runtime);
330   // save the live input values
331   __ push(store_addr);











332 #ifdef _LP64
333   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
334 #else
335   __ push(thread);
336   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
337   __ pop(thread);
338 #endif
339   __ pop(store_addr);











340 
341   __ bind(done);
342 }
343 
344 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
345                                          Address dst, Register val, Register tmp1, Register tmp2) {
346   bool in_heap = (decorators & IN_HEAP) != 0;
347   bool as_normal = (decorators & AS_NORMAL) != 0;
348   assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported");
349 
350   bool needs_pre_barrier = as_normal;
351   bool needs_post_barrier = val != noreg && in_heap;
352 
353   Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);


354   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
355   // flatten object address if needed
356   // We do it regardless of precise because we need the registers
357   if (dst.index() == noreg && dst.disp() == 0) {
358     if (dst.base() != tmp1) {
359       __ movptr(tmp1, dst.base());
360     }
361   } else {
362     __ lea(tmp1, dst);
363   }
364 
365 #ifndef _LP64
366   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
367 #endif
368 
369   NOT_LP64(__ get_thread(rcx));
370   NOT_LP64(imasm->save_bcp());
371 
372   if (needs_pre_barrier) {
373     g1_write_barrier_pre(masm /*masm*/,
374                          tmp1 /* obj */,
375                          tmp2 /* pre_val */,
376                          rthread /* thread */,
377                          tmp3  /* tmp */,
378                          val != noreg /* tosca_live */,
379                          false /* expand_call */);
380   }
381   if (val == noreg) {
382     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
383   } else {
384     Register new_val = val;
385     if (needs_post_barrier) {
386       // G1 barrier needs uncompressed oop for region cross check.
387       if (UseCompressedOops) {
388         new_val = tmp2;
389         __ movptr(new_val, val);
390       }
391     }
392     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
393     if (needs_post_barrier) {
394       g1_write_barrier_post(masm /*masm*/,
395                             tmp1 /* store_adr */,
396                             new_val /* new_val */,
397                             rthread /* thread */,
398                             tmp3 /* tmp */,
399                             tmp2 /* tmp2 */);
400     }
401   }
402   NOT_LP64(imasm->restore_bcp());
403 }
404 
405 #ifdef COMPILER1
406 
407 #undef __
408 #define __ ce->masm()->
409 
410 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
411   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
412   // At this point we know that marking is in progress.

173   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
174     __ cmpl(in_progress, 0);
175   } else {
176     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
177     __ cmpb(in_progress, 0);
178   }
179   __ jcc(Assembler::equal, done);
180 
181   // Do we need to load the previous value?
182   if (obj != noreg) {
183     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
184   }
185 
186   // Is the previous value null?
187   __ cmpptr(pre_val, (int32_t) NULL_WORD);
188   __ jcc(Assembler::equal, done);
189 
190   // Can we store original value in the thread's buffer?
191   // Is index == 0?
192   // (The index field is typed as size_t.)

193   __ movptr(tmp, index);                   // tmp := *index_adr
194   __ cmpptr(tmp, 0);                       // tmp == 0?
195   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
196 
197   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
198   __ movptr(index, tmp);                   // *index_adr := tmp
199   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
200 
201   // Record the previous value
202   __ movptr(Address(tmp, 0), pre_val);
203   __ jmp(done);
204 
205   __ bind(runtime);
206   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
207   // types. Save all argument registers before calling into the runtime.
208   __ pusha();
209   __ subptr(rsp, 64);
210   __ movdbl(Address(rsp, 0),  j_farg0);
211   __ movdbl(Address(rsp, 8),  j_farg1);
212   __ movdbl(Address(rsp, 16), j_farg2);
213   __ movdbl(Address(rsp, 24), j_farg3);
214   __ movdbl(Address(rsp, 32), j_farg4);
215   __ movdbl(Address(rsp, 40), j_farg5);
216   __ movdbl(Address(rsp, 48), j_farg6);
217   __ movdbl(Address(rsp, 56), j_farg7);
218 
219   // Calling the runtime using the regular call_VM_leaf mechanism generates
220   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
221   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
222   //
223   // If we care generating the pre-barrier without a frame (e.g. in the
224   // intrinsified Reference.get() routine) then ebp might be pointing to
225   // the caller frame and so this check will most likely fail at runtime.
226   //
227   // Expanding the call directly bypasses the generation of the check.
228   // So when we do not have have a full interpreter frame on the stack
229   // expand_call should be passed true.
230 


231   if (expand_call) {
232     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
233 #ifdef _LP64
234     if (c_rarg1 != thread) {
235       __ mov(c_rarg1, thread);
236     }
237     if (c_rarg0 != pre_val) {
238       __ mov(c_rarg0, pre_val);
239     }
240 #else
241     __ push(thread);
242     __ push(pre_val);
243 #endif
244     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
245   } else {
246     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
247   }
248 
249   // Restore registers
250   __ movdbl(j_farg0, Address(rsp, 0));
251   __ movdbl(j_farg1, Address(rsp, 8));
252   __ movdbl(j_farg2, Address(rsp, 16));
253   __ movdbl(j_farg3, Address(rsp, 24));
254   __ movdbl(j_farg4, Address(rsp, 32));
255   __ movdbl(j_farg5, Address(rsp, 40));
256   __ movdbl(j_farg6, Address(rsp, 48));
257   __ movdbl(j_farg7, Address(rsp, 56));
258   __ addptr(rsp, 64);
259   __ popa();
260 
261   __ bind(done);
262 }
263 
264 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
265                                                   Register store_addr,
266                                                   Register new_val,
267                                                   Register thread,
268                                                   Register tmp,
269                                                   Register tmp2) {
270   // Generated code assumes that buffer index is pointer sized.
271   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
272 #ifdef _LP64
273   assert(thread == r15_thread, "must be");
274 #endif // _LP64
275 
276   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
277   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
278 
279   CardTableBarrierSet* ct =

312   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
313   __ cmpb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
314   __ jcc(Assembler::equal, done);
315 
316 
317   // storing a region crossing, non-NULL oop, card is clean.
318   // dirty card and log.
319 
320   __ movb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
321 
322   __ movptr(tmp2, queue_index);
323   __ testptr(tmp2, tmp2);
324   __ jcc(Assembler::zero, runtime);
325   __ subptr(tmp2, wordSize);
326   __ movptr(queue_index, tmp2);
327   __ addptr(tmp2, buffer);
328   __ movptr(Address(tmp2, 0), card_addr);
329   __ jmp(done);
330 
331   __ bind(runtime);
332   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
333   // types. Save all argument registers before calling into the runtime.
334   __ pusha();
335   __ subptr(rsp, 64);
336   __ movdbl(Address(rsp, 0),  j_farg0);
337   __ movdbl(Address(rsp, 8),  j_farg1);
338   __ movdbl(Address(rsp, 16), j_farg2);
339   __ movdbl(Address(rsp, 24), j_farg3);
340   __ movdbl(Address(rsp, 32), j_farg4);
341   __ movdbl(Address(rsp, 40), j_farg5);
342   __ movdbl(Address(rsp, 48), j_farg6);
343   __ movdbl(Address(rsp, 56), j_farg7);
344 
345 #ifdef _LP64
346   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
347 #else
348   __ push(thread);
349   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
350   __ pop(thread);
351 #endif
352 
353   // Restore registers
354   __ movdbl(j_farg0, Address(rsp, 0));
355   __ movdbl(j_farg1, Address(rsp, 8));
356   __ movdbl(j_farg2, Address(rsp, 16));
357   __ movdbl(j_farg3, Address(rsp, 24));
358   __ movdbl(j_farg4, Address(rsp, 32));
359   __ movdbl(j_farg5, Address(rsp, 40));
360   __ movdbl(j_farg6, Address(rsp, 48));
361   __ movdbl(j_farg7, Address(rsp, 56));
362   __ addptr(rsp, 64);
363   __ popa();
364 
365   __ bind(done);
366 }
367 
368 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
369                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
370   bool in_heap = (decorators & IN_HEAP) != 0;
371   bool as_normal = (decorators & AS_NORMAL) != 0;
372   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
373 
374   bool needs_pre_barrier = as_normal && !dest_uninitialized;
375   bool needs_post_barrier = val != noreg && in_heap;
376 
377   if (tmp3 == noreg) {
378     tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
379   }
380   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
381   // flatten object address if needed
382   // We do it regardless of precise because we need the registers
383   if (dst.index() == noreg && dst.disp() == 0) {
384     if (dst.base() != tmp1) {
385       __ movptr(tmp1, dst.base());
386     }
387   } else {
388     __ lea(tmp1, dst);
389   }
390 
391 #ifndef _LP64
392   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
393 #endif
394 
395   NOT_LP64(__ get_thread(rcx));
396   NOT_LP64(imasm->save_bcp());
397 
398   if (needs_pre_barrier) {
399     g1_write_barrier_pre(masm /*masm*/,
400                          tmp1 /* obj */,
401                          tmp2 /* pre_val */,
402                          rthread /* thread */,
403                          tmp3  /* tmp */,
404                          val != noreg /* tosca_live */,
405                          false /* expand_call */);
406   }
407   if (val == noreg) {
408     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
409   } else {
410     Register new_val = val;
411     if (needs_post_barrier) {
412       // G1 barrier needs uncompressed oop for region cross check.
413       if (UseCompressedOops) {
414         new_val = tmp2;
415         __ movptr(new_val, val);
416       }
417     }
418     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
419     if (needs_post_barrier) {
420       g1_write_barrier_post(masm /*masm*/,
421                             tmp1 /* store_adr */,
422                             new_val /* new_val */,
423                             rthread /* thread */,
424                             tmp3 /* tmp */,
425                             tmp2 /* tmp2 */);
426     }
427   }
428   NOT_LP64(imasm->restore_bcp());
429 }
430 
431 #ifdef COMPILER1
432 
433 #undef __
434 #define __ ce->masm()->
435 
436 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
437   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
438   // At this point we know that marking is in progress.
< prev index next >