< prev index next >

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp

Print this page

138 
139   // Can we store original value in the thread's buffer?
140   // Is index == 0?
141   // (The index field is typed as size_t.)
142 
143   __ ldr(tmp1, index);                      // tmp := *index_adr
144   __ cbz(tmp1, runtime);                    // tmp == 0?
145                                         // If yes, goto runtime
146 
147   __ sub(tmp1, tmp1, wordSize);             // tmp := tmp - wordSize
148   __ str(tmp1, index);                      // *index_adr := tmp
149   __ ldr(tmp2, buffer);
150   __ add(tmp1, tmp1, tmp2);                 // tmp := tmp + *buffer_adr
151 
152   // Record the previous value
153   __ str(pre_val, Address(tmp1, 0));
154   __ b(done);
155 
156   __ bind(runtime);
157 
158   __ push_call_clobbered_registers();




















159 
160   // Calling the runtime using the regular call_VM_leaf mechanism generates
161   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
162   // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
163   //
164   // If we care generating the pre-barrier without a frame (e.g. in the
165   // intrinsified Reference.get() routine) then rfp might be pointing to
166   // the caller frame and so this check will most likely fail at runtime.
167   //
168   // Expanding the call directly bypasses the generation of the check.
169   // So when we do not have have a full interpreter frame on the stack
170   // expand_call should be passed true.
171 
172   if (expand_call) {
173     assert(pre_val != c_rarg1, "smashed arg");
174     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
175   } else {
176     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
177   }
178 
179   __ pop_call_clobbered_registers();





180 
181   __ bind(done);
182 
183 }
184 
185 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
186                                                   Register store_addr,
187                                                   Register new_val,
188                                                   Register thread,
189                                                   Register tmp1,
190                                                   Register tmp2) {
191   assert(thread == rthread, "must be");
192   assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
193                              rscratch1);
194   assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
195          && tmp2 != noreg, "expecting a register");
196 
197   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
198   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
199 
200   BarrierSet* bs = BarrierSet::barrier_set();
201   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
202   CardTable* ct = ctbs->card_table();
203 
204   Label done;
205   Label runtime;
206 
207   // Does store cross heap regions?
208 
209   __ eor(tmp1, store_addr, new_val);
210   __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
211   __ cbz(tmp1, done);
212 
213   // crosses regions, storing null?
214 
215   __ cbz(new_val, done);
216 
217   // storing region crossing non-null, is card already dirty?
218 


219   const Register card_addr = tmp1;
220 
221   __ lsr(card_addr, store_addr, CardTable::card_shift());
222 
223   // get the address of the card
224   __ load_byte_map_base(tmp2);
225   __ add(card_addr, card_addr, tmp2);
226   __ ldrb(tmp2, Address(card_addr));
227   __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
228   __ br(Assembler::EQ, done);
229 
230   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
231 
232   __ membar(Assembler::StoreLoad);
233 
234   __ ldrb(tmp2, Address(card_addr));
235   __ cbzw(tmp2, done);
236 
237   // storing a region crossing, non-null oop, card is clean.
238   // dirty card and log.
239 
240   __ strb(zr, Address(card_addr));
241 
242   __ ldr(rscratch1, queue_index);
243   __ cbz(rscratch1, runtime);
244   __ sub(rscratch1, rscratch1, wordSize);
245   __ str(rscratch1, queue_index);
246 
247   __ ldr(tmp2, buffer);
248   __ str(card_addr, Address(tmp2, rscratch1));
249   __ b(done);
250 
251   __ bind(runtime);

252   // save the live input values
253   RegSet saved = RegSet::of(store_addr);













254   __ push(saved, sp);

255   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);

256   __ pop(saved, sp);
257 
258   __ bind(done);
259 }
260 
261 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
262                                     Register dst, Address src, Register tmp1, Register tmp2) {
263   bool on_oop = is_reference_type(type);
264   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
265   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
266   bool on_reference = on_weak || on_phantom;
267   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
268   if (on_oop && on_reference) {
269     // LR is live.  It must be saved around calls.
270     __ enter(/*strip_ret_addr*/true); // barrier may call runtime
271     // Generate the G1 pre-barrier code to log the value of
272     // the referent field in an SATB buffer.
273     g1_write_barrier_pre(masm /* masm */,
274                          noreg /* obj */,
275                          dst /* pre_val */,
276                          rthread /* thread */,
277                          tmp1 /* tmp1 */,
278                          tmp2 /* tmp2 */,
279                          true /* tosca_live */,
280                          true /* expand_call */);
281     __ leave();
282   }
283 }
284 
285 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
286                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {










287   // flatten object address if needed
288   if (dst.index() == noreg && dst.offset() == 0) {
289     if (dst.base() != tmp3) {
290       __ mov(tmp3, dst.base());
291     }
292   } else {
293     __ lea(tmp3, dst);
294   }
295 
296   g1_write_barrier_pre(masm,
297                        tmp3 /* obj */,
298                        tmp2 /* pre_val */,
299                        rthread /* thread */,
300                        tmp1  /* tmp1 */,
301                        rscratch2  /* tmp2 */,
302                        val != noreg /* tosca_live */,
303                        false /* expand_call */);


304 
305   if (val == noreg) {
306     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
307   } else {
308     // G1 barrier needs uncompressed oop for region cross check.
309     Register new_val = val;
310     if (UseCompressedOops) {
311       new_val = rscratch2;
312       __ mov(new_val, val);


313     }

314     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
315     g1_write_barrier_post(masm,
316                           tmp3 /* store_adr */,
317                           new_val /* new_val */,
318                           rthread /* thread */,
319                           tmp1 /* tmp1 */,
320                           tmp2 /* tmp2 */);


321   }
322 
323 }
324 
325 #ifdef COMPILER1
326 
327 #undef __
328 #define __ ce->masm()->
329 
330 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
331   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
332   // At this point we know that marking is in progress.
333   // If do_load() is true then we have to emit the
334   // load of the previous value; otherwise it has already
335   // been loaded into _pre_val.
336 
337   __ bind(*stub->entry());
338 
339   assert(stub->pre_val()->is_register(), "Precondition.");
340 

138 
139   // Can we store original value in the thread's buffer?
140   // Is index == 0?
141   // (The index field is typed as size_t.)
142 
143   __ ldr(tmp1, index);                      // tmp := *index_adr
144   __ cbz(tmp1, runtime);                    // tmp == 0?
145                                         // If yes, goto runtime
146 
147   __ sub(tmp1, tmp1, wordSize);             // tmp := tmp - wordSize
148   __ str(tmp1, index);                      // *index_adr := tmp
149   __ ldr(tmp2, buffer);
150   __ add(tmp1, tmp1, tmp2);                 // tmp := tmp + *buffer_adr
151 
152   // Record the previous value
153   __ str(pre_val, Address(tmp1, 0));
154   __ b(done);
155 
156   __ bind(runtime);
157 
158   // save the live input values
159   RegSet saved = RegSet::of(pre_val);
160   FloatRegSet fsaved;
161 
162   // Barriers might be emitted when converting between (scalarized) calling
163   // conventions for inline types. Save all argument registers before calling
164   // into the runtime.
165   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
166     if (tosca_live) saved += RegSet::of(r0);
167     if (obj != noreg) saved += RegSet::of(obj);
168     saved += RegSet::of(j_rarg0, j_rarg1, j_rarg2, j_rarg3);
169     saved += RegSet::of(j_rarg4, j_rarg5, j_rarg6, j_rarg7);
170 
171     fsaved += FloatRegSet::of(j_farg0, j_farg1, j_farg2, j_farg3);
172     fsaved += FloatRegSet::of(j_farg4, j_farg5, j_farg6, j_farg7);
173 
174     __ push(saved, sp);
175     __ push_fp(fsaved, sp);
176   } else {
177     __ push_call_clobbered_registers();
178   }
179 
180   // Calling the runtime using the regular call_VM_leaf mechanism generates
181   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
182   // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
183   //
184   // If we care generating the pre-barrier without a frame (e.g. in the
185   // intrinsified Reference.get() routine) then rfp might be pointing to
186   // the caller frame and so this check will most likely fail at runtime.
187   //
188   // Expanding the call directly bypasses the generation of the check.
189   // So when we do not have have a full interpreter frame on the stack
190   // expand_call should be passed true.
191 
192   if (expand_call) {
193     assert(pre_val != c_rarg1, "smashed arg");
194     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
195   } else {
196     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
197   }
198 
199   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
200   __ pop_fp(fsaved, sp);
201   __ pop(saved, sp);
202   } else {
203     __ pop_call_clobbered_registers();
204   }
205 
206   __ bind(done);
207 
208 }
209 
210 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
211                                                   Register store_addr,
212                                                   Register new_val,
213                                                   Register thread,
214                                                   Register tmp1,
215                                                   Register tmp2) {
216   assert(thread == rthread, "must be");
217   assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
218                              rscratch1);
219   assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
220          && tmp2 != noreg, "expecting a register");
221 
222   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
223   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
224 
225   BarrierSet* bs = BarrierSet::barrier_set();
226   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
227   CardTable* ct = ctbs->card_table();
228 
229   Label done;
230   Label runtime;
231 
232   // Does store cross heap regions?
233 
234   __ eor(tmp1, store_addr, new_val);
235   __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
236   __ cbz(tmp1, done);
237 
238   // crosses regions, storing null?
239 
240   __ cbz(new_val, done);
241 
242   // storing region crossing non-null, is card already dirty?
243 
244   assert_different_registers(store_addr, thread, tmp1, tmp2, rscratch1);
245 
246   const Register card_addr = tmp1;
247 
248   __ lsr(card_addr, store_addr, CardTable::card_shift());
249 
250   // get the address of the card
251   __ load_byte_map_base(tmp2);
252   __ add(card_addr, card_addr, tmp2);
253   __ ldrb(tmp2, Address(card_addr));
254   __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
255   __ br(Assembler::EQ, done);
256 
257   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
258 
259   __ membar(Assembler::StoreLoad);
260 
261   __ ldrb(tmp2, Address(card_addr));
262   __ cbzw(tmp2, done);
263 
264   // storing a region crossing, non-null oop, card is clean.
265   // dirty card and log.
266 
267   __ strb(zr, Address(card_addr));
268 
269   __ ldr(rscratch1, queue_index);
270   __ cbz(rscratch1, runtime);
271   __ sub(rscratch1, rscratch1, wordSize);
272   __ str(rscratch1, queue_index);
273 
274   __ ldr(tmp2, buffer);
275   __ str(card_addr, Address(tmp2, rscratch1));
276   __ b(done);
277 
278   __ bind(runtime);
279 
280   // save the live input values
281   RegSet saved = RegSet::of(store_addr);
282   FloatRegSet fsaved;
283 
284   // Barriers might be emitted when converting between (scalarized) calling
285   // conventions for inline types. Save all argument registers before calling
286   // into the runtime.
287   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
288     saved += RegSet::of(j_rarg0, j_rarg1, j_rarg2, j_rarg3);
289     saved += RegSet::of(j_rarg4, j_rarg5, j_rarg6, j_rarg7);
290 
291     fsaved += FloatRegSet::of(j_farg0, j_farg1, j_farg2, j_farg3);
292     fsaved += FloatRegSet::of(j_farg4, j_farg5, j_farg6, j_farg7);
293   }
294 
295   __ push(saved, sp);
296   __ push_fp(fsaved, sp);
297   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
298   __ pop_fp(fsaved, sp);
299   __ pop(saved, sp);
300 
301   __ bind(done);
302 }
303 
304 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
305                                     Register dst, Address src, Register tmp1, Register tmp2) {
306   bool on_oop = is_reference_type(type);
307   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
308   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
309   bool on_reference = on_weak || on_phantom;
310   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
311   if (on_oop && on_reference) {
312     // LR is live.  It must be saved around calls.
313     __ enter(/*strip_ret_addr*/true); // barrier may call runtime
314     // Generate the G1 pre-barrier code to log the value of
315     // the referent field in an SATB buffer.
316     g1_write_barrier_pre(masm /* masm */,
317                          noreg /* obj */,
318                          dst /* pre_val */,
319                          rthread /* thread */,
320                          tmp1 /* tmp1 */,
321                          tmp2 /* tmp2 */,
322                          true /* tosca_live */,
323                          true /* expand_call */);
324     __ leave();
325   }
326 }
327 
328 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
329                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
330 
331   bool in_heap = (decorators & IN_HEAP) != 0;
332   bool as_normal = (decorators & AS_NORMAL) != 0;
333   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
334 
335   bool needs_pre_barrier = as_normal && !dest_uninitialized;
336   bool needs_post_barrier = (val != noreg && in_heap);
337 
338   assert_different_registers(val, tmp1, tmp2, tmp3);
339 
340   // flatten object address if needed
341   if (dst.index() == noreg && dst.offset() == 0) {
342     if (dst.base() != tmp3) {
343       __ mov(tmp3, dst.base());
344     }
345   } else {
346     __ lea(tmp3, dst);
347   }
348 
349   if (needs_pre_barrier) {
350     g1_write_barrier_pre(masm,
351                          tmp3 /* obj */,
352                          tmp2 /* pre_val */,
353                          rthread /* thread */,
354                          tmp1  /* tmp1 */,
355                          rscratch2  /* tmp2 */,
356                          val != noreg /* tosca_live */,
357                          false /* expand_call */);
358   }
359 
360   if (val == noreg) {
361     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
362   } else {
363     // G1 barrier needs uncompressed oop for region cross check.
364     Register new_val = val;
365     if (needs_post_barrier) {
366       if (UseCompressedOops) {
367         new_val = rscratch2;
368         __ mov(new_val, val);
369       }
370     }
371 
372     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
373     if (needs_post_barrier) {
374       g1_write_barrier_post(masm,
375                             tmp3 /* store_adr */,
376                             new_val /* new_val */,
377                             rthread /* thread */,
378                             tmp1 /* tmp1 */,
379                             tmp2 /* tmp2 */);
380     }
381   }
382 
383 }
384 
385 #ifdef COMPILER1
386 
387 #undef __
388 #define __ ce->masm()->
389 
390 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
391   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
392   // At this point we know that marking is in progress.
393   // If do_load() is true then we have to emit the
394   // load of the previous value; otherwise it has already
395   // been loaded into _pre_val.
396 
397   __ bind(*stub->entry());
398 
399   assert(stub->pre_val()->is_register(), "Precondition.");
400 
< prev index next >