193 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
194 __ cmpl(in_progress, 0);
195 } else {
196 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
197 __ cmpb(in_progress, 0);
198 }
199 __ jcc(Assembler::equal, done);
200
201 // Do we need to load the previous value?
202 if (obj != noreg) {
203 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
204 }
205
206 // Is the previous value null?
207 __ cmpptr(pre_val, NULL_WORD);
208 __ jcc(Assembler::equal, done);
209
210 // Can we store original value in the thread's buffer?
211 // Is index == 0?
212 // (The index field is typed as size_t.)
213
214 __ movptr(tmp, index); // tmp := *index_adr
215 __ cmpptr(tmp, 0); // tmp == 0?
216 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
217
218 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
219 __ movptr(index, tmp); // *index_adr := tmp
220 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
221
222 // Record the previous value
223 __ movptr(Address(tmp, 0), pre_val);
224 __ jmp(done);
225
226 __ bind(runtime);
227
228 // Determine and save the live input values
229 __ push_call_clobbered_registers();
230
231 // Calling the runtime using the regular call_VM_leaf mechanism generates
232 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
233 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
234 //
235 // If we care generating the pre-barrier without a frame (e.g. in the
236 // intrinsified Reference.get() routine) then ebp might be pointing to
237 // the caller frame and so this check will most likely fail at runtime.
238 //
239 // Expanding the call directly bypasses the generation of the check.
240 // So when we do not have have a full interpreter frame on the stack
241 // expand_call should be passed true.
242
243 if (expand_call) {
244 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
245 #ifdef _LP64
246 if (c_rarg1 != thread) {
247 __ mov(c_rarg1, thread);
248 }
249 if (c_rarg0 != pre_val) {
250 __ mov(c_rarg0, pre_val);
251 }
252 #else
253 __ push(thread);
254 __ push(pre_val);
255 #endif
256 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
257 } else {
258 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
259 }
260
261 __ pop_call_clobbered_registers();
262
263 __ bind(done);
264 }
265
266 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
267 Register store_addr,
268 Register new_val,
269 Register thread,
270 Register tmp,
271 Register tmp2) {
272 // Generated code assumes that buffer index is pointer sized.
273 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
274 #ifdef _LP64
275 assert(thread == r15_thread, "must be");
276 #endif // _LP64
277
278 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
279 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
280
281 CardTableBarrierSet* ct =
314 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
315 __ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
316 __ jcc(Assembler::equal, done);
317
318
319 // storing a region crossing, non-null oop, card is clean.
320 // dirty card and log.
321
322 __ movb(Address(card_addr, 0), G1CardTable::dirty_card_val());
323
324 __ movptr(tmp2, queue_index);
325 __ testptr(tmp2, tmp2);
326 __ jcc(Assembler::zero, runtime);
327 __ subptr(tmp2, wordSize);
328 __ movptr(queue_index, tmp2);
329 __ addptr(tmp2, buffer);
330 __ movptr(Address(tmp2, 0), card_addr);
331 __ jmp(done);
332
333 __ bind(runtime);
334 // save the live input values
335 RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
336 __ push_set(saved);
337 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
338 __ pop_set(saved);
339
340 __ bind(done);
341 }
342
343 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
344 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
345 bool in_heap = (decorators & IN_HEAP) != 0;
346 bool as_normal = (decorators & AS_NORMAL) != 0;
347
348 bool needs_pre_barrier = as_normal;
349 bool needs_post_barrier = val != noreg && in_heap;
350
351 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
352 // flatten object address if needed
353 // We do it regardless of precise because we need the registers
354 if (dst.index() == noreg && dst.disp() == 0) {
355 if (dst.base() != tmp1) {
356 __ movptr(tmp1, dst.base());
357 }
358 } else {
359 __ lea(tmp1, dst);
360 }
361
362 #ifndef _LP64
363 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
364 #endif
365
366 NOT_LP64(__ get_thread(rcx));
367 NOT_LP64(imasm->save_bcp());
368
|
193 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
194 __ cmpl(in_progress, 0);
195 } else {
196 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
197 __ cmpb(in_progress, 0);
198 }
199 __ jcc(Assembler::equal, done);
200
201 // Do we need to load the previous value?
202 if (obj != noreg) {
203 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
204 }
205
206 // Is the previous value null?
207 __ cmpptr(pre_val, NULL_WORD);
208 __ jcc(Assembler::equal, done);
209
210 // Can we store original value in the thread's buffer?
211 // Is index == 0?
212 // (The index field is typed as size_t.)
213 __ movptr(tmp, index); // tmp := *index_adr
214 __ cmpptr(tmp, 0); // tmp == 0?
215 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
216
217 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
218 __ movptr(index, tmp); // *index_adr := tmp
219 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
220
221 // Record the previous value
222 __ movptr(Address(tmp, 0), pre_val);
223 __ jmp(done);
224
225 __ bind(runtime);
226
227 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
228 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
229 // types. Save all argument registers before calling into the runtime.
230 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
231 __ pusha();
232 __ subptr(rsp, 64);
233 __ movdbl(Address(rsp, 0), j_farg0);
234 __ movdbl(Address(rsp, 8), j_farg1);
235 __ movdbl(Address(rsp, 16), j_farg2);
236 __ movdbl(Address(rsp, 24), j_farg3);
237 __ movdbl(Address(rsp, 32), j_farg4);
238 __ movdbl(Address(rsp, 40), j_farg5);
239 __ movdbl(Address(rsp, 48), j_farg6);
240 __ movdbl(Address(rsp, 56), j_farg7);
241 } else {
242 // Determine and save the live input values
243 __ push_call_clobbered_registers();
244 }
245
246 // Calling the runtime using the regular call_VM_leaf mechanism generates
247 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
248 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
249 //
250 // If we care generating the pre-barrier without a frame (e.g. in the
251 // intrinsified Reference.get() routine) then ebp might be pointing to
252 // the caller frame and so this check will most likely fail at runtime.
253 //
254 // Expanding the call directly bypasses the generation of the check.
255 // So when we do not have have a full interpreter frame on the stack
256 // expand_call should be passed true.
257
258 if (expand_call) {
259 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
260 #ifdef _LP64
261 if (c_rarg1 != thread) {
262 __ mov(c_rarg1, thread);
263 }
264 if (c_rarg0 != pre_val) {
265 __ mov(c_rarg0, pre_val);
266 }
267 #else
268 __ push(thread);
269 __ push(pre_val);
270 #endif
271 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
272 } else {
273 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
274 }
275
276 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
277 // Restore registers
278 __ movdbl(j_farg0, Address(rsp, 0));
279 __ movdbl(j_farg1, Address(rsp, 8));
280 __ movdbl(j_farg2, Address(rsp, 16));
281 __ movdbl(j_farg3, Address(rsp, 24));
282 __ movdbl(j_farg4, Address(rsp, 32));
283 __ movdbl(j_farg5, Address(rsp, 40));
284 __ movdbl(j_farg6, Address(rsp, 48));
285 __ movdbl(j_farg7, Address(rsp, 56));
286 __ addptr(rsp, 64);
287 __ popa();
288 } else {
289 __ pop_call_clobbered_registers();
290 }
291
292 __ bind(done);
293 }
294
295 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
296 Register store_addr,
297 Register new_val,
298 Register thread,
299 Register tmp,
300 Register tmp2) {
301 // Generated code assumes that buffer index is pointer sized.
302 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
303 #ifdef _LP64
304 assert(thread == r15_thread, "must be");
305 #endif // _LP64
306
307 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
308 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
309
310 CardTableBarrierSet* ct =
343 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
344 __ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
345 __ jcc(Assembler::equal, done);
346
347
348 // storing a region crossing, non-null oop, card is clean.
349 // dirty card and log.
350
351 __ movb(Address(card_addr, 0), G1CardTable::dirty_card_val());
352
353 __ movptr(tmp2, queue_index);
354 __ testptr(tmp2, tmp2);
355 __ jcc(Assembler::zero, runtime);
356 __ subptr(tmp2, wordSize);
357 __ movptr(queue_index, tmp2);
358 __ addptr(tmp2, buffer);
359 __ movptr(Address(tmp2, 0), card_addr);
360 __ jmp(done);
361
362 __ bind(runtime);
363 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
364 // types. Save all argument registers before calling into the runtime.
365 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
366 __ pusha();
367 __ subptr(rsp, 64);
368 __ movdbl(Address(rsp, 0), j_farg0);
369 __ movdbl(Address(rsp, 8), j_farg1);
370 __ movdbl(Address(rsp, 16), j_farg2);
371 __ movdbl(Address(rsp, 24), j_farg3);
372 __ movdbl(Address(rsp, 32), j_farg4);
373 __ movdbl(Address(rsp, 40), j_farg5);
374 __ movdbl(Address(rsp, 48), j_farg6);
375 __ movdbl(Address(rsp, 56), j_farg7);
376
377 #ifdef _LP64
378 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
379 #else
380 __ push(thread);
381 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
382 __ pop(thread);
383 #endif
384
385 // Restore registers
386 __ movdbl(j_farg0, Address(rsp, 0));
387 __ movdbl(j_farg1, Address(rsp, 8));
388 __ movdbl(j_farg2, Address(rsp, 16));
389 __ movdbl(j_farg3, Address(rsp, 24));
390 __ movdbl(j_farg4, Address(rsp, 32));
391 __ movdbl(j_farg5, Address(rsp, 40));
392 __ movdbl(j_farg6, Address(rsp, 48));
393 __ movdbl(j_farg7, Address(rsp, 56));
394 __ addptr(rsp, 64);
395 __ popa();
396
397 __ bind(done);
398 }
399
400 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
401 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
402 bool in_heap = (decorators & IN_HEAP) != 0;
403 bool as_normal = (decorators & AS_NORMAL) != 0;
404 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
405
406 bool needs_pre_barrier = as_normal && !dest_uninitialized;
407 bool needs_post_barrier = val != noreg && in_heap;
408
409 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
410 // flatten object address if needed
411 // We do it regardless of precise because we need the registers
412 if (dst.index() == noreg && dst.disp() == 0) {
413 if (dst.base() != tmp1) {
414 __ movptr(tmp1, dst.base());
415 }
416 } else {
417 __ lea(tmp1, dst);
418 }
419
420 #ifndef _LP64
421 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
422 #endif
423
424 NOT_LP64(__ get_thread(rcx));
425 NOT_LP64(imasm->save_bcp());
426
|