115 __ add(start, start, scratch);
116
117 __ bind(loop);
118 if (UseCondCardMark) {
119 __ ldrb(scratch, Address(start, count));
120 // Instead of loading clean_card_val and comparing, we exploit the fact that
121 // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
122 __ tbz(scratch, 0, next);
123 }
124 static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
125 __ strb(zr, Address(start, count));
126 __ bind(next);
127 __ subs(count, count, 1);
128 __ br(Assembler::GE, loop);
129
130 __ bind(done);
131 }
132
133 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
134 const Register thread, const Register value, const Register temp1, const Register temp2) {
135 // Can we store a value in the given thread's buffer?
136 // (The index field is typed as size_t.)
137 __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
138 __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
139 // The buffer is not full, store value into it.
140 __ sub(temp1, temp1, wordSize); // temp1 := next index
141 __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
142 __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
143 __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
144 }
145
146 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
147 const Register thread,
148 const Register tmp1) {
149 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
150 // Is marking active?
151 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
152 __ ldrw(tmp1, in_progress);
153 } else {
154 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
188 bool expand_call) {
189 // If expand_call is true then we expand the call_VM_leaf macro
190 // directly to skip generating the check by
191 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
192
193 assert(thread == rthread, "must be");
194
195 Label done;
196 Label runtime;
197
198 assert_different_registers(obj, pre_val, tmp1, tmp2);
199 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
200
201 generate_pre_barrier_fast_path(masm, thread, tmp1);
202 // If marking is not active (*(mark queue active address) == 0), jump to done
203 __ cbzw(tmp1, done);
204 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
205
206 __ bind(runtime);
207
208 __ push_call_clobbered_registers();
209
210 // Calling the runtime using the regular call_VM_leaf mechanism generates
211 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
212 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
213 //
214 // If we care generating the pre-barrier without a frame (e.g. in the
215 // intrinsified Reference.get() routine) then rfp might be pointing to
216 // the caller frame and so this check will most likely fail at runtime.
217 //
218 // Expanding the call directly bypasses the generation of the check.
219 // So when we do not have have a full interpreter frame on the stack
220 // expand_call should be passed true.
221
222 if (expand_call) {
223 assert(pre_val != c_rarg1, "smashed arg");
224 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
225 } else {
226 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
227 }
228
229 __ pop_call_clobbered_registers();
230
231 __ bind(done);
232
233 }
234
235 static void generate_post_barrier(MacroAssembler* masm,
236 const Register store_addr,
237 const Register new_val,
238 const Register thread,
239 const Register tmp1,
240 const Register tmp2,
241 Label& done,
242 bool new_val_may_be_null) {
243 assert(thread == rthread, "must be");
244 assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
245
246 // Does store cross heap regions?
247 __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
248 __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
249 __ cbz(tmp1, done);
350 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
351 if (on_oop && on_reference) {
352 // LR is live. It must be saved around calls.
353 __ enter(/*strip_ret_addr*/true); // barrier may call runtime
354 // Generate the G1 pre-barrier code to log the value of
355 // the referent field in an SATB buffer.
356 g1_write_barrier_pre(masm /* masm */,
357 noreg /* obj */,
358 dst /* pre_val */,
359 rthread /* thread */,
360 tmp1 /* tmp1 */,
361 tmp2 /* tmp2 */,
362 true /* tosca_live */,
363 true /* expand_call */);
364 __ leave();
365 }
366 }
367
368 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
369 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
370 // flatten object address if needed
371 if (dst.index() == noreg && dst.offset() == 0) {
372 if (dst.base() != tmp3) {
373 __ mov(tmp3, dst.base());
374 }
375 } else {
376 __ lea(tmp3, dst);
377 }
378
379 g1_write_barrier_pre(masm,
380 tmp3 /* obj */,
381 tmp2 /* pre_val */,
382 rthread /* thread */,
383 tmp1 /* tmp1 */,
384 rscratch2 /* tmp2 */,
385 val != noreg /* tosca_live */,
386 false /* expand_call */);
387
388 if (val == noreg) {
389 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
390 } else {
391 // G1 barrier needs uncompressed oop for region cross check.
392 Register new_val = val;
393 if (UseCompressedOops) {
394 new_val = rscratch2;
395 __ mov(new_val, val);
396 }
397 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
398 g1_write_barrier_post(masm,
399 tmp3 /* store_adr */,
400 new_val /* new_val */,
401 rthread /* thread */,
402 tmp1 /* tmp1 */,
403 tmp2 /* tmp2 */);
404 }
405
406 }
407
408 #ifdef COMPILER1
409
410 #undef __
411 #define __ ce->masm()->
412
413 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
414 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
415 // At this point we know that marking is in progress.
416 // If do_load() is true then we have to emit the
417 // load of the previous value; otherwise it has already
418 // been loaded into _pre_val.
419
420 __ bind(*stub->entry());
421
422 assert(stub->pre_val()->is_register(), "Precondition.");
423
|
115 __ add(start, start, scratch);
116
117 __ bind(loop);
118 if (UseCondCardMark) {
119 __ ldrb(scratch, Address(start, count));
120 // Instead of loading clean_card_val and comparing, we exploit the fact that
121 // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
122 __ tbz(scratch, 0, next);
123 }
124 static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
125 __ strb(zr, Address(start, count));
126 __ bind(next);
127 __ subs(count, count, 1);
128 __ br(Assembler::GE, loop);
129
130 __ bind(done);
131 }
132
133 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
134 const Register thread, const Register value, const Register temp1, const Register temp2) {
135 assert_different_registers(value, temp1, temp2);
136 // Can we store a value in the given thread's buffer?
137 // (The index field is typed as size_t.)
138 __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
139 __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
140 // The buffer is not full, store value into it.
141 __ sub(temp1, temp1, wordSize); // temp1 := next index
142 __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
143 __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
144 __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
145 }
146
147 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
148 const Register thread,
149 const Register tmp1) {
150 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
151 // Is marking active?
152 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
153 __ ldrw(tmp1, in_progress);
154 } else {
155 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
189 bool expand_call) {
190 // If expand_call is true then we expand the call_VM_leaf macro
191 // directly to skip generating the check by
192 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
193
194 assert(thread == rthread, "must be");
195
196 Label done;
197 Label runtime;
198
199 assert_different_registers(obj, pre_val, tmp1, tmp2);
200 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
201
202 generate_pre_barrier_fast_path(masm, thread, tmp1);
203 // If marking is not active (*(mark queue active address) == 0), jump to done
204 __ cbzw(tmp1, done);
205 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
206
207 __ bind(runtime);
208
209 // save the live input values
210 RegSet saved = RegSet::of(pre_val);
211 FloatRegSet fsaved;
212
213 // Barriers might be emitted when converting between (scalarized) calling
214 // conventions for inline types. Save all argument registers before calling
215 // into the runtime.
216
217 // TODO 8366717 This came with 8284161: Implementation of Virtual Threads (Preview) later in May 2022
218 // Check if it's sufficient
219 //__ push_call_clobbered_registers();
220 assert_different_registers(rscratch1, pre_val); // push_CPU_state trashes rscratch1
221 __ push_CPU_state(true);
222
223 // Calling the runtime using the regular call_VM_leaf mechanism generates
224 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
225 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
226 //
227 // If we care generating the pre-barrier without a frame (e.g. in the
228 // intrinsified Reference.get() routine) then rfp might be pointing to
229 // the caller frame and so this check will most likely fail at runtime.
230 //
231 // Expanding the call directly bypasses the generation of the check.
232 // So when we do not have have a full interpreter frame on the stack
233 // expand_call should be passed true.
234
235 if (expand_call) {
236 assert(pre_val != c_rarg1, "smashed arg");
237 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
238 } else {
239 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
240 }
241
242 __ pop_CPU_state(true);
243
244 __ bind(done);
245
246 }
247
248 static void generate_post_barrier(MacroAssembler* masm,
249 const Register store_addr,
250 const Register new_val,
251 const Register thread,
252 const Register tmp1,
253 const Register tmp2,
254 Label& done,
255 bool new_val_may_be_null) {
256 assert(thread == rthread, "must be");
257 assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
258
259 // Does store cross heap regions?
260 __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
261 __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
262 __ cbz(tmp1, done);
363 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
364 if (on_oop && on_reference) {
365 // LR is live. It must be saved around calls.
366 __ enter(/*strip_ret_addr*/true); // barrier may call runtime
367 // Generate the G1 pre-barrier code to log the value of
368 // the referent field in an SATB buffer.
369 g1_write_barrier_pre(masm /* masm */,
370 noreg /* obj */,
371 dst /* pre_val */,
372 rthread /* thread */,
373 tmp1 /* tmp1 */,
374 tmp2 /* tmp2 */,
375 true /* tosca_live */,
376 true /* expand_call */);
377 __ leave();
378 }
379 }
380
381 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
382 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
383
384 bool in_heap = (decorators & IN_HEAP) != 0;
385 bool as_normal = (decorators & AS_NORMAL) != 0;
386 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
387
388 bool needs_pre_barrier = as_normal && !dest_uninitialized;
389 bool needs_post_barrier = (val != noreg && in_heap);
390
391 assert_different_registers(val, tmp1, tmp2, tmp3);
392
393 // flatten object address if needed
394 if (dst.index() == noreg && dst.offset() == 0) {
395 if (dst.base() != tmp3) {
396 __ mov(tmp3, dst.base());
397 }
398 } else {
399 __ lea(tmp3, dst);
400 }
401
402 if (needs_pre_barrier) {
403 g1_write_barrier_pre(masm,
404 tmp3 /* obj */,
405 tmp2 /* pre_val */,
406 rthread /* thread */,
407 tmp1 /* tmp1 */,
408 rscratch2 /* tmp2 */,
409 val != noreg /* tosca_live */,
410 false /* expand_call */);
411 }
412
413 if (val == noreg) {
414 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
415 } else {
416 // G1 barrier needs uncompressed oop for region cross check.
417 Register new_val = val;
418 if (needs_post_barrier) {
419 if (UseCompressedOops) {
420 new_val = rscratch2;
421 __ mov(new_val, val);
422 }
423 }
424
425 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
426 if (needs_post_barrier) {
427 g1_write_barrier_post(masm,
428 tmp3 /* store_adr */,
429 new_val /* new_val */,
430 rthread /* thread */,
431 tmp1 /* tmp1 */,
432 tmp2 /* tmp2 */);
433 }
434 }
435
436 }
437
438 #ifdef COMPILER1
439
440 #undef __
441 #define __ ce->masm()->
442
443 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
444 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
445 // At this point we know that marking is in progress.
446 // If do_load() is true then we have to emit the
447 // load of the previous value; otherwise it has already
448 // been loaded into _pre_val.
449
450 __ bind(*stub->entry());
451
452 assert(stub->pre_val()->is_register(), "Precondition.");
453
|