1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
115 __ add(start, start, scratch);
116
117 __ bind(loop);
118 if (UseCondCardMark) {
119 __ ldrb(scratch, Address(start, count));
120 // Instead of loading clean_card_val and comparing, we exploit the fact that
121 // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
122 __ tbz(scratch, 0, next);
123 }
124 static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
125 __ strb(zr, Address(start, count));
126 __ bind(next);
127 __ subs(count, count, 1);
128 __ br(Assembler::GE, loop);
129
130 __ bind(done);
131 }
132
133 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
134 const Register thread, const Register value, const Register temp1, const Register temp2) {
135 // Can we store a value in the given thread's buffer?
136 // (The index field is typed as size_t.)
137 __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
138 __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
139 // The buffer is not full, store value into it.
140 __ sub(temp1, temp1, wordSize); // temp1 := next index
141 __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
142 __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
143 __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
144 }
145
146 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
147 const Register thread,
148 const Register tmp1) {
149 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
150 // Is marking active?
151 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
152 __ ldrw(tmp1, in_progress);
153 } else {
154 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
188 bool expand_call) {
189 // If expand_call is true then we expand the call_VM_leaf macro
190 // directly to skip generating the check by
191 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
192
193 assert(thread == rthread, "must be");
194
195 Label done;
196 Label runtime;
197
198 assert_different_registers(obj, pre_val, tmp1, tmp2);
199 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
200
201 generate_pre_barrier_fast_path(masm, thread, tmp1);
202 // If marking is not active (*(mark queue active address) == 0), jump to done
203 __ cbzw(tmp1, done);
204 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
205
206 __ bind(runtime);
207
208 __ push_call_clobbered_registers();
209
210 // Calling the runtime using the regular call_VM_leaf mechanism generates
211 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
212 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
213 //
214 // If we care generating the pre-barrier without a frame (e.g. in the
215 // intrinsified Reference.get() routine) then rfp might be pointing to
216 // the caller frame and so this check will most likely fail at runtime.
217 //
218 // Expanding the call directly bypasses the generation of the check.
219 // So when we do not have have a full interpreter frame on the stack
220 // expand_call should be passed true.
221
222 if (expand_call) {
223 assert(pre_val != c_rarg1, "smashed arg");
224 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
225 } else {
226 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
227 }
366 CardTableBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
367 if (on_oop && on_reference) {
368 // LR is live. It must be saved around calls.
369 __ enter(/*strip_ret_addr*/true); // barrier may call runtime
370 // Generate the G1 pre-barrier code to log the value of
371 // the referent field in an SATB buffer.
372 g1_write_barrier_pre(masm /* masm */,
373 noreg /* obj */,
374 dst /* pre_val */,
375 rthread /* thread */,
376 tmp1 /* tmp1 */,
377 tmp2 /* tmp2 */,
378 true /* tosca_live */,
379 true /* expand_call */);
380 __ leave();
381 }
382 }
383
384 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
385 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
386 // flatten object address if needed
387 if (dst.index() == noreg && dst.offset() == 0) {
388 if (dst.base() != tmp3) {
389 __ mov(tmp3, dst.base());
390 }
391 } else {
392 __ lea(tmp3, dst);
393 }
394
395 g1_write_barrier_pre(masm,
396 tmp3 /* obj */,
397 tmp2 /* pre_val */,
398 rthread /* thread */,
399 tmp1 /* tmp1 */,
400 rscratch2 /* tmp2 */,
401 val != noreg /* tosca_live */,
402 false /* expand_call */);
403
404 if (val == noreg) {
405 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
406 } else {
407 // G1 barrier needs uncompressed oop for region cross check.
408 Register new_val = val;
409 if (UseCompressedOops) {
410 new_val = rscratch2;
411 __ mov(new_val, val);
412 }
413 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
414 g1_write_barrier_post(masm,
415 tmp3 /* store_adr */,
416 new_val /* new_val */,
417 rthread /* thread */,
418 tmp1 /* tmp1 */,
419 tmp2 /* tmp2 */);
420 }
421
422 }
423
424 #ifdef COMPILER1
425
426 #undef __
427 #define __ ce->masm()->
428
429 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
430 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
431 // At this point we know that marking is in progress.
432 // If do_load() is true then we have to emit the
433 // load of the previous value; otherwise it has already
434 // been loaded into _pre_val.
435
436 __ bind(*stub->entry());
437
438 assert(stub->pre_val()->is_register(), "Precondition.");
439
|
1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
115 __ add(start, start, scratch);
116
117 __ bind(loop);
118 if (UseCondCardMark) {
119 __ ldrb(scratch, Address(start, count));
120 // Instead of loading clean_card_val and comparing, we exploit the fact that
121 // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
122 __ tbz(scratch, 0, next);
123 }
124 static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
125 __ strb(zr, Address(start, count));
126 __ bind(next);
127 __ subs(count, count, 1);
128 __ br(Assembler::GE, loop);
129
130 __ bind(done);
131 }
132
133 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
134 const Register thread, const Register value, const Register temp1, const Register temp2) {
135 assert_different_registers(value, temp1, temp2);
136 // Can we store a value in the given thread's buffer?
137 // (The index field is typed as size_t.)
138 __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
139 __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
140 // The buffer is not full, store value into it.
141 __ sub(temp1, temp1, wordSize); // temp1 := next index
142 __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
143 __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
144 __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
145 }
146
147 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
148 const Register thread,
149 const Register tmp1) {
150 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
151 // Is marking active?
152 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
153 __ ldrw(tmp1, in_progress);
154 } else {
155 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
189 bool expand_call) {
190 // If expand_call is true then we expand the call_VM_leaf macro
191 // directly to skip generating the check by
192 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
193
194 assert(thread == rthread, "must be");
195
196 Label done;
197 Label runtime;
198
199 assert_different_registers(obj, pre_val, tmp1, tmp2);
200 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
201
202 generate_pre_barrier_fast_path(masm, thread, tmp1);
203 // If marking is not active (*(mark queue active address) == 0), jump to done
204 __ cbzw(tmp1, done);
205 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
206
207 __ bind(runtime);
208
209 assert_different_registers(rscratch1, pre_val); // push_call_clobbered_registers trashes rscratch1
210 __ push_call_clobbered_registers();
211
212 // Calling the runtime using the regular call_VM_leaf mechanism generates
213 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
214 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
215 //
216 // If we care generating the pre-barrier without a frame (e.g. in the
217 // intrinsified Reference.get() routine) then rfp might be pointing to
218 // the caller frame and so this check will most likely fail at runtime.
219 //
220 // Expanding the call directly bypasses the generation of the check.
221 // So when we do not have have a full interpreter frame on the stack
222 // expand_call should be passed true.
223
224 if (expand_call) {
225 assert(pre_val != c_rarg1, "smashed arg");
226 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
227 } else {
228 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
229 }
368 CardTableBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
369 if (on_oop && on_reference) {
370 // LR is live. It must be saved around calls.
371 __ enter(/*strip_ret_addr*/true); // barrier may call runtime
372 // Generate the G1 pre-barrier code to log the value of
373 // the referent field in an SATB buffer.
374 g1_write_barrier_pre(masm /* masm */,
375 noreg /* obj */,
376 dst /* pre_val */,
377 rthread /* thread */,
378 tmp1 /* tmp1 */,
379 tmp2 /* tmp2 */,
380 true /* tosca_live */,
381 true /* expand_call */);
382 __ leave();
383 }
384 }
385
386 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
387 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
388
389 bool in_heap = (decorators & IN_HEAP) != 0;
390 bool as_normal = (decorators & AS_NORMAL) != 0;
391 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
392
393 bool needs_pre_barrier = as_normal && !dest_uninitialized;
394 bool needs_post_barrier = (val != noreg && in_heap);
395
396 assert_different_registers(val, tmp1, tmp2, tmp3);
397
398 // flatten object address if needed
399 if (dst.index() == noreg && dst.offset() == 0) {
400 if (dst.base() != tmp3) {
401 __ mov(tmp3, dst.base());
402 }
403 } else {
404 __ lea(tmp3, dst);
405 }
406
407 if (needs_pre_barrier) {
408 g1_write_barrier_pre(masm,
409 tmp3 /* obj */,
410 tmp2 /* pre_val */,
411 rthread /* thread */,
412 tmp1 /* tmp1 */,
413 rscratch2 /* tmp2 */,
414 val != noreg /* tosca_live */,
415 false /* expand_call */);
416 }
417
418 if (val == noreg) {
419 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
420 } else {
421 // G1 barrier needs uncompressed oop for region cross check.
422 Register new_val = val;
423 if (needs_post_barrier) {
424 if (UseCompressedOops) {
425 new_val = rscratch2;
426 __ mov(new_val, val);
427 }
428 }
429
430 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
431 if (needs_post_barrier) {
432 g1_write_barrier_post(masm,
433 tmp3 /* store_adr */,
434 new_val /* new_val */,
435 rthread /* thread */,
436 tmp1 /* tmp1 */,
437 tmp2 /* tmp2 */);
438 }
439 }
440
441 }
442
443 #ifdef COMPILER1
444
445 #undef __
446 #define __ ce->masm()->
447
448 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
449 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
450 // At this point we know that marking is in progress.
451 // If do_load() is true then we have to emit the
452 // load of the previous value; otherwise it has already
453 // been loaded into _pre_val.
454
455 __ bind(*stub->entry());
456
457 assert(stub->pre_val()->is_register(), "Precondition.");
458
|