1 /*
2 * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
137 __ cbz(pre_val, done);
138
139 // Can we store original value in the thread's buffer?
140 // Is index == 0?
141 // (The index field is typed as size_t.)
142
143 __ ldr(tmp, index); // tmp := *index_adr
144 __ cbz(tmp, runtime); // tmp == 0?
145 // If yes, goto runtime
146
147 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
148 __ str(tmp, index); // *index_adr := tmp
149 __ ldr(rscratch1, buffer);
150 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
151
152 // Record the previous value
153 __ str(pre_val, Address(tmp, 0));
154 __ b(done);
155
156 __ bind(runtime);
157 // save the live input values
158 RegSet saved = RegSet::of(pre_val);
159 if (tosca_live) saved += RegSet::of(r0);
160 if (obj != noreg) saved += RegSet::of(obj);
161
162 __ push(saved, sp);
163
164 // Calling the runtime using the regular call_VM_leaf mechanism generates
165 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
166 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
167 //
168 // If we care generating the pre-barrier without a frame (e.g. in the
169 // intrinsified Reference.get() routine) then ebp might be pointing to
170 // the caller frame and so this check will most likely fail at runtime.
171 //
172 // Expanding the call directly bypasses the generation of the check.
173 // So when we do not have have a full interpreter frame on the stack
174 // expand_call should be passed true.
175
176 if (expand_call) {
177 assert(pre_val != c_rarg1, "smashed arg");
178 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
179 } else {
180 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
181 }
182
183 __ pop(saved, sp);
184
185 __ bind(done);
186
187 }
188
189 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
190 Register store_addr,
191 Register new_val,
192 Register thread,
193 Register tmp,
194 Register tmp2) {
195 assert(thread == rthread, "must be");
196 assert_different_registers(store_addr, new_val, thread, tmp, tmp2,
197 rscratch1);
198 assert(store_addr != noreg && new_val != noreg && tmp != noreg
199 && tmp2 != noreg, "expecting a register");
200
201 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
202 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
203
204 BarrierSet* bs = BarrierSet::barrier_set();
205 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
206 CardTable* ct = ctbs->card_table();
207
208 Label done;
209 Label runtime;
210
211 // Does store cross heap regions?
212
213 __ eor(tmp, store_addr, new_val);
214 __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
215 __ cbz(tmp, done);
216
217 // crosses regions, storing NULL?
218
219 __ cbz(new_val, done);
220
221 // storing region crossing non-NULL, is card already dirty?
222
223 const Register card_addr = tmp;
224
225 __ lsr(card_addr, store_addr, CardTable::card_shift);
226
227 // get the address of the card
228 __ load_byte_map_base(tmp2);
229 __ add(card_addr, card_addr, tmp2);
230 __ ldrb(tmp2, Address(card_addr));
231 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
232 __ br(Assembler::EQ, done);
233
234 assert((int)CardTable::dirty_card_val() == 0, "must be 0");
235
236 __ membar(Assembler::StoreLoad);
237
238 __ ldrb(tmp2, Address(card_addr));
239 __ cbzw(tmp2, done);
240
241 // storing a region crossing, non-NULL oop, card is clean.
242 // dirty card and log.
243
244 __ strb(zr, Address(card_addr));
245
246 __ ldr(rscratch1, queue_index);
247 __ cbz(rscratch1, runtime);
248 __ sub(rscratch1, rscratch1, wordSize);
249 __ str(rscratch1, queue_index);
250
251 __ ldr(tmp2, buffer);
252 __ str(card_addr, Address(tmp2, rscratch1));
253 __ b(done);
254
255 __ bind(runtime);
256 // save the live input values
257 RegSet saved = RegSet::of(store_addr);
258 __ push(saved, sp);
259 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
260 __ pop(saved, sp);
261
262 __ bind(done);
263 }
264
265 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
266 Register dst, Address src, Register tmp1, Register tmp_thread) {
267 bool on_oop = is_reference_type(type);
268 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
269 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
270 bool on_reference = on_weak || on_phantom;
271 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
272 if (on_oop && on_reference) {
273 // LR is live. It must be saved around calls.
274 __ enter(); // barrier may call runtime
275 // Generate the G1 pre-barrier code to log the value of
276 // the referent field in an SATB buffer.
277 g1_write_barrier_pre(masm /* masm */,
278 noreg /* obj */,
279 dst /* pre_val */,
280 rthread /* thread */,
281 tmp1 /* tmp */,
282 true /* tosca_live */,
283 true /* expand_call */);
284 __ leave();
285 }
286 }
287
288 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
289 Address dst, Register val, Register tmp1, Register tmp2) {
290 // flatten object address if needed
291 if (dst.index() == noreg && dst.offset() == 0) {
292 if (dst.base() != r3) {
293 __ mov(r3, dst.base());
294 }
295 } else {
296 __ lea(r3, dst);
297 }
298
299 g1_write_barrier_pre(masm,
300 r3 /* obj */,
301 tmp2 /* pre_val */,
302 rthread /* thread */,
303 tmp1 /* tmp */,
304 val != noreg /* tosca_live */,
305 false /* expand_call */);
306
307 if (val == noreg) {
308 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
309 } else {
310 // G1 barrier needs uncompressed oop for region cross check.
311 Register new_val = val;
312 if (UseCompressedOops) {
313 new_val = rscratch2;
314 __ mov(new_val, val);
315 }
316 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
317 g1_write_barrier_post(masm,
318 r3 /* store_adr */,
319 new_val /* new_val */,
320 rthread /* thread */,
321 tmp1 /* tmp */,
322 tmp2 /* tmp2 */);
323 }
324
325 }
326
327 #ifdef COMPILER1
328
329 #undef __
330 #define __ ce->masm()->
331
332 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
333 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
334 // At this point we know that marking is in progress.
335 // If do_load() is true then we have to emit the
336 // load of the previous value; otherwise it has already
337 // been loaded into _pre_val.
338
339 __ bind(*stub->entry());
340
341 assert(stub->pre_val()->is_register(), "Precondition.");
342
|
1 /*
2 * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
137 __ cbz(pre_val, done);
138
139 // Can we store original value in the thread's buffer?
140 // Is index == 0?
141 // (The index field is typed as size_t.)
142
143 __ ldr(tmp, index); // tmp := *index_adr
144 __ cbz(tmp, runtime); // tmp == 0?
145 // If yes, goto runtime
146
147 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
148 __ str(tmp, index); // *index_adr := tmp
149 __ ldr(rscratch1, buffer);
150 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
151
152 // Record the previous value
153 __ str(pre_val, Address(tmp, 0));
154 __ b(done);
155
156 __ bind(runtime);
157
158 // save the live input values
159 RegSet saved = RegSet::of(pre_val);
160 FloatRegSet fsaved;
161 if (tosca_live) saved += RegSet::of(r0);
162 if (obj != noreg) saved += RegSet::of(obj);
163
164 // Barriers might be emitted when converting between (scalarized) calling
165 // conventions for inline types. Save all argument registers before calling
166 // into the runtime.
167 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
168 saved += RegSet::of(j_rarg0, j_rarg1, j_rarg2, j_rarg3);
169 saved += RegSet::of(j_rarg4, j_rarg5, j_rarg6, j_rarg7);
170
171 fsaved += FloatRegSet::of(j_farg0, j_farg1, j_farg2, j_farg3);
172 fsaved += FloatRegSet::of(j_farg4, j_farg5, j_farg6, j_farg7);
173 }
174
175 __ push(saved, sp);
176 __ push_fp(fsaved, sp);
177
178 // Calling the runtime using the regular call_VM_leaf mechanism generates
179 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
180 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
181 //
182 // If we care generating the pre-barrier without a frame (e.g. in the
183 // intrinsified Reference.get() routine) then ebp might be pointing to
184 // the caller frame and so this check will most likely fail at runtime.
185 //
186 // Expanding the call directly bypasses the generation of the check.
187 // So when we do not have have a full interpreter frame on the stack
188 // expand_call should be passed true.
189
190 if (expand_call) {
191 assert(pre_val != c_rarg1, "smashed arg");
192 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
193 } else {
194 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
195 }
196
197 __ pop_fp(fsaved, sp);
198 __ pop(saved, sp);
199
200 __ bind(done);
201
202 }
203
204 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
205 Register store_addr,
206 Register new_val,
207 Register thread,
208 Register tmp,
209 Register tmp2) {
210 assert(thread == rthread, "must be");
211 assert_different_registers(store_addr, new_val, thread, tmp, rscratch1);
212 assert(store_addr != noreg && new_val != noreg && tmp != noreg
213 && tmp2 != noreg, "expecting a register");
214
215 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
216 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
217
218 BarrierSet* bs = BarrierSet::barrier_set();
219 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
220 CardTable* ct = ctbs->card_table();
221
222 Label done;
223 Label runtime;
224
225 // Does store cross heap regions?
226
227 __ eor(tmp, store_addr, new_val);
228 __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
229 __ cbz(tmp, done);
230
231 // crosses regions, storing NULL?
232
233 __ cbz(new_val, done);
234
235 // storing region crossing non-NULL, is card already dirty?
236
237 assert_different_registers(store_addr, thread, tmp, tmp2, rscratch1);
238
239 const Register card_addr = tmp;
240
241 __ lsr(card_addr, store_addr, CardTable::card_shift);
242
243 // get the address of the card
244 __ load_byte_map_base(tmp2);
245 __ add(card_addr, card_addr, tmp2);
246 __ ldrb(tmp2, Address(card_addr));
247 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
248 __ br(Assembler::EQ, done);
249
250 assert((int)CardTable::dirty_card_val() == 0, "must be 0");
251
252 __ membar(Assembler::StoreLoad);
253
254 __ ldrb(tmp2, Address(card_addr));
255 __ cbzw(tmp2, done);
256
257 // storing a region crossing, non-NULL oop, card is clean.
258 // dirty card and log.
259
260 __ strb(zr, Address(card_addr));
261
262 __ ldr(rscratch1, queue_index);
263 __ cbz(rscratch1, runtime);
264 __ sub(rscratch1, rscratch1, wordSize);
265 __ str(rscratch1, queue_index);
266
267 __ ldr(tmp2, buffer);
268 __ str(card_addr, Address(tmp2, rscratch1));
269 __ b(done);
270
271 __ bind(runtime);
272
273 // save the live input values
274 RegSet saved = RegSet::of(store_addr);
275 FloatRegSet fsaved;
276
277 // Barriers might be emitted when converting between (scalarized) calling
278 // conventions for inline types. Save all argument registers before calling
279 // into the runtime.
280 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
281 saved += RegSet::of(j_rarg0, j_rarg1, j_rarg2, j_rarg3);
282 saved += RegSet::of(j_rarg4, j_rarg5, j_rarg6, j_rarg7);
283
284 fsaved += FloatRegSet::of(j_farg0, j_farg1, j_farg2, j_farg3);
285 fsaved += FloatRegSet::of(j_farg4, j_farg5, j_farg6, j_farg7);
286 }
287
288 __ push(saved, sp);
289 __ push_fp(fsaved, sp);
290 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
291 __ pop_fp(fsaved, sp);
292 __ pop(saved, sp);
293
294 __ bind(done);
295 }
296
297 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
298 Register dst, Address src, Register tmp1, Register tmp_thread) {
299 bool on_oop = is_reference_type(type);
300 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
301 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
302 bool on_reference = on_weak || on_phantom;
303 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
304 if (on_oop && on_reference) {
305 // LR is live. It must be saved around calls.
306 __ enter(); // barrier may call runtime
307 // Generate the G1 pre-barrier code to log the value of
308 // the referent field in an SATB buffer.
309 g1_write_barrier_pre(masm /* masm */,
310 noreg /* obj */,
311 dst /* pre_val */,
312 rthread /* thread */,
313 tmp1 /* tmp */,
314 true /* tosca_live */,
315 true /* expand_call */);
316 __ leave();
317 }
318 }
319
320 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
321 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
322
323 bool in_heap = (decorators & IN_HEAP) != 0;
324 bool as_normal = (decorators & AS_NORMAL) != 0;
325 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
326
327 bool needs_pre_barrier = as_normal && !dest_uninitialized;
328 bool needs_post_barrier = (val != noreg && in_heap);
329
330 if (tmp3 == noreg) {
331 tmp3 = rscratch2;
332 }
333 assert_different_registers(val, tmp1, tmp2, tmp3);
334
335 // flatten object address if needed
336 if (dst.index() == noreg && dst.offset() == 0) {
337 if (dst.base() != tmp1) {
338 __ mov(tmp1, dst.base());
339 }
340 } else {
341 __ lea(tmp1, dst);
342 }
343
344 if (needs_pre_barrier) {
345 g1_write_barrier_pre(masm,
346 tmp1 /* obj */,
347 tmp2 /* pre_val */,
348 rthread /* thread */,
349 tmp3 /* tmp */,
350 val != noreg /* tosca_live */,
351 false /* expand_call */);
352 }
353
354 if (val == noreg) {
355 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), noreg, noreg, noreg, noreg);
356 } else {
357 // G1 barrier needs uncompressed oop for region cross check.
358 Register new_val = val;
359 if (needs_post_barrier) {
360 if (UseCompressedOops) {
361 new_val = tmp3;
362 __ mov(new_val, val);
363 }
364 }
365
366 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
367
368 if (needs_post_barrier) {
369 g1_write_barrier_post(masm,
370 tmp1 /* store_adr */,
371 new_val /* new_val */,
372 rthread /* thread */,
373 tmp2 /* tmp */,
374 tmp3 /* tmp2 */);
375 }
376 }
377
378 }
379
380 #ifdef COMPILER1
381
382 #undef __
383 #define __ ce->masm()->
384
385 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
386 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
387 // At this point we know that marking is in progress.
388 // If do_load() is true then we have to emit the
389 // load of the previous value; otherwise it has already
390 // been loaded into _pre_val.
391
392 __ bind(*stub->entry());
393
394 assert(stub->pre_val()->is_register(), "Precondition.");
395
|