1 /*
2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #ifdef COMPILER1
39 #include "c1/c1_LIRAssembler.hpp"
40 #include "c1/c1_MacroAssembler.hpp"
41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
42 #endif
43
44 #define __ masm->
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
47 Register src, Register dst, Register count, RegSet saved_regs) {
48 if (is_oop) {
49 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
51
52 Label done;
53
54 // Avoid calling runtime if count == 0
55 __ cbz(count, done);
56
57 // Is GC active?
58 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
59 __ ldrb(rscratch1, gc_state);
60 if (ShenandoahSATBBarrier && dest_uninitialized) {
61 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
62 } else {
63 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
64 __ tst(rscratch1, rscratch2);
65 __ br(Assembler::EQ, done);
66 }
67
68 __ push(saved_regs, sp);
69 if (UseCompressedOops) {
70 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count);
71 } else {
72 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
73 }
74 __ pop(saved_regs, sp);
75 __ bind(done);
76 }
77 }
78 }
79
80 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
81 Register obj,
82 Register pre_val,
83 Register thread,
84 Register tmp,
85 bool tosca_live,
86 bool expand_call) {
87 if (ShenandoahSATBBarrier) {
88 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, rscratch1, tosca_live, expand_call);
89 }
90 }
91
92 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
93 Register obj,
94 Register pre_val,
95 Register thread,
96 Register tmp1,
97 Register tmp2,
98 bool tosca_live,
99 bool expand_call) {
100 // If expand_call is true then we expand the call_VM_leaf macro
101 // directly to skip generating the check by
102 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
103
104 assert(thread == rthread, "must be");
105
106 Label done;
107 Label runtime;
108
109 assert_different_registers(obj, pre_val, tmp1, tmp2);
110 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
111
112 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
113 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
114
115 // Is marking active?
116 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
117 __ ldrb(tmp1, gc_state);
118 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
119
120 // Do we need to load the previous value?
121 if (obj != noreg) {
122 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
123 }
124
125 // Is the previous value null?
126 __ cbz(pre_val, done);
127
128 // Can we store original value in the thread's buffer?
129 // Is index == 0?
130 // (The index field is typed as size_t.)
131
132 __ ldr(tmp1, index); // tmp := *index_adr
133 __ cbz(tmp1, runtime); // tmp == 0?
134 // If yes, goto runtime
135
136 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
137 __ str(tmp1, index); // *index_adr := tmp
138 __ ldr(tmp2, buffer);
139 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
140
141 // Record the previous value
142 __ str(pre_val, Address(tmp1, 0));
143 __ b(done);
144
145 __ bind(runtime);
146 // save the live input values
147 RegSet saved = RegSet::of(pre_val);
148 if (tosca_live) saved += RegSet::of(r0);
149 if (obj != noreg) saved += RegSet::of(obj);
150
151 __ push(saved, sp);
152
153 // Calling the runtime using the regular call_VM_leaf mechanism generates
154 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
155 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
156 //
157 // If we care generating the pre-barrier without a frame (e.g. in the
158 // intrinsified Reference.get() routine) then rfp might be pointing to
159 // the caller frame and so this check will most likely fail at runtime.
160 //
161 // Expanding the call directly bypasses the generation of the check.
162 // So when we do not have have a full interpreter frame on the stack
163 // expand_call should be passed true.
164
165 if (expand_call) {
166 assert(pre_val != c_rarg1, "smashed arg");
167 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread);
168 } else {
169 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread);
170 }
171
172 __ pop(saved, sp);
173
174 __ bind(done);
175 }
176
177 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
178 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
179 Label is_null;
180 __ cbz(dst, is_null);
181 resolve_forward_pointer_not_null(masm, dst, tmp);
182 __ bind(is_null);
183 }
184
185 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitly
186 // passed in.
187 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
188 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
189 // The below loads the mark word, checks if the lowest two bits are
190 // set, and if so, clear the lowest two bits and copy the result
191 // to dst. Otherwise it leaves dst alone.
192 // Implementing this is surprisingly awkward. I do it here by:
193 // - Inverting the mark word
194 // - Test lowest two bits == 0
195 // - If so, set the lowest two bits
196 // - Invert the result back, and copy to dst
197
198 bool borrow_reg = (tmp == noreg);
199 if (borrow_reg) {
200 // No free registers available. Make one useful.
201 tmp = rscratch1;
202 if (tmp == dst) {
203 tmp = rscratch2;
204 }
205 __ push(RegSet::of(tmp), sp);
206 }
207
208 assert_different_registers(tmp, dst);
209
210 Label done;
211 __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
212 __ eon(tmp, tmp, zr);
213 __ ands(zr, tmp, markWord::lock_mask_in_place);
214 __ br(Assembler::NE, done);
215 __ orr(tmp, tmp, markWord::marked_value);
216 __ eon(dst, tmp, zr);
217 __ bind(done);
218
219 if (borrow_reg) {
220 __ pop(RegSet::of(tmp), sp);
221 }
222 }
223
224 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
225 assert(ShenandoahLoadRefBarrier, "Should be enabled");
226 assert(dst != rscratch2, "need rscratch2");
227 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
228
229 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
230 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
231 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
232 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
233 bool is_narrow = UseCompressedOops && !is_native;
234
235 Label heap_stable, not_cset;
236 __ enter(/*strip_ret_addr*/true);
237 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
238 __ ldrb(rscratch2, gc_state);
239
240 // Check for heap stability
241 if (is_strong) {
242 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
243 } else {
244 Label lrb;
245 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
246 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
247 __ bind(lrb);
248 }
249
250 // use r1 for load address
251 Register result_dst = dst;
252 if (dst == r1) {
253 __ mov(rscratch1, dst);
254 dst = rscratch1;
255 }
256
257 // Save r0 and r1, unless it is an output register
258 RegSet to_save = RegSet::of(r0, r1) - result_dst;
259 __ push(to_save, sp);
260 __ lea(r1, load_addr);
261 __ mov(r0, dst);
262
263 // Test for in-cset
264 if (is_strong) {
265 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
266 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
267 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
268 __ tbz(rscratch2, 0, not_cset);
269 }
270
271 __ push_call_clobbered_registers();
272 if (is_strong) {
273 if (is_narrow) {
274 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
275 } else {
276 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
277 }
278 } else if (is_weak) {
279 if (is_narrow) {
280 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
281 } else {
282 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
283 }
284 } else {
285 assert(is_phantom, "only remaining strength");
286 assert(!is_narrow, "phantom access cannot be narrow");
287 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
288 }
289 __ blr(lr);
290 __ mov(rscratch1, r0);
291 __ pop_call_clobbered_registers();
292 __ mov(r0, rscratch1);
293
294 __ bind(not_cset);
295
296 __ mov(result_dst, r0);
297 __ pop(to_save, sp);
298
299 __ bind(heap_stable);
300 __ leave();
301 }
302
303 //
304 // Arguments:
305 //
306 // Inputs:
307 // src: oop location to load from, might be clobbered
308 //
309 // Output:
310 // dst: oop loaded from src location
311 //
312 // Kill:
313 // rscratch1 (scratch reg)
314 //
315 // Alias:
316 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
317 //
318 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
319 Register dst, Address src, Register tmp1, Register tmp2) {
320 // 1: non-reference load, no additional barrier is needed
321 if (!is_reference_type(type)) {
322 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
323 return;
324 }
325
326 // 2: load a reference from src location and apply LRB if needed
327 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
328 Register result_dst = dst;
329
330 // Preserve src location for LRB
331 if (dst == src.base() || dst == src.index()) {
332 dst = rscratch1;
333 }
334 assert_different_registers(dst, src.base(), src.index());
335
336 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
337
338 load_reference_barrier(masm, dst, src, decorators);
339
340 if (dst != result_dst) {
341 __ mov(result_dst, dst);
342 dst = result_dst;
343 }
344 } else {
345 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
346 }
347
348 // 3: apply keep-alive barrier if needed
349 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
350 __ enter(/*strip_ret_addr*/true);
351 __ push_call_clobbered_registers();
352 satb_write_barrier_pre(masm /* masm */,
353 noreg /* obj */,
354 dst /* pre_val */,
355 rthread /* thread */,
356 tmp1 /* tmp1 */,
357 tmp2 /* tmp2 */,
358 true /* tosca_live */,
359 true /* expand_call */);
360 __ pop_call_clobbered_registers();
361 __ leave();
362 }
363 }
364
365 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
366 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
367 bool on_oop = is_reference_type(type);
368 if (!on_oop) {
369 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
370 return;
371 }
372
373 // flatten object address if needed
374 if (dst.index() == noreg && dst.offset() == 0) {
375 if (dst.base() != tmp3) {
376 __ mov(tmp3, dst.base());
377 }
378 } else {
379 __ lea(tmp3, dst);
380 }
381
382 shenandoah_write_barrier_pre(masm,
383 tmp3 /* obj */,
384 tmp2 /* pre_val */,
385 rthread /* thread */,
386 tmp1 /* tmp */,
387 val != noreg /* tosca_live */,
388 false /* expand_call */);
389
390 if (val == noreg) {
391 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
392 } else {
393 // Barrier needs uncompressed oop for region cross check.
394 Register new_val = val;
395 if (UseCompressedOops) {
396 new_val = rscratch2;
397 __ mov(new_val, val);
398 }
399 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
400 }
401
402 }
403
404 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
405 Register obj, Register tmp, Label& slowpath) {
406 Label done;
407 // Resolve jobject
408 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
409
410 // Check for null.
411 __ cbz(obj, done);
412
413 assert(obj != rscratch2, "need rscratch2");
414 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
415 __ lea(rscratch2, gc_state);
416 __ ldrb(rscratch2, Address(rscratch2));
417
418 // Check for heap in evacuation phase
419 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
420
421 __ bind(done);
422 }
423
424 // Special Shenandoah CAS implementation that handles false negatives due
425 // to concurrent evacuation. The service is more complex than a
426 // traditional CAS operation because the CAS operation is intended to
427 // succeed if the reference at addr exactly matches expected or if the
428 // reference at addr holds a pointer to a from-space object that has
429 // been relocated to the location named by expected. There are two
430 // races that must be addressed:
431 // a) A parallel thread may mutate the contents of addr so that it points
432 // to a different object. In this case, the CAS operation should fail.
433 // b) A parallel thread may heal the contents of addr, replacing a
434 // from-space pointer held in addr with the to-space pointer
435 // representing the new location of the object.
436 // Upon entry to cmpxchg_oop, it is assured that new_val equals null
437 // or it refers to an object that is not being evacuated out of
438 // from-space, or it refers to the to-space version of an object that
439 // is being evacuated out of from-space.
440 //
441 // By default the value held in the result register following execution
442 // of the generated code sequence is 0 to indicate failure of CAS,
443 // non-zero to indicate success. If is_cae, the result is the value most
444 // recently fetched from addr rather than a boolean success indicator.
445 //
446 // Clobbers rscratch1, rscratch2
447 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
448 Register addr,
449 Register expected,
450 Register new_val,
451 bool acquire, bool release,
452 bool is_cae,
453 Register result) {
454 Register tmp1 = rscratch1;
455 Register tmp2 = rscratch2;
456 bool is_narrow = UseCompressedOops;
457 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
458
459 assert_different_registers(addr, expected, tmp1, tmp2);
460 assert_different_registers(addr, new_val, tmp1, tmp2);
461
462 Label step4, done;
463
464 // There are two ways to reach this label. Initial entry into the
465 // cmpxchg_oop code expansion starts at step1 (which is equivalent
466 // to label step4). Additionally, in the rare case that four steps
467 // are required to perform the requested operation, the fourth step
468 // is the same as the first. On a second pass through step 1,
469 // control may flow through step 2 on its way to failure. It will
470 // not flow from step 2 to step 3 since we are assured that the
471 // memory at addr no longer holds a from-space pointer.
472 //
473 // The comments that immediately follow the step4 label apply only
474 // to the case in which control reaches this label by branch from
475 // step 3.
476
477 __ bind (step4);
478
479 // Step 4. CAS has failed because the value most recently fetched
480 // from addr is no longer the from-space pointer held in tmp2. If a
481 // different thread replaced the in-memory value with its equivalent
482 // to-space pointer, then CAS may still be able to succeed. The
483 // value held in the expected register has not changed.
484 //
485 // It is extremely rare we reach this point. For this reason, the
486 // implementation opts for smaller rather than potentially faster
487 // code. Ultimately, smaller code for this rare case most likely
488 // delivers higher overall throughput by enabling improved icache
489 // performance.
490
491 // Step 1. Fast-path.
492 //
493 // Try to CAS with given arguments. If successful, then we are done.
494 //
495 // No label required for step 1.
496
497 __ cmpxchg(addr, expected, new_val, size, acquire, release, false, tmp2);
498 // EQ flag set iff success. tmp2 holds value fetched.
499
500 // If expected equals null but tmp2 does not equal null, the
501 // following branches to done to report failure of CAS. If both
502 // expected and tmp2 equal null, the following branches to done to
503 // report success of CAS. There's no need for a special test of
504 // expected equal to null.
505
506 __ br(Assembler::EQ, done);
507 // if CAS failed, fall through to step 2
508
509 // Step 2. CAS has failed because the value held at addr does not
510 // match expected. This may be a false negative because the value fetched
511 // from addr (now held in tmp2) may be a from-space pointer to the
512 // original copy of same object referenced by to-space pointer expected.
513 //
514 // To resolve this, it suffices to find the forward pointer associated
515 // with fetched value. If this matches expected, retry CAS with new
516 // parameters. If this mismatches, then we have a legitimate
517 // failure, and we're done.
518 //
519 // No need for step2 label.
520
521 // overwrite tmp1 with from-space pointer fetched from memory
522 __ mov(tmp1, tmp2);
523
524 if (is_narrow) {
525 // Decode tmp1 in order to resolve its forward pointer
526 __ decode_heap_oop(tmp1, tmp1);
527 }
528 resolve_forward_pointer(masm, tmp1);
529 // Encode tmp1 to compare against expected.
530 __ encode_heap_oop(tmp1, tmp1);
531
532 // Does forwarded value of fetched from-space pointer match original
533 // value of expected? If tmp1 holds null, this comparison will fail
534 // because we know from step1 that expected is not null. There is
535 // no need for a separate test for tmp1 (the value originally held
536 // in memory) equal to null.
537 __ cmp(tmp1, expected);
538
539 // If not, then the failure was legitimate and we're done.
540 // Branching to done with NE condition denotes failure.
541 __ br(Assembler::NE, done);
542
543 // Fall through to step 3. No need for step3 label.
544
545 // Step 3. We've confirmed that the value originally held in memory
546 // (now held in tmp2) pointed to from-space version of original
547 // expected value. Try the CAS again with the from-space expected
548 // value. If it now succeeds, we're good.
549 //
550 // Note: tmp2 holds encoded from-space pointer that matches to-space
551 // object residing at expected. tmp2 is the new "expected".
552
553 // Note that macro implementation of __cmpxchg cannot use same register
554 // tmp2 for result and expected since it overwrites result before it
555 // compares result with expected.
556 __ cmpxchg(addr, tmp2, new_val, size, acquire, release, false, noreg);
557 // EQ flag set iff success. tmp2 holds value fetched, tmp1 (rscratch1) clobbered.
558
559 // If fetched value did not equal the new expected, this could
560 // still be a false negative because some other thread may have
561 // newly overwritten the memory value with its to-space equivalent.
562 __ br(Assembler::NE, step4);
563
564 if (is_cae) {
565 // We're falling through to done to indicate success. Success
566 // with is_cae is denoted by returning the value of expected as
567 // result.
568 __ mov(tmp2, expected);
569 }
570
571 __ bind(done);
572 // At entry to done, the Z (EQ) flag is on iff if the CAS
573 // operation was successful. Additionally, if is_cae, tmp2 holds
574 // the value most recently fetched from addr. In this case, success
575 // is denoted by tmp2 matching expected.
576
577 if (is_cae) {
578 __ mov(result, tmp2);
579 } else {
580 __ cset(result, Assembler::EQ);
581 }
582 }
583
584 #undef __
585
586 #ifdef COMPILER1
587
588 #define __ ce->masm()->
589
590 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
591 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
592 // At this point we know that marking is in progress.
593 // If do_load() is true then we have to emit the
594 // load of the previous value; otherwise it has already
595 // been loaded into _pre_val.
596
597 __ bind(*stub->entry());
598
599 assert(stub->pre_val()->is_register(), "Precondition.");
600
601 Register pre_val_reg = stub->pre_val()->as_register();
602
603 if (stub->do_load()) {
604 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
605 }
606 __ cbz(pre_val_reg, *stub->continuation());
607 ce->store_parameter(stub->pre_val()->as_register(), 0);
608 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
609 __ b(*stub->continuation());
610 }
611
612 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
613 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
614 __ bind(*stub->entry());
615
616 DecoratorSet decorators = stub->decorators();
617 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
618 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
619 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
620 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
621
622 Register obj = stub->obj()->as_register();
623 Register res = stub->result()->as_register();
624 Register addr = stub->addr()->as_pointer_register();
625 Register tmp1 = stub->tmp1()->as_register();
626 Register tmp2 = stub->tmp2()->as_register();
627
628 assert(res == r0, "result must arrive in r0");
629
630 if (res != obj) {
631 __ mov(res, obj);
632 }
633
634 if (is_strong) {
635 // Check for object in cset.
636 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
637 __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
638 __ ldrb(tmp2, Address(tmp2, tmp1));
639 __ cbz(tmp2, *stub->continuation());
640 }
641
642 ce->store_parameter(res, 0);
643 ce->store_parameter(addr, 1);
644 if (is_strong) {
645 if (is_native) {
646 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
647 } else {
648 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
649 }
650 } else if (is_weak) {
651 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
652 } else {
653 assert(is_phantom, "only remaining strength");
654 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
655 }
656
657 __ b(*stub->continuation());
658 }
659
660 #undef __
661
662 #define __ sasm->
663
664 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
665 __ prologue("shenandoah_pre_barrier", false);
666
667 // arg0 : previous value of memory
668
669 BarrierSet* bs = BarrierSet::barrier_set();
670
671 const Register pre_val = r0;
672 const Register thread = rthread;
673 const Register tmp = rscratch1;
674
675 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
676 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
677
678 Label done;
679 Label runtime;
680
681 // Is marking still active?
682 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
683 __ ldrb(tmp, gc_state);
684 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
685
686 // Can we store original value in the thread's buffer?
687 __ ldr(tmp, queue_index);
688 __ cbz(tmp, runtime);
689
690 __ sub(tmp, tmp, wordSize);
691 __ str(tmp, queue_index);
692 __ ldr(rscratch2, buffer);
693 __ add(tmp, tmp, rscratch2);
694 __ load_parameter(0, rscratch2);
695 __ str(rscratch2, Address(tmp, 0));
696 __ b(done);
697
698 __ bind(runtime);
699 __ push_call_clobbered_registers();
700 __ load_parameter(0, pre_val);
701 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread);
702 __ pop_call_clobbered_registers();
703 __ bind(done);
704
705 __ epilogue();
706 }
707
708 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
709 __ prologue("shenandoah_load_reference_barrier", false);
710 // arg0 : object to be resolved
711
712 __ push_call_clobbered_registers();
713 __ load_parameter(0, r0);
714 __ load_parameter(1, r1);
715
716 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
717 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
718 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
719 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
720 if (is_strong) {
721 if (is_native) {
722 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
723 } else {
724 if (UseCompressedOops) {
725 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
726 } else {
727 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
728 }
729 }
730 } else if (is_weak) {
731 assert(!is_native, "weak must not be called off-heap");
732 if (UseCompressedOops) {
733 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
734 } else {
735 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
736 }
737 } else {
738 assert(is_phantom, "only remaining strength");
739 assert(is_native, "phantom must only be called off-heap");
740 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
741 }
742 __ blr(lr);
743 __ mov(rscratch1, r0);
744 __ pop_call_clobbered_registers();
745 __ mov(r0, rscratch1);
746
747 __ epilogue();
748 }
749
750 #undef __
751
752 #endif // COMPILER1