1 /*
2 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 #ifdef PRODUCT
51 #define BLOCK_COMMENT(str) /* nothing */
52 #else
53 #define BLOCK_COMMENT(str) __ block_comment(str)
54 #endif
55
56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
57 Register src, Register dst, Register count, RegSet saved_regs) {
58 if (is_oop) {
59 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
60 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
61
62 Label done;
63
64 // Avoid calling runtime if count == 0
65 __ cbz(count, done);
66
67 // Is GC active?
68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
69 __ ldrb(rscratch1, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
72 } else {
73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
74 __ tst(rscratch1, rscratch2);
75 __ br(Assembler::EQ, done);
76 }
77
78 __ push(saved_regs, sp);
79 if (UseCompressedOops) {
80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count);
81 } else {
82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
83 }
84 __ pop(saved_regs, sp);
85 __ bind(done);
86 }
87 }
88 }
89
90 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
91 Register start, Register count, Register tmp) {
92 if (ShenandoahCardBarrier && is_oop) {
93 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
94 }
95 }
96
97 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
98 Register obj,
99 Register pre_val,
100 Register thread,
101 Register tmp1,
102 Register tmp2,
103 bool tosca_live,
104 bool expand_call) {
105 assert(ShenandoahSATBBarrier, "Should be checked by caller");
106
107 // If expand_call is true then we expand the call_VM_leaf macro
108 // directly to skip generating the check by
109 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
110
111 assert(thread == rthread, "must be");
112
113 Label done;
114 Label runtime;
115
116 assert_different_registers(obj, pre_val, tmp1, tmp2);
117 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
118
119 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
120 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
121
122 // Is marking active?
123 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
124 __ ldrb(tmp1, gc_state);
125 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
126
127 // Do we need to load the previous value?
128 if (obj != noreg) {
129 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
130 }
131
132 // Is the previous value null?
133 __ cbz(pre_val, done);
134
135 // Can we store original value in the thread's buffer?
136 // Is index == 0?
137 // (The index field is typed as size_t.)
138
139 __ ldr(tmp1, index); // tmp := *index_adr
140 __ cbz(tmp1, runtime); // tmp == 0?
141 // If yes, goto runtime
142
143 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
144 __ str(tmp1, index); // *index_adr := tmp
145 __ ldr(tmp2, buffer);
146 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
147
148 // Record the previous value
149 __ str(pre_val, Address(tmp1, 0));
150 __ b(done);
151
152 __ bind(runtime);
153 // save the live input values
154 RegSet saved = RegSet::of(pre_val);
155 if (tosca_live) saved += RegSet::of(r0);
156 if (obj != noreg) saved += RegSet::of(obj);
157
158 __ push(saved, sp);
159
160 // Calling the runtime using the regular call_VM_leaf mechanism generates
161 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
162 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
163 //
164 // If we care generating the pre-barrier without a frame (e.g. in the
165 // intrinsified Reference.get() routine) then rfp might be pointing to
166 // the caller frame and so this check will most likely fail at runtime.
167 //
168 // Expanding the call directly bypasses the generation of the check.
169 // So when we do not have have a full interpreter frame on the stack
170 // expand_call should be passed true.
171
172 if (expand_call) {
173 assert(pre_val != c_rarg1, "smashed arg");
174 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
175 } else {
176 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
177 }
178
179 __ pop(saved, sp);
180
181 __ bind(done);
182 }
183
184 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
185 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
186 Label is_null;
187 __ cbz(dst, is_null);
188 resolve_forward_pointer_not_null(masm, dst, tmp);
189 __ bind(is_null);
190 }
191
192 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitly
193 // passed in.
194 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
195 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
196 // The below loads the mark word, checks if the lowest two bits are
197 // set, and if so, clear the lowest two bits and copy the result
198 // to dst. Otherwise it leaves dst alone.
199 // Implementing this is surprisingly awkward. I do it here by:
200 // - Inverting the mark word
201 // - Test lowest two bits == 0
202 // - If so, set the lowest two bits
203 // - Invert the result back, and copy to dst
204
205 bool borrow_reg = (tmp == noreg);
206 if (borrow_reg) {
207 // No free registers available. Make one useful.
208 tmp = rscratch1;
209 if (tmp == dst) {
210 tmp = rscratch2;
211 }
212 __ push(RegSet::of(tmp), sp);
213 }
214
215 assert_different_registers(tmp, dst);
216
217 Label done;
218 __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
219 __ eon(tmp, tmp, zr);
220 __ ands(zr, tmp, markWord::lock_mask_in_place);
221 __ br(Assembler::NE, done);
222 __ orr(tmp, tmp, markWord::marked_value);
223 __ eon(dst, tmp, zr);
224 __ bind(done);
225
226 if (borrow_reg) {
227 __ pop(RegSet::of(tmp), sp);
228 }
229 }
230
231 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
232 assert(ShenandoahLoadRefBarrier, "Should be enabled");
233 assert(dst != rscratch2, "need rscratch2");
234 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
235
236 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
237 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
238 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
239 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
240 bool is_narrow = UseCompressedOops && !is_native;
241
242 Label heap_stable, not_cset;
243 __ enter(/*strip_ret_addr*/true);
244 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
245 __ ldrb(rscratch2, gc_state);
246
247 // Check for heap stability
248 if (is_strong) {
249 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
250 } else {
251 Label lrb;
252 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
253 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
254 __ bind(lrb);
255 }
256
257 // use r1 for load address
258 Register result_dst = dst;
259 if (dst == r1) {
260 __ mov(rscratch1, dst);
261 dst = rscratch1;
262 }
263
264 // Save r0 and r1, unless it is an output register
265 RegSet to_save = RegSet::of(r0, r1) - result_dst;
266 __ push(to_save, sp);
267 __ lea(r1, load_addr);
268 __ mov(r0, dst);
269
270 // Test for in-cset
271 if (is_strong) {
272 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
273 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
274 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
275 __ tbz(rscratch2, 0, not_cset);
276 }
277
278 __ push_call_clobbered_registers();
279 if (is_strong) {
280 if (is_narrow) {
281 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
282 } else {
283 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
284 }
285 } else if (is_weak) {
286 if (is_narrow) {
287 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
288 } else {
289 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
290 }
291 } else {
292 assert(is_phantom, "only remaining strength");
293 assert(!is_narrow, "phantom access cannot be narrow");
294 // AOT saved adapters need relocation for this call.
295 __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
296 }
297 __ blr(lr);
298 __ mov(rscratch1, r0);
299 __ pop_call_clobbered_registers();
300 __ mov(r0, rscratch1);
301
302 __ bind(not_cset);
303
304 __ mov(result_dst, r0);
305 __ pop(to_save, sp);
306
307 __ bind(heap_stable);
308 __ leave();
309 }
310
311 //
312 // Arguments:
313 //
314 // Inputs:
315 // src: oop location to load from, might be clobbered
316 //
317 // Output:
318 // dst: oop loaded from src location
319 //
320 // Kill:
321 // rscratch1 (scratch reg)
322 //
323 // Alias:
324 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
325 //
326 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
327 Register dst, Address src, Register tmp1, Register tmp2) {
328 // 1: non-reference load, no additional barrier is needed
329 if (!is_reference_type(type)) {
330 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
331 return;
332 }
333
334 // 2: load a reference from src location and apply LRB if needed
335 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
336 Register result_dst = dst;
337
338 // Preserve src location for LRB
339 if (dst == src.base() || dst == src.index()) {
340 dst = rscratch1;
341 }
342 assert_different_registers(dst, src.base(), src.index());
343
344 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
345
346 load_reference_barrier(masm, dst, src, decorators);
347
348 if (dst != result_dst) {
349 __ mov(result_dst, dst);
350 dst = result_dst;
351 }
352 } else {
353 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
354 }
355
356 // 3: apply keep-alive barrier if needed
357 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
358 __ enter(/*strip_ret_addr*/true);
359 __ push_call_clobbered_registers();
360 satb_barrier(masm /* masm */,
361 noreg /* obj */,
362 dst /* pre_val */,
363 rthread /* thread */,
364 tmp1 /* tmp1 */,
365 tmp2 /* tmp2 */,
366 true /* tosca_live */,
367 true /* expand_call */);
368 __ pop_call_clobbered_registers();
369 __ leave();
370 }
371 }
372
373 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
374 assert(ShenandoahCardBarrier, "Should have been checked by caller");
375
376 __ lsr(obj, obj, CardTable::card_shift());
377
378 assert(CardTable::dirty_card_val() == 0, "must be");
379
380 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
381 __ ldr(rscratch1, curr_ct_holder_addr);
382
383 if (UseCondCardMark) {
384 Label L_already_dirty;
385 __ ldrb(rscratch2, Address(obj, rscratch1));
386 __ cbz(rscratch2, L_already_dirty);
387 __ strb(zr, Address(obj, rscratch1));
388 __ bind(L_already_dirty);
389 } else {
390 __ strb(zr, Address(obj, rscratch1));
391 }
392 }
393
394 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
395 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
396 // 1: non-reference types require no barriers
397 if (!is_reference_type(type)) {
398 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
399 return;
400 }
401
402 // Flatten object address right away for simplicity: likely needed by barriers
403 if (dst.index() == noreg && dst.offset() == 0) {
404 if (dst.base() != tmp3) {
405 __ mov(tmp3, dst.base());
406 }
407 } else {
408 __ lea(tmp3, dst);
409 }
410
411 bool storing_non_null = (val != noreg);
412
413 // 2: pre-barrier: SATB needs the previous value
414 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
415 satb_barrier(masm,
416 tmp3 /* obj */,
417 tmp2 /* pre_val */,
418 rthread /* thread */,
419 tmp1 /* tmp */,
420 rscratch1 /* tmp2 */,
421 storing_non_null /* tosca_live */,
422 false /* expand_call */);
423 }
424
425 // Store!
426 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
427
428 // 3: post-barrier: card barrier needs store address
429 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
430 card_barrier(masm, tmp3);
431 }
432 }
433
434 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
435 Register obj, Register tmp, Label& slowpath) {
436 Label done;
437 // Resolve jobject
438 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
439
440 // Check for null.
441 __ cbz(obj, done);
442
443 assert(obj != rscratch2, "need rscratch2");
444 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
445 __ lea(rscratch2, gc_state);
446 __ ldrb(rscratch2, Address(rscratch2));
447
448 // Check for heap in evacuation phase
449 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
450
451 __ bind(done);
452 }
453
454 // Special Shenandoah CAS implementation that handles false negatives due
455 // to concurrent evacuation. The service is more complex than a
456 // traditional CAS operation because the CAS operation is intended to
457 // succeed if the reference at addr exactly matches expected or if the
458 // reference at addr holds a pointer to a from-space object that has
459 // been relocated to the location named by expected. There are two
460 // races that must be addressed:
461 // a) A parallel thread may mutate the contents of addr so that it points
462 // to a different object. In this case, the CAS operation should fail.
463 // b) A parallel thread may heal the contents of addr, replacing a
464 // from-space pointer held in addr with the to-space pointer
465 // representing the new location of the object.
466 // Upon entry to cmpxchg_oop, it is assured that new_val equals null
467 // or it refers to an object that is not being evacuated out of
468 // from-space, or it refers to the to-space version of an object that
469 // is being evacuated out of from-space.
470 //
471 // By default the value held in the result register following execution
472 // of the generated code sequence is 0 to indicate failure of CAS,
473 // non-zero to indicate success. If is_cae, the result is the value most
474 // recently fetched from addr rather than a boolean success indicator.
475 //
476 // Clobbers rscratch1, rscratch2
477 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
478 Register addr,
479 Register expected,
480 Register new_val,
481 bool acquire, bool release,
482 bool is_cae,
483 Register result) {
484 Register tmp1 = rscratch1;
485 Register tmp2 = rscratch2;
486 bool is_narrow = UseCompressedOops;
487 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
488
489 assert_different_registers(addr, expected, tmp1, tmp2);
490 assert_different_registers(addr, new_val, tmp1, tmp2);
491
492 Label step4, done;
493
494 // There are two ways to reach this label. Initial entry into the
495 // cmpxchg_oop code expansion starts at step1 (which is equivalent
496 // to label step4). Additionally, in the rare case that four steps
497 // are required to perform the requested operation, the fourth step
498 // is the same as the first. On a second pass through step 1,
499 // control may flow through step 2 on its way to failure. It will
500 // not flow from step 2 to step 3 since we are assured that the
501 // memory at addr no longer holds a from-space pointer.
502 //
503 // The comments that immediately follow the step4 label apply only
504 // to the case in which control reaches this label by branch from
505 // step 3.
506
507 __ bind (step4);
508
509 // Step 4. CAS has failed because the value most recently fetched
510 // from addr is no longer the from-space pointer held in tmp2. If a
511 // different thread replaced the in-memory value with its equivalent
512 // to-space pointer, then CAS may still be able to succeed. The
513 // value held in the expected register has not changed.
514 //
515 // It is extremely rare we reach this point. For this reason, the
516 // implementation opts for smaller rather than potentially faster
517 // code. Ultimately, smaller code for this rare case most likely
518 // delivers higher overall throughput by enabling improved icache
519 // performance.
520
521 // Step 1. Fast-path.
522 //
523 // Try to CAS with given arguments. If successful, then we are done.
524 //
525 // No label required for step 1.
526
527 __ cmpxchg(addr, expected, new_val, size, acquire, release, false, tmp2);
528 // EQ flag set iff success. tmp2 holds value fetched.
529
530 // If expected equals null but tmp2 does not equal null, the
531 // following branches to done to report failure of CAS. If both
532 // expected and tmp2 equal null, the following branches to done to
533 // report success of CAS. There's no need for a special test of
534 // expected equal to null.
535
536 __ br(Assembler::EQ, done);
537 // if CAS failed, fall through to step 2
538
539 // Step 2. CAS has failed because the value held at addr does not
540 // match expected. This may be a false negative because the value fetched
541 // from addr (now held in tmp2) may be a from-space pointer to the
542 // original copy of same object referenced by to-space pointer expected.
543 //
544 // To resolve this, it suffices to find the forward pointer associated
545 // with fetched value. If this matches expected, retry CAS with new
546 // parameters. If this mismatches, then we have a legitimate
547 // failure, and we're done.
548 //
549 // No need for step2 label.
550
551 // overwrite tmp1 with from-space pointer fetched from memory
552 __ mov(tmp1, tmp2);
553
554 if (is_narrow) {
555 // Decode tmp1 in order to resolve its forward pointer
556 __ decode_heap_oop(tmp1, tmp1);
557 }
558 resolve_forward_pointer(masm, tmp1);
559 // Encode tmp1 to compare against expected.
560 __ encode_heap_oop(tmp1, tmp1);
561
562 // Does forwarded value of fetched from-space pointer match original
563 // value of expected? If tmp1 holds null, this comparison will fail
564 // because we know from step1 that expected is not null. There is
565 // no need for a separate test for tmp1 (the value originally held
566 // in memory) equal to null.
567 __ cmp(tmp1, expected);
568
569 // If not, then the failure was legitimate and we're done.
570 // Branching to done with NE condition denotes failure.
571 __ br(Assembler::NE, done);
572
573 // Fall through to step 3. No need for step3 label.
574
575 // Step 3. We've confirmed that the value originally held in memory
576 // (now held in tmp2) pointed to from-space version of original
577 // expected value. Try the CAS again with the from-space expected
578 // value. If it now succeeds, we're good.
579 //
580 // Note: tmp2 holds encoded from-space pointer that matches to-space
581 // object residing at expected. tmp2 is the new "expected".
582
583 // Note that macro implementation of __cmpxchg cannot use same register
584 // tmp2 for result and expected since it overwrites result before it
585 // compares result with expected.
586 __ cmpxchg(addr, tmp2, new_val, size, acquire, release, false, noreg);
587 // EQ flag set iff success. tmp2 holds value fetched, tmp1 (rscratch1) clobbered.
588
589 // If fetched value did not equal the new expected, this could
590 // still be a false negative because some other thread may have
591 // newly overwritten the memory value with its to-space equivalent.
592 __ br(Assembler::NE, step4);
593
594 if (is_cae) {
595 // We're falling through to done to indicate success. Success
596 // with is_cae is denoted by returning the value of expected as
597 // result.
598 __ mov(tmp2, expected);
599 }
600
601 __ bind(done);
602 // At entry to done, the Z (EQ) flag is on iff if the CAS
603 // operation was successful. Additionally, if is_cae, tmp2 holds
604 // the value most recently fetched from addr. In this case, success
605 // is denoted by tmp2 matching expected.
606
607 if (is_cae) {
608 __ mov(result, tmp2);
609 } else {
610 __ cset(result, Assembler::EQ);
611 }
612 }
613
614 #ifdef COMPILER2
615 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) {
616 assert_different_registers(obj, addr);
617 BLOCK_COMMENT("load_ref_barrier_c2 {");
618 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
619 return;
620 }
621 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
622 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow);
623
624 // Don't preserve the obj across the runtime call, we override it from the
625 // return value anyway.
626 stub->dont_preserve(obj);
627 stub->dont_preserve(gc_state);
628
629 // Check if GC marking is in progress or we are handling a weak reference,
630 // otherwise we don't have to do anything. The code below was optimized to
631 // use less registers and instructions as possible at the expense of always
632 // having a branch instruction. The reason why we use this particular branch
633 // scheme is because the stub entry may be too far for the tbnz to jump to.
634 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
635 if (is_strong) {
636 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
637 __ b(*stub->entry());
638 } else {
639 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
640 __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
641 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
642 __ b(*stub->entry());
643 }
644
645 __ bind(*stub->continuation());
646 BLOCK_COMMENT("} load_ref_barrier_c2");
647 }
648
649 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
650 Register dst, bool dst_narrow,
651 Register src, bool src_narrow,
652 Register tmp, Register pre_val,
653 bool is_volatile) {
654
655 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
656 __ ldrb(tmp, gcs_addr);
657
658 satb_barrier_c2(node, masm, dst, pre_val, tmp, src_narrow);
659
660 card_barrier_c2(node, masm, dst, tmp);
661
662 // Need to encode into tmp, because we cannot clobber src.
663 // TODO: Maybe there is a matcher way to test that src is unused after this?
664 if (dst_narrow && !src_narrow) {
665 __ mov(tmp, src);
666 if (ShenandoahStoreBarrierStubC2::src_not_null(node)) {
667 __ encode_heap_oop_not_null(tmp);
668 } else {
669 __ encode_heap_oop(tmp);
670 }
671 src = tmp;
672 }
673
674 // Do the actual store
675 if (dst_narrow) {
676 if (is_volatile) {
677 __ stlrw(src, dst);
678 } else {
679 __ strw(src, dst);
680 }
681 } else {
682 if (is_volatile) {
683 __ stlr(src, dst);
684 } else {
685 __ str(src, dst);
686 }
687 }
688 }
689
690 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val,
691 Register gc_state, bool encoded_preval) {
692 BLOCK_COMMENT("satb_barrier_c2 {");
693 assert_different_registers(addr, pre_val, rscratch1, rscratch2);
694 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
695 return;
696 }
697 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
698 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval);
699
700 // Check if GC marking is in progress, otherwise we don't have to do
701 // anything.
702 __ tstw(gc_state, ShenandoahHeap::MARKING);
703 __ br(Assembler::NE, *stub->entry());
704 __ bind(*stub->continuation());
705 BLOCK_COMMENT("} satb_barrier_c2");
706 }
707
708 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
709 if (!ShenandoahCardBarrier ||
710 (node->barrier_data() & ShenandoahBarrierCardMark) == 0) {
711 return;
712 }
713
714 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
715 __ lsr(tmp, addr, CardTable::card_shift());
716
717 assert(CardTable::dirty_card_val() == 0, "must be");
718
719 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
720 __ ldr(rscratch1, curr_ct_holder_addr);
721
722 if (UseCondCardMark) {
723 Label L_already_dirty;
724 __ ldrb(rscratch2, Address(tmp, rscratch1));
725 __ cbz(rscratch2, L_already_dirty);
726 __ strb(zr, Address(tmp, rscratch1));
727 __ bind(L_already_dirty);
728 } else {
729 __ strb(zr, Address(tmp, rscratch1));
730 }
731 }
732
733 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
734 MacroAssembler* masm,
735 Register addr, Register oldval,
736 Register newval, Register res,
737 Register gc_state, Register tmp,
738 bool acquire, bool release, bool weak, bool exchange) {
739 assert(res != noreg, "need result register");
740 assert_different_registers(oldval, addr, res, gc_state, tmp);
741 assert_different_registers(newval, addr, res, gc_state, tmp);
742
743 // Fast-path: Try to CAS optimistically. If successful, then we are done.
744 // EQ flag set iff success. 'tmp' holds value fetched.
745 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
746 __ cmpxchg(addr, oldval, newval, size, acquire, release, weak, tmp);
747
748 // If we need a boolean result out of CAS, set the flag appropriately. This
749 // would be the final result if we do not go slow.
750 if (!exchange) {
751 __ cset(res, Assembler::EQ);
752 } else {
753 __ mov(res, tmp);
754 }
755
756 if (ShenandoahCASBarrier) {
757 ShenandoahCASBarrierSlowStubC2* const slow_stub =
758 ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, gc_state, tmp, exchange, acquire, release, weak);
759
760 slow_stub->preserve(gc_state); // this really need to be preserved as we
761 // try to use it in subsequent barriers
762
763 slow_stub->dont_preserve(res); // set at the end, no need to save
764 slow_stub->dont_preserve(oldval); // saved explicitly
765 slow_stub->dont_preserve(tmp); // temp, no need to save
766
767 // On success, we do not need any additional handling.
768 __ br(Assembler::EQ, *slow_stub->continuation());
769
770 // If GC is in progress, it is likely we need additional handling for false negatives.
771 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *slow_stub->continuation());
772 __ b(*slow_stub->entry());
773
774 // Slow stub re-enters with result set correctly.
775 __ bind(*slow_stub->continuation());
776 }
777 }
778
779 #undef __
780 #define __ masm.
781
782 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
783 BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {");
784 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
785 __ bind(*entry());
786 Register obj = _obj;
787 if (_narrow) {
788 __ decode_heap_oop(_tmp1, _obj);
789 obj = _tmp1;
790 }
791 // Weak/phantom loads always need to go to runtime.
792 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
793 // Check for object in cset.
794 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
795 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
796 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
797 __ cbz(rscratch2, *continuation());
798 }
799 {
800 SaveLiveRegisters save_registers(&masm, this);
801 if (c_rarg0 != obj) {
802 if (c_rarg0 == _addr) {
803 __ mov(rscratch1, _addr);
804 _addr = rscratch1;
805 }
806 __ mov(c_rarg0, obj);
807 }
808 __ mov(c_rarg1, _addr);
809
810 if (_narrow) {
811 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
812 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
813 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
814 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
815 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
816 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
817 }
818 } else {
819 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
820 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
821 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
822 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
823 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
824 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
825 }
826 }
827 __ blr(rscratch1);
828 __ mov(_obj, r0);
829 }
830 if (_narrow) {
831 __ encode_heap_oop(_obj);
832 }
833 __ b(*continuation());
834 BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code");
835 }
836
837 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
838 BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {");
839 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
840 __ bind(*entry());
841
842 // The tmp register that we receive is usually a register holding the
843 // "gc_state" which may be required by subsequent memory operations in their
844 // fastpath.
845 RegSet saved = RegSet::of(_tmp);
846 __ push(saved, sp);
847
848 // Do we need to load the previous value?
849 if (_addr != noreg) {
850 __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW);
851 } else {
852 if (_encoded_preval) {
853 __ decode_heap_oop(_tmp, _preval);
854 } else {
855 _tmp = _preval;
856 }
857 }
858
859 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
860 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
861 Label runtime;
862 __ ldr(rscratch1, index);
863 // If buffer is full, call into runtime.
864 __ cbz(rscratch1, runtime);
865
866 // The buffer is not full, store value into it.
867 __ sub(rscratch1, rscratch1, wordSize);
868 __ str(rscratch1, index);
869 __ ldr(rscratch2, buffer);
870 __ str(_tmp, Address(rscratch2, rscratch1));
871 __ pop(saved, sp);
872 __ b(*continuation());
873
874 // Runtime call
875 __ bind(runtime);
876 {
877 SaveLiveRegisters save_registers(&masm, this);
878 __ mov(c_rarg0, _tmp);
879 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
880 __ blr(rscratch1);
881 }
882 __ pop(saved, sp);
883 __ b(*continuation());
884 BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code");
885 }
886
887 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
888 Unimplemented();
889 }
890
891 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
892 Unimplemented();
893 }
894
895 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
896 __ bind(*entry());
897
898 // CAS has failed because the value held at addr does not match expected.
899 // This may be a false negative because the version in memory might be
900 // the from-space version of the same object we currently hold to-space
901 // reference for.
902 //
903 // To resolve this, we need to pass the location through the LRB fixup,
904 // this will make sure that the location has only to-space pointers.
905 // To avoid calling into runtime often, we cset-check the object first.
906 // We can inline most of the work here, but there is little point,
907 // as CAS failures over cset locations must be rare. This fast-slow split
908 // matches what we do for normal LRB.
909
910 // Non-strong references should always go to runtime. We do not expect
911 // CASes over non-strong locations.
912 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
913
914 Label L_final;
915
916 // (Compressed) failure witness is in _tmp2.
917 // Unpack it and check if it is in collection set.
918 // We need to backup the compressed version to use in the LRB.
919 __ mov(_result, _tmp2);
920 if (UseCompressedOops) {
921 __ decode_heap_oop(_tmp2);
922 }
923
924 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
925 __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
926 __ ldrb(_tmp1, Address(_tmp1, _tmp2));
927 __ cbz(_tmp1, L_final);
928
929 {
930 SaveLiveRegisters save_registers(&masm, this);
931 // Load up failure witness again.
932 __ mov(c_rarg0, _result);
933 if (UseCompressedOops) {
934 __ decode_heap_oop(c_rarg0);
935 }
936 __ mov(c_rarg1, _addr_reg);
937
938 if (UseCompressedOops) {
939 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
940 } else {
941 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
942 }
943 // We have called LRB to fix up the heap location. We do not care about its
944 // result, as we will just try to CAS the location again.
945 }
946
947 __ bind(L_final);
948
949 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
950 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
951
952 if (!_cae) {
953 __ cset(_result, Assembler::EQ);
954 }
955 __ b(*continuation());
956 }
957 #undef __
958 #define __ masm->
959 #endif // COMPILER2
960
961 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
962 Register start, Register count, Register scratch) {
963 assert(ShenandoahCardBarrier, "Should have been checked by caller");
964
965 Label L_loop, L_done;
966 const Register end = count;
967
968 // Zero count? Nothing to do.
969 __ cbz(count, L_done);
970
971 // end = start + count << LogBytesPerHeapOop
972 // last element address to make inclusive
973 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
974 __ sub(end, end, BytesPerHeapOop);
975 __ lsr(start, start, CardTable::card_shift());
976 __ lsr(end, end, CardTable::card_shift());
977
978 // number of bytes to copy
979 __ sub(count, end, start);
980
981 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
982 __ ldr(scratch, curr_ct_holder_addr);
983 __ add(start, start, scratch);
984 __ bind(L_loop);
985 __ strb(zr, Address(start, count));
986 __ subs(count, count, 1);
987 __ br(Assembler::GE, L_loop);
988 __ bind(L_done);
989 }
990
991 #undef __
992
993 #ifdef COMPILER1
994
995 #define __ ce->masm()->
996
997 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
998 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
999 // At this point we know that marking is in progress.
1000 // If do_load() is true then we have to emit the
1001 // load of the previous value; otherwise it has already
1002 // been loaded into _pre_val.
1003
1004 __ bind(*stub->entry());
1005
1006 assert(stub->pre_val()->is_register(), "Precondition.");
1007
1008 Register pre_val_reg = stub->pre_val()->as_register();
1009
1010 if (stub->do_load()) {
1011 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
1012 }
1013 __ cbz(pre_val_reg, *stub->continuation());
1014 ce->store_parameter(stub->pre_val()->as_register(), 0);
1015 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
1016 __ b(*stub->continuation());
1017 }
1018
1019 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
1020 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
1021 __ bind(*stub->entry());
1022
1023 DecoratorSet decorators = stub->decorators();
1024 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1025 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1026 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1027 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1028
1029 Register obj = stub->obj()->as_register();
1030 Register res = stub->result()->as_register();
1031 Register addr = stub->addr()->as_pointer_register();
1032 Register tmp1 = stub->tmp1()->as_register();
1033 Register tmp2 = stub->tmp2()->as_register();
1034
1035 assert(res == r0, "result must arrive in r0");
1036
1037 if (res != obj) {
1038 __ mov(res, obj);
1039 }
1040
1041 if (is_strong) {
1042 // Check for object in cset.
1043 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
1044 __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1045 __ ldrb(tmp2, Address(tmp2, tmp1));
1046 __ cbz(tmp2, *stub->continuation());
1047 }
1048
1049 ce->store_parameter(res, 0);
1050 ce->store_parameter(addr, 1);
1051 if (is_strong) {
1052 if (is_native) {
1053 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
1054 } else {
1055 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
1056 }
1057 } else if (is_weak) {
1058 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
1059 } else {
1060 assert(is_phantom, "only remaining strength");
1061 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
1062 }
1063
1064 __ b(*stub->continuation());
1065 }
1066
1067 #undef __
1068
1069 #define __ sasm->
1070
1071 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
1072 __ prologue("shenandoah_pre_barrier", false);
1073
1074 // arg0 : previous value of memory
1075
1076 BarrierSet* bs = BarrierSet::barrier_set();
1077
1078 const Register pre_val = r0;
1079 const Register thread = rthread;
1080 const Register tmp = rscratch1;
1081
1082 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1083 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1084
1085 Label done;
1086 Label runtime;
1087
1088 // Is marking still active?
1089 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1090 __ ldrb(tmp, gc_state);
1091 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
1092
1093 // Can we store original value in the thread's buffer?
1094 __ ldr(tmp, queue_index);
1095 __ cbz(tmp, runtime);
1096
1097 __ sub(tmp, tmp, wordSize);
1098 __ str(tmp, queue_index);
1099 __ ldr(rscratch2, buffer);
1100 __ add(tmp, tmp, rscratch2);
1101 __ load_parameter(0, rscratch2);
1102 __ str(rscratch2, Address(tmp, 0));
1103 __ b(done);
1104
1105 __ bind(runtime);
1106 __ push_call_clobbered_registers();
1107 __ load_parameter(0, pre_val);
1108 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
1109 __ pop_call_clobbered_registers();
1110 __ bind(done);
1111
1112 __ epilogue();
1113 }
1114
1115 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
1116 __ prologue("shenandoah_load_reference_barrier", false);
1117 // arg0 : object to be resolved
1118
1119 __ push_call_clobbered_registers();
1120 __ load_parameter(0, r0);
1121 __ load_parameter(1, r1);
1122
1123 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1124 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1125 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1126 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1127 if (is_strong) {
1128 if (is_native) {
1129 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
1130 } else {
1131 if (UseCompressedOops) {
1132 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
1133 } else {
1134 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
1135 }
1136 }
1137 } else if (is_weak) {
1138 assert(!is_native, "weak must not be called off-heap");
1139 if (UseCompressedOops) {
1140 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
1141 } else {
1142 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
1143 }
1144 } else {
1145 assert(is_phantom, "only remaining strength");
1146 assert(is_native, "phantom must only be called off-heap");
1147 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
1148 }
1149 __ blr(lr);
1150 __ mov(rscratch1, r0);
1151 __ pop_call_clobbered_registers();
1152 __ mov(r0, rscratch1);
1153
1154 __ epilogue();
1155 }
1156
1157 #undef __
1158
1159 #endif // COMPILER1