1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
49 Register src, Register dst, Register count) {
50
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52
53 if (is_reference_type(type)) {
54 if (ShenandoahCardBarrier) {
55 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
56 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
57 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
58
59 // We need to save the original element count because the array copy stub
60 // will destroy the value and we need it for the card marking barrier.
61 if (!checkcast) {
62 if (!obj_int) {
63 // Save count for barrier
64 __ movptr(r11, count);
65 } else if (disjoint) {
66 // Save dst in r11 in the disjoint case
67 __ movq(r11, dst);
68 }
69 }
70 }
71
72 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
73 Register thread = r15_thread;
74 assert_different_registers(src, dst, count, thread);
75
76 Label L_done;
77 // Short-circuit if count == 0.
78 __ testptr(count, count);
79 __ jcc(Assembler::zero, L_done);
80
81 // Avoid runtime call when not active.
82 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
83 int flags;
84 if (ShenandoahSATBBarrier && dest_uninitialized) {
85 flags = ShenandoahHeap::HAS_FORWARDED;
86 } else {
87 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
88 }
89 __ testb(gc_state, flags);
90 __ jcc(Assembler::zero, L_done);
91
92 __ push_call_clobbered_registers(/* save_fpu = */ false);
93 // If arguments are not in proper places, shuffle them.
94 // Doing this via the stack is the most straight-forward way to avoid
95 // accidentally smashing any register.
96 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
97 __ push(src);
98 __ push(dst);
99 __ push(count);
100 __ pop(c_rarg2);
101 __ pop(c_rarg1);
102 __ pop(c_rarg0);
103 }
104 address target = nullptr;
105 if (UseCompressedOops) {
106 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
107 } else {
108 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
109 }
110 __ call_VM_leaf(target, 3);
111
112 __ pop_call_clobbered_registers(/* restore_fpu = */ false);
113
114 __ bind(L_done);
115 }
116 }
117
118 }
119
120 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
121 Register src, Register dst, Register count) {
122
123 if (ShenandoahCardBarrier && is_reference_type(type)) {
124 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
125 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
126 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
127 Register tmp = rax;
128
129 if (!checkcast) {
130 if (!obj_int) {
131 // Save count for barrier
132 count = r11;
133 } else if (disjoint) {
134 // Use the saved dst in the disjoint case
135 dst = r11;
136 }
137 } else {
138 tmp = rscratch1;
139 }
140 gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
141 }
142 }
143
144 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
145 Register obj,
146 Register pre_val,
147 Register tmp) {
148 assert(ShenandoahSATBBarrier, "Should be checked by caller");
149 const Register thread = r15_thread;
150
151 Label done;
152 Label runtime;
153
154 assert(pre_val != noreg, "check this code");
155
156 if (obj != noreg) {
157 assert_different_registers(obj, pre_val, tmp);
158 assert(pre_val != rax, "check this code");
159 }
160
161 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
162 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
163
164 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
165 __ testb(gc_state, ShenandoahHeap::MARKING);
166 __ jcc(Assembler::zero, done);
167
168 // Do we need to load the previous value?
169 if (obj != noreg) {
170 if (UseCompressedOops) {
171 __ movl(pre_val, Address(obj, 0));
172 __ decode_heap_oop(pre_val);
173 } else {
174 __ movq(pre_val, Address(obj, 0));
175 }
176 }
177
178 // Is the previous value null?
179 __ cmpptr(pre_val, NULL_WORD);
180 __ jcc(Assembler::equal, done);
181
182 // Can we store original value in the thread's buffer?
183 // Is index == 0?
184 // (The index field is typed as size_t.)
185
186 __ movptr(tmp, index); // tmp := *index_adr
187 __ cmpptr(tmp, 0); // tmp == 0?
188 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
189
190 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
191 __ movptr(index, tmp); // *index_adr := tmp
192 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
193
194 // Record the previous value
195 __ movptr(Address(tmp, 0), pre_val);
196 __ jmp(done);
197
198 __ bind(runtime);
199
200 // Slow-path call.
201 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
202 __ enter();
203 __ push_call_clobbered_registers(/* save_fpu = */ true);
204
205 assert(thread != c_rarg0, "smashed arg");
206 if (c_rarg0 != pre_val) {
207 __ mov(c_rarg0, pre_val);
208 }
209
210 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
211 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
212
213 __ pop_call_clobbered_registers(/* restore_fpu = */ true);
214 __ leave();
215
216 __ bind(done);
217 }
218
219 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) {
220 assert(ShenandoahLoadRefBarrier, "Should be enabled");
221
222 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
223 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
224 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
225 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
226 bool is_narrow = UseCompressedOops && !is_native;
227
228 Label heap_stable, not_cset;
229
230 __ block_comment("load_reference_barrier { ");
231
232 // Check if GC is active
233 Register thread = r15_thread;
234
235 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
236 int flags = ShenandoahHeap::HAS_FORWARDED;
237 if (!is_strong) {
238 flags |= ShenandoahHeap::WEAK_ROOTS;
239 }
240 __ testb(gc_state, flags);
241 __ jcc(Assembler::zero, heap_stable);
242
243 Register tmp1 = noreg, tmp2 = noreg;
244 if (is_strong) {
245 // Test for object in cset
246 // Allocate temporary registers
247 for (int i = 0; i < Register::available_gp_registers(); i++) {
248 Register r = as_Register(i);
249 if (r != rsp && r != rbp && r != rcx && r != dst && r != src.base() && r != src.index() ) {
250 if (tmp1 == noreg) {
251 tmp1 = r;
252 } else {
253 tmp2 = r;
254 break;
255 }
256 }
257 }
258 assert(tmp1 != noreg, "tmp1 allocated");
259 assert(tmp2 != noreg, "tmp2 allocated");
260 assert_different_registers(tmp1, tmp2, src.base(), src.index());
261 assert_different_registers(tmp1, tmp2, dst);
262
263 __ push(tmp1);
264 __ push(tmp2);
265
266 // Optimized cset-test
267 __ movptr(tmp1, dst);
268 if (AOTCodeCache::is_on_for_dump()) {
269 assert_different_registers(tmp1, tmp2, rcx);
270 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
271 __ push(rcx);
272 __ movb(rcx, Address(tmp2));
273 __ shrptr(tmp1);
274 __ pop(rcx);
275 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
276 __ movptr(tmp2, Address(tmp2));
277 } else {
278 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
279 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
280 }
281 __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1));
282 __ testbool(tmp1);
283 __ jcc(Assembler::zero, not_cset);
284 }
285
286 // Slow-path call.
287 // Save registers that can be clobbered by call.
288 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
289 __ enter();
290 if (dst != rax) {
291 __ push(rax);
292 }
293 __ push_call_clobbered_registers_except(rax, /* save_fpu = */ true);
294
295 // Shuffle registers such that dst is in c_rarg0 and addr in c_rarg1.
296 if (dst == c_rarg1) {
297 __ lea(c_rarg0, src);
298 __ xchgptr(c_rarg1, c_rarg0);
299 } else {
300 __ lea(c_rarg1, src);
301 __ movptr(c_rarg0, dst);
302 }
303
304 address target = nullptr;
305 if (is_strong) {
306 if (is_narrow) {
307 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
308 } else {
309 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
310 }
311 } else if (is_weak) {
312 if (is_narrow) {
313 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
314 } else {
315 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
316 }
317 } else {
318 assert(is_phantom, "only remaining strength");
319 assert(!is_narrow, "phantom access cannot be narrow");
320 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
321 }
322
323 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
324 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
325 __ pop_call_clobbered_registers_except(rax, /* restore_fpu = */ true);
326 if (dst != rax) {
327 __ movptr(dst, rax);
328 __ pop(rax);
329 }
330 __ leave();
331
332 __ bind(not_cset);
333
334 if (is_strong) {
335 __ pop(tmp2);
336 __ pop(tmp1);
337 }
338
339 __ bind(heap_stable);
340
341 __ block_comment("} load_reference_barrier");
342 }
343
344 //
345 // Arguments:
346 //
347 // Inputs:
348 // src: oop location, might be clobbered
349 // tmp1: scratch register, might not be valid.
350 //
351 // Output:
352 // dst: oop loaded from src location
353 //
354 // Kill:
355 // tmp1 (if it is valid)
356 //
357 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
358 Register dst, Address src, Register tmp1) {
359 // 1: non-reference load, no additional barrier is needed
360 if (!is_reference_type(type)) {
361 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
362 return;
363 }
364
365 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected");
366
367 // 2: load a reference from src location and apply LRB if needed
368 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
369 Register result_dst = dst;
370 bool use_tmp1_for_dst = false;
371
372 // Preserve src location for LRB
373 if (dst == src.base() || dst == src.index()) {
374 // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
375 if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
376 dst = tmp1;
377 use_tmp1_for_dst = true;
378 } else {
379 dst = rdi;
380 __ push(dst);
381 }
382 assert_different_registers(dst, src.base(), src.index());
383 }
384
385 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
386
387 load_reference_barrier(masm, dst, src, decorators);
388
389 // Move loaded oop to final destination
390 if (dst != result_dst) {
391 __ movptr(result_dst, dst);
392
393 if (!use_tmp1_for_dst) {
394 __ pop(dst);
395 }
396
397 dst = result_dst;
398 }
399 } else {
400 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
401 }
402
403 // 3: apply keep-alive barrier if needed
404 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
405 satb_barrier(masm /* masm */,
406 noreg /* obj */,
407 dst /* pre_val */,
408 tmp1 /* tmp */);
409 }
410 }
411
412 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
413 assert(ShenandoahCardBarrier, "Should have been checked by caller");
414
415 // Does a store check for the oop in register obj. The content of
416 // register obj is destroyed afterwards.
417 __ shrptr(obj, CardTable::card_shift());
418
419 // We'll use this register as the TLS base address and also later on
420 // to hold the byte_map_base.
421 Register thread = r15_thread;
422 Register tmp = rscratch1;
423
424 Address curr_ct_holder_addr(thread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
425 __ movptr(tmp, curr_ct_holder_addr);
426 Address card_addr(tmp, obj, Address::times_1);
427
428 int dirty = CardTable::dirty_card_val();
429 if (UseCondCardMark) {
430 Label L_already_dirty;
431 __ cmpb(card_addr, dirty);
432 __ jccb(Assembler::equal, L_already_dirty);
433 __ movb(card_addr, dirty);
434 __ bind(L_already_dirty);
435 } else {
436 __ movb(card_addr, dirty);
437 }
438 }
439
440 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
441 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
442
443 // 1: non-reference types require no barriers
444 if (!is_reference_type(type)) {
445 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
446 return;
447 }
448
449 // Flatten object address right away for simplicity: likely needed by barriers
450 assert_different_registers(val, tmp1, tmp2, tmp3, r15_thread);
451 if (dst.index() == noreg && dst.disp() == 0) {
452 if (dst.base() != tmp1) {
453 __ movptr(tmp1, dst.base());
454 }
455 } else {
456 __ lea(tmp1, dst);
457 }
458
459 // 2: pre-barrier: SATB needs the previous value
460 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
461 satb_barrier(masm,
462 tmp1 /* obj */,
463 tmp2 /* pre_val */,
464 tmp3 /* tmp */);
465 }
466
467 // Store!
468 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
469
470 // 3: post-barrier: card barrier needs store address
471 bool storing_non_null = (val != noreg);
472 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
473 card_barrier(masm, tmp1);
474 }
475 }
476
477 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
478 Register obj, Register tmp, Label& slowpath) {
479 Label done;
480 // Resolve jobject
481 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
482
483 // Check for null.
484 __ testptr(obj, obj);
485 __ jcc(Assembler::zero, done);
486
487 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
488 __ testb(gc_state, ShenandoahHeap::EVACUATION);
489 __ jccb(Assembler::notZero, slowpath);
490 __ bind(done);
491 }
492
493 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slowpath) {
494 Label done;
495
496 // Peek weak handle using the standard implementation.
497 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, slowpath);
498
499 // Check if the reference is null, and if it is, take the fast path.
500 __ testptr(obj, obj);
501 __ jcc(Assembler::zero, done);
502
503 Address gc_state(r15_thread, ShenandoahThreadLocalData::gc_state_offset());
504
505 // Check if the heap is under weak-reference/roots processing, in
506 // which case we need to take the slow path.
507 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
508 __ jcc(Assembler::notZero, slowpath);
509 __ bind(done);
510 }
511
512 // Special Shenandoah CAS implementation that handles false negatives
513 // due to concurrent evacuation.
514 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
515 Register res, Address addr, Register oldval, Register newval,
516 bool exchange, Register tmp1, Register tmp2) {
517 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
518 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
519 assert_different_registers(oldval, tmp1, tmp2);
520 assert_different_registers(newval, tmp1, tmp2);
521
522 Label L_success, L_failure;
523
524 // Remember oldval for retry logic below
525 if (UseCompressedOops) {
526 __ movl(tmp1, oldval);
527 } else {
528 __ movptr(tmp1, oldval);
529 }
530
531 // Step 1. Fast-path.
532 //
533 // Try to CAS with given arguments. If successful, then we are done.
534
535 if (UseCompressedOops) {
536 __ lock();
537 __ cmpxchgl(newval, addr);
538 } else {
539 __ lock();
540 __ cmpxchgptr(newval, addr);
541 }
542 __ jcc(Assembler::equal, L_success);
543
544 // Step 2. CAS had failed. This may be a false negative.
545 //
546 // The trouble comes when we compare the to-space pointer with the from-space
547 // pointer to the same object. To resolve this, it will suffice to resolve
548 // the value from memory -- this will give both to-space pointers.
549 // If they mismatch, then it was a legitimate failure.
550 //
551 // Before reaching to resolve sequence, see if we can avoid the whole shebang
552 // with filters.
553
554 // Filter: when offending in-memory value is null, the failure is definitely legitimate
555 __ testptr(oldval, oldval);
556 __ jcc(Assembler::zero, L_failure);
557
558 // Filter: when heap is stable, the failure is definitely legitimate
559 const Register thread = r15_thread;
560 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
561 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
562 __ jcc(Assembler::zero, L_failure);
563
564 if (UseCompressedOops) {
565 __ movl(tmp2, oldval);
566 __ decode_heap_oop(tmp2);
567 } else {
568 __ movptr(tmp2, oldval);
569 }
570
571 // Decode offending in-memory value.
572 // Test if-forwarded
573 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
574 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded
575 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded
576
577 // Load and mask forwarding pointer
578 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
579 __ shrptr(tmp2, 2);
580 __ shlptr(tmp2, 2);
581
582 if (UseCompressedOops) {
583 __ decode_heap_oop(tmp1); // decode for comparison
584 }
585
586 // Now we have the forwarded offender in tmp2.
587 // Compare and if they don't match, we have legitimate failure
588 __ cmpptr(tmp1, tmp2);
589 __ jcc(Assembler::notEqual, L_failure);
590
591 // Step 3. Need to fix the memory ptr before continuing.
592 //
593 // At this point, we have from-space oldval in the register, and its to-space
594 // address is in tmp2. Let's try to update it into memory. We don't care if it
595 // succeeds or not. If it does, then the retrying CAS would see it and succeed.
596 // If this fixup fails, this means somebody else beat us to it, and necessarily
597 // with to-space ptr store. We still have to do the retry, because the GC might
598 // have updated the reference for us.
599
600 if (UseCompressedOops) {
601 __ encode_heap_oop(tmp2); // previously decoded at step 2.
602 }
603
604 if (UseCompressedOops) {
605 __ lock();
606 __ cmpxchgl(tmp2, addr);
607 } else {
608 __ lock();
609 __ cmpxchgptr(tmp2, addr);
610 }
611
612 // Step 4. Try to CAS again.
613 //
614 // This is guaranteed not to have false negatives, because oldval is definitely
615 // to-space, and memory pointer is to-space as well. Nothing is able to store
616 // from-space ptr into memory anymore. Make sure oldval is restored, after being
617 // garbled during retries.
618 //
619 if (UseCompressedOops) {
620 __ movl(oldval, tmp2);
621 } else {
622 __ movptr(oldval, tmp2);
623 }
624
625 if (UseCompressedOops) {
626 __ lock();
627 __ cmpxchgl(newval, addr);
628 } else {
629 __ lock();
630 __ cmpxchgptr(newval, addr);
631 }
632 if (!exchange) {
633 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
634 }
635
636 // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
637 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
638 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
639
640 if (exchange) {
641 __ bind(L_failure);
642 __ bind(L_success);
643 } else {
644 assert(res != noreg, "need result register");
645
646 Label exit;
647 __ bind(L_failure);
648 __ xorptr(res, res);
649 __ jmpb(exit);
650
651 __ bind(L_success);
652 __ movptr(res, 1);
653 __ bind(exit);
654 }
655 }
656
657 #ifdef PRODUCT
658 #define BLOCK_COMMENT(str) /* nothing */
659 #else
660 #define BLOCK_COMMENT(str) __ block_comment(str)
661 #endif
662
663 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
664
665 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
666
667 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
668 Register addr, Register count,
669 Register tmp) {
670 assert(ShenandoahCardBarrier, "Should have been checked by caller");
671
672 Label L_loop, L_done;
673 const Register end = count;
674 assert_different_registers(addr, end);
675
676 // Zero count? Nothing to do.
677 __ testl(count, count);
678 __ jccb(Assembler::zero, L_done);
679
680 const Register thread = r15_thread;
681 Address curr_ct_holder_addr(thread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
682 __ movptr(tmp, curr_ct_holder_addr);
683
684 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
685 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
686 __ shrptr(addr, CardTable::card_shift());
687 __ shrptr(end, CardTable::card_shift());
688 __ subptr(end, addr); // end --> cards count
689
690 __ addptr(addr, tmp);
691
692 __ BIND(L_loop);
693 __ movb(Address(addr, count, Address::times_1), 0);
694 __ decrement(count);
695 __ jccb(Assembler::greaterEqual, L_loop);
696
697 __ BIND(L_done);
698 }
699
700 #undef __
701
702 #ifdef COMPILER1
703
704 #define __ ce->masm()->
705
706 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
707 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
708 // At this point we know that marking is in progress.
709 // If do_load() is true then we have to emit the
710 // load of the previous value; otherwise it has already
711 // been loaded into _pre_val.
712
713 __ bind(*stub->entry());
714 assert(stub->pre_val()->is_register(), "Precondition.");
715
716 Register pre_val_reg = stub->pre_val()->as_register();
717
718 if (stub->do_load()) {
719 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
720 }
721
722 __ cmpptr(pre_val_reg, NULL_WORD);
723 __ jcc(Assembler::equal, *stub->continuation());
724 ce->store_parameter(stub->pre_val()->as_register(), 0);
725 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
726 __ jmp(*stub->continuation());
727
728 }
729
730 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
731 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
732 __ bind(*stub->entry());
733
734 DecoratorSet decorators = stub->decorators();
735 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
736 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
737 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
738 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
739
740 Register obj = stub->obj()->as_register();
741 Register res = stub->result()->as_register();
742 Register addr = stub->addr()->as_pointer_register();
743 Register tmp1 = stub->tmp1()->as_register();
744 Register tmp2 = stub->tmp2()->as_register();
745 assert_different_registers(obj, res, addr, tmp1, tmp2);
746
747 Label slow_path;
748
749 assert(res == rax, "result must arrive in rax");
750
751 if (res != obj) {
752 __ mov(res, obj);
753 }
754
755 if (is_strong) {
756 // Check for object being in the collection set.
757 __ mov(tmp1, res);
758 if (AOTCodeCache::is_on_for_dump()) {
759 __ push(rcx);
760 __ lea(rcx, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
761 __ movl(rcx, Address(rcx));
762 if (tmp1 != rcx) {
763 __ mov(tmp1, res);
764 __ shrptr(tmp1);
765 __ pop(rcx);
766 } else {
767 assert_different_registers(tmp2, rcx);
768 __ mov(tmp2, res);
769 __ shrptr(tmp2);
770 __ pop(rcx);
771 __ movptr(tmp1, tmp2);
772 }
773 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
774 __ movptr(tmp2, Address(tmp2));
775 } else {
776 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
777 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
778 }
779 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
780 __ testbool(tmp2);
781 __ jcc(Assembler::zero, *stub->continuation());
782 }
783
784 __ bind(slow_path);
785 ce->store_parameter(res, 0);
786 ce->store_parameter(addr, 1);
787 if (is_strong) {
788 if (is_native) {
789 __ call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
790 } else {
791 __ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
792 }
793 } else if (is_weak) {
794 __ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
795 } else {
796 assert(is_phantom, "only remaining strength");
797 __ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
798 }
799 __ jmp(*stub->continuation());
800 }
801
802 #undef __
803
804 #define __ sasm->
805
806 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
807 __ prologue("shenandoah_pre_barrier", false);
808 // arg0 : previous value of memory
809
810 __ push(rax);
811 __ push(rdx);
812
813 const Register pre_val = rax;
814 const Register thread = r15_thread;
815 const Register tmp = rdx;
816
817 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
818 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
819
820 Label done;
821 Label runtime;
822
823 // Is SATB still active?
824 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
825 __ testb(gc_state, ShenandoahHeap::MARKING);
826 __ jcc(Assembler::zero, done);
827
828 // Can we store original value in the thread's buffer?
829
830 __ movptr(tmp, queue_index);
831 __ testptr(tmp, tmp);
832 __ jcc(Assembler::zero, runtime);
833 __ subptr(tmp, wordSize);
834 __ movptr(queue_index, tmp);
835 __ addptr(tmp, buffer);
836
837 // prev_val (rax)
838 __ load_parameter(0, pre_val);
839 __ movptr(Address(tmp, 0), pre_val);
840 __ jmp(done);
841
842 __ bind(runtime);
843
844 __ save_live_registers_no_oop_map(true);
845
846 // load the pre-value
847 __ load_parameter(0, rcx);
848 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), rcx);
849
850 __ restore_live_registers(true);
851
852 __ bind(done);
853
854 __ pop(rdx);
855 __ pop(rax);
856
857 __ epilogue();
858 }
859
860 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
861 __ prologue("shenandoah_load_reference_barrier", false);
862 // arg0 : object to be resolved
863
864 __ save_live_registers_no_oop_map(true);
865
866 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
867 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
868 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
869 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
870
871 __ load_parameter(0, c_rarg0);
872 __ load_parameter(1, c_rarg1);
873 if (is_strong) {
874 if (is_native) {
875 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
876 } else {
877 if (UseCompressedOops) {
878 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0, c_rarg1);
879 } else {
880 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
881 }
882 }
883 } else if (is_weak) {
884 assert(!is_native, "weak must not be called off-heap");
885 if (UseCompressedOops) {
886 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
887 } else {
888 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
889 }
890 } else {
891 assert(is_phantom, "only remaining strength");
892 assert(is_native, "phantom must only be called off-heap");
893 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
894 }
895
896 __ restore_live_registers_except_rax(true);
897
898 __ epilogue();
899 }
900
901 #undef __
902
903 #endif // COMPILER1