1 /*
2 * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "code/codeBlob.hpp"
28 #include "code/vmreg.inline.hpp"
29 #include "gc/z/zAddress.hpp"
30 #include "gc/z/zBarrier.inline.hpp"
31 #include "gc/z/zBarrierSet.hpp"
32 #include "gc/z/zBarrierSetAssembler.hpp"
33 #include "gc/z/zBarrierSetRuntime.hpp"
34 #include "gc/z/zThreadLocalData.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "runtime/jniHandles.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/z/c1/zBarrierSetC1.hpp"
44 #endif // COMPILER1
45 #ifdef COMPILER2
46 #include "gc/z/c2/zBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif // COMPILER2
49
50 #ifdef PRODUCT
51 #define BLOCK_COMMENT(str) /* nothing */
52 #else
53 #define BLOCK_COMMENT(str) __ block_comment(str)
54 #endif
55
56 #undef __
57 #define __ masm->
58
59 // Helper for saving and restoring registers across a runtime call that does
60 // not have any live vector registers.
61 class ZRuntimeCallSpill {
62 private:
63 MacroAssembler* _masm;
64 Register _result;
65
66 void save() {
67 MacroAssembler* masm = _masm;
68
69 __ enter();
70 if (_result != noreg) {
71 __ push_call_clobbered_registers_except(RegSet::of(_result));
72 } else {
73 __ push_call_clobbered_registers();
74 }
75 }
76
77 void restore() {
78 MacroAssembler* masm = _masm;
79
80 if (_result != noreg) {
81 // Make sure _result has the return value.
82 if (_result != x10) {
83 __ mv(_result, x10);
84 }
85
86 __ pop_call_clobbered_registers_except(RegSet::of(_result));
87 } else {
88 __ pop_call_clobbered_registers();
89 }
90 __ leave();
91 }
92
93 public:
94 ZRuntimeCallSpill(MacroAssembler* masm, Register result)
95 : _masm(masm),
96 _result(result) {
97 save();
98 }
99
100 ~ZRuntimeCallSpill() {
101 restore();
102 }
103 };
104
105 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
106 DecoratorSet decorators,
107 BasicType type,
108 Register dst,
109 Address src,
110 Register tmp1,
111 Register tmp2) {
112 if (!ZBarrierSet::barrier_needed(decorators, type)) {
113 // Barrier not needed
114 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
115 return;
116 }
117
118 assert_different_registers(tmp1, tmp2, src.base(), noreg);
119 assert_different_registers(tmp1, tmp2, dst, noreg);
120 assert_different_registers(tmp2, t0);
121
122 Label done;
123 Label uncolor;
124
125 // Load bad mask into scratch register.
126 const bool on_non_strong =
127 (decorators & ON_WEAK_OOP_REF) != 0 ||
128 (decorators & ON_PHANTOM_OOP_REF) != 0;
129
130 if (on_non_strong) {
131 __ ld(tmp1, mark_bad_mask_from_thread(xthread));
132 } else {
133 __ ld(tmp1, load_bad_mask_from_thread(xthread));
134 }
135
136 __ la(tmp2, src);
137 __ ld(dst, tmp2);
138
139 // Test reference against bad mask. If mask bad, then we need to fix it up.
140 __ andr(tmp1, dst, tmp1);
141 __ beqz(tmp1, uncolor);
142
143 {
144 // Call VM
145 ZRuntimeCallSpill rsc(masm, dst);
146
147 if (c_rarg0 != dst) {
148 __ mv(c_rarg0, dst);
149 }
150 __ mv(c_rarg1, tmp2);
151
152 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
153 }
154
155 // Slow-path has already uncolored
156 __ j(done);
157
158 __ bind(uncolor);
159
160 // Remove the color bits
161 __ srli(dst, dst, ZPointerLoadShift);
162
163 __ bind(done);
164 }
165
166 void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
167 Address ref_addr,
168 Register rnew_zaddress,
169 Register rnew_zpointer,
170 Register rtmp,
171 bool in_nmethod,
172 bool is_atomic,
173 Label& medium_path,
174 Label& medium_path_continuation) const {
175 assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
176 assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);
177
178 if (in_nmethod) {
179 if (is_atomic) {
180 __ lhu(rtmp, ref_addr);
181 // Atomic operations must ensure that the contents of memory are store-good before
182 // an atomic opertion can execute.
183 // A non-relocatable object could have spurious raw null pointers in its fields after
184 // getting promoted to the old generation.
185 __ relocate(barrier_Relocation::spec(), [&] {
186 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
187 }, ZBarrierRelocationFormatStoreGoodBits);
188 __ bne(rtmp, rnew_zpointer, medium_path, true /* is_far */);
189 } else {
190 __ ld(rtmp, ref_addr);
191 // Stores on relocatable objects never need to deal with raw null pointers in fields.
192 // Raw null pointers may only exists in the young generation, as they get pruned when
193 // the object is relocated to old. And no pre-write barrier needs to perform any action
194 // in the young generation.
195 __ relocate(barrier_Relocation::spec(), [&] {
196 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
197 }, ZBarrierRelocationFormatStoreBadMask);
198 __ andr(rtmp, rtmp, rnew_zpointer);
199 __ bnez(rtmp, medium_path, true /* is_far */);
200 }
201 __ bind(medium_path_continuation);
202 __ relocate(barrier_Relocation::spec(), [&] {
203 __ li16u(rtmp, barrier_Relocation::unpatched);
204 }, ZBarrierRelocationFormatStoreGoodBits);
205 __ slli(rnew_zpointer, rnew_zaddress, ZPointerLoadShift);
206 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
207 } else {
208 assert(!is_atomic, "atomic outside of nmethods not supported");
209 __ la(rtmp, ref_addr);
210 __ ld(rtmp, rtmp);
211 __ ld(rnew_zpointer, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
212 __ andr(rtmp, rtmp, rnew_zpointer);
213 __ bnez(rtmp, medium_path, true /* is_far */);
214 __ bind(medium_path_continuation);
215 if (rnew_zaddress == noreg) {
216 __ mv(rnew_zpointer, zr);
217 } else {
218 __ mv(rnew_zpointer, rnew_zaddress);
219 }
220
221 // Load the current good shift, and add the color bits
222 __ slli(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
223 __ ld(rtmp, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
224 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
225 }
226 }
227
228 static void store_barrier_buffer_add(MacroAssembler* masm,
229 Address ref_addr,
230 Register tmp1,
231 Register tmp2,
232 Label& slow_path) {
233 Address buffer(xthread, ZThreadLocalData::store_barrier_buffer_offset());
234 assert_different_registers(ref_addr.base(), tmp1, tmp2);
235
236 __ ld(tmp1, buffer);
237
238 // Combined pointer bump and check if the buffer is disabled or full
239 __ ld(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
240 __ beqz(tmp2, slow_path);
241
242 // Bump the pointer
243 __ subi(tmp2, tmp2, sizeof(ZStoreBarrierEntry));
244 __ sd(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
245
246 // Compute the buffer entry address
247 __ la(tmp2, Address(tmp2, ZStoreBarrierBuffer::buffer_offset()));
248 __ add(tmp2, tmp2, tmp1);
249
250 // Compute and log the store address
251 __ la(tmp1, ref_addr);
252 __ sd(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset())));
253
254 // Load and log the prev value
255 __ ld(tmp1, tmp1);
256 __ sd(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset())));
257 }
258
259 void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
260 Address ref_addr,
261 Register rtmp1,
262 Register rtmp2,
263 Register rtmp3,
264 bool is_native,
265 bool is_atomic,
266 Label& medium_path_continuation,
267 Label& slow_path,
268 Label& slow_path_continuation) const {
269 assert_different_registers(ref_addr.base(), rtmp1, rtmp2, rtmp3);
270
271 // The reason to end up in the medium path is that the pre-value was not 'good'.
272 if (is_native) {
273 __ j(slow_path);
274 __ bind(slow_path_continuation);
275 __ j(medium_path_continuation);
276 } else if (is_atomic) {
277 // Atomic accesses can get to the medium fast path because the value was a
278 // raw null value. If it was not null, then there is no doubt we need to take a slow path.
279
280 __ la(rtmp2, ref_addr);
281 __ ld(rtmp1, rtmp2);
282 __ bnez(rtmp1, slow_path);
283
284 // If we get this far, we know there is a young raw null value in the field.
285 __ relocate(barrier_Relocation::spec(), [&] {
286 __ li16u(rtmp1, barrier_Relocation::unpatched);
287 }, ZBarrierRelocationFormatStoreGoodBits);
288 __ weak_cmpxchg(rtmp2, zr, rtmp1,
289 Assembler::int64,
290 Assembler::relaxed /* acquire */, Assembler::relaxed /* release */,
291 rtmp3);
292 __ beqz(rtmp3, slow_path);
293 __ bind(slow_path_continuation);
294 __ j(medium_path_continuation);
295 } else {
296 // A non-atomic relocatable object wont't get to the medium fast path due to a
297 // raw null in the young generation. We only get here because the field is bad.
298 // In this path we don't need any self healing, so we can avoid a runtime call
299 // most of the time by buffering the store barrier to be applied lazily.
300 store_barrier_buffer_add(masm,
301 ref_addr,
302 rtmp1,
303 rtmp2,
304 slow_path);
305 __ bind(slow_path_continuation);
306 __ j(medium_path_continuation);
307 }
308 }
309
310 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
311 DecoratorSet decorators,
312 BasicType type,
313 Address dst,
314 Register val,
315 Register tmp1,
316 Register tmp2,
317 Register tmp3) {
318 if (!ZBarrierSet::barrier_needed(decorators, type)) {
319 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
320 return;
321 }
322
323 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
324
325 assert_different_registers(val, tmp1, dst.base());
326
327 if (dest_uninitialized) {
328 if (val == noreg) {
329 __ mv(tmp1, zr);
330 } else {
331 __ mv(tmp1, val);
332 }
333 // Add the color bits
334 __ slli(tmp1, tmp1, ZPointerLoadShift);
335 __ ld(tmp2, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
336 __ orr(tmp1, tmp2, tmp1);
337 } else {
338 Label done;
339 Label medium;
340 Label medium_continuation;
341 Label slow;
342 Label slow_continuation;
343 store_barrier_fast(masm, dst, val, tmp1, tmp2, false, false, medium, medium_continuation);
344
345 __ j(done);
346 __ bind(medium);
347 store_barrier_medium(masm,
348 dst,
349 tmp1,
350 tmp2,
351 noreg /* tmp3 */,
352 false /* is_native */,
353 false /* is_atomic */,
354 medium_continuation,
355 slow,
356 slow_continuation);
357
358 __ bind(slow);
359 {
360 // Call VM
361 ZRuntimeCallSpill rcs(masm, noreg);
362 __ la(c_rarg0, dst);
363 __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
364 }
365
366 __ j(slow_continuation);
367 __ bind(done);
368 }
369
370 // Store value
371 BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, tmp2, tmp3, noreg);
372 }
373
374 class ZCopyRuntimeCallSpill {
375 private:
376 MacroAssembler* _masm;
377 Register _result;
378
379 void save() {
380 MacroAssembler* masm = _masm;
381
382 __ enter();
383 if (_result != noreg) {
384 __ push_call_clobbered_registers_except(RegSet::of(_result));
385 } else {
386 __ push_call_clobbered_registers();
387 }
388 }
389
390 void restore() {
391 MacroAssembler* masm = _masm;
392
393 if (_result != noreg) {
394 if (_result != x10) {
395 __ mv(_result, x10);
396 }
397 __ pop_call_clobbered_registers_except(RegSet::of(_result));
398 } else {
399 __ pop_call_clobbered_registers();
400 }
401 __ leave();
402 }
403
404 public:
405 ZCopyRuntimeCallSpill(MacroAssembler* masm, Register result)
406 : _masm(masm),
407 _result(result) {
408 save();
409 }
410
411 ~ZCopyRuntimeCallSpill() {
412 restore();
413 }
414 };
415
416 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
417 DecoratorSet decorators,
418 bool is_oop,
419 Register src,
420 Register dst,
421 Register count,
422 RegSet saved_regs) {}
423
424 static void copy_load_barrier(MacroAssembler* masm,
425 Register ref,
426 Address src,
427 Register tmp) {
428 Label done;
429
430 __ ld(tmp, Address(xthread, ZThreadLocalData::load_bad_mask_offset()));
431
432 // Test reference against bad mask. If mask bad, then we need to fix it up
433 __ andr(tmp, ref, tmp);
434 __ beqz(tmp, done);
435
436 {
437 // Call VM
438 ZCopyRuntimeCallSpill rsc(masm, ref);
439
440 __ la(c_rarg1, src);
441
442 if (c_rarg0 != ref) {
443 __ mv(c_rarg0, ref);
444 }
445
446 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(IN_HEAP | ON_STRONG_OOP_REF), 2);
447 }
448
449 // Slow-path has uncolored; revert
450 __ slli(ref, ref, ZPointerLoadShift);
451
452 __ bind(done);
453 }
454
455 static void copy_store_barrier(MacroAssembler* masm,
456 Register pre_ref,
457 Register new_ref,
458 Address src,
459 Register tmp1,
460 Register tmp2) {
461 Label done;
462 Label slow;
463
464 // Test reference against bad mask. If mask bad, then we need to fix it up.
465 __ ld(tmp1, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
466 __ andr(tmp1, pre_ref, tmp1);
467 __ beqz(tmp1, done);
468
469 store_barrier_buffer_add(masm, src, tmp1, tmp2, slow);
470 __ j(done);
471
472 __ bind(slow);
473 {
474 // Call VM
475 ZCopyRuntimeCallSpill rcs(masm, noreg);
476
477 __ la(c_rarg0, src);
478
479 __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
480 }
481
482 __ bind(done);
483
484 if (new_ref != noreg) {
485 // Set store-good color, replacing whatever color was there before
486 __ ld(tmp1, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
487 __ srli(new_ref, new_ref, 16);
488 __ slli(new_ref, new_ref, 16);
489 __ orr(new_ref, new_ref, tmp1);
490 }
491 }
492
493 void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
494 DecoratorSet decorators,
495 BasicType type,
496 size_t bytes,
497 Register dst,
498 Address src,
499 Register tmp) {
500 if (!is_reference_type(type)) {
501 BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, noreg);
502 return;
503 }
504
505 BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, noreg);
506
507 assert(bytes == 8, "unsupported copy step");
508 copy_load_barrier(masm, dst, src, tmp);
509
510 if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
511 __ srli(dst, dst, ZPointerLoadShift);
512 }
513 }
514
515 void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
516 DecoratorSet decorators,
517 BasicType type,
518 size_t bytes,
519 Address dst,
520 Register src,
521 Register tmp1,
522 Register tmp2,
523 Register tmp3) {
524 if (!is_reference_type(type)) {
525 BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, noreg, noreg, noreg);
526 return;
527 }
528
529 if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
530 __ slli(src, src, ZPointerLoadShift);
531 }
532
533 bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
534
535 assert(bytes == 8, "unsupported copy step");
536 if (is_dest_uninitialized) {
537 __ ld(tmp1, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
538 __ srli(src, src, 16);
539 __ slli(src, src, 16);
540 __ orr(src, src, tmp1);
541 } else {
542 // Store barrier pre values and color new values
543 __ ld(tmp1, dst);
544 copy_store_barrier(masm, tmp1, src, dst, tmp2, tmp3);
545 }
546
547 // Store new values
548 BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, noreg, noreg, noreg);
549 }
550
551 bool ZBarrierSetAssembler::supports_rvv_arraycopy() {
552 return false;
553 }
554
555 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
556 Register jni_env,
557 Register robj,
558 Register tmp,
559 Label& slowpath) {
560 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
561
562 Label done, tagged, weak_tagged, uncolor;
563
564 // Test for tag
565 __ andi(tmp, robj, JNIHandles::tag_mask);
566 __ bnez(tmp, tagged);
567
568 // Resolve local handle
569 __ ld(robj, robj);
570 __ j(done);
571
572 __ bind(tagged);
573
574 // Test for weak tag
575 __ andi(tmp, robj, JNIHandles::TypeTag::weak_global);
576 __ bnez(tmp, weak_tagged);
577
578 // Resolve global handle
579 __ ld(robj, Address(robj, -JNIHandles::TypeTag::global));
580 __ la(tmp, load_bad_mask_from_jni_env(jni_env));
581 __ ld(tmp, tmp);
582 __ andr(tmp, robj, tmp);
583 __ bnez(tmp, slowpath);
584 __ j(uncolor);
585
586 __ bind(weak_tagged);
587
588 // Resolve weak handle
589 __ ld(robj, Address(robj, -JNIHandles::TypeTag::weak_global));
590 __ la(tmp, mark_bad_mask_from_jni_env(jni_env));
591 __ ld(tmp, tmp);
592 __ andr(tmp, robj, tmp);
593 __ bnez(tmp, slowpath);
594
595 __ bind(uncolor);
596
597 // Uncolor
598 __ srli(robj, robj, ZPointerLoadShift);
599
600 __ bind(done);
601
602 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
603 }
604
605 void ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
606 Register tmp, Label& slow_path) {
607 BLOCK_COMMENT("ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod {");
608
609 assert_different_registers(weak_handle, tmp, noreg);
610 assert_different_registers(obj, tmp, noreg);
611
612
613 // Peek weak handle using the standard implementation.
614 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
615
616 // Check if the oop is bad, in which case we need to take the slow path.
617 __ relocate(barrier_Relocation::spec(), [&] {
618 __ li16u(tmp, barrier_Relocation::unpatched);
619 }, ZBarrierRelocationFormatMarkBadMask);
620 __ andr(tmp, obj, tmp);
621 __ bnez(tmp, slow_path);
622
623 // Oop is okay, so we uncolor it.
624 __ srli(obj, obj, ZPointerLoadShift);
625
626 BLOCK_COMMENT("} ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod");
627 }
628
629 static uint16_t patch_barrier_relocation_value(int format) {
630 switch (format) {
631 case ZBarrierRelocationFormatLoadBadMask:
632 return (uint16_t)ZPointerLoadBadMask;
633 case ZBarrierRelocationFormatMarkBadMask:
634 return (uint16_t)ZPointerMarkBadMask;
635 case ZBarrierRelocationFormatStoreGoodBits:
636 return (uint16_t)ZPointerStoreGoodMask;
637 case ZBarrierRelocationFormatStoreBadMask:
638 return (uint16_t)ZPointerStoreBadMask;
639
640 default:
641 ShouldNotReachHere();
642 return 0;
643 }
644 }
645
646 void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
647 const uint16_t value = patch_barrier_relocation_value(format);
648
649 int bytes;
650 switch (format) {
651 case ZBarrierRelocationFormatLoadBadMask:
652 case ZBarrierRelocationFormatMarkBadMask:
653 case ZBarrierRelocationFormatStoreGoodBits:
654 case ZBarrierRelocationFormatStoreBadMask:
655 assert(MacroAssembler::is_li16u_at(addr), "invalide zgc barrier");
656 bytes = MacroAssembler::pd_patch_instruction_size(addr, (address)(uintptr_t)value);
657 break;
658 default:
659 ShouldNotReachHere();
660 }
661
662 // If we are using UseCtxFencei no ICache invalidation is needed here.
663 // Instead every hart will preform an fence.i either by a Java thread
664 // (due to patching epoch will take it to slow path),
665 // or by the kernel when a Java thread is moved to a hart.
666 // The instruction streams changes must only happen before the disarm of
667 // the nmethod barrier. Where the disarm have a leading full two way fence.
668 // If this is performed during a safepoint, all Java threads will emit a fence.i
669 // before transitioning to 'Java', e.g. leaving native or the safepoint wait barrier.
670 if (!UseCtxFencei) {
671 // ICache invalidation is a serialization point.
672 // The above patching of instructions happens before the invalidation.
673 // Hence it have a leading full two way fence (wr, wr).
674 ICache::invalidate_range(addr, bytes);
675 }
676 }
677
678 #ifdef COMPILER2
679
680 #undef __
681 #define __ _masm->
682
683 class ZSetupArguments {
684 private:
685 MacroAssembler* const _masm;
686 const Register _ref;
687 const Address _ref_addr;
688
689 public:
690 ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub)
691 : _masm(masm),
692 _ref(stub->ref()),
693 _ref_addr(stub->ref_addr()) {
694
695 // Setup arguments
696 if (_ref_addr.base() == noreg) {
697 // No self healing
698 if (_ref != c_rarg0) {
699 __ mv(c_rarg0, _ref);
700 }
701 __ mv(c_rarg1, zr);
702 } else {
703 // Self healing
704 if (_ref == c_rarg0) {
705 // _ref is already at correct place
706 __ la(c_rarg1, _ref_addr);
707 } else if (_ref != c_rarg1) {
708 // _ref is in wrong place, but not in c_rarg1, so fix it first
709 __ la(c_rarg1, _ref_addr);
710 __ mv(c_rarg0, _ref);
711 } else if (_ref_addr.base() != c_rarg0) {
712 assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
713 __ mv(c_rarg0, _ref);
714 __ la(c_rarg1, _ref_addr);
715 } else {
716 assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
717 if (_ref_addr.base() == c_rarg0) {
718 __ mv(t1, c_rarg1);
719 __ la(c_rarg1, _ref_addr);
720 __ mv(c_rarg0, t1);
721 } else {
722 ShouldNotReachHere();
723 }
724 }
725 }
726 }
727
728 ~ZSetupArguments() {
729 // Transfer result
730 if (_ref != x10) {
731 __ mv(_ref, x10);
732 }
733 }
734 };
735
736 #undef __
737 #define __ masm->
738
739 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
740 Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
741 BLOCK_COMMENT("ZLoadBarrierStubC2");
742
743 // Stub entry
744 if (!Compile::current()->output()->in_scratch_emit_size()) {
745 __ bind(*stub->entry());
746 }
747
748 {
749 SaveLiveRegisters save_live_registers(masm, stub);
750 ZSetupArguments setup_arguments(masm, stub);
751 __ mv(t1, stub->slow_path());
752 __ jalr(t1);
753 }
754
755 // Stub exit
756 __ j(*stub->continuation());
757 }
758
759 void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
760 Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
761 BLOCK_COMMENT("ZStoreBarrierStubC2");
762
763 // Stub entry
764 __ bind(*stub->entry());
765
766 Label slow;
767 Label slow_continuation;
768 store_barrier_medium(masm,
769 stub->ref_addr(),
770 stub->new_zpointer(),
771 t1,
772 t0,
773 stub->is_native(),
774 stub->is_atomic(),
775 *stub->continuation(),
776 slow,
777 slow_continuation);
778
779 __ bind(slow);
780
781 {
782 SaveLiveRegisters save_live_registers(masm, stub);
783 __ la(c_rarg0, stub->ref_addr());
784
785 if (stub->is_native()) {
786 __ rt_call(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr());
787 } else if (stub->is_atomic()) {
788 __ rt_call(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr());
789 } else if (stub->is_nokeepalive()) {
790 __ rt_call(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr());
791 } else {
792 __ rt_call(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr());
793 }
794 }
795
796 // Stub exit
797 __ j(slow_continuation);
798 }
799
800 #undef __
801
802 #endif // COMPILER2
803
804 #ifdef COMPILER1
805 #undef __
806 #define __ ce->masm()->
807
808 static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
809 __ relocate(barrier_Relocation::spec(), [&] {
810 __ li16u(t1, barrier_Relocation::unpatched);
811 }, ZBarrierRelocationFormatStoreGoodBits);
812 __ slli(ref->as_register(), ref->as_register(), ZPointerLoadShift);
813 __ orr(ref->as_register(), ref->as_register(), t1);
814 }
815
816 static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
817 __ srli(ref->as_register(), ref->as_register(), ZPointerLoadShift);
818 }
819
820 static void check_color(LIR_Assembler* ce, LIR_Opr ref, bool on_non_strong) {
821 assert_different_registers(t0, xthread, ref->as_register());
822 int format = on_non_strong ? ZBarrierRelocationFormatMarkBadMask
823 : ZBarrierRelocationFormatLoadBadMask;
824 Label good;
825 __ relocate(barrier_Relocation::spec(), [&] {
826 __ li16u(t0, barrier_Relocation::unpatched);
827 }, format);
828 __ andr(t0, ref->as_register(), t0);
829 }
830
831 void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
832 z_color(ce, ref);
833 }
834
835 void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
836 z_uncolor(ce, ref);
837 }
838
839 void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
840 LIR_Opr ref,
841 ZLoadBarrierStubC1* stub,
842 bool on_non_strong) const {
843 Label good;
844 check_color(ce, ref, on_non_strong);
845 __ beqz(t0, good);
846 __ j(*stub->entry());
847
848 __ bind(good);
849 z_uncolor(ce, ref);
850 __ bind(*stub->continuation());
851 }
852
853 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
854 ZLoadBarrierStubC1* stub) const {
855 // Stub entry
856 __ bind(*stub->entry());
857
858 Register ref = stub->ref()->as_register();
859 Register ref_addr = noreg;
860 Register tmp = noreg;
861
862 if (stub->tmp()->is_valid()) {
863 // Load address into tmp register
864 ce->leal(stub->ref_addr(), stub->tmp());
865 ref_addr = tmp = stub->tmp()->as_pointer_register();
866 } else {
867 // Address already in register
868 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
869 }
870
871 assert_different_registers(ref, ref_addr, noreg);
872
873 // Save x10 unless it is the result or tmp register
874 // Set up SP to accommdate parameters and maybe x10.
875 if (ref != x10 && tmp != x10) {
876 __ subi(sp, sp, 32);
877 __ sd(x10, Address(sp, 16));
878 } else {
879 __ subi(sp, sp, 16);
880 }
881
882 // Setup arguments and call runtime stub
883 ce->store_parameter(ref_addr, 1);
884 ce->store_parameter(ref, 0);
885
886 __ far_call(stub->runtime_stub());
887
888 // Verify result
889 __ verify_oop(x10);
890
891 // Move result into place
892 if (ref != x10) {
893 __ mv(ref, x10);
894 }
895
896 // Restore x10 unless it is the result or tmp register
897 if (ref != x10 && tmp != x10) {
898 __ ld(x10, Address(sp, 16));
899 __ addi(sp, sp, 32);
900 } else {
901 __ addi(sp, sp, 16);
902 }
903
904 // Stub exit
905 __ j(*stub->continuation());
906 }
907
908 #undef __
909 #define __ sasm->
910
911 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
912 DecoratorSet decorators) const {
913 __ prologue("zgc_load_barrier stub", false);
914
915 __ push_call_clobbered_registers_except(RegSet::of(x10));
916
917 // Setup arguments
918 __ load_parameter(0, c_rarg0);
919 __ load_parameter(1, c_rarg1);
920
921 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
922
923 __ pop_call_clobbered_registers_except(RegSet::of(x10));
924
925 __ epilogue();
926 }
927
928 void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
929 bool self_healing) const {
930 __ prologue("zgc_store_barrier stub", false);
931
932 __ push_call_clobbered_registers();
933
934 // Setup arguments
935 __ load_parameter(0, c_rarg0);
936
937 if (self_healing) {
938 __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), 1);
939 } else {
940 __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
941 }
942
943 __ pop_call_clobbered_registers();
944
945 __ epilogue();
946 }
947
948 #undef __
949 #define __ ce->masm()->
950
951 void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
952 LIR_Address* addr,
953 LIR_Opr new_zaddress,
954 LIR_Opr new_zpointer,
955 ZStoreBarrierStubC1* stub) const {
956 Register rnew_zaddress = new_zaddress->as_register();
957 Register rnew_zpointer = new_zpointer->as_register();
958
959 store_barrier_fast(ce->masm(),
960 ce->as_Address(addr),
961 rnew_zaddress,
962 rnew_zpointer,
963 t1,
964 true,
965 stub->is_atomic(),
966 *stub->entry(),
967 *stub->continuation());
968 }
969
970 void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
971 ZStoreBarrierStubC1* stub) const {
972 // Stub entry
973 __ bind(*stub->entry());
974 Label slow;
975 Label slow_continuation;
976 store_barrier_medium(ce->masm(),
977 ce->as_Address(stub->ref_addr()->as_address_ptr()),
978 t1,
979 stub->new_zpointer()->as_register(),
980 stub->tmp()->as_pointer_register(),
981 false /* is_native */,
982 stub->is_atomic(),
983 *stub->continuation(),
984 slow,
985 slow_continuation);
986
987 __ bind(slow);
988
989 __ la(stub->new_zpointer()->as_register(), ce->as_Address(stub->ref_addr()->as_address_ptr()));
990
991 __ subi(sp, sp, 16);
992 //Setup arguments and call runtime stub
993 assert(stub->new_zpointer()->is_valid(), "invariant");
994 ce->store_parameter(stub->new_zpointer()->as_register(), 0);
995 __ far_call(stub->runtime_stub());
996 __ addi(sp, sp, 16);
997
998 // Stub exit
999 __ j(slow_continuation);
1000 }
1001
1002 #undef __
1003
1004 #endif // COMPILER1
1005
1006 #undef __
1007 #define __ masm->
1008
1009 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
1010 // C1 calls verify_oop in the middle of barriers, before they have been uncolored
1011 // and after being colored. Therefore, we must deal with colored oops as well.
1012 Label done;
1013 Label check_oop;
1014 Label check_zaddress;
1015 int color_bits = ZPointerRemappedShift + ZPointerRemappedBits;
1016
1017 uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1;
1018 uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1;
1019 uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask;
1020
1021 uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1;
1022 uintptr_t shifted_address_mask = shifted_base_end_mask ^ (uintptr_t)CONST64(-1);
1023
1024 // Check colored null
1025 __ mv(tmp1, shifted_address_mask);
1026 __ andr(tmp1, tmp1, obj);
1027 __ beqz(tmp1, done);
1028
1029 // Check for zpointer
1030 __ mv(tmp1, shifted_base_mask);
1031 __ andr(tmp1, tmp1, obj);
1032 __ beqz(tmp1, check_oop);
1033
1034 // Uncolor presumed zpointer
1035 __ srli(obj, obj, ZPointerLoadShift);
1036
1037 __ j(check_zaddress);
1038
1039 __ bind(check_oop);
1040
1041 // Make sure klass is 'reasonable', which is not zero
1042 __ load_klass(tmp1, obj, tmp2);
1043 __ beqz(tmp1, error);
1044
1045 __ bind(check_zaddress);
1046 // Check if the oop is the right area of memory
1047 __ mv(tmp1, (intptr_t) Universe::verify_oop_mask());
1048 __ andr(tmp1, tmp1, obj);
1049 __ mv(obj, (intptr_t) Universe::verify_oop_bits());
1050 __ bne(tmp1, obj, error);
1051
1052 __ bind(done);
1053 }
1054
1055 #undef __