1 /*
2 * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/codeBlob.hpp"
27 #include "code/vmreg.inline.hpp"
28 #include "gc/z/zBarrier.inline.hpp"
29 #include "gc/z/zBarrierSet.hpp"
30 #include "gc/z/zBarrierSetAssembler.hpp"
31 #include "gc/z/zBarrierSetRuntime.hpp"
32 #include "gc/z/zThreadLocalData.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/z/c1/zBarrierSetC1.hpp"
40 #endif // COMPILER1
41 #ifdef COMPILER2
42 #include "gc/z/c2/zBarrierSetC2.hpp"
43 #endif // COMPILER2
44
45 #ifdef PRODUCT
46 #define BLOCK_COMMENT(str) /* nothing */
47 #else
48 #define BLOCK_COMMENT(str) __ block_comment(str)
49 #endif
50
51 #undef __
52 #define __ masm->
53
54 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
55 DecoratorSet decorators,
56 BasicType type,
57 Register dst,
58 Address src,
59 Register tmp1,
60 Register tmp_thread) {
61 if (!ZBarrierSet::barrier_needed(decorators, type)) {
62 // Barrier not needed
63 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
64 return;
65 }
66
67 assert_different_registers(rscratch1, rscratch2, src.base());
68 assert_different_registers(rscratch1, rscratch2, dst);
69
70 Label done;
71
72 // Load bad mask into scratch register.
73 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
74 __ lea(rscratch2, src);
75 __ ldr(dst, src);
76
77 // Test reference against bad mask. If mask bad, then we need to fix it up.
78 __ tst(dst, rscratch1);
79 __ br(Assembler::EQ, done);
80
81 __ enter(/*strip_ret_addr*/true);
82
83 __ push_call_clobbered_registers_except(RegSet::of(dst));
84
85 if (c_rarg0 != dst) {
86 __ mov(c_rarg0, dst);
87 }
88 __ mov(c_rarg1, rscratch2);
89
90 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
91
92 // Make sure dst has the return value.
93 if (dst != r0) {
94 __ mov(dst, r0);
95 }
96
97 __ pop_call_clobbered_registers_except(RegSet::of(dst));
98 __ leave();
99
100 __ bind(done);
101 }
102
103 #ifdef ASSERT
104
105 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
106 DecoratorSet decorators,
107 BasicType type,
108 Address dst,
109 Register val,
110 Register tmp1,
111 Register tmp2,
112 Register tmp3) {
113 // Verify value
114 if (is_reference_type(type)) {
115 // Note that src could be noreg, which means we
116 // are storing null and can skip verification.
117 if (val != noreg) {
118 Label done;
119
120 // tmp1 and tmp2 are often set to noreg.
121 RegSet savedRegs = RegSet::of(rscratch1);
122 __ push(savedRegs, sp);
123
124 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
125 __ tst(val, rscratch1);
126 __ br(Assembler::EQ, done);
127 __ stop("Verify oop store failed");
128 __ should_not_reach_here();
129 __ bind(done);
130 __ pop(savedRegs, sp);
131 }
132 }
133
134 // Store value
135 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
136 }
137
138 #endif // ASSERT
139
140 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
141 DecoratorSet decorators,
142 bool is_oop,
143 Register src,
144 Register dst,
145 Register count,
146 RegSet saved_regs) {
147 if (!is_oop) {
148 // Barrier not needed
149 return;
150 }
151
152 BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
153
154 assert_different_registers(src, count, rscratch1);
155
156 __ push(saved_regs, sp);
157
158 if (count == c_rarg0) {
159 if (src == c_rarg1) {
160 // exactly backwards!!
161 __ mov(rscratch1, c_rarg0);
162 __ mov(c_rarg0, c_rarg1);
163 __ mov(c_rarg1, rscratch1);
164 } else {
165 __ mov(c_rarg1, count);
166 __ mov(c_rarg0, src);
167 }
168 } else {
169 __ mov(c_rarg0, src);
170 __ mov(c_rarg1, count);
171 }
172
173 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
174
175 __ pop(saved_regs, sp);
176
177 BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
178 }
179
180 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
181 Register jni_env,
182 Register robj,
183 Register tmp,
184 Label& slowpath) {
185 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
186
187 assert_different_registers(jni_env, robj, tmp);
188
189 // Resolve jobject
190 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
191
192 // The Address offset is too large to direct load - -784. Our range is +127, -128.
193 __ mov(tmp, (int64_t)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
194 in_bytes(JavaThread::jni_environment_offset())));
195
196 // Load address bad mask
197 __ add(tmp, jni_env, tmp);
198 __ ldr(tmp, Address(tmp));
199
200 // Check address bad mask
201 __ tst(robj, tmp);
202 __ br(Assembler::NE, slowpath);
203
204 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
205 }
206
207 #ifdef COMPILER1
208
209 #undef __
210 #define __ ce->masm()->
211
212 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
213 LIR_Opr ref) const {
214 assert_different_registers(rscratch1, rthread, ref->as_register());
215
216 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
217 __ tst(ref->as_register(), rscratch1);
218 }
219
220 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
221 ZLoadBarrierStubC1* stub) const {
222 // Stub entry
223 __ bind(*stub->entry());
224
225 Register ref = stub->ref()->as_register();
226 Register ref_addr = noreg;
227 Register tmp = noreg;
228
229 if (stub->tmp()->is_valid()) {
230 // Load address into tmp register
231 ce->leal(stub->ref_addr(), stub->tmp());
232 ref_addr = tmp = stub->tmp()->as_pointer_register();
233 } else {
234 // Address already in register
235 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
236 }
237
238 assert_different_registers(ref, ref_addr, noreg);
239
240 // Save r0 unless it is the result or tmp register
241 // Set up SP to accomodate parameters and maybe r0..
242 if (ref != r0 && tmp != r0) {
243 __ sub(sp, sp, 32);
244 __ str(r0, Address(sp, 16));
245 } else {
246 __ sub(sp, sp, 16);
247 }
248
249 // Setup arguments and call runtime stub
250 ce->store_parameter(ref_addr, 1);
251 ce->store_parameter(ref, 0);
252
253 __ far_call(stub->runtime_stub());
254
255 // Verify result
256 __ verify_oop(r0, "Bad oop");
257
258 // Move result into place
259 if (ref != r0) {
260 __ mov(ref, r0);
261 }
262
263 // Restore r0 unless it is the result or tmp register
264 if (ref != r0 && tmp != r0) {
265 __ ldr(r0, Address(sp, 16));
266 __ add(sp, sp, 32);
267 } else {
268 __ add(sp, sp, 16);
269 }
270
271 // Stub exit
272 __ b(*stub->continuation());
273 }
274
275 #undef __
276 #define __ sasm->
277
278 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
279 DecoratorSet decorators) const {
280 __ prologue("zgc_load_barrier stub", false);
281
282 __ push_call_clobbered_registers_except(RegSet::of(r0));
283
284 // Setup arguments
285 __ load_parameter(0, c_rarg0);
286 __ load_parameter(1, c_rarg1);
287
288 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
289
290 __ pop_call_clobbered_registers_except(RegSet::of(r0));
291
292 __ epilogue();
293 }
294 #endif // COMPILER1
295
296 #ifdef COMPILER2
297
298 OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
299 if (!OptoReg::is_reg(opto_reg)) {
300 return OptoReg::Bad;
301 }
302
303 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
304 if (vm_reg->is_FloatRegister()) {
305 return opto_reg & ~1;
306 }
307
308 return opto_reg;
309 }
310
311 #undef __
312 #define __ _masm->
313
314 class ZSaveLiveRegisters {
315 private:
316 MacroAssembler* const _masm;
317 RegSet _gp_regs;
318 FloatRegSet _fp_regs;
319 PRegSet _p_regs;
320
321 public:
322 void initialize(ZLoadBarrierStubC2* stub) {
323 // Record registers that needs to be saved/restored
324 RegMaskIterator rmi(stub->live());
325 while (rmi.has_next()) {
326 const OptoReg::Name opto_reg = rmi.next();
327 if (OptoReg::is_reg(opto_reg)) {
328 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
329 if (vm_reg->is_Register()) {
330 _gp_regs += RegSet::of(vm_reg->as_Register());
331 } else if (vm_reg->is_FloatRegister()) {
332 _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
333 } else if (vm_reg->is_PRegister()) {
334 _p_regs += PRegSet::of(vm_reg->as_PRegister());
335 } else {
336 fatal("Unknown register type");
337 }
338 }
339 }
340
341 // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
342 _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
343 }
344
345 ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
346 _masm(masm),
347 _gp_regs(),
348 _fp_regs(),
349 _p_regs() {
350
351 // Figure out what registers to save/restore
352 initialize(stub);
353
354 // Save registers
355 __ push(_gp_regs, sp);
356 __ push_fp(_fp_regs, sp);
357 __ push_p(_p_regs, sp);
358 }
359
360 ~ZSaveLiveRegisters() {
361 // Restore registers
362 __ pop_p(_p_regs, sp);
363 __ pop_fp(_fp_regs, sp);
364
365 // External runtime call may clobber ptrue reg
366 __ reinitialize_ptrue();
367
368 __ pop(_gp_regs, sp);
369 }
370 };
371
372 #undef __
373 #define __ _masm->
374
375 class ZSetupArguments {
376 private:
377 MacroAssembler* const _masm;
378 const Register _ref;
379 const Address _ref_addr;
380
381 public:
382 ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
383 _masm(masm),
384 _ref(stub->ref()),
385 _ref_addr(stub->ref_addr()) {
386
387 // Setup arguments
388 if (_ref_addr.base() == noreg) {
389 // No self healing
390 if (_ref != c_rarg0) {
391 __ mov(c_rarg0, _ref);
392 }
393 __ mov(c_rarg1, 0);
394 } else {
395 // Self healing
396 if (_ref == c_rarg0) {
397 // _ref is already at correct place
398 __ lea(c_rarg1, _ref_addr);
399 } else if (_ref != c_rarg1) {
400 // _ref is in wrong place, but not in c_rarg1, so fix it first
401 __ lea(c_rarg1, _ref_addr);
402 __ mov(c_rarg0, _ref);
403 } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
404 assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
405 __ mov(c_rarg0, _ref);
406 __ lea(c_rarg1, _ref_addr);
407 } else {
408 assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
409 if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
410 __ mov(rscratch2, c_rarg1);
411 __ lea(c_rarg1, _ref_addr);
412 __ mov(c_rarg0, rscratch2);
413 } else {
414 ShouldNotReachHere();
415 }
416 }
417 }
418 }
419
420 ~ZSetupArguments() {
421 // Transfer result
422 if (_ref != r0) {
423 __ mov(_ref, r0);
424 }
425 }
426 };
427
428 #undef __
429 #define __ masm->
430
431 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
432 BLOCK_COMMENT("ZLoadBarrierStubC2");
433
434 // Stub entry
435 __ bind(*stub->entry());
436
437 {
438 ZSaveLiveRegisters save_live_registers(masm, stub);
439 ZSetupArguments setup_arguments(masm, stub);
440 __ mov(rscratch1, stub->slow_path());
441 __ blr(rscratch1);
442 }
443 // Stub exit
444 __ b(*stub->continuation());
445 }
446
447 #undef __
448
449 #endif // COMPILER2