1 /*
  2  * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "asm/macroAssembler.inline.hpp"
 26 #include "code/codeBlob.hpp"
 27 #include "code/vmreg.inline.hpp"
 28 #include "gc/z/zBarrier.inline.hpp"
 29 #include "gc/z/zBarrierSet.hpp"
 30 #include "gc/z/zBarrierSetAssembler.hpp"
 31 #include "gc/z/zBarrierSetRuntime.hpp"
 32 #include "gc/z/zThreadLocalData.hpp"
 33 #include "memory/resourceArea.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "utilities/macros.hpp"
 36 #ifdef COMPILER1
 37 #include "c1/c1_LIRAssembler.hpp"
 38 #include "c1/c1_MacroAssembler.hpp"
 39 #include "gc/z/c1/zBarrierSetC1.hpp"
 40 #endif // COMPILER1
 41 #ifdef COMPILER2
 42 #include "gc/z/c2/zBarrierSetC2.hpp"
 43 #endif // COMPILER2
 44 
 45 #ifdef PRODUCT
 46 #define BLOCK_COMMENT(str) /* nothing */
 47 #else
 48 #define BLOCK_COMMENT(str) __ block_comment(str)
 49 #endif
 50 
 51 #undef __
 52 #define __ masm->
 53 
 54 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
 55                                    DecoratorSet decorators,
 56                                    BasicType type,
 57                                    Register dst,
 58                                    Address src,
 59                                    Register tmp1,
 60                                    Register tmp_thread) {
 61   if (!ZBarrierSet::barrier_needed(decorators, type)) {
 62     // Barrier not needed
 63     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 64     return;
 65   }
 66 
 67   assert_different_registers(rscratch1, rscratch2, src.base());
 68   assert_different_registers(rscratch1, rscratch2, dst);
 69 
 70   Label done;
 71 
 72   // Load bad mask into scratch register.
 73   __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
 74   __ lea(rscratch2, src);
 75   __ ldr(dst, src);
 76 
 77   // Test reference against bad mask. If mask bad, then we need to fix it up.
 78   __ tst(dst, rscratch1);
 79   __ br(Assembler::EQ, done);
 80 
 81   __ enter();
 82 
 83   __ push_call_clobbered_registers_except(RegSet::of(dst));
 84 
 85   if (c_rarg0 != dst) {
 86     __ mov(c_rarg0, dst);
 87   }
 88   __ mov(c_rarg1, rscratch2);
 89 
 90   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
 91 
 92   // Make sure dst has the return value.
 93   if (dst != r0) {
 94     __ mov(dst, r0);
 95   }
 96 
 97   __ pop_call_clobbered_registers_except(RegSet::of(dst));
 98   __ leave();
 99 
100   __ bind(done);
101 }
102 
103 #ifdef ASSERT
104 
105 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
106                                         DecoratorSet decorators,
107                                         BasicType type,
108                                         Address dst,
109                                         Register val,
110                                         Register tmp1,
111                                         Register tmp2,
112                                         Register tmp3) {
113   // Verify value
114   if (is_reference_type(type)) {
115     // Note that src could be noreg, which means we
116     // are storing null and can skip verification.
117     if (val != noreg) {
118       Label done;
119 
120       // tmp1 and tmp2 are often set to noreg.
121       RegSet savedRegs = RegSet::of(rscratch1);
122       __ push(savedRegs, sp);
123 
124       __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
125       __ tst(val, rscratch1);
126       __ br(Assembler::EQ, done);
127       __ stop("Verify oop store failed");
128       __ should_not_reach_here();
129       __ bind(done);
130       __ pop(savedRegs, sp);
131     }
132   }
133 
134   // Store value
135   BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
136 }
137 
138 #endif // ASSERT
139 
140 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
141                                               DecoratorSet decorators,
142                                               bool is_oop,
143                                               Register src,
144                                               Register dst,
145                                               Register count,
146                                               RegSet saved_regs) {
147   if (!is_oop) {
148     // Barrier not needed
149     return;
150   }
151 
152   BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
153 
154   assert_different_registers(src, count, rscratch1);
155 
156   __ push(saved_regs, sp);
157 
158   if (count == c_rarg0) {
159     if (src == c_rarg1) {
160       // exactly backwards!!
161       __ mov(rscratch1, c_rarg0);
162       __ mov(c_rarg0, c_rarg1);
163       __ mov(c_rarg1, rscratch1);
164     } else {
165       __ mov(c_rarg1, count);
166       __ mov(c_rarg0, src);
167     }
168   } else {
169     __ mov(c_rarg0, src);
170     __ mov(c_rarg1, count);
171   }
172 
173   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
174 
175   __ pop(saved_regs, sp);
176 
177   BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
178 }
179 
180 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
181                                                          Register jni_env,
182                                                          Register robj,
183                                                          Register tmp,
184                                                          Label& slowpath) {
185   BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
186 
187   assert_different_registers(jni_env, robj, tmp);
188 
189   // Resolve jobject
190   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
191 
192   // The Address offset is too large to direct load - -784. Our range is +127, -128.
193   __ mov(tmp, (int64_t)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
194               in_bytes(JavaThread::jni_environment_offset())));
195 
196   // Load address bad mask
197   __ add(tmp, jni_env, tmp);
198   __ ldr(tmp, Address(tmp));
199 
200   // Check address bad mask
201   __ tst(robj, tmp);
202   __ br(Assembler::NE, slowpath);
203 
204   BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
205 }
206 
207 #ifdef COMPILER1
208 
209 #undef __
210 #define __ ce->masm()->
211 
212 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
213                                                          LIR_Opr ref) const {
214   assert_different_registers(rscratch1, rthread, ref->as_register());
215 
216   __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
217   __ tst(ref->as_register(), rscratch1);
218 }
219 
220 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
221                                                          ZLoadBarrierStubC1* stub) const {
222   // Stub entry
223   __ bind(*stub->entry());
224 
225   Register ref = stub->ref()->as_register();
226   Register ref_addr = noreg;
227   Register tmp = noreg;
228 
229   if (stub->tmp()->is_valid()) {
230     // Load address into tmp register
231     ce->leal(stub->ref_addr(), stub->tmp());
232     ref_addr = tmp = stub->tmp()->as_pointer_register();
233   } else {
234     // Address already in register
235     ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
236   }
237 
238   assert_different_registers(ref, ref_addr, noreg);
239 
240   // Save r0 unless it is the result or tmp register
241   // Set up SP to accomodate parameters and maybe r0..
242   if (ref != r0 && tmp != r0) {
243     __ sub(sp, sp, 32);
244     __ str(r0, Address(sp, 16));
245   } else {
246     __ sub(sp, sp, 16);
247   }
248 
249   // Setup arguments and call runtime stub
250   ce->store_parameter(ref_addr, 1);
251   ce->store_parameter(ref, 0);
252 
253   __ far_call(stub->runtime_stub());
254 
255   // Verify result
256   __ verify_oop(r0, "Bad oop");
257 
258   // Move result into place
259   if (ref != r0) {
260     __ mov(ref, r0);
261   }
262 
263   // Restore r0 unless it is the result or tmp register
264   if (ref != r0 && tmp != r0) {
265     __ ldr(r0, Address(sp, 16));
266     __ add(sp, sp, 32);
267   } else {
268     __ add(sp, sp, 16);
269   }
270 
271   // Stub exit
272   __ b(*stub->continuation());
273 }
274 
275 #undef __
276 #define __ sasm->
277 
278 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
279                                                                  DecoratorSet decorators) const {
280   __ prologue("zgc_load_barrier stub", false);
281 
282   __ push_call_clobbered_registers_except(RegSet::of(r0));
283 
284   // Setup arguments
285   __ load_parameter(0, c_rarg0);
286   __ load_parameter(1, c_rarg1);
287 
288   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
289 
290   __ pop_call_clobbered_registers_except(RegSet::of(r0));
291 
292   __ epilogue();
293 }
294 #endif // COMPILER1
295 
296 #ifdef COMPILER2
297 
298 OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
299   if (!OptoReg::is_reg(opto_reg)) {
300     return OptoReg::Bad;
301   }
302 
303   const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
304   if (vm_reg->is_FloatRegister()) {
305     return opto_reg & ~1;
306   }
307 
308   return opto_reg;
309 }
310 
311 #undef __
312 #define __ _masm->
313 
314 class ZSaveLiveRegisters {
315 private:
316   MacroAssembler* const _masm;
317   RegSet                _gp_regs;
318   FloatRegSet           _fp_regs;
319 
320 public:
321   void initialize(ZLoadBarrierStubC2* stub) {
322     // Record registers that needs to be saved/restored
323     RegMaskIterator rmi(stub->live());
324     while (rmi.has_next()) {
325       const OptoReg::Name opto_reg = rmi.next();
326       if (OptoReg::is_reg(opto_reg)) {
327         const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
328         if (vm_reg->is_Register()) {
329           _gp_regs += RegSet::of(vm_reg->as_Register());
330         } else if (vm_reg->is_FloatRegister()) {
331           _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
332         } else {
333           fatal("Unknown register type");
334         }
335       }
336     }
337 
338     // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
339     _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
340   }
341 
342   ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
343       _masm(masm),
344       _gp_regs(),
345       _fp_regs() {
346 
347     // Figure out what registers to save/restore
348     initialize(stub);
349 
350     // Save registers
351     __ push(_gp_regs, sp);
352     __ push_fp(_fp_regs, sp);
353   }
354 
355   ~ZSaveLiveRegisters() {
356     // Restore registers
357     __ pop_fp(_fp_regs, sp);
358 
359     // External runtime call may clobber ptrue reg
360     __ reinitialize_ptrue();
361 
362     __ pop(_gp_regs, sp);
363   }
364 };
365 
366 #undef __
367 #define __ _masm->
368 
369 class ZSetupArguments {
370 private:
371   MacroAssembler* const _masm;
372   const Register        _ref;
373   const Address         _ref_addr;
374 
375 public:
376   ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
377       _masm(masm),
378       _ref(stub->ref()),
379       _ref_addr(stub->ref_addr()) {
380 
381     // Setup arguments
382     if (_ref_addr.base() == noreg) {
383       // No self healing
384       if (_ref != c_rarg0) {
385         __ mov(c_rarg0, _ref);
386       }
387       __ mov(c_rarg1, 0);
388     } else {
389       // Self healing
390       if (_ref == c_rarg0) {
391         // _ref is already at correct place
392         __ lea(c_rarg1, _ref_addr);
393       } else if (_ref != c_rarg1) {
394         // _ref is in wrong place, but not in c_rarg1, so fix it first
395         __ lea(c_rarg1, _ref_addr);
396         __ mov(c_rarg0, _ref);
397       } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
398         assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
399         __ mov(c_rarg0, _ref);
400         __ lea(c_rarg1, _ref_addr);
401       } else {
402         assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
403         if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
404           __ mov(rscratch2, c_rarg1);
405           __ lea(c_rarg1, _ref_addr);
406           __ mov(c_rarg0, rscratch2);
407         } else {
408           ShouldNotReachHere();
409         }
410       }
411     }
412   }
413 
414   ~ZSetupArguments() {
415     // Transfer result
416     if (_ref != r0) {
417       __ mov(_ref, r0);
418     }
419   }
420 };
421 
422 #undef __
423 #define __ masm->
424 
425 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
426   BLOCK_COMMENT("ZLoadBarrierStubC2");
427 
428   // Stub entry
429   __ bind(*stub->entry());
430 
431   {
432     ZSaveLiveRegisters save_live_registers(masm, stub);
433     ZSetupArguments setup_arguments(masm, stub);
434     __ mov(rscratch1, stub->slow_path());
435     __ blr(rscratch1);
436   }
437   // Stub exit
438   __ b(*stub->continuation());
439 }
440 
441 #undef __
442 
443 #endif // COMPILER2