1 /*
  2  * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/classLoaderData.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "gc/shared/barrierSetAssembler.hpp"
 29 #include "gc/shared/barrierSetNMethod.hpp"
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "interpreter/interp_masm.hpp"
 32 #include "memory/universe.hpp"
 33 #include "runtime/jniHandles.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "runtime/stubRoutines.hpp"
 36 #include "runtime/thread.hpp"
 37 
 38 
 39 #define __ masm->
 40 
 41 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 42                                   Register dst, Address src, Register tmp1, Register tmp_thread) {
 43 
 44   // LR is live.  It must be saved around calls.
 45 
 46   bool in_heap = (decorators & IN_HEAP) != 0;
 47   bool in_native = (decorators & IN_NATIVE) != 0;
 48   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 49   switch (type) {
 50   case T_OBJECT:
 51   case T_ARRAY: {
 52     if (in_heap) {
 53       if (UseCompressedOops) {
 54         __ ldrw(dst, src);
 55         if (is_not_null) {
 56           __ decode_heap_oop_not_null(dst);
 57         } else {
 58           __ decode_heap_oop(dst);
 59         }
 60       } else {
 61         __ ldr(dst, src);
 62       }
 63     } else {
 64       assert(in_native, "why else?");
 65       __ ldr(dst, src);
 66     }
 67     break;
 68   }
 69   case T_BOOLEAN: __ load_unsigned_byte (dst, src); break;
 70   case T_BYTE:    __ load_signed_byte   (dst, src); break;
 71   case T_CHAR:    __ load_unsigned_short(dst, src); break;
 72   case T_SHORT:   __ load_signed_short  (dst, src); break;
 73   case T_INT:     __ ldrw               (dst, src); break;
 74   case T_LONG:    __ ldr                (dst, src); break;
 75   case T_ADDRESS: __ ldr                (dst, src); break;
 76   case T_FLOAT:   __ ldrs               (v0, src);  break;
 77   case T_DOUBLE:  __ ldrd               (v0, src);  break;
 78   default: Unimplemented();
 79   }
 80 }
 81 
 82 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 83                                    Address dst, Register val, Register tmp1, Register tmp2) {
 84   bool in_heap = (decorators & IN_HEAP) != 0;
 85   bool in_native = (decorators & IN_NATIVE) != 0;
 86   switch (type) {
 87   case T_OBJECT:
 88   case T_ARRAY: {
 89     val = val == noreg ? zr : val;
 90     if (in_heap) {
 91       if (UseCompressedOops) {
 92         assert(!dst.uses(val), "not enough registers");
 93         if (val != zr) {
 94           __ encode_heap_oop(val);
 95         }
 96         __ strw(val, dst);
 97       } else {
 98         __ str(val, dst);
 99       }
100     } else {
101       assert(in_native, "why else?");
102       __ str(val, dst);
103     }
104     break;
105   }
106   case T_BOOLEAN:
107     __ andw(val, val, 0x1);  // boolean is true if LSB is 1
108     __ strb(val, dst);
109     break;
110   case T_BYTE:    __ strb(val, dst); break;
111   case T_CHAR:    __ strh(val, dst); break;
112   case T_SHORT:   __ strh(val, dst); break;
113   case T_INT:     __ strw(val, dst); break;
114   case T_LONG:    __ str (val, dst); break;
115   case T_ADDRESS: __ str (val, dst); break;
116   case T_FLOAT:   __ strs(v0,  dst); break;
117   case T_DOUBLE:  __ strd(v0,  dst); break;
118   default: Unimplemented();
119   }
120 }
121 
122 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
123                                                         Register obj, Register tmp, Label& slowpath) {
124   // If mask changes we need to ensure that the inverse is still encodable as an immediate
125   STATIC_ASSERT(JNIHandles::weak_tag_mask == 1);
126   __ andr(obj, obj, ~JNIHandles::weak_tag_mask);
127   __ ldr(obj, Address(obj, 0));             // *obj
128 }
129 
130 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
131 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
132                                         Register var_size_in_bytes,
133                                         int con_size_in_bytes,
134                                         Register t1,
135                                         Register t2,
136                                         Label& slow_case) {
137   assert_different_registers(obj, t2);
138   assert_different_registers(obj, var_size_in_bytes);
139   Register end = t2;
140 
141   // verify_tlab();
142 
143   __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
144   if (var_size_in_bytes == noreg) {
145     __ lea(end, Address(obj, con_size_in_bytes));
146   } else {
147     __ lea(end, Address(obj, var_size_in_bytes));
148   }
149   __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
150   __ cmp(end, rscratch1);
151   __ br(Assembler::HI, slow_case);
152 
153   // update the tlab top pointer
154   __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
155 
156   // recover var_size_in_bytes if necessary
157   if (var_size_in_bytes == end) {
158     __ sub(var_size_in_bytes, var_size_in_bytes, obj);
159   }
160   // verify_tlab();
161 }
162 
163 // Defines obj, preserves var_size_in_bytes
164 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj,
165                                         Register var_size_in_bytes,
166                                         int con_size_in_bytes,
167                                         Register t1,
168                                         Label& slow_case) {
169   assert_different_registers(obj, var_size_in_bytes, t1);
170   if (!Universe::heap()->supports_inline_contig_alloc()) {
171     __ b(slow_case);
172   } else {
173     Register end = t1;
174     Register heap_end = rscratch2;
175     Label retry;
176     __ bind(retry);
177     {
178       uint64_t offset;
179       __ adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
180       __ ldr(heap_end, Address(rscratch1, offset));
181     }
182 
183     ExternalAddress heap_top((address) Universe::heap()->top_addr());
184 
185     // Get the current top of the heap
186     {
187       uint64_t offset;
188       __ adrp(rscratch1, heap_top, offset);
189       // Use add() here after ARDP, rather than lea().
190       // lea() does not generate anything if its offset is zero.
191       // However, relocs expect to find either an ADD or a load/store
192       // insn after an ADRP.  add() always generates an ADD insn, even
193       // for add(Rn, Rn, 0).
194       __ add(rscratch1, rscratch1, offset);
195       __ ldaxr(obj, rscratch1);
196     }
197 
198     // Adjust it my the size of our new object
199     if (var_size_in_bytes == noreg) {
200       __ lea(end, Address(obj, con_size_in_bytes));
201     } else {
202       __ lea(end, Address(obj, var_size_in_bytes));
203     }
204 
205     // if end < obj then we wrapped around high memory
206     __ cmp(end, obj);
207     __ br(Assembler::LO, slow_case);
208 
209     __ cmp(end, heap_end);
210     __ br(Assembler::HI, slow_case);
211 
212     // If heap_top hasn't been changed by some other thread, update it.
213     __ stlxr(rscratch2, end, rscratch1);
214     __ cbnzw(rscratch2, retry);
215 
216     incr_allocated_bytes(masm, var_size_in_bytes, con_size_in_bytes, t1);
217   }
218 }
219 
220 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm,
221                                                Register var_size_in_bytes,
222                                                int con_size_in_bytes,
223                                                Register t1) {
224   assert(t1->is_valid(), "need temp reg");
225 
226   __ ldr(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
227   if (var_size_in_bytes->is_valid()) {
228     __ add(t1, t1, var_size_in_bytes);
229   } else {
230     __ add(t1, t1, con_size_in_bytes);
231   }
232   __ str(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
233 }
234 
235 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
236   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
237 
238   if (bs_nm == NULL) {
239     return;
240   }
241 
242   Label skip, guard;
243   Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_offset()));
244 
245   __ ldrw(rscratch1, guard);
246 
247   // Subsequent loads of oops must occur after load of guard value.
248   // BarrierSetNMethod::disarm sets guard with release semantics.
249   __ membar(__ LoadLoad);
250   __ ldrw(rscratch2, thread_disarmed_addr);
251   __ cmpw(rscratch1, rscratch2);
252   __ br(Assembler::EQ, skip);
253 
254   __ movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier());
255   __ blr(rscratch1);
256   __ b(skip);
257 
258   __ bind(guard);
259 
260   __ emit_int32(0);   // nmethod guard value. Skipped over in common case.
261 
262   __ bind(skip);
263 }
264 
265 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
266   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
267   if (bs == NULL) {
268     return;
269   }
270 
271   Label bad_call;
272   __ cbz(rmethod, bad_call);
273 
274   // Pointer chase to the method holder to find out if the method is concurrently unloading.
275   Label method_live;
276   __ load_method_holder_cld(rscratch1, rmethod);
277 
278   // Is it a strong CLD?
279   __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
280   __ cbnz(rscratch2, method_live);
281 
282   // Is it a weak but alive CLD?
283   __ stp(r10, r11, Address(__ pre(sp, -2 * wordSize)));
284   __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));
285 
286   // Uses rscratch1 & rscratch2, so we must pass new temporaries.
287   __ resolve_weak_handle(r10, r11);
288   __ mov(rscratch1, r10);
289   __ ldp(r10, r11, Address(__ post(sp, 2 * wordSize)));
290   __ cbnz(rscratch1, method_live);
291 
292   __ bind(bad_call);
293 
294   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
295   __ bind(method_live);
296 }
297