1 /*
  2  * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "classfile/classLoaderData.hpp"
 28 #include "gc/shared/barrierSet.hpp"
 29 #include "gc/shared/barrierSetAssembler.hpp"
 30 #include "gc/shared/barrierSetNMethod.hpp"
 31 #include "gc/shared/barrierSetRuntime.hpp"
 32 #include "gc/shared/collectedHeap.hpp"
 33 #include "interpreter/interp_masm.hpp"
 34 #include "memory/universe.hpp"
 35 #include "runtime/jniHandles.hpp"
 36 #include "runtime/sharedRuntime.hpp"
 37 #include "runtime/stubRoutines.hpp"
 38 #include "runtime/thread.hpp"
 39 
 40 #define __ masm->
 41 
 42 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 43                                   Register dst, Address src, Register tmp1, Register tmp_thread) {
 44   bool in_heap = (decorators & IN_HEAP) != 0;
 45   bool in_native = (decorators & IN_NATIVE) != 0;
 46   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 47   bool atomic = (decorators & MO_RELAXED) != 0;
 48 
 49   assert(type != T_INLINE_TYPE, "Not supported yet");
 50   switch (type) {
 51   case T_OBJECT:
 52   case T_ARRAY: {
 53     if (in_heap) {
 54 #ifdef _LP64
 55       if (UseCompressedOops) {
 56         __ movl(dst, src);
 57         if (is_not_null) {
 58           __ decode_heap_oop_not_null(dst);
 59         } else {
 60           __ decode_heap_oop(dst);
 61         }
 62       } else
 63 #endif
 64       {
 65         __ movptr(dst, src);
 66       }
 67     } else {
 68       assert(in_native, "why else?");
 69       __ movptr(dst, src);
 70     }
 71     break;
 72   }
 73   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
 74   case T_BYTE:    __ load_signed_byte(dst, src);    break;
 75   case T_CHAR:    __ load_unsigned_short(dst, src); break;
 76   case T_SHORT:   __ load_signed_short(dst, src);   break;
 77   case T_INT:     __ movl  (dst, src);              break;
 78   case T_ADDRESS: __ movptr(dst, src);              break;
 79   case T_FLOAT:
 80     assert(dst == noreg, "only to ftos");
 81     __ load_float(src);
 82     break;
 83   case T_DOUBLE:
 84     assert(dst == noreg, "only to dtos");
 85     __ load_double(src);
 86     break;
 87   case T_LONG:
 88     assert(dst == noreg, "only to ltos");
 89 #ifdef _LP64
 90     __ movq(rax, src);
 91 #else
 92     if (atomic) {
 93       __ fild_d(src);               // Must load atomically
 94       __ subptr(rsp,2*wordSize);    // Make space for store
 95       __ fistp_d(Address(rsp,0));
 96       __ pop(rax);
 97       __ pop(rdx);
 98     } else {
 99       __ movl(rax, src);
100       __ movl(rdx, src.plus_disp(wordSize));
101     }
102 #endif
103     break;
104   default: Unimplemented();
105   }
106 }
107 
108 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
109                                    Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
110   bool in_heap = (decorators & IN_HEAP) != 0;
111   bool in_native = (decorators & IN_NATIVE) != 0;
112   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
113   bool atomic = (decorators & MO_RELAXED) != 0;
114 
115   assert(type != T_INLINE_TYPE, "Not supported yet");
116   switch (type) {
117   case T_OBJECT:
118   case T_ARRAY: {
119     if (in_heap) {
120       if (val == noreg) {
121         assert(!is_not_null, "inconsistent access");
122 #ifdef _LP64
123         if (UseCompressedOops) {
124           __ movl(dst, (int32_t)NULL_WORD);
125         } else {
126           __ movslq(dst, (int32_t)NULL_WORD);
127         }
128 #else
129         __ movl(dst, (int32_t)NULL_WORD);
130 #endif
131       } else {
132 #ifdef _LP64
133         if (UseCompressedOops) {
134           assert(!dst.uses(val), "not enough registers");
135           if (is_not_null) {
136             __ encode_heap_oop_not_null(val);
137           } else {
138             __ encode_heap_oop(val);
139           }
140           __ movl(dst, val);
141         } else
142 #endif
143         {
144           __ movptr(dst, val);
145         }
146       }
147     } else {
148       assert(in_native, "why else?");
149       assert(val != noreg, "not supported");
150       __ movptr(dst, val);
151     }
152     break;
153   }
154   case T_BOOLEAN:
155     __ andl(val, 0x1);  // boolean is true if LSB is 1
156     __ movb(dst, val);
157     break;
158   case T_BYTE:
159     __ movb(dst, val);
160     break;
161   case T_SHORT:
162     __ movw(dst, val);
163     break;
164   case T_CHAR:
165     __ movw(dst, val);
166     break;
167   case T_INT:
168     __ movl(dst, val);
169     break;
170   case T_LONG:
171     assert(val == noreg, "only tos");
172 #ifdef _LP64
173     __ movq(dst, rax);
174 #else
175     if (atomic) {
176       __ push(rdx);
177       __ push(rax);                 // Must update atomically with FIST
178       __ fild_d(Address(rsp,0));    // So load into FPU register
179       __ fistp_d(dst);              // and put into memory atomically
180       __ addptr(rsp, 2*wordSize);
181     } else {
182       __ movptr(dst, rax);
183       __ movptr(dst.plus_disp(wordSize), rdx);
184     }
185 #endif
186     break;
187   case T_FLOAT:
188     assert(val == noreg, "only tos");
189     __ store_float(dst);
190     break;
191   case T_DOUBLE:
192     assert(val == noreg, "only tos");
193     __ store_double(dst);
194     break;
195   case T_ADDRESS:
196     __ movptr(dst, val);
197     break;
198   default: Unimplemented();
199   }
200 }
201 
202 void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators,
203                                      Register src, Register dst, Register value_klass) {
204   // value_copy implementation is fairly complex, and there are not any
205   // "short-cuts" to be made from asm. What there is, appears to have the same
206   // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
207   // of hand-rolled instructions...
208   if (decorators & IS_DEST_UNINITIALIZED) {
209     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass);
210   } else {
211     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass);
212   }
213 }
214 
215 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
216                                                         Register obj, Register tmp, Label& slowpath) {
217   __ clear_jweak_tag(obj);
218   __ movptr(obj, Address(obj, 0));
219 }
220 
221 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
222                                         Register thread, Register obj,
223                                         Register var_size_in_bytes,
224                                         int con_size_in_bytes,
225                                         Register t1,
226                                         Register t2,
227                                         Label& slow_case) {
228   assert_different_registers(obj, t1, t2);
229   assert_different_registers(obj, var_size_in_bytes, t1);
230   Register end = t2;
231   if (!thread->is_valid()) {
232 #ifdef _LP64
233     thread = r15_thread;
234 #else
235     assert(t1->is_valid(), "need temp reg");
236     thread = t1;
237     __ get_thread(thread);
238 #endif
239   }
240 
241   __ verify_tlab();
242 
243   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
244   if (var_size_in_bytes == noreg) {
245     __ lea(end, Address(obj, con_size_in_bytes));
246   } else {
247     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
248   }
249   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
250   __ jcc(Assembler::above, slow_case);
251 
252   // update the tlab top pointer
253   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
254 
255   // recover var_size_in_bytes if necessary
256   if (var_size_in_bytes == end) {
257     __ subptr(var_size_in_bytes, obj);
258   }
259   __ verify_tlab();
260 }
261 
262 // Defines obj, preserves var_size_in_bytes
263 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm,
264                                         Register thread, Register obj,
265                                         Register var_size_in_bytes,
266                                         int con_size_in_bytes,
267                                         Register t1,
268                                         Label& slow_case) {
269   assert(obj == rax, "obj must be in rax, for cmpxchg");
270   assert_different_registers(obj, var_size_in_bytes, t1);
271   if (!Universe::heap()->supports_inline_contig_alloc()) {
272     __ jmp(slow_case);
273   } else {
274     Register end = t1;
275     Label retry;
276     __ bind(retry);
277     ExternalAddress heap_top((address) Universe::heap()->top_addr());
278     __ movptr(obj, heap_top);
279     if (var_size_in_bytes == noreg) {
280       __ lea(end, Address(obj, con_size_in_bytes));
281     } else {
282       __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
283     }
284     // if end < obj then we wrapped around => object too long => slow case
285     __ cmpptr(end, obj);
286     __ jcc(Assembler::below, slow_case);
287     __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
288     __ jcc(Assembler::above, slow_case);
289     // Compare obj with the top addr, and if still equal, store the new top addr in
290     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
291     // it otherwise. Use lock prefix for atomicity on MPs.
292     __ locked_cmpxchgptr(end, heap_top);
293     __ jcc(Assembler::notEqual, retry);
294     incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1);
295   }
296 }
297 
298 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
299                                                Register var_size_in_bytes,
300                                                int con_size_in_bytes,
301                                                Register t1) {
302   if (!thread->is_valid()) {
303 #ifdef _LP64
304     thread = r15_thread;
305 #else
306     assert(t1->is_valid(), "need temp reg");
307     thread = t1;
308     __ get_thread(thread);
309 #endif
310   }
311 
312 #ifdef _LP64
313   if (var_size_in_bytes->is_valid()) {
314     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
315   } else {
316     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
317   }
318 #else
319   if (var_size_in_bytes->is_valid()) {
320     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
321   } else {
322     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
323   }
324   __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
325 #endif
326 }
327 
328 #ifdef _LP64
329 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
330   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
331   if (bs_nm == NULL) {
332     return;
333   }
334   Label continuation;
335   Register thread = r15_thread;
336   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset()));
337   __ align(8);
338   __ cmpl(disarmed_addr, 0);
339   __ jcc(Assembler::equal, continuation);
340   __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
341   __ bind(continuation);
342 }
343 #else
344 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
345   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
346   if (bs_nm == NULL) {
347     return;
348   }
349 
350   Label continuation;
351 
352   Register tmp = rdi;
353   __ push(tmp);
354   __ movptr(tmp, (intptr_t)bs_nm->disarmed_value_address());
355   Address disarmed_addr(tmp, 0);
356   __ align(4);
357   __ cmpl(disarmed_addr, 0);
358   __ pop(tmp);
359   __ jcc(Assembler::equal, continuation);
360   __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
361   __ bind(continuation);
362 }
363 #endif
364 
365 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
366   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
367   if (bs == NULL) {
368     return;
369   }
370 
371   Label bad_call;
372   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
373   __ jcc(Assembler::equal, bad_call);
374 
375 #ifdef _LP64
376   Register tmp1 = rscratch1;
377   Register tmp2 = rscratch2;
378 #else
379   Register tmp1 = rax;
380   Register tmp2 = rcx;
381   __ push(tmp1);
382   __ push(tmp2);
383 #endif // _LP64
384 
385   // Pointer chase to the method holder to find out if the method is concurrently unloading.
386   Label method_live;
387   __ load_method_holder_cld(tmp1, rbx);
388 
389    // Is it a strong CLD?
390   __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_offset()), 0);
391   __ jcc(Assembler::greater, method_live);
392 
393    // Is it a weak but alive CLD?
394   __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
395   __ resolve_weak_handle(tmp1, tmp2);
396   __ cmpptr(tmp1, 0);
397   __ jcc(Assembler::notEqual, method_live);
398 
399 #ifndef _LP64
400   __ pop(tmp2);
401   __ pop(tmp1);
402 #endif
403 
404   __ bind(bad_call);
405   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
406   __ bind(method_live);
407 
408 #ifndef _LP64
409   __ pop(tmp2);
410   __ pop(tmp1);
411 #endif
412 }