1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/classLoaderData.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "gc/shared/barrierSetAssembler.hpp"
 29 #include "gc/shared/barrierSetNMethod.hpp"
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "interpreter/interp_masm.hpp"
 32 #include "memory/universe.hpp"
 33 #include "runtime/javaThread.hpp"
 34 #include "runtime/jniHandles.hpp"
 35 #include "runtime/sharedRuntime.hpp"
 36 #include "runtime/stubRoutines.hpp"
 37 
 38 #define __ masm->
 39 
 40 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 41                                   Register dst, Address src, Register tmp1, Register tmp_thread) {
 42   bool in_heap = (decorators & IN_HEAP) != 0;
 43   bool in_native = (decorators & IN_NATIVE) != 0;
 44   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 45   bool atomic = (decorators & MO_RELAXED) != 0;
 46 
 47   switch (type) {
 48   case T_OBJECT:
 49   case T_ARRAY: {
 50     if (in_heap) {
 51 #ifdef _LP64
 52       if (UseCompressedOops) {
 53         __ movl(dst, src);
 54         if (is_not_null) {
 55           __ decode_heap_oop_not_null(dst);
 56         } else {
 57           __ decode_heap_oop(dst);
 58         }
 59       } else
 60 #endif
 61       {
 62         __ movptr(dst, src);
 63       }
 64     } else {
 65       assert(in_native, "why else?");
 66       __ movptr(dst, src);
 67     }
 68     break;
 69   }
 70   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
 71   case T_BYTE:    __ load_signed_byte(dst, src);    break;
 72   case T_CHAR:    __ load_unsigned_short(dst, src); break;
 73   case T_SHORT:   __ load_signed_short(dst, src);   break;
 74   case T_INT:     __ movl  (dst, src);              break;
 75   case T_ADDRESS: __ movptr(dst, src);              break;
 76   case T_FLOAT:
 77     assert(dst == noreg, "only to ftos");
 78     __ load_float(src);
 79     break;
 80   case T_DOUBLE:
 81     assert(dst == noreg, "only to dtos");
 82     __ load_double(src);
 83     break;
 84   case T_LONG:
 85     assert(dst == noreg, "only to ltos");
 86 #ifdef _LP64
 87     __ movq(rax, src);
 88 #else
 89     if (atomic) {
 90       __ fild_d(src);               // Must load atomically
 91       __ subptr(rsp,2*wordSize);    // Make space for store
 92       __ fistp_d(Address(rsp,0));
 93       __ pop(rax);
 94       __ pop(rdx);
 95     } else {
 96       __ movl(rax, src);
 97       __ movl(rdx, src.plus_disp(wordSize));
 98     }
 99 #endif
100     break;
101   default: Unimplemented();
102   }
103 }
104 
105 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
106                                    Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
107   bool in_heap = (decorators & IN_HEAP) != 0;
108   bool in_native = (decorators & IN_NATIVE) != 0;
109   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
110   bool atomic = (decorators & MO_RELAXED) != 0;
111 
112   switch (type) {
113   case T_OBJECT:
114   case T_ARRAY: {
115     if (in_heap) {
116       if (val == noreg) {
117         assert(!is_not_null, "inconsistent access");
118 #ifdef _LP64
119         if (UseCompressedOops) {
120           __ movl(dst, NULL_WORD);
121         } else {
122           __ movslq(dst, NULL_WORD);
123         }
124 #else
125         __ movl(dst, NULL_WORD);
126 #endif
127       } else {
128 #ifdef _LP64
129         if (UseCompressedOops) {
130           assert(!dst.uses(val), "not enough registers");
131           if (is_not_null) {
132             __ encode_heap_oop_not_null(val);
133           } else {
134             __ encode_heap_oop(val);
135           }
136           __ movl(dst, val);
137         } else
138 #endif
139         {
140           __ movptr(dst, val);
141         }
142       }
143     } else {
144       assert(in_native, "why else?");
145       assert(val != noreg, "not supported");
146       __ movptr(dst, val);
147     }
148     break;
149   }
150   case T_BOOLEAN:
151     __ andl(val, 0x1);  // boolean is true if LSB is 1
152     __ movb(dst, val);
153     break;
154   case T_BYTE:
155     __ movb(dst, val);
156     break;
157   case T_SHORT:
158     __ movw(dst, val);
159     break;
160   case T_CHAR:
161     __ movw(dst, val);
162     break;
163   case T_INT:
164     __ movl(dst, val);
165     break;
166   case T_LONG:
167     assert(val == noreg, "only tos");
168 #ifdef _LP64
169     __ movq(dst, rax);
170 #else
171     if (atomic) {
172       __ push(rdx);
173       __ push(rax);                 // Must update atomically with FIST
174       __ fild_d(Address(rsp,0));    // So load into FPU register
175       __ fistp_d(dst);              // and put into memory atomically
176       __ addptr(rsp, 2*wordSize);
177     } else {
178       __ movptr(dst, rax);
179       __ movptr(dst.plus_disp(wordSize), rdx);
180     }
181 #endif
182     break;
183   case T_FLOAT:
184     assert(val == noreg, "only tos");
185     __ store_float(dst);
186     break;
187   case T_DOUBLE:
188     assert(val == noreg, "only tos");
189     __ store_double(dst);
190     break;
191   case T_ADDRESS:
192     __ movptr(dst, val);
193     break;
194   default: Unimplemented();
195   }
196 }
197 
198 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
199                                        DecoratorSet decorators,
200                                        BasicType type,
201                                        size_t bytes,
202                                        Register dst,
203                                        Address src,
204                                        Register tmp) {
205   assert(bytes <= 8, "can only deal with non-vector registers");
206   switch (bytes) {
207   case 1:
208     __ movb(dst, src);
209     break;
210   case 2:
211     __ movw(dst, src);
212     break;
213   case 4:
214     __ movl(dst, src);
215     break;
216   case 8:
217 #ifdef _LP64
218     __ movq(dst, src);
219 #else
220     fatal("No support for 8 bytes copy");
221 #endif
222     break;
223   default:
224     fatal("Unexpected size");
225   }
226 #ifdef _LP64
227   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
228     __ decode_heap_oop(dst);
229   }
230 #endif
231 }
232 
233 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
234                                         DecoratorSet decorators,
235                                         BasicType type,
236                                         size_t bytes,
237                                         Address dst,
238                                         Register src,
239                                         Register tmp) {
240 #ifdef _LP64
241   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
242     __ encode_heap_oop(src);
243   }
244 #endif
245   assert(bytes <= 8, "can only deal with non-vector registers");
246   switch (bytes) {
247   case 1:
248     __ movb(dst, src);
249     break;
250   case 2:
251     __ movw(dst, src);
252     break;
253   case 4:
254     __ movl(dst, src);
255     break;
256   case 8:
257 #ifdef _LP64
258     __ movq(dst, src);
259 #else
260     fatal("No support for 8 bytes copy");
261 #endif
262     break;
263   default:
264     fatal("Unexpected size");
265   }
266 }
267 
268 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
269                                        DecoratorSet decorators,
270                                        BasicType type,
271                                        size_t bytes,
272                                        XMMRegister dst,
273                                        Address src,
274                                        Register tmp,
275                                        XMMRegister xmm_tmp) {
276   assert(bytes > 8, "can only deal with vector registers");
277   if (bytes == 16) {
278     __ movdqu(dst, src);
279   } else if (bytes == 32) {
280     __ vmovdqu(dst, src);
281   } else {
282     fatal("No support for >32 bytes copy");
283   }
284 }
285 
286 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
287                                         DecoratorSet decorators,
288                                         BasicType type,
289                                         size_t bytes,
290                                         Address dst,
291                                         XMMRegister src,
292                                         Register tmp1,
293                                         Register tmp2,
294                                         XMMRegister xmm_tmp) {
295   assert(bytes > 8, "can only deal with vector registers");
296   if (bytes == 16) {
297     __ movdqu(dst, src);
298   } else if (bytes == 32) {
299     __ vmovdqu(dst, src);
300   } else {
301     fatal("No support for >32 bytes copy");
302   }
303 }
304 
305 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
306                                                         Register obj, Register tmp, Label& slowpath) {
307   __ clear_jobject_tag(obj);
308   __ movptr(obj, Address(obj, 0));
309 }
310 
311 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
312                                         Register thread, Register obj,
313                                         Register var_size_in_bytes,
314                                         int con_size_in_bytes,
315                                         Register t1,
316                                         Register t2,
317                                         Label& slow_case) {
318   assert_different_registers(obj, t1, t2);
319   assert_different_registers(obj, var_size_in_bytes, t1);
320   Register end = t2;
321   if (!thread->is_valid()) {
322 #ifdef _LP64
323     thread = r15_thread;
324 #else
325     assert(t1->is_valid(), "need temp reg");
326     thread = t1;
327     __ get_thread(thread);
328 #endif
329   }
330 
331   __ verify_tlab();
332 
333   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
334   if (var_size_in_bytes == noreg) {
335     __ lea(end, Address(obj, con_size_in_bytes));
336   } else {
337     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
338   }
339   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
340   __ jcc(Assembler::above, slow_case);
341 
342   // update the tlab top pointer
343   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
344 
345   // recover var_size_in_bytes if necessary
346   if (var_size_in_bytes == end) {
347     __ subptr(var_size_in_bytes, obj);
348   }
349   __ verify_tlab();
350 }
351 
352 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
353                                                Register var_size_in_bytes,
354                                                int con_size_in_bytes,
355                                                Register t1) {
356   if (!thread->is_valid()) {
357 #ifdef _LP64
358     thread = r15_thread;
359 #else
360     assert(t1->is_valid(), "need temp reg");
361     thread = t1;
362     __ get_thread(thread);
363 #endif
364   }
365 
366 #ifdef _LP64
367   if (var_size_in_bytes->is_valid()) {
368     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
369   } else {
370     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
371   }
372 #else
373   if (var_size_in_bytes->is_valid()) {
374     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
375   } else {
376     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
377   }
378   __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
379 #endif
380 }
381 
382 #ifdef _LP64
383 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
384   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
385   if (bs_nm == nullptr) {
386     return;
387   }
388   Register thread = r15_thread;
389   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
390   // The immediate is the last 4 bytes, so if we align the start of the cmp
391   // instruction to 4 bytes, we know that the second half of it is also 4
392   // byte aligned, which means that the immediate will not cross a cache line
393   __ align(4);
394   uintptr_t before_cmp = (uintptr_t)__ pc();
395   __ cmpl_imm32(disarmed_addr, 0);
396   uintptr_t after_cmp = (uintptr_t)__ pc();
397   guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
398 
399   if (slow_path != nullptr) {
400     __ jcc(Assembler::notEqual, *slow_path);
401     __ bind(*continuation);
402   } else {
403     Label done;
404     __ jccb(Assembler::equal, done);
405     __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
406     __ bind(done);
407   }
408 }
409 #else
410 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label*, Label*) {
411   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
412   if (bs_nm == nullptr) {
413     return;
414   }
415 
416   Label continuation;
417 
418   Register tmp = rdi;
419   __ push(tmp);
420   __ movptr(tmp, (intptr_t)bs_nm->disarmed_guard_value_address());
421   Address disarmed_addr(tmp, 0);
422   __ align(4);
423   __ cmpl_imm32(disarmed_addr, 0);
424   __ pop(tmp);
425   __ jcc(Assembler::equal, continuation);
426   __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
427   __ bind(continuation);
428 }
429 #endif
430 
431 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
432   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
433   if (bs == nullptr) {
434     return;
435   }
436 
437   Label bad_call;
438   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
439   __ jcc(Assembler::equal, bad_call);
440 
441   Register tmp1 = LP64_ONLY( rscratch1 ) NOT_LP64( rax );
442   Register tmp2 = LP64_ONLY( rscratch2 ) NOT_LP64( rcx );
443 #ifndef _LP64
444   __ push(tmp1);
445   __ push(tmp2);
446 #endif // !_LP64
447 
448   // Pointer chase to the method holder to find out if the method is concurrently unloading.
449   Label method_live;
450   __ load_method_holder_cld(tmp1, rbx);
451 
452    // Is it a strong CLD?
453   __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_offset()), 0);
454   __ jcc(Assembler::greater, method_live);
455 
456    // Is it a weak but alive CLD?
457   __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
458   __ resolve_weak_handle(tmp1, tmp2);
459   __ cmpptr(tmp1, 0);
460   __ jcc(Assembler::notEqual, method_live);
461 
462 #ifndef _LP64
463   __ pop(tmp2);
464   __ pop(tmp1);
465 #endif
466 
467   __ bind(bad_call);
468   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
469   __ bind(method_live);
470 
471 #ifndef _LP64
472   __ pop(tmp2);
473   __ pop(tmp1);
474 #endif
475 }
476 
477 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
478   // Check if the oop is in the right area of memory
479   __ movptr(tmp1, obj);
480   __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
481   __ andptr(tmp1, tmp2);
482   __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
483   __ cmpptr(tmp1, tmp2);
484   __ jcc(Assembler::notZero, error);
485 
486   // make sure klass is 'reasonable', which is not zero.
487   __ load_klass(obj, obj, tmp1);  // get klass
488   __ testptr(obj, obj);
489   __ jcc(Assembler::zero, error); // if klass is null it is broken
490 }