1 /*
  2  * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
  5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  6  *
  7  * This code is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License version 2 only, as
  9  * published by the Free Software Foundation.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  *
 25  */
 26 
 27 #include "precompiled.hpp"
 28 #include "c1/c1_LIR.hpp"
 29 #include "c1/c1_MacroAssembler.hpp"
 30 #include "c1/c1_Runtime1.hpp"
 31 #include "classfile/systemDictionary.hpp"
 32 #include "gc/shared/barrierSetAssembler.hpp"
 33 #include "gc/shared/collectedHeap.hpp"
 34 #include "interpreter/interpreter.hpp"
 35 #include "oops/arrayOop.hpp"
 36 #include "oops/markWord.hpp"
 37 #include "runtime/basicLock.hpp"
 38 #include "runtime/os.hpp"
 39 #include "runtime/sharedRuntime.hpp"
 40 #include "runtime/stubRoutines.hpp"
 41 
 42 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 43                                   FloatRegister freg0, FloatRegister freg1,
 44                                   Register result) {
 45   if (is_float) {
 46     float_compare(result, freg0, freg1, unordered_result);
 47   } else {
 48     double_compare(result, freg0, freg1, unordered_result);
 49   }
 50 }
 51 
 52 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
 53   const int aligned_mask = BytesPerWord - 1;
 54   const int hdr_offset = oopDesc::mark_offset_in_bytes();
 55   assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
 56   int null_check_offset = -1;
 57 
 58   verify_oop(obj);
 59 
 60   // save object being locked into the BasicObjectLock
 61   sd(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 62 
 63   null_check_offset = offset();
 64 
 65   if (DiagnoseSyncOnValueBasedClasses != 0) {
 66     load_klass(hdr, obj);
 67     lwu(hdr, Address(hdr, Klass::access_flags_offset()));
 68     test_bit(temp, hdr, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
 69     bnez(temp, slow_case, true /* is_far */);
 70   }
 71 
 72   if (LockingMode == LM_LIGHTWEIGHT) {
 73     lightweight_lock(obj, hdr, temp, t1, slow_case);
 74   } else if (LockingMode == LM_LEGACY) {
 75     Label done;
 76     // Load object header
 77     ld(hdr, Address(obj, hdr_offset));
 78     // and mark it as unlocked
 79     ori(hdr, hdr, markWord::unlocked_value);
 80     // save unlocked object header into the displaced header location on the stack
 81     sd(hdr, Address(disp_hdr, 0));
 82     // test if object header is still the same (i.e. unlocked), and if so, store the
 83     // displaced header address in the object header - if it is not the same, get the
 84     // object header instead
 85     la(temp, Address(obj, hdr_offset));
 86     cmpxchgptr(hdr, disp_hdr, temp, t1, done, /*fallthough*/nullptr);
 87     // if the object header was the same, we're done
 88     // if the object header was not the same, it is now in the hdr register
 89     // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
 90     //
 91     // 1) (hdr & aligned_mask) == 0
 92     // 2) sp <= hdr
 93     // 3) hdr <= sp + page_size
 94     //
 95     // these 3 tests can be done by evaluating the following expression:
 96     //
 97     // (hdr -sp) & (aligned_mask - page_size)
 98     //
 99     // assuming both the stack pointer and page_size have their least
100     // significant 2 bits cleared and page_size is a power of 2
101     sub(hdr, hdr, sp);
102     mv(temp, aligned_mask - (int)os::vm_page_size());
103     andr(hdr, hdr, temp);
104     // for recursive locking, the result is zero => save it in the displaced header
105     // location (null in the displaced hdr location indicates recursive locking)
106     sd(hdr, Address(disp_hdr, 0));
107     // otherwise we don't care about the result and handle locking via runtime call
108     bnez(hdr, slow_case, /* is_far */ true);
109     // done
110     bind(done);
111   }
112 
113   increment(Address(xthread, JavaThread::held_monitor_count_offset()));
114   return null_check_offset;
115 }
116 
117 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
118   const int aligned_mask = BytesPerWord - 1;
119   const int hdr_offset = oopDesc::mark_offset_in_bytes();
120   assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
121   Label done;
122 
123   if (LockingMode != LM_LIGHTWEIGHT) {
124     // load displaced header
125     ld(hdr, Address(disp_hdr, 0));
126     // if the loaded hdr is null we had recursive locking
127     // if we had recursive locking, we are done
128     beqz(hdr, done);
129   }
130 
131   // load object
132   ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
133   verify_oop(obj);
134 
135   if (LockingMode == LM_LIGHTWEIGHT) {
136     lightweight_unlock(obj, hdr, temp, t1, slow_case);
137   } else if (LockingMode == LM_LEGACY) {
138     // test if object header is pointing to the displaced header, and if so, restore
139     // the displaced header in the object - if the object header is not pointing to
140     // the displaced header, get the object header instead
141     // if the object header was not pointing to the displaced header,
142     // we do unlocking via runtime call
143     if (hdr_offset) {
144       la(temp, Address(obj, hdr_offset));
145       cmpxchgptr(disp_hdr, hdr, temp, t1, done, &slow_case);
146     } else {
147       cmpxchgptr(disp_hdr, hdr, obj, t1, done, &slow_case);
148     }
149     // done
150     bind(done);
151   }
152 
153   decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
154 }
155 
156 // Defines obj, preserves var_size_in_bytes
157 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, Label& slow_case) {
158   if (UseTLAB) {
159     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, /* is_far */ true);
160   } else {
161     j(slow_case);
162   }
163 }
164 
165 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2) {
166   assert_different_registers(obj, klass, len, tmp1, tmp2);
167   // This assumes that all prototype bits fitr in an int32_t
168   mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value());
169   sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
170 
171   if (UseCompressedClassPointers) { // Take care not to kill klass
172     encode_klass_not_null(tmp1, klass, tmp2);
173     sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
174   } else {
175     sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
176   }
177 
178   if (len->is_valid()) {
179     sw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
180     int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
181     if (!is_aligned(base_offset, BytesPerWord)) {
182       assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
183       // Clear gap/first 4 bytes following the length field.
184       sw(zr, Address(obj, base_offset));
185     }
186   } else if (UseCompressedClassPointers) {
187     store_klass_gap(obj, zr);
188   }
189 }
190 
191 // preserves obj, destroys len_in_bytes
192 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register tmp) {
193   assert(hdr_size_in_bytes >= 0, "header size must be positive or 0");
194   Label done;
195 
196   // len_in_bytes is positive and ptr sized
197   sub(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
198   beqz(len_in_bytes, done);
199 
200   // Preserve obj
201   if (hdr_size_in_bytes) {
202     add(obj, obj, hdr_size_in_bytes);
203   }
204   zero_memory(obj, len_in_bytes, tmp);
205   if (hdr_size_in_bytes) {
206     sub(obj, obj, hdr_size_in_bytes);
207   }
208 
209   bind(done);
210 }
211 
212 void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, int header_size, int object_size, Register klass, Label& slow_case) {
213   assert_different_registers(obj, tmp1, tmp2);
214   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
215 
216   try_allocate(obj, noreg, object_size * BytesPerWord, tmp1, tmp2, slow_case);
217 
218   initialize_object(obj, klass, noreg, object_size * HeapWordSize, tmp1, tmp2, UseTLAB);
219 }
220 
221 void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, bool is_tlab_allocated) {
222   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
223          "con_size_in_bytes is not multiple of alignment");
224   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
225 
226   initialize_header(obj, klass, noreg, tmp1, tmp2);
227 
228   if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
229     // clear rest of allocated space
230     const Register index = tmp2;
231     // 16: multiplier for threshold
232     const int threshold = 16 * BytesPerWord;    // approximate break even point for code size (see comments below)
233     if (var_size_in_bytes != noreg) {
234       mv(index, var_size_in_bytes);
235       initialize_body(obj, index, hdr_size_in_bytes, tmp1);
236     } else if (con_size_in_bytes <= threshold) {
237       // use explicit null stores
238       int i = hdr_size_in_bytes;
239       if (i < con_size_in_bytes && (con_size_in_bytes % (2 * BytesPerWord))) { // 2: multiplier for BytesPerWord
240         sd(zr, Address(obj, i));
241         i += BytesPerWord;
242       }
243       for (; i < con_size_in_bytes; i += BytesPerWord) {
244         sd(zr, Address(obj, i));
245       }
246     } else if (con_size_in_bytes > hdr_size_in_bytes) {
247       block_comment("zero memory");
248       // use loop to null out the fields
249       int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
250       mv(index, words / 8); // 8: byte size
251 
252       const int unroll = 8; // Number of sd(zr) instructions we'll unroll
253       int remainder = words % unroll;
254       la(t0, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
255 
256       Label entry_point, loop;
257       j(entry_point);
258 
259       bind(loop);
260       sub(index, index, 1);
261       for (int i = -unroll; i < 0; i++) {
262         if (-i == remainder) {
263           bind(entry_point);
264         }
265         sd(zr, Address(t0, i * wordSize));
266       }
267       if (remainder == 0) {
268         bind(entry_point);
269       }
270       add(t0, t0, unroll * wordSize);
271       bnez(index, loop);
272     }
273   }
274 
275   membar(MacroAssembler::StoreStore);
276 
277   if (CURRENT_ENV->dtrace_alloc_probes()) {
278     assert(obj == x10, "must be");
279     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
280   }
281 
282   verify_oop(obj);
283 }
284 
285 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case, bool zero_array) {
286   assert_different_registers(obj, len, tmp1, tmp2, klass);
287 
288   // determine alignment mask
289   assert(!(BytesPerWord & 1), "must be multiple of 2 for masking code to work");
290 
291   // check for negative or excessive length
292   mv(t0, (int32_t)max_array_allocation_length);
293   bgeu(len, t0, slow_case, /* is_far */ true);
294 
295   const Register arr_size = tmp2; // okay to be the same
296   // align object end
297   mv(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
298   shadd(arr_size, len, arr_size, t0, f);
299   andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
300 
301   try_allocate(obj, arr_size, 0, tmp1, tmp2, slow_case);
302 
303   initialize_header(obj, klass, len, tmp1, tmp2);
304 
305   // Align-up to word boundary, because we clear the 4 bytes potentially
306   // following the length field in initialize_header().
307   int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
308 
309   // clear rest of allocated space
310   const Register len_zero = len;
311   if (zero_array) {
312     initialize_body(obj, arr_size, base_offset, len_zero);
313   }
314 
315   membar(MacroAssembler::StoreStore);
316 
317   if (CURRENT_ENV->dtrace_alloc_probes()) {
318     assert(obj == x10, "must be");
319     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
320   }
321 
322   verify_oop(obj);
323 }
324 
325 void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
326   assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
327   // Make sure there is enough stack space for this method's activation.
328   // Note that we do this before creating a frame.
329   generate_stack_overflow_check(bang_size_in_bytes);
330   MacroAssembler::build_frame(framesize);
331 
332   // Insert nmethod entry barrier into frame.
333   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
334   bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */, nullptr /* guard */);
335 }
336 
337 void C1_MacroAssembler::remove_frame(int framesize) {
338   MacroAssembler::remove_frame(framesize);
339 }
340 
341 
342 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
343   // If we have to make this method not-entrant we'll overwrite its
344   // first instruction with a jump. For this action to be legal we
345   // must ensure that this first instruction is a J, JAL or NOP.
346   // Make it a NOP.
347   IncompressibleRegion ir(this);  // keep the nop as 4 bytes for patching.
348   assert_alignment(pc());
349   nop();  // 4 bytes
350 }
351 
352 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) {
353   //  fp + -2: link
354   //     + -1: return address
355   //     +  0: argument with offset 0
356   //     +  1: argument with offset 1
357   //     +  2: ...
358   ld(reg, Address(fp, offset_in_words * BytesPerWord));
359 }
360 
361 #ifndef PRODUCT
362 
363 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
364   if (!VerifyOops) {
365     return;
366   }
367   verify_oop_addr(Address(sp, stack_offset));
368 }
369 
370 void C1_MacroAssembler::verify_not_null_oop(Register r) {
371   if (!VerifyOops) return;
372   Label not_null;
373   bnez(r, not_null);
374   stop("non-null oop required");
375   bind(not_null);
376   verify_oop(r);
377 }
378 
379 void C1_MacroAssembler::invalidate_registers(bool inv_x10, bool inv_x9, bool inv_x12, bool inv_x13, bool inv_x14, bool inv_x15) {
380 #ifdef ASSERT
381   static int nn;
382   if (inv_x10) { mv(x10, 0xDEAD); }
383   if (inv_x9)  { mv(x9, 0xDEAD);  }
384   if (inv_x12) { mv(x12, nn++);   }
385   if (inv_x13) { mv(x13, 0xDEAD); }
386   if (inv_x14) { mv(x14, 0xDEAD); }
387   if (inv_x15) { mv(x15, 0xDEAD); }
388 #endif // ASSERT
389 }
390 #endif // ifndef PRODUCT
391 
392 typedef void (C1_MacroAssembler::*c1_cond_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
393 typedef void (C1_MacroAssembler::*c1_float_cond_branch_insn)(FloatRegister op1, FloatRegister op2,
394               Label& label, bool is_far, bool is_unordered);
395 
396 static c1_cond_branch_insn c1_cond_branch[] =
397 {
398   /* SHORT branches */
399   (c1_cond_branch_insn)&MacroAssembler::beq,
400   (c1_cond_branch_insn)&MacroAssembler::bne,
401   (c1_cond_branch_insn)&MacroAssembler::blt,
402   (c1_cond_branch_insn)&MacroAssembler::ble,
403   (c1_cond_branch_insn)&MacroAssembler::bge,
404   (c1_cond_branch_insn)&MacroAssembler::bgt,
405   (c1_cond_branch_insn)&MacroAssembler::bleu, // lir_cond_belowEqual
406   (c1_cond_branch_insn)&MacroAssembler::bgeu  // lir_cond_aboveEqual
407 };
408 
409 static c1_float_cond_branch_insn c1_float_cond_branch[] =
410 {
411   /* FLOAT branches */
412   (c1_float_cond_branch_insn)&MacroAssembler::float_beq,
413   (c1_float_cond_branch_insn)&MacroAssembler::float_bne,
414   (c1_float_cond_branch_insn)&MacroAssembler::float_blt,
415   (c1_float_cond_branch_insn)&MacroAssembler::float_ble,
416   (c1_float_cond_branch_insn)&MacroAssembler::float_bge,
417   (c1_float_cond_branch_insn)&MacroAssembler::float_bgt,
418   nullptr, // lir_cond_belowEqual
419   nullptr, // lir_cond_aboveEqual
420 
421   /* DOUBLE branches */
422   (c1_float_cond_branch_insn)&MacroAssembler::double_beq,
423   (c1_float_cond_branch_insn)&MacroAssembler::double_bne,
424   (c1_float_cond_branch_insn)&MacroAssembler::double_blt,
425   (c1_float_cond_branch_insn)&MacroAssembler::double_ble,
426   (c1_float_cond_branch_insn)&MacroAssembler::double_bge,
427   (c1_float_cond_branch_insn)&MacroAssembler::double_bgt,
428   nullptr, // lir_cond_belowEqual
429   nullptr  // lir_cond_aboveEqual
430 };
431 
432 void C1_MacroAssembler::c1_cmp_branch(int cmpFlag, Register op1, Register op2, Label& label,
433                                       BasicType type, bool is_far) {
434   if (type == T_OBJECT || type == T_ARRAY) {
435     assert(cmpFlag == lir_cond_equal || cmpFlag == lir_cond_notEqual, "Should be equal or notEqual");
436     if (cmpFlag == lir_cond_equal) {
437       beq(op1, op2, label, is_far);
438     } else {
439       bne(op1, op2, label, is_far);
440     }
441   } else {
442     assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(c1_cond_branch) / sizeof(c1_cond_branch[0])),
443            "invalid c1 conditional branch index");
444     (this->*c1_cond_branch[cmpFlag])(op1, op2, label, is_far);
445   }
446 }
447 
448 void C1_MacroAssembler::c1_float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label,
449                                             bool is_far, bool is_unordered) {
450   assert(cmpFlag >= 0 &&
451          cmpFlag < (int)(sizeof(c1_float_cond_branch) / sizeof(c1_float_cond_branch[0])),
452          "invalid c1 float conditional branch index");
453   (this->*c1_float_cond_branch[cmpFlag])(op1, op2, label, is_far, is_unordered);
454 }