1 /*
  2  * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
  5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  6  *
  7  * This code is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License version 2 only, as
  9  * published by the Free Software Foundation.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  *
 25  */
 26 
 27 #include "precompiled.hpp"
 28 #include "c1/c1_LIR.hpp"
 29 #include "c1/c1_MacroAssembler.hpp"
 30 #include "c1/c1_Runtime1.hpp"
 31 #include "classfile/systemDictionary.hpp"
 32 #include "gc/shared/barrierSetAssembler.hpp"
 33 #include "gc/shared/collectedHeap.hpp"
 34 #include "interpreter/interpreter.hpp"
 35 #include "oops/arrayOop.hpp"
 36 #include "oops/markWord.hpp"
 37 #include "runtime/basicLock.hpp"
 38 #include "runtime/os.hpp"
 39 #include "runtime/sharedRuntime.hpp"
 40 #include "runtime/stubRoutines.hpp"
 41 
 42 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 43                                   FloatRegister freg0, FloatRegister freg1,
 44                                   Register result)
 45 {
 46   if (is_float) {
 47     float_compare(result, freg0, freg1, unordered_result);
 48   } else {
 49     double_compare(result, freg0, freg1, unordered_result);
 50   }
 51 }
 52 
 53 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
 54   const int aligned_mask = BytesPerWord - 1;
 55   const int hdr_offset = oopDesc::mark_offset_in_bytes();
 56   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
 57   Label done;
 58   int null_check_offset = -1;
 59 
 60   verify_oop(obj);
 61 
 62   null_check_offset = offset();
 63 
 64   if (DiagnoseSyncOnValueBasedClasses != 0) {
 65     load_klass(hdr, obj);
 66     lwu(hdr, Address(hdr, Klass::access_flags_offset()));
 67     andi(t0, hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
 68     bnez(t0, slow_case, true /* is_far */);
 69   }
 70 
 71   ld(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
 72   fast_lock(obj, hdr, disp_hdr, t0, t1, slow_case);
 73 
 74   return null_check_offset;
 75 }
 76 
 77 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
 78   const int aligned_mask = BytesPerWord - 1;
 79   const int hdr_offset = oopDesc::mark_offset_in_bytes();
 80   assert_different_registers(hdr, obj, disp_hdr);
 81 
 82   verify_oop(obj);
 83 
 84   ld(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
 85   fast_unlock(obj, hdr, t0, t1, slow_case);
 86 }
 87 
 88 // Defines obj, preserves var_size_in_bytes
 89 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, Label& slow_case) {
 90   if (UseTLAB) {
 91     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, /* is_far */ true);
 92   } else {
 93     j(slow_case);
 94   }
 95 }
 96 
 97 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2) {
 98   assert_different_registers(obj, klass, len);
 99   // This assumes that all prototype bits fitr in an int32_t
100   mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value());
101   sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
102 
103   if (UseCompressedClassPointers) { // Take care not to kill klass
104     encode_klass_not_null(tmp1, klass);
105     sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
106   } else {
107     sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
108   }
109 
110   if (len->is_valid()) {
111     sw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
112   } else if (UseCompressedClassPointers) {
113     store_klass_gap(obj, zr);
114   }
115 }
116 
117 // preserves obj, destroys len_in_bytes
118 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register tmp) {
119   assert(hdr_size_in_bytes >= 0, "header size must be positive or 0");
120   Label done;
121 
122   // len_in_bytes is positive and ptr sized
123   sub(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
124   beqz(len_in_bytes, done);
125 
126   // Preserve obj
127   if (hdr_size_in_bytes) {
128     add(obj, obj, hdr_size_in_bytes);
129   }
130   zero_memory(obj, len_in_bytes, tmp);
131   if (hdr_size_in_bytes) {
132     sub(obj, obj, hdr_size_in_bytes);
133   }
134 
135   bind(done);
136 }
137 
138 void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, int header_size, int object_size, Register klass, Label& slow_case) {
139   assert_different_registers(obj, tmp1, tmp2);
140   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
141 
142   try_allocate(obj, noreg, object_size * BytesPerWord, tmp1, tmp2, slow_case);
143 
144   initialize_object(obj, klass, noreg, object_size * HeapWordSize, tmp1, tmp2, UseTLAB);
145 }
146 
147 void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, bool is_tlab_allocated) {
148   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
149          "con_size_in_bytes is not multiple of alignment");
150   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
151 
152   initialize_header(obj, klass, noreg, tmp1, tmp2);
153 
154   if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
155     // clear rest of allocated space
156     const Register index = tmp2;
157     // 16: multiplier for threshold
158     const int threshold = 16 * BytesPerWord;    // approximate break even point for code size (see comments below)
159     if (var_size_in_bytes != noreg) {
160       mv(index, var_size_in_bytes);
161       initialize_body(obj, index, hdr_size_in_bytes, tmp1);
162     } else if (con_size_in_bytes <= threshold) {
163       // use explicit null stores
164       int i = hdr_size_in_bytes;
165       if (i < con_size_in_bytes && (con_size_in_bytes % (2 * BytesPerWord))) { // 2: multiplier for BytesPerWord
166         sd(zr, Address(obj, i));
167         i += BytesPerWord;
168       }
169       for (; i < con_size_in_bytes; i += BytesPerWord) {
170         sd(zr, Address(obj, i));
171       }
172     } else if (con_size_in_bytes > hdr_size_in_bytes) {
173       block_comment("zero memory");
174       // use loop to null out the fields
175       int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
176       mv(index, words / 8); // 8: byte size
177 
178       const int unroll = 8; // Number of sd(zr) instructions we'll unroll
179       int remainder = words % unroll;
180       la(t0, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
181 
182       Label entry_point, loop;
183       j(entry_point);
184 
185       bind(loop);
186       sub(index, index, 1);
187       for (int i = -unroll; i < 0; i++) {
188         if (-i == remainder) {
189           bind(entry_point);
190         }
191         sd(zr, Address(t0, i * wordSize));
192       }
193       if (remainder == 0) {
194         bind(entry_point);
195       }
196       add(t0, t0, unroll * wordSize);
197       bnez(index, loop);
198     }
199   }
200 
201   membar(MacroAssembler::StoreStore);
202 
203   if (CURRENT_ENV->dtrace_alloc_probes()) {
204     assert(obj == x10, "must be");
205     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
206   }
207 
208   verify_oop(obj);
209 }
210 
211 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) {
212   assert_different_registers(obj, len, tmp1, tmp2, klass);
213 
214   // determine alignment mask
215   assert(!(BytesPerWord & 1), "must be multiple of 2 for masking code to work");
216 
217   // check for negative or excessive length
218   mv(t0, (int32_t)max_array_allocation_length);
219   bgeu(len, t0, slow_case, /* is_far */ true);
220 
221   const Register arr_size = tmp2; // okay to be the same
222   // align object end
223   mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
224   shadd(arr_size, len, arr_size, t0, f);
225   andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
226 
227   try_allocate(obj, arr_size, 0, tmp1, tmp2, slow_case);
228 
229   initialize_header(obj, klass, len, tmp1, tmp2);
230 
231   // clear rest of allocated space
232   const Register len_zero = len;
233   initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
234 
235   membar(MacroAssembler::StoreStore);
236 
237   if (CURRENT_ENV->dtrace_alloc_probes()) {
238     assert(obj == x10, "must be");
239     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
240   }
241 
242   verify_oop(obj);
243 }
244 
245 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache, Label &L) {
246   verify_oop(receiver);
247   // explicit NULL check not needed since load from [klass_offset] causes a trap
248   // check against inline cache
249   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
250   cmp_klass(receiver, iCache, t0, L);
251 }
252 
253 void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
254   assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
255   // Make sure there is enough stack space for this method's activation.
256   // Note that we do this before creating a frame.
257   generate_stack_overflow_check(bang_size_in_bytes);
258   MacroAssembler::build_frame(framesize);
259 
260   // Insert nmethod entry barrier into frame.
261   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
262   bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */);
263 }
264 
265 void C1_MacroAssembler::remove_frame(int framesize) {
266   MacroAssembler::remove_frame(framesize);
267 }
268 
269 
270 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
271   // If we have to make this method not-entrant we'll overwrite its
272   // first instruction with a jump. For this action to be legal we
273   // must ensure that this first instruction is a J, JAL or NOP.
274   // Make it a NOP.
275   assert_alignment(pc());
276   nop();
277 }
278 
279 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) {
280   //  fp + -2: link
281   //     + -1: return address
282   //     +  0: argument with offset 0
283   //     +  1: argument with offset 1
284   //     +  2: ...
285   ld(reg, Address(fp, offset_in_words * BytesPerWord));
286 }
287 
288 #ifndef PRODUCT
289 
290 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
291   if (!VerifyOops) {
292     return;
293   }
294   verify_oop_addr(Address(sp, stack_offset));
295 }
296 
297 void C1_MacroAssembler::verify_not_null_oop(Register r) {
298   if (!VerifyOops) return;
299   Label not_null;
300   bnez(r, not_null);
301   stop("non-null oop required");
302   bind(not_null);
303   verify_oop(r);
304 }
305 
306 void C1_MacroAssembler::invalidate_registers(bool inv_x10, bool inv_x9, bool inv_x12, bool inv_x13, bool inv_x14, bool inv_x15) {
307 #ifdef ASSERT
308   static int nn;
309   if (inv_x10) { mv(x10, 0xDEAD); }
310   if (inv_x9)  { mv(x9, 0xDEAD);  }
311   if (inv_x12) { mv(x12, nn++);   }
312   if (inv_x13) { mv(x13, 0xDEAD); }
313   if (inv_x14) { mv(x14, 0xDEAD); }
314   if (inv_x15) { mv(x15, 0xDEAD); }
315 #endif // ASSERT
316 }
317 #endif // ifndef PRODUCT
318 
319 typedef void (C1_MacroAssembler::*c1_cond_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
320 typedef void (C1_MacroAssembler::*c1_float_cond_branch_insn)(FloatRegister op1, FloatRegister op2,
321               Label& label, bool is_far, bool is_unordered);
322 
323 static c1_cond_branch_insn c1_cond_branch[] =
324 {
325   /* SHORT branches */
326   (c1_cond_branch_insn)&Assembler::beq,
327   (c1_cond_branch_insn)&Assembler::bne,
328   (c1_cond_branch_insn)&Assembler::blt,
329   (c1_cond_branch_insn)&Assembler::ble,
330   (c1_cond_branch_insn)&Assembler::bge,
331   (c1_cond_branch_insn)&Assembler::bgt,
332   (c1_cond_branch_insn)&Assembler::bleu, // lir_cond_belowEqual
333   (c1_cond_branch_insn)&Assembler::bgeu  // lir_cond_aboveEqual
334 };
335 
336 static c1_float_cond_branch_insn c1_float_cond_branch[] =
337 {
338   /* FLOAT branches */
339   (c1_float_cond_branch_insn)&MacroAssembler::float_beq,
340   (c1_float_cond_branch_insn)&MacroAssembler::float_bne,
341   (c1_float_cond_branch_insn)&MacroAssembler::float_blt,
342   (c1_float_cond_branch_insn)&MacroAssembler::float_ble,
343   (c1_float_cond_branch_insn)&MacroAssembler::float_bge,
344   (c1_float_cond_branch_insn)&MacroAssembler::float_bgt,
345   NULL, // lir_cond_belowEqual
346   NULL, // lir_cond_aboveEqual
347 
348   /* DOUBLE branches */
349   (c1_float_cond_branch_insn)&MacroAssembler::double_beq,
350   (c1_float_cond_branch_insn)&MacroAssembler::double_bne,
351   (c1_float_cond_branch_insn)&MacroAssembler::double_blt,
352   (c1_float_cond_branch_insn)&MacroAssembler::double_ble,
353   (c1_float_cond_branch_insn)&MacroAssembler::double_bge,
354   (c1_float_cond_branch_insn)&MacroAssembler::double_bgt,
355   NULL, // lir_cond_belowEqual
356   NULL  // lir_cond_aboveEqual
357 };
358 
359 void C1_MacroAssembler::c1_cmp_branch(int cmpFlag, Register op1, Register op2, Label& label,
360                                       BasicType type, bool is_far) {
361   if (type == T_OBJECT || type == T_ARRAY) {
362     assert(cmpFlag == lir_cond_equal || cmpFlag == lir_cond_notEqual, "Should be equal or notEqual");
363     if (cmpFlag == lir_cond_equal) {
364       beq(op1, op2, label, is_far);
365     } else {
366       bne(op1, op2, label, is_far);
367     }
368   } else {
369     assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(c1_cond_branch) / sizeof(c1_cond_branch[0])),
370            "invalid c1 conditional branch index");
371     (this->*c1_cond_branch[cmpFlag])(op1, op2, label, is_far);
372   }
373 }
374 
375 void C1_MacroAssembler::c1_float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label,
376                                             bool is_far, bool is_unordered) {
377   assert(cmpFlag >= 0 &&
378          cmpFlag < (int)(sizeof(c1_float_cond_branch) / sizeof(c1_float_cond_branch[0])),
379          "invalid c1 float conditional branch index");
380   (this->*c1_float_cond_branch[cmpFlag])(op1, op2, label, is_far, is_unordered);
381 }