1 /*
  2  * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 #include "asm/macroAssembler.inline.hpp"
 28 #include "c1/c1_MacroAssembler.hpp"
 29 #include "c1/c1_Runtime1.hpp"
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "gc/shared/tlab_globals.hpp"
 32 #include "interpreter/interpreter.hpp"
 33 #include "oops/arrayOop.hpp"
 34 #include "oops/markWord.hpp"
 35 #include "runtime/basicLock.hpp"
 36 #include "runtime/os.hpp"
 37 #include "runtime/sharedRuntime.hpp"
 38 #include "runtime/stubRoutines.hpp"
 39 #include "utilities/align.hpp"
 40 #include "utilities/macros.hpp"
 41 #include "utilities/powerOfTwo.hpp"
 42 
 43 
 44 void C1_MacroAssembler::explicit_null_check(Register base) {
 45   Unimplemented();
 46 }
 47 
 48 
 49 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
 50   // Avoid stack bang as first instruction. It may get overwritten by patch_verified_entry.
 51   const Register return_pc = R20;
 52   mflr(return_pc);
 53 
 54   // Make sure there is enough stack space for this method's activation.
 55   assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
 56   generate_stack_overflow_check(bang_size_in_bytes);
 57 
 58   std(return_pc, _abi0(lr), R1_SP);     // SP->lr = return_pc
 59   push_frame(frame_size_in_bytes, R0); // SP -= frame_size_in_bytes
 60 
 61   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 62   bs->nmethod_entry_barrier(this, R20);
 63 }
 64 
 65 
 66 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
 67   if (breakAtEntry) illtrap();
 68   // build frame
 69 }
 70 
 71 
 72 void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) {
 73   assert_different_registers(Rmark, Roop, Rbox, Rscratch);
 74 
 75   Label done, cas_failed, slow_int;
 76 
 77   // The following move must be the first instruction of emitted since debug
 78   // information may be generated for it.
 79   // Load object header.
 80   ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
 81 
 82   verify_oop(Roop, FILE_AND_LINE);
 83 
 84   // Save object being locked into the BasicObjectLock...
 85   std(Roop, in_bytes(BasicObjectLock::obj_offset()), Rbox);
 86 
 87   if (DiagnoseSyncOnValueBasedClasses != 0) {
 88     load_klass(Rscratch, Roop);
 89     lbz(Rscratch, in_bytes(Klass::misc_flags_offset()), Rscratch);
 90     testbitdi(CCR0, R0, Rscratch, exact_log2(KlassFlags::_misc_is_value_based_class));
 91     bne(CCR0, slow_int);
 92   }
 93 
 94   if (LockingMode == LM_LIGHTWEIGHT) {
 95     lightweight_lock(Rbox, Roop, Rmark, Rscratch, slow_int);
 96   } else if (LockingMode == LM_LEGACY) {
 97     // ... and mark it unlocked.
 98     ori(Rmark, Rmark, markWord::unlocked_value);
 99 
100     // Save unlocked object header into the displaced header location on the stack.
101     std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
102 
103     // Compare object markWord with Rmark and if equal exchange Rscratch with object markWord.
104     assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
105     cmpxchgd(/*flag=*/CCR0,
106              /*current_value=*/Rscratch,
107              /*compare_value=*/Rmark,
108              /*exchange_value=*/Rbox,
109              /*where=*/Roop/*+0==mark_offset_in_bytes*/,
110              MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
111              MacroAssembler::cmpxchgx_hint_acquire_lock(),
112              noreg,
113              &cas_failed,
114              /*check without membar and ldarx first*/true);
115     // If compare/exchange succeeded we found an unlocked object and we now have locked it
116     // hence we are done.
117   } else {
118     assert(false, "Unhandled LockingMode:%d", LockingMode);
119   }
120   b(done);
121 
122   bind(slow_int);
123   b(slow_case); // far
124 
125   if (LockingMode == LM_LEGACY) {
126     bind(cas_failed);
127     // We did not find an unlocked object so see if this is a recursive case.
128     sub(Rscratch, Rscratch, R1_SP);
129     load_const_optimized(R0, (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
130     and_(R0/*==0?*/, Rscratch, R0);
131     std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
132     bne(CCR0, slow_int);
133   }
134 
135   bind(done);
136   if (LockingMode == LM_LEGACY) {
137     inc_held_monitor_count(Rmark /*tmp*/);
138   }
139 }
140 
141 
142 void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rbox, Label& slow_case) {
143   assert_different_registers(Rmark, Roop, Rbox);
144 
145   Label slow_int, done;
146 
147   Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
148   assert(mark_addr.disp() == 0, "cas must take a zero displacement");
149 
150   if (LockingMode != LM_LIGHTWEIGHT) {
151     // Test first if it is a fast recursive unlock.
152     ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
153     cmpdi(CCR0, Rmark, 0);
154     beq(CCR0, done);
155   }
156 
157   // Load object.
158   ld(Roop, in_bytes(BasicObjectLock::obj_offset()), Rbox);
159   verify_oop(Roop, FILE_AND_LINE);
160 
161   if (LockingMode == LM_LIGHTWEIGHT) {
162     lightweight_unlock(Roop, Rmark, slow_int);
163   } else if (LockingMode == LM_LEGACY) {
164     // Check if it is still a light weight lock, this is is true if we see
165     // the stack address of the basicLock in the markWord of the object.
166     cmpxchgd(/*flag=*/CCR0,
167              /*current_value=*/R0,
168              /*compare_value=*/Rbox,
169              /*exchange_value=*/Rmark,
170              /*where=*/Roop,
171              MacroAssembler::MemBarRel,
172              MacroAssembler::cmpxchgx_hint_release_lock(),
173              noreg,
174              &slow_int);
175   } else {
176     assert(false, "Unhandled LockingMode:%d", LockingMode);
177   }
178   b(done);
179   bind(slow_int);
180   b(slow_case); // far
181 
182   // Done
183   bind(done);
184   if (LockingMode == LM_LEGACY) {
185     dec_held_monitor_count(Rmark /*tmp*/);
186   }
187 }
188 
189 
190 void C1_MacroAssembler::try_allocate(
191   Register obj,                        // result: pointer to object after successful allocation
192   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
193   int      con_size_in_bytes,          // object size in bytes if   known at compile time
194   Register t1,                         // temp register
195   Register t2,                         // temp register
196   Label&   slow_case                   // continuation point if fast allocation fails
197 ) {
198   if (UseTLAB) {
199     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
200   } else {
201     b(slow_case);
202   }
203 }
204 
205 
206 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
207   assert_different_registers(obj, klass, len, t1, t2);
208   load_const_optimized(t1, (intx)markWord::prototype().value());
209   std(t1, oopDesc::mark_offset_in_bytes(), obj);
210   store_klass(obj, klass);
211   if (len->is_valid()) {
212     stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
213   } else if (UseCompressedClassPointers) {
214     // Otherwise length is in the class gap.
215     store_klass_gap(obj);
216   }
217 }
218 
219 
220 void C1_MacroAssembler::initialize_body(Register base, Register index) {
221   assert_different_registers(base, index);
222   srdi(index, index, LogBytesPerWord);
223   clear_memory_doubleword(base, index);
224 }
225 
226 void C1_MacroAssembler::initialize_body(Register obj, Register tmp1, Register tmp2,
227                                         int obj_size_in_bytes, int hdr_size_in_bytes) {
228   const int index = (obj_size_in_bytes - hdr_size_in_bytes) / HeapWordSize;
229 
230   // 2x unrolled loop is shorter with more than 9 HeapWords.
231   if (index <= 9) {
232     clear_memory_unrolled(obj, index, R0, hdr_size_in_bytes);
233   } else {
234     const Register base_ptr = tmp1,
235                    cnt_dwords = tmp2;
236 
237     addi(base_ptr, obj, hdr_size_in_bytes); // Compute address of first element.
238     clear_memory_doubleword(base_ptr, cnt_dwords, R0, index);
239   }
240 }
241 
242 void C1_MacroAssembler::allocate_object(
243   Register obj,                        // result: pointer to object after successful allocation
244   Register t1,                         // temp register
245   Register t2,                         // temp register
246   Register t3,                         // temp register
247   int      hdr_size,                   // object header size in words
248   int      obj_size,                   // object size in words
249   Register klass,                      // object klass
250   Label&   slow_case                   // continuation point if fast allocation fails
251 ) {
252   assert_different_registers(obj, t1, t2, t3, klass);
253 
254   // allocate space & initialize header
255   if (!is_simm16(obj_size * wordSize)) {
256     // Would need to use extra register to load
257     // object size => go the slow case for now.
258     b(slow_case);
259     return;
260   }
261   try_allocate(obj, noreg, obj_size * wordSize, t2, t3, slow_case);
262 
263   initialize_object(obj, klass, noreg, obj_size * HeapWordSize, t1, t2);
264 }
265 
266 void C1_MacroAssembler::initialize_object(
267   Register obj,                        // result: pointer to object after successful allocation
268   Register klass,                      // object klass
269   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
270   int      con_size_in_bytes,          // object size in bytes if   known at compile time
271   Register t1,                         // temp register
272   Register t2                          // temp register
273   ) {
274   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
275 
276   initialize_header(obj, klass, noreg, t1, t2);
277 
278 #ifdef ASSERT
279   {
280     lwz(t1, in_bytes(Klass::layout_helper_offset()), klass);
281     if (var_size_in_bytes != noreg) {
282       cmpw(CCR0, t1, var_size_in_bytes);
283     } else {
284       cmpwi(CCR0, t1, con_size_in_bytes);
285     }
286     asm_assert_eq("bad size in initialize_object");
287   }
288 #endif
289 
290   // Initialize body.
291   if (var_size_in_bytes != noreg) {
292     // Use a loop.
293     addi(t1, obj, hdr_size_in_bytes);                // Compute address of first element.
294     addi(t2, var_size_in_bytes, -hdr_size_in_bytes); // Compute size of body.
295     initialize_body(t1, t2);
296   } else if (con_size_in_bytes > hdr_size_in_bytes) {
297     // Use a loop.
298     initialize_body(obj, t1, t2, con_size_in_bytes, hdr_size_in_bytes);
299   }
300 
301   if (CURRENT_ENV->dtrace_alloc_probes()) {
302     Unimplemented();
303 //    assert(obj == O0, "must be");
304 //    call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(C1StubId::dtrace_object_alloc_id)),
305 //         relocInfo::runtime_call_type);
306   }
307 
308   verify_oop(obj, FILE_AND_LINE);
309 }
310 
311 
312 void C1_MacroAssembler::allocate_array(
313   Register obj,                        // result: pointer to array after successful allocation
314   Register len,                        // array length
315   Register t1,                         // temp register
316   Register t2,                         // temp register
317   Register t3,                         // temp register
318   int      base_offset_in_bytes,       // elements offset in bytes
319   int      elt_size,                   // element size in bytes
320   Register klass,                      // object klass
321   Label&   slow_case,                  // continuation point if fast allocation fails
322   bool     zero_array                  // zero the allocated array or not
323 ) {
324   assert_different_registers(obj, len, t1, t2, t3, klass);
325 
326   // Determine alignment mask.
327   assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
328   int log2_elt_size = exact_log2(elt_size);
329 
330   // Check for negative or excessive length.
331   size_t max_length = max_array_allocation_length >> log2_elt_size;
332   if (UseTLAB) {
333     size_t max_tlab = align_up(ThreadLocalAllocBuffer::max_size() >> log2_elt_size, 64*K);
334     if (max_tlab < max_length) { max_length = max_tlab; }
335   }
336   load_const_optimized(t1, max_length);
337   cmpld(CCR0, len, t1);
338   bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
339 
340   // compute array size
341   // note: If 0 <= len <= max_length, len*elt_size + header + alignment is
342   //       smaller or equal to the largest integer; also, since top is always
343   //       aligned, we can do the alignment here instead of at the end address
344   //       computation.
345   const Register arr_size = t1;
346   Register arr_len_in_bytes = len;
347   if (elt_size != 1) {
348     sldi(t1, len, log2_elt_size);
349     arr_len_in_bytes = t1;
350   }
351   addi(arr_size, arr_len_in_bytes, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
352   clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes);                              // Align array size.
353 
354   // Allocate space & initialize header.
355   try_allocate(obj, arr_size, 0, t2, t3, slow_case);
356   initialize_header(obj, klass, len, t2, t3);
357 
358   if (zero_array) {
359     // Initialize body.
360     const Register base  = t2;
361     const Register index = t3;
362     addi(base, obj, base_offset_in_bytes);               // compute address of first element
363     addi(index, arr_size, -(base_offset_in_bytes));      // compute index = number of bytes to clear
364 
365     // Zero first 4 bytes, if start offset is not word aligned.
366     if (!is_aligned(base_offset_in_bytes, BytesPerWord)) {
367       assert(is_aligned(base_offset_in_bytes, BytesPerInt), "must be 4-byte aligned");
368       li(t1, 0);
369       stw(t1, 0, base);
370       addi(base, base, BytesPerInt);
371       // Note: initialize_body will align index down, no need to correct it here.
372     }
373 
374     initialize_body(base, index);
375   }
376 
377   if (CURRENT_ENV->dtrace_alloc_probes()) {
378     Unimplemented();
379     //assert(obj == O0, "must be");
380     //call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(C1StubId::dtrace_object_alloc_id)),
381     //     relocInfo::runtime_call_type);
382   }
383 
384   verify_oop(obj, FILE_AND_LINE);
385 }
386 
387 
388 #ifndef PRODUCT
389 
390 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
391   verify_oop_addr((RegisterOrConstant)stack_offset, R1_SP, "broken oop in stack slot");
392 }
393 
394 void C1_MacroAssembler::verify_not_null_oop(Register r) {
395   Label not_null;
396   cmpdi(CCR0, r, 0);
397   bne(CCR0, not_null);
398   stop("non-null oop required");
399   bind(not_null);
400   verify_oop(r, FILE_AND_LINE);
401 }
402 
403 #endif // PRODUCT
404 
405 void C1_MacroAssembler::null_check(Register r, Label* Lnull) {
406   if (TrapBasedNullChecks) { // SIGTRAP based
407     trap_null_check(r);
408   } else { // explicit
409     //const address exception_entry = Runtime1::entry_for(C1StubId::throw_null_pointer_exception_id);
410     assert(Lnull != nullptr, "must have Label for explicit check");
411     cmpdi(CCR0, r, 0);
412     bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::equal), *Lnull);
413   }
414 }