1 /*
  2  * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 #include "asm/macroAssembler.inline.hpp"
 28 #include "c1/c1_MacroAssembler.hpp"
 29 #include "c1/c1_Runtime1.hpp"
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "gc/shared/tlab_globals.hpp"
 32 #include "interpreter/interpreter.hpp"
 33 #include "oops/arrayOop.hpp"
 34 #include "oops/markWord.hpp"
 35 #include "runtime/basicLock.hpp"
 36 #include "runtime/os.hpp"
 37 #include "runtime/sharedRuntime.hpp"
 38 #include "runtime/stubRoutines.hpp"
 39 #include "utilities/align.hpp"
 40 #include "utilities/macros.hpp"
 41 #include "utilities/powerOfTwo.hpp"
 42 
 43 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
 44   const Register temp_reg = R12_scratch2;
 45   Label Lmiss;
 46 
 47   verify_oop(receiver, FILE_AND_LINE);
 48   MacroAssembler::null_check(receiver, oopDesc::klass_offset_in_bytes(), &Lmiss);
 49   load_klass(temp_reg, receiver);
 50 
 51   if (TrapBasedICMissChecks && TrapBasedNullChecks) {
 52     trap_ic_miss_check(temp_reg, iCache);
 53   } else {
 54     Label Lok;
 55     cmpd(CCR0, temp_reg, iCache);
 56     beq(CCR0, Lok);
 57     bind(Lmiss);
 58     //load_const_optimized(temp_reg, SharedRuntime::get_ic_miss_stub(), R0);
 59     calculate_address_from_global_toc(temp_reg, SharedRuntime::get_ic_miss_stub(), true, true, false);
 60     mtctr(temp_reg);
 61     bctr();
 62     align(32, 12);
 63     bind(Lok);
 64   }
 65 }
 66 
 67 
 68 void C1_MacroAssembler::explicit_null_check(Register base) {
 69   Unimplemented();
 70 }
 71 
 72 
 73 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
 74   // Avoid stack bang as first instruction. It may get overwritten by patch_verified_entry.
 75   const Register return_pc = R20;
 76   mflr(return_pc);
 77 
 78   // Make sure there is enough stack space for this method's activation.
 79   assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
 80   generate_stack_overflow_check(bang_size_in_bytes);
 81 
 82   std(return_pc, _abi0(lr), R1_SP);     // SP->lr = return_pc
 83   push_frame(frame_size_in_bytes, R0); // SP -= frame_size_in_bytes
 84 
 85   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 86   bs->nmethod_entry_barrier(this, R20);
 87 }
 88 
 89 
 90 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {
 91   if (breakAtEntry) illtrap();
 92   // build frame
 93 }
 94 
 95 
 96 void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) {
 97   assert_different_registers(Rmark, Roop, Rbox, Rscratch);
 98 
 99   Label done, cas_failed, slow_int;
100 
101   // The following move must be the first instruction of emitted since debug
102   // information may be generated for it.
103   // Load object header.
104   ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
105 
106   verify_oop(Roop, FILE_AND_LINE);
107 
108   // Save object being locked into the BasicObjectLock...
109   std(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
110 
111   if (DiagnoseSyncOnValueBasedClasses != 0) {
112     load_klass(Rscratch, Roop);
113     lwz(Rscratch, in_bytes(Klass::access_flags_offset()), Rscratch);
114     testbitdi(CCR0, R0, Rscratch, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
115     bne(CCR0, slow_int);
116   }
117 
118   // ... and mark it unlocked.
119   ori(Rmark, Rmark, markWord::unlocked_value);
120 
121   // Save unlocked object header into the displaced header location on the stack.
122   std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
123 
124   // Compare object markWord with Rmark and if equal exchange Rscratch with object markWord.
125   assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
126   cmpxchgd(/*flag=*/CCR0,
127            /*current_value=*/Rscratch,
128            /*compare_value=*/Rmark,
129            /*exchange_value=*/Rbox,
130            /*where=*/Roop/*+0==mark_offset_in_bytes*/,
131            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
132            MacroAssembler::cmpxchgx_hint_acquire_lock(),
133            noreg,
134            &cas_failed,
135            /*check without membar and ldarx first*/true);
136   // If compare/exchange succeeded we found an unlocked object and we now have locked it
137   // hence we are done.
138   b(done);
139 
140   bind(slow_int);
141   b(slow_case); // far
142 
143   bind(cas_failed);
144   // We did not find an unlocked object so see if this is a recursive case.
145   sub(Rscratch, Rscratch, R1_SP);
146   load_const_optimized(R0, (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
147   and_(R0/*==0?*/, Rscratch, R0);
148   std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
149   bne(CCR0, slow_int);
150 
151   bind(done);
152 }
153 
154 
155 void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rbox, Label& slow_case) {
156   assert_different_registers(Rmark, Roop, Rbox);
157 
158   Label slow_int, done;
159 
160   Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
161   assert(mark_addr.disp() == 0, "cas must take a zero displacement");
162 
163   // Test first it it is a fast recursive unlock.
164   ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
165   cmpdi(CCR0, Rmark, 0);
166   beq(CCR0, done);
167 
168   // Load object.
169   ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
170   verify_oop(Roop, FILE_AND_LINE);
171 
172   // Check if it is still a light weight lock, this is is true if we see
173   // the stack address of the basicLock in the markWord of the object.
174   cmpxchgd(/*flag=*/CCR0,
175            /*current_value=*/R0,
176            /*compare_value=*/Rbox,
177            /*exchange_value=*/Rmark,
178            /*where=*/Roop,
179            MacroAssembler::MemBarRel,
180            MacroAssembler::cmpxchgx_hint_release_lock(),
181            noreg,
182            &slow_int);
183   b(done);
184   bind(slow_int);
185   b(slow_case); // far
186 
187   // Done
188   bind(done);
189 }
190 
191 
192 void C1_MacroAssembler::try_allocate(
193   Register obj,                        // result: pointer to object after successful allocation
194   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
195   int      con_size_in_bytes,          // object size in bytes if   known at compile time
196   Register t1,                         // temp register, must be global register for incr_allocated_bytes
197   Register t2,                         // temp register
198   Label&   slow_case                   // continuation point if fast allocation fails
199 ) {
200   if (UseTLAB) {
201     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
202   } else {
203     b(slow_case);
204   }
205 }
206 
207 
208 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
209   assert_different_registers(obj, klass, len, t1, t2);
210   load_const_optimized(t1, (intx)markWord::prototype().value());
211   std(t1, oopDesc::mark_offset_in_bytes(), obj);
212   store_klass(obj, klass);
213   if (len->is_valid()) {
214     stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
215   } else if (UseCompressedClassPointers) {
216     // Otherwise length is in the class gap.
217     store_klass_gap(obj);
218   }
219 }
220 
221 
222 void C1_MacroAssembler::initialize_body(Register base, Register index) {
223   assert_different_registers(base, index);
224   srdi(index, index, LogBytesPerWord);
225   clear_memory_doubleword(base, index);
226 }
227 
228 void C1_MacroAssembler::initialize_body(Register obj, Register tmp1, Register tmp2,
229                                         int obj_size_in_bytes, int hdr_size_in_bytes) {
230   const int index = (obj_size_in_bytes - hdr_size_in_bytes) / HeapWordSize;
231 
232   // 2x unrolled loop is shorter with more than 9 HeapWords.
233   if (index <= 9) {
234     clear_memory_unrolled(obj, index, R0, hdr_size_in_bytes);
235   } else {
236     const Register base_ptr = tmp1,
237                    cnt_dwords = tmp2;
238 
239     addi(base_ptr, obj, hdr_size_in_bytes); // Compute address of first element.
240     clear_memory_doubleword(base_ptr, cnt_dwords, R0, index);
241   }
242 }
243 
244 void C1_MacroAssembler::allocate_object(
245   Register obj,                        // result: pointer to object after successful allocation
246   Register t1,                         // temp register
247   Register t2,                         // temp register
248   Register t3,                         // temp register
249   int      hdr_size,                   // object header size in words
250   int      obj_size,                   // object size in words
251   Register klass,                      // object klass
252   Label&   slow_case                   // continuation point if fast allocation fails
253 ) {
254   assert_different_registers(obj, t1, t2, t3, klass);
255 
256   // allocate space & initialize header
257   if (!is_simm16(obj_size * wordSize)) {
258     // Would need to use extra register to load
259     // object size => go the slow case for now.
260     b(slow_case);
261     return;
262   }
263   try_allocate(obj, noreg, obj_size * wordSize, t2, t3, slow_case);
264 
265   initialize_object(obj, klass, noreg, obj_size * HeapWordSize, t1, t2);
266 }
267 
268 void C1_MacroAssembler::initialize_object(
269   Register obj,                        // result: pointer to object after successful allocation
270   Register klass,                      // object klass
271   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
272   int      con_size_in_bytes,          // object size in bytes if   known at compile time
273   Register t1,                         // temp register
274   Register t2                          // temp register
275   ) {
276   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
277 
278   initialize_header(obj, klass, noreg, t1, t2);
279 
280 #ifdef ASSERT
281   {
282     lwz(t1, in_bytes(Klass::layout_helper_offset()), klass);
283     if (var_size_in_bytes != noreg) {
284       cmpw(CCR0, t1, var_size_in_bytes);
285     } else {
286       cmpwi(CCR0, t1, con_size_in_bytes);
287     }
288     asm_assert_eq("bad size in initialize_object");
289   }
290 #endif
291 
292   // Initialize body.
293   if (var_size_in_bytes != noreg) {
294     // Use a loop.
295     addi(t1, obj, hdr_size_in_bytes);                // Compute address of first element.
296     addi(t2, var_size_in_bytes, -hdr_size_in_bytes); // Compute size of body.
297     initialize_body(t1, t2);
298   } else if (con_size_in_bytes > hdr_size_in_bytes) {
299     // Use a loop.
300     initialize_body(obj, t1, t2, con_size_in_bytes, hdr_size_in_bytes);
301   }
302 
303   if (CURRENT_ENV->dtrace_alloc_probes()) {
304     Unimplemented();
305 //    assert(obj == O0, "must be");
306 //    call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)),
307 //         relocInfo::runtime_call_type);
308   }
309 
310   verify_oop(obj, FILE_AND_LINE);
311 }
312 
313 
314 void C1_MacroAssembler::allocate_array(
315   Register obj,                        // result: pointer to array after successful allocation
316   Register len,                        // array length
317   Register t1,                         // temp register
318   Register t2,                         // temp register
319   Register t3,                         // temp register
320   int      hdr_size,                   // object header size in words
321   int      elt_size,                   // element size in bytes
322   Register klass,                      // object klass
323   Label&   slow_case                   // continuation point if fast allocation fails
324 ) {
325   assert_different_registers(obj, len, t1, t2, t3, klass);
326 
327   // Determine alignment mask.
328   assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
329   int log2_elt_size = exact_log2(elt_size);
330 
331   // Check for negative or excessive length.
332   size_t max_length = max_array_allocation_length >> log2_elt_size;
333   if (UseTLAB) {
334     size_t max_tlab = align_up(ThreadLocalAllocBuffer::max_size() >> log2_elt_size, 64*K);
335     if (max_tlab < max_length) { max_length = max_tlab; }
336   }
337   load_const_optimized(t1, max_length);
338   cmpld(CCR0, len, t1);
339   bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
340 
341   // compute array size
342   // note: If 0 <= len <= max_length, len*elt_size + header + alignment is
343   //       smaller or equal to the largest integer; also, since top is always
344   //       aligned, we can do the alignment here instead of at the end address
345   //       computation.
346   const Register arr_size = t1;
347   Register arr_len_in_bytes = len;
348   if (elt_size != 1) {
349     sldi(t1, len, log2_elt_size);
350     arr_len_in_bytes = t1;
351   }
352   addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
353   clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes);                              // Align array size.
354 
355   // Allocate space & initialize header.
356   try_allocate(obj, arr_size, 0, t2, t3, slow_case);
357   initialize_header(obj, klass, len, t2, t3);
358 
359   // Initialize body.
360   const Register base  = t2;
361   const Register index = t3;
362   addi(base, obj, hdr_size * wordSize);               // compute address of first element
363   addi(index, arr_size, -(hdr_size * wordSize));      // compute index = number of bytes to clear
364   initialize_body(base, index);
365 
366   if (CURRENT_ENV->dtrace_alloc_probes()) {
367     Unimplemented();
368     //assert(obj == O0, "must be");
369     //call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)),
370     //     relocInfo::runtime_call_type);
371   }
372 
373   verify_oop(obj, FILE_AND_LINE);
374 }
375 
376 
377 #ifndef PRODUCT
378 
379 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
380   verify_oop_addr((RegisterOrConstant)stack_offset, R1_SP, "broken oop in stack slot");
381 }
382 
383 void C1_MacroAssembler::verify_not_null_oop(Register r) {
384   Label not_null;
385   cmpdi(CCR0, r, 0);
386   bne(CCR0, not_null);
387   stop("non-null oop required");
388   bind(not_null);
389   verify_oop(r, FILE_AND_LINE);
390 }
391 
392 #endif // PRODUCT
393 
394 void C1_MacroAssembler::null_check(Register r, Label* Lnull) {
395   if (TrapBasedNullChecks) { // SIGTRAP based
396     trap_null_check(r);
397   } else { // explicit
398     //const address exception_entry = Runtime1::entry_for(Runtime1::throw_null_pointer_exception_id);
399     assert(Lnull != NULL, "must have Label for explicit check");
400     cmpdi(CCR0, r, 0);
401     bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::equal), *Lnull);
402   }
403 }
404 
405 address C1_MacroAssembler::call_c_with_frame_resize(address dest, int frame_resize) {
406   if (frame_resize) { resize_frame(-frame_resize, R0); }
407 #if defined(ABI_ELFv2)
408   address return_pc = call_c(dest, relocInfo::runtime_call_type);
409 #else
410   address return_pc = call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, dest), relocInfo::runtime_call_type);
411 #endif
412   if (frame_resize) { resize_frame(frame_resize, R0); }
413   return return_pc;
414 }