1 /*
   2  * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_MacroAssembler.hpp"
  29 #include "c1/c1_Runtime1.hpp"
  30 #include "classfile/systemDictionary.hpp"
  31 #include "gc/shared/collectedHeap.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "oops/arrayOop.hpp"
  34 #include "oops/markOop.hpp"
  35 #include "runtime/basicLock.hpp"
  36 #include "runtime/biasedLocking.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "utilities/align.hpp"
  41 
  42 
  43 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
  44   const Register temp_reg = R12_scratch2;
  45   Label Lmiss;
  46 
  47   verify_oop(receiver);
  48   MacroAssembler::null_check(receiver, oopDesc::klass_offset_in_bytes(), &Lmiss);
  49   load_klass(temp_reg, receiver);
  50 
  51   if (TrapBasedICMissChecks && TrapBasedNullChecks) {
  52     trap_ic_miss_check(temp_reg, iCache);
  53   } else {
  54     Label Lok;
  55     cmpd(CCR0, temp_reg, iCache);
  56     beq(CCR0, Lok);
  57     bind(Lmiss);
  58     //load_const_optimized(temp_reg, SharedRuntime::get_ic_miss_stub(), R0);
  59     calculate_address_from_global_toc(temp_reg, SharedRuntime::get_ic_miss_stub(), true, true, false);
  60     mtctr(temp_reg);
  61     bctr();
  62     align(32, 12);
  63     bind(Lok);
  64   }
  65 }
  66 
  67 
  68 void C1_MacroAssembler::explicit_null_check(Register base) {
  69   Unimplemented();
  70 }
  71 
  72 
  73 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
  74   // Avoid stack bang as first instruction. It may get overwritten by patch_verified_entry.
  75   const Register return_pc = R20;
  76   mflr(return_pc);
  77 
  78   // Make sure there is enough stack space for this method's activation.
  79   assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
  80   generate_stack_overflow_check(bang_size_in_bytes);
  81 
  82   std(return_pc, _abi(lr), R1_SP);     // SP->lr = return_pc
  83   push_frame(frame_size_in_bytes, R0); // SP -= frame_size_in_bytes
  84 }
  85 
  86 
  87 void C1_MacroAssembler::verified_entry() {
  88   if (C1Breakpoint) illtrap();
  89   // build frame
  90 }
  91 
  92 
  93 void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) {
  94   assert_different_registers(Rmark, Roop, Rbox, Rscratch);
  95 
  96   Label done, cas_failed, slow_int;
  97 
  98   // The following move must be the first instruction of emitted since debug
  99   // information may be generated for it.
 100   // Load object header.
 101   ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
 102 
 103   verify_oop(Roop);
 104 
 105   // Save object being locked into the BasicObjectLock...
 106   std(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
 107 
 108   if (UseBiasedLocking) {
 109     biased_locking_enter(CCR0, Roop, Rmark, Rscratch, R0, done, &slow_int);
 110   }
 111 
 112   // ... and mark it unlocked.
 113   ori(Rmark, Rmark, markOopDesc::unlocked_value);
 114 
 115   // Save unlocked object header into the displaced header location on the stack.
 116   std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
 117 
 118   // Compare object markOop with Rmark and if equal exchange Rscratch with object markOop.
 119   assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
 120   cmpxchgd(/*flag=*/CCR0,
 121            /*current_value=*/Rscratch,
 122            /*compare_value=*/Rmark,
 123            /*exchange_value=*/Rbox,
 124            /*where=*/Roop/*+0==mark_offset_in_bytes*/,
 125            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
 126            MacroAssembler::cmpxchgx_hint_acquire_lock(),
 127            noreg,
 128            &cas_failed,
 129            /*check without membar and ldarx first*/true);
 130   // If compare/exchange succeeded we found an unlocked object and we now have locked it
 131   // hence we are done.
 132   b(done);
 133 
 134   bind(slow_int);
 135   b(slow_case); // far
 136 
 137   bind(cas_failed);
 138   // We did not find an unlocked object so see if this is a recursive case.
 139   sub(Rscratch, Rscratch, R1_SP);
 140   load_const_optimized(R0, (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
 141   and_(R0/*==0?*/, Rscratch, R0);
 142   std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
 143   bne(CCR0, slow_int);
 144 
 145   bind(done);
 146 }
 147 
 148 
 149 void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rbox, Label& slow_case) {
 150   assert_different_registers(Rmark, Roop, Rbox);
 151 
 152   Label slow_int, done;
 153 
 154   Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
 155   assert(mark_addr.disp() == 0, "cas must take a zero displacement");
 156 
 157   if (UseBiasedLocking) {
 158     // Load the object out of the BasicObjectLock.
 159     ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
 160     verify_oop(Roop);
 161     biased_locking_exit(CCR0, Roop, R0, done);
 162   }
 163   // Test first it it is a fast recursive unlock.
 164   ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
 165   cmpdi(CCR0, Rmark, 0);
 166   beq(CCR0, done);
 167   if (!UseBiasedLocking) {
 168     // Load object.
 169     ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
 170     verify_oop(Roop);
 171   }
 172 
 173   // Check if it is still a light weight lock, this is is true if we see
 174   // the stack address of the basicLock in the markOop of the object.
 175   cmpxchgd(/*flag=*/CCR0,
 176            /*current_value=*/R0,
 177            /*compare_value=*/Rbox,
 178            /*exchange_value=*/Rmark,
 179            /*where=*/Roop,
 180            MacroAssembler::MemBarRel,
 181            MacroAssembler::cmpxchgx_hint_release_lock(),
 182            noreg,
 183            &slow_int);
 184   b(done);
 185   bind(slow_int);
 186   b(slow_case); // far
 187 
 188   // Done
 189   bind(done);
 190 }
 191 
 192 
 193 void C1_MacroAssembler::try_allocate(
 194   Register obj,                        // result: pointer to object after successful allocation
 195   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
 196   int      con_size_in_bytes,          // object size in bytes if   known at compile time
 197   Register t1,                         // temp register, must be global register for incr_allocated_bytes
 198   Register t2,                         // temp register
 199   Label&   slow_case                   // continuation point if fast allocation fails
 200 ) {
 201   if (UseTLAB) {
 202     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
 203   } else {
 204     eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 205     RegisterOrConstant size_in_bytes = var_size_in_bytes->is_valid()
 206                                        ? RegisterOrConstant(var_size_in_bytes)
 207                                        : RegisterOrConstant(con_size_in_bytes);
 208     incr_allocated_bytes(size_in_bytes, t1, t2);
 209   }
 210 }
 211 
 212 
 213 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
 214   assert_different_registers(obj, klass, len, t1, t2);
 215   if (UseBiasedLocking && !len->is_valid()) {
 216     ld(t1, in_bytes(Klass::prototype_header_offset()), klass);
 217   } else {
 218     load_const_optimized(t1, (intx)markOopDesc::prototype());
 219   }
 220   std(t1, oopDesc::mark_offset_in_bytes(), obj);
 221   store_klass(obj, klass);
 222   if (len->is_valid()) {
 223     stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
 224   } else if (UseCompressedClassPointers) {
 225     // Otherwise length is in the class gap.
 226     store_klass_gap(obj);
 227   }
 228 }
 229 
 230 
 231 void C1_MacroAssembler::initialize_body(Register base, Register index) {
 232   assert_different_registers(base, index);
 233   srdi(index, index, LogBytesPerWord);
 234   clear_memory_doubleword(base, index);
 235 }
 236 
 237 void C1_MacroAssembler::initialize_body(Register obj, Register tmp1, Register tmp2,
 238                                         int obj_size_in_bytes, int hdr_size_in_bytes) {
 239   const int index = (obj_size_in_bytes - hdr_size_in_bytes) / HeapWordSize;
 240 
 241   // 2x unrolled loop is shorter with more than 9 HeapWords.
 242   if (index <= 9) {
 243     clear_memory_unrolled(obj, index, R0, hdr_size_in_bytes);
 244   } else {
 245     const Register base_ptr = tmp1,
 246                    cnt_dwords = tmp2;
 247 
 248     addi(base_ptr, obj, hdr_size_in_bytes); // Compute address of first element.
 249     clear_memory_doubleword(base_ptr, cnt_dwords, R0, index);
 250   }
 251 }
 252 
 253 void C1_MacroAssembler::allocate_object(
 254   Register obj,                        // result: pointer to object after successful allocation
 255   Register t1,                         // temp register
 256   Register t2,                         // temp register
 257   Register t3,                         // temp register
 258   int      hdr_size,                   // object header size in words
 259   int      obj_size,                   // object size in words
 260   Register klass,                      // object klass
 261   Label&   slow_case                   // continuation point if fast allocation fails
 262 ) {
 263   assert_different_registers(obj, t1, t2, t3, klass);
 264 
 265   // allocate space & initialize header
 266   if (!is_simm16(obj_size * wordSize)) {
 267     // Would need to use extra register to load
 268     // object size => go the slow case for now.
 269     b(slow_case);
 270     return;
 271   }
 272   try_allocate(obj, noreg, obj_size * wordSize, t2, t3, slow_case);
 273 
 274   initialize_object(obj, klass, noreg, obj_size * HeapWordSize, t1, t2);
 275 }
 276 
 277 void C1_MacroAssembler::initialize_object(
 278   Register obj,                        // result: pointer to object after successful allocation
 279   Register klass,                      // object klass
 280   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
 281   int      con_size_in_bytes,          // object size in bytes if   known at compile time
 282   Register t1,                         // temp register
 283   Register t2                          // temp register
 284   ) {
 285   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
 286 
 287   initialize_header(obj, klass, noreg, t1, t2);
 288 
 289 #ifdef ASSERT
 290   {
 291     lwz(t1, in_bytes(Klass::layout_helper_offset()), klass);
 292     if (var_size_in_bytes != noreg) {
 293       cmpw(CCR0, t1, var_size_in_bytes);
 294     } else {
 295       cmpwi(CCR0, t1, con_size_in_bytes);
 296     }
 297     asm_assert_eq("bad size in initialize_object", 0x753);
 298   }
 299 #endif
 300 
 301   // Initialize body.
 302   if (var_size_in_bytes != noreg) {
 303     // Use a loop.
 304     addi(t1, obj, hdr_size_in_bytes);                // Compute address of first element.
 305     addi(t2, var_size_in_bytes, -hdr_size_in_bytes); // Compute size of body.
 306     initialize_body(t1, t2);
 307   } else if (con_size_in_bytes > hdr_size_in_bytes) {
 308     // Use a loop.
 309     initialize_body(obj, t1, t2, con_size_in_bytes, hdr_size_in_bytes);
 310   }
 311 
 312   if (CURRENT_ENV->dtrace_alloc_probes()) {
 313     Unimplemented();
 314 //    assert(obj == O0, "must be");
 315 //    call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)),
 316 //         relocInfo::runtime_call_type);
 317   }
 318 
 319   verify_oop(obj);
 320 }
 321 
 322 
 323 void C1_MacroAssembler::allocate_array(
 324   Register obj,                        // result: pointer to array after successful allocation
 325   Register len,                        // array length
 326   Register t1,                         // temp register
 327   Register t2,                         // temp register
 328   Register t3,                         // temp register
 329   int      hdr_size,                   // object header size in words
 330   int      elt_size,                   // element size in bytes
 331   Register klass,                      // object klass
 332   Label&   slow_case                   // continuation point if fast allocation fails
 333 ) {
 334   assert_different_registers(obj, len, t1, t2, t3, klass);
 335 
 336   // Determine alignment mask.
 337   assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
 338   int log2_elt_size = exact_log2(elt_size);
 339 
 340   // Check for negative or excessive length.
 341   size_t max_length = max_array_allocation_length >> log2_elt_size;
 342   if (UseTLAB) {
 343     size_t max_tlab = align_up(ThreadLocalAllocBuffer::max_size() >> log2_elt_size, 64*K);
 344     if (max_tlab < max_length) { max_length = max_tlab; }
 345   }
 346   load_const_optimized(t1, max_length);
 347   cmpld(CCR0, len, t1);
 348   bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
 349 
 350   // compute array size
 351   // note: If 0 <= len <= max_length, len*elt_size + header + alignment is
 352   //       smaller or equal to the largest integer; also, since top is always
 353   //       aligned, we can do the alignment here instead of at the end address
 354   //       computation.
 355   const Register arr_size = t1;
 356   Register arr_len_in_bytes = len;
 357   if (elt_size != 1) {
 358     sldi(t1, len, log2_elt_size);
 359     arr_len_in_bytes = t1;
 360   }
 361   addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
 362   clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes);                              // Align array size.
 363 
 364   // Allocate space & initialize header.
 365   if (UseTLAB) {
 366     tlab_allocate(obj, arr_size, 0, t2, slow_case);
 367   } else {
 368     eden_allocate(obj, arr_size, 0, t2, t3, slow_case);
 369   }
 370   initialize_header(obj, klass, len, t2, t3);
 371 
 372   // Initialize body.
 373   const Register base  = t2;
 374   const Register index = t3;
 375   addi(base, obj, hdr_size * wordSize);               // compute address of first element
 376   addi(index, arr_size, -(hdr_size * wordSize));      // compute index = number of bytes to clear
 377   initialize_body(base, index);
 378 
 379   if (CURRENT_ENV->dtrace_alloc_probes()) {
 380     Unimplemented();
 381     //assert(obj == O0, "must be");
 382     //call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)),
 383     //     relocInfo::runtime_call_type);
 384   }
 385 
 386   verify_oop(obj);
 387 }
 388 
 389 
 390 #ifndef PRODUCT
 391 
 392 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
 393   verify_oop_addr((RegisterOrConstant)(stack_offset + STACK_BIAS), R1_SP, "broken oop in stack slot");
 394 }
 395 
 396 void C1_MacroAssembler::verify_not_null_oop(Register r) {
 397   Label not_null;
 398   cmpdi(CCR0, r, 0);
 399   bne(CCR0, not_null);
 400   stop("non-null oop required");
 401   bind(not_null);
 402   if (!VerifyOops) return;
 403   verify_oop(r);
 404 }
 405 
 406 #endif // PRODUCT
 407 
 408 void C1_MacroAssembler::null_check(Register r, Label* Lnull) {
 409   if (TrapBasedNullChecks) { // SIGTRAP based
 410     trap_null_check(r);
 411   } else { // explicit
 412     //const address exception_entry = Runtime1::entry_for(Runtime1::throw_null_pointer_exception_id);
 413     assert(Lnull != NULL, "must have Label for explicit check");
 414     cmpdi(CCR0, r, 0);
 415     bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::equal), *Lnull);
 416   }
 417 }
 418 
 419 address C1_MacroAssembler::call_c_with_frame_resize(address dest, int frame_resize) {
 420   if (frame_resize) { resize_frame(-frame_resize, R0); }
 421 #if defined(ABI_ELFv2)
 422   address return_pc = call_c(dest, relocInfo::runtime_call_type);
 423 #else
 424   address return_pc = call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, dest), relocInfo::runtime_call_type);
 425 #endif
 426   if (frame_resize) { resize_frame(frame_resize, R0); }
 427   return return_pc;
 428 }